Watch
0
0
Fork
You've already forked hyperhive
0

Compare commits

..
11 changed files with 272 additions and 388 deletions

View file

@ -219,12 +219,10 @@ and failure series on `collector` appear once something fails, and the
collector and of the swarm's collector, which both carry the collector collector and of the swarm's collector, which both carry the collector
binary's default `service.name`, `otelcol-contrib`. The timeseries panels binary's default `service.name`, `otelcol-contrib`. The timeseries panels
split by `server.port`: 8888 is the hive collector, 8889 the swarm split by `server.port`: 8888 is the hive collector, 8889 the swarm
collector. Every per-container collector (the secret store, the other service collector. The per-container collectors (the secret store, NATS, Grafana,
containers and the agents' forwarders) also scrapes its own endpoint on Authelia, the Victoria stores and the agents' forwarders) don't report
loopback and pushes the result up its normal hop, so each has a line of its their own metrics yet, so they have no lines here; a separate issue on
own, on its own port: agents all report on 8904, service containers on the forge covers that. `scripts/check-dashboard-service-names.sh` runs in CI
8890 to 8903. The series carry `container` (services) or `agent` (agents)
to tell them apart. `scripts/check-dashboard-service-names.sh` runs in CI
and in `scripts/pre-push`. It fails when a dashboard query selects a and in `scripts/pre-push`. It fails when a dashboard query selects a
`service.name` that nothing emits. `service.name` that nothing emits.

View file

@ -72,11 +72,6 @@ let
OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE = "cumulative"; OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE = "cumulative";
}; };
# Where the collector serves its own metrics, inside the container's own
# netns. Not the binary's default 8888, which the collector dashboard
# reads as the hive collector's.
selfTelemetryPort = 8904;
# The exporter's NAME picks the wire protocol, so `protocol` has to be # The exporter's NAME picks the wire protocol, so `protocol` has to be
# read here as well as handed to the SDKs above — same expression as the # read here as well as handed to the SDKs above — same expression as the
# swarm tier's upstream exporter in nix/host-modules/swarm-otel.nix. # swarm tier's upstream exporter in nix/host-modules/swarm-otel.nix.
@ -215,101 +210,91 @@ in
# and the collector then dies at startup. Same caveat as both other # and the collector then dies at startup. Same caveat as both other
# tiers; see nix/host-modules/otel.nix for the measurement. # tiers; see nix/host-modules/otel.nix for the measurement.
validateConfigFile = true; validateConfigFile = true;
settings = lib.mkMerge [ settings = {
{ # `journalctl --follow --lines=0` (what this receiver runs under
# `journalctl --follow --lines=0` (what this receiver runs under # the hood) ships only what's written *after* it starts — every
# the hood) ships only what's written *after* it starts — every # collector start has a silent gap at the front. `storage:` below
# collector start has a silent gap at the front. `storage:` below # persists the read cursor, so a restart resumes from where it
# persists the read cursor, so a restart resumes from where it # left off instead of re-opening that gap; `start_at` stays `end`
# left off instead of re-opening that gap; `start_at` stays `end` # (its default) because a cursor already covers every run after
# (its default) because a cursor already covers every run after # the first, and `beginning` without one would re-ship the whole
# the first, and `beginning` without one would re-ship the whole # journal on every restart. The cursor lives in the unit's own
# journal on every restart. The cursor lives in the unit's own # `StateDirectory` (nixpkgs' module already sets one, `%S`, below
# `StateDirectory` (nixpkgs' module already sets one, `%S`, below # as `WorkingDirectory` too) — container-lifetime, not the
# as `WorkingDirectory` too) — container-lifetime, not the # host-persisted state bind mount: it indexes /var/log/journal,
# host-persisted state bind mount: it indexes /var/log/journal, # itself fresh on every container start, so the two must share
# itself fresh on every container start, so the two must share # a lifetime or the cursor outlives the journal it points into.
# a lifetime or the cursor outlives the journal it points into. # `create_directory` matters here for a build-time-vs-runtime
# `create_directory` matters here for a build-time-vs-runtime # reason, not a convenience one: `validateConfigFile` above runs
# reason, not a convenience one: `validateConfigFile` above runs # `otelcol validate` in the nix build sandbox, before systemd's
# `otelcol validate` in the nix build sandbox, before systemd's # `StateDirectory=` has ever created this path. Without it the
# `StateDirectory=` has ever created this path. Without it the # validator refuses with "directory must exist" and the build fails.
# validator refuses with "directory must exist" and the build fails. extensions.file_storage = {
extensions.file_storage = { directory = "/var/lib/opentelemetry-collector";
directory = "/var/lib/opentelemetry-collector"; create_directory = true;
create_directory = true; };
};
receivers.journald = { receivers.journald = {
# ⚠️ STATED, and it must stay stated: the receiver's own default # ⚠️ STATED, and it must stay stated: the receiver's own default
# is the RUNTIME journal (`/run/log/journal`), which in an agent # is the RUNTIME journal (`/run/log/journal`), which in an agent
# container is empty — journald stores persistently here, so # container is empty — journald stores persistently here, so
# every entry is under /var/log/journal. Dropping this line # every entry is under /var/log/journal. Dropping this line
# leaves a collector that validates, starts, reports healthy and # leaves a collector that validates, starts, reports healthy and
# forwards nothing. # forwards nothing.
directory = "/var/log/journal"; directory = "/var/log/journal";
storage = "file_storage"; storage = "file_storage";
# No `units` allowlist, and no user-session filter like the swarm # No `units` allowlist, and no user-session filter like the swarm
# tier's: a container's journal is the harness and what the # tier's: a container's journal is the harness and what the
# harness spawns, so there is no foreign traffic to filter out and # harness spawns, so there is no foreign traffic to filter out and
# an allowlist would only be a list to forget to update. # an allowlist would only be a list to forget to update.
# journald's PRIORITY carries the level every line already has; # journald's PRIORITY carries the level every line already has;
# without this the record reaches VictoriaLogs with # without this the record reaches VictoriaLogs with
# `severity_text: Unspecified` and the store cannot tell an error # `severity_text: Unspecified` and the store cannot tell an error
# from a debug line. The table — and the inverted direction that # from a debug line. The table — and the inverted direction that
# makes it worth a file of its own — lives in # makes it worth a file of its own — lives in
# ../journald-severity.nix, shared with the swarm tier's receiver. # ../journald-severity.nix, shared with the swarm tier's receiver.
operators = import ../journald-severity.nix; operators = import ../journald-severity.nix;
}; };
# Identity the hop above cannot supply. A host-side reader can say # Identity the hop above cannot supply. A host-side reader can say
# which machine a line came from; only a collector running as the # which machine a line came from; only a collector running as the
# agent can say which agent. `hive` and `swarm` are deliberately # agent can say which agent. `hive` and `swarm` are deliberately
# NOT stamped here: the swarm tier upserts both from the receiver # NOT stamped here: the swarm tier upserts both from the receiver
# that accepted the sample, precisely so the labels come from # that accepted the sample, precisely so the labels come from
# something the sender cannot write. # something the sender cannot write.
processors.resource.attributes = [ processors.resource.attributes = [
{ {
key = "service.name"; key = "service.name";
value = "hyperhive-agent"; value = "hyperhive-agent";
action = "upsert"; action = "upsert";
}
{
key = "agent";
value = userName;
action = "upsert";
}
];
# The hive's collector, at the same base address every in-process
# producer already exports to. `endpoint` is a BASE the exporter
# appends `/v1/logs` to — the path an OTLP/HTTP receiver serves.
exporters.${logExporter} = {
endpoint = cfg.endpoint;
} }
// lib.optionalAttrs (cfg.protocol == "http/json") { encoding = "json"; }; {
key = "agent";
value = userName;
action = "upsert";
}
];
service.pipelines.logs = { # The hive's collector, at the same base address every in-process
receivers = [ "journald" ]; # producer already exports to. `endpoint` is a BASE the exporter
processors = [ "resource" ]; # appends `/v1/logs` to — the path an OTLP/HTTP receiver serves.
exporters = [ logExporter ]; exporters.${logExporter} = {
}; endpoint = cfg.endpoint;
# An extension configured but not listed here is INERT — the
# journald receiver's `storage: file_storage` above would name a
# component the collector never starts.
service.extensions = [ "file_storage" ];
} }
# Every agent container has a network namespace of its own, so they can all // lib.optionalAttrs (cfg.protocol == "http/json") { encoding = "json"; };
# use the same port.
(import ../collector-self-scrape.nix { service.pipelines.logs = {
inherit lib; receivers = [ "journald" ];
port = selfTelemetryPort; processors = [ "resource" ];
exporter = logExporter; exporters = [ logExporter ];
attributes.agent = userName; };
})
]; # An extension configured but not listed here is INERT — the
# journald receiver's `storage: file_storage` above would name a
# component the collector never starts.
service.extensions = [ "file_storage" ];
};
}; };
}; };
} }

View file

@ -1,52 +0,0 @@
# Collector settings that make a container's OpenTelemetry collector report its
# own health (refused and failed records, exporter queue, memory), imported by
# every container collector: the agent forwarder, the service journal
# forwarder and the secret store's collector. The two host-side tiers scrape
# themselves through their `scrapeTargets` instead.
#
# One function renders both halves, so the endpoint and the job that reads it
# cannot name different ports. The endpoint binds loopback: the collector
# scrapes itself, and nothing outside the container has a reason to.
#
# The job runs in its own pipeline. A container's other metrics pipeline may
# carry a `resource` processor that upserts `service.name`, which would
# relabel these series away from the `otelcol-contrib` the collector dashboard
# selects on. The attributes stamped here tell the container's collectors
# apart.
#
# `readers`, not `metrics.address`: this collector version rejects `address`
# at startup.
{
lib,
port,
exporter,
attributes,
}:
{
service.telemetry.metrics.readers = [
{
pull.exporter.prometheus = {
host = "127.0.0.1";
inherit port;
};
}
];
receivers."prometheus/self".config.scrape_configs = [
{
job_name = "collector";
static_configs = [ { targets = [ "127.0.0.1:${toString port}" ]; } ];
}
];
processors."resource/self".attributes = lib.mapAttrsToList (key: value: {
inherit key value;
action = "upsert";
}) attributes;
service.pipelines."metrics/self" = {
receivers = [ "prometheus/self" ];
processors = [ "resource/self" ];
exporters = [ exporter ];
};
}

View file

@ -96,9 +96,8 @@ in
type = lib.types.port; type = lib.types.port;
description = '' description = ''
Loopback port for the collector's own metrics (refused and dropped Loopback port for the collector's own metrics (refused and dropped
records, exporter failures), which it scrapes and pushes to the swarm records, exporter failures). Required because the container may share
collector. Required because the container may share the host netns, the host netns, where the binary's default 8888 is the hive tier's.
where the binary's default 8888 is the hive tier's.
''; '';
}; };
@ -243,6 +242,17 @@ in
exporters = [ "otlphttp" ]; exporters = [ "otlphttp" ];
}; };
# ⚠️ `readers`, not `metrics.address`: this collector version rejects
# `address` at startup.
service.telemetry.metrics.readers = [
{
pull.exporter.prometheus = {
host = "127.0.0.1";
port = cfg.telemetryPort;
};
}
];
# An extension not listed here is inert: the receiver's storage would # An extension not listed here is inert: the receiver's storage would
# name a component that never starts, and the exporter would push # name a component that never starts, and the exporter would push
# unauthenticated. # unauthenticated.
@ -251,12 +261,6 @@ in
authName authName
]; ];
} }
(import ../collector-self-scrape.nix {
inherit lib;
port = cfg.telemetryPort;
exporter = "otlphttp";
attributes.container = cfg.serviceName;
})
# A separate set the module system merges in: joining it to the first with # A separate set the module system merges in: joining it to the first with
# `//` would replace the first set's whole `service` attrset. # `//` would replace the first set's whole `service` attrset.
(lib.mkIf (cfg.otlpMetricsPort != null) { (lib.mkIf (cfg.otlpMetricsPort != null) {

View file

@ -4438,163 +4438,172 @@ in
# caveat as every other tier; see nix/host-modules/otel.nix for # caveat as every other tier; see nix/host-modules/otel.nix for
# the measurement. # the measurement.
validateConfigFile = true; validateConfigFile = true;
settings = lib.mkMerge [ settings = {
{ # `journalctl --follow --lines=0` (what the receiver runs under
# `journalctl --follow --lines=0` (what the receiver runs under # the hood) ships only what is written *after* it starts, so
# the hood) ships only what is written *after* it starts, so # every collector start has a silent gap at the front. This
# every collector start has a silent gap at the front. This # persists the read cursor, so a restart resumes instead of
# persists the read cursor, so a restart resumes instead of # re-opening that gap; `start_at` stays `end` (its default),
# re-opening that gap; `start_at` stays `end` (its default), # since a cursor covers every run after the first and
# since a cursor covers every run after the first and # `beginning` without one would re-ship the whole journal on
# `beginning` without one would re-ship the whole journal on # every restart. `create_directory` is required for a
# every restart. `create_directory` is required for a # build-time reason rather than a convenience one:
# build-time reason rather than a convenience one: # `validateConfigFile` above runs `otelcol validate` in the nix
# `validateConfigFile` above runs `otelcol validate` in the nix # sandbox, before systemd's `StateDirectory=` has created the
# sandbox, before systemd's `StateDirectory=` has created the # path, and the validator refuses with "directory must exist".
# path, and the validator refuses with "directory must exist". extensions.file_storage = {
extensions.file_storage = { directory = "/var/lib/opentelemetry-collector";
directory = "/var/lib/opentelemetry-collector"; create_directory = true;
create_directory = true; };
};
receivers.journald = { receivers.journald = {
# ⚠️ STATED, and it must stay stated: the receiver's own # ⚠️ STATED, and it must stay stated: the receiver's own
# default is the RUNTIME journal (`/run/log/journal`), which # default is the RUNTIME journal (`/run/log/journal`), which
# in this container is empty — journald stores persistently # in this container is empty — journald stores persistently
# here, so every entry is under /var/log/journal. Dropping # here, so every entry is under /var/log/journal. Dropping
# this line leaves a collector that validates, starts, # this line leaves a collector that validates, starts,
# reports healthy and forwards nothing. # reports healthy and forwards nothing.
directory = "/var/log/journal"; directory = "/var/log/journal";
# ⚠️ `--merge` is a guard: an unqualified # ⚠️ `--merge` is a guard: an unqualified
# `journalctl --follow` over a journal with no entry for the # `journalctl --follow` over a journal with no entry for the
# current boot (a host-linked one) crash-loops the receiver # current boot (a host-linked one) crash-loops the receiver
# every ~2s with "No journal boot entry found for the # every ~2s with "No journal boot entry found for the
# specified boot (+0)". This journal is container-local, so # specified boot (+0)". This journal is container-local, so
# the current boot has an entry and the flag changes nothing. # the current boot has an entry and the flag changes nothing.
merge = true; merge = true;
storage = "file_storage"; storage = "file_storage";
}; };
# The store's scrape, run by this container's own collector. # The store's scrape, run by this container's own collector.
# #
# ⚠️ `metrics_path` and `params` are STATED, and both have to # ⚠️ `metrics_path` and `params` are STATED, and both have to
# be: openbao serves no `/metrics` at all, and # be: openbao serves no `/metrics` at all, and
# `/v1/sys/metrics` answers JSON unless the format is asked # `/v1/sys/metrics` answers JSON unless the format is asked
# for. A scrape of the default path 404s and a scrape without # for. A scrape of the default path 404s and a scrape without
# the query parses as nothing — each reads as a dead exporter # the query parses as nothing — each reads as a dead exporter
# rather than as a wrong request. The spelling matches # rather than as a wrong request. The spelling matches
# ./swarm-otel.nix's `loopbackScrapeConfig`, the renderer for # ./swarm-otel.nix's `loopbackScrapeConfig`, the renderer for
# every other service's target. # every other service's target.
# #
# No credential, unlike the swarm collector's # No credential, unlike the swarm collector's
# `publishedScrapeTargets` jobs: the target is the loopback # `publishedScrapeTargets` jobs: the target is the loopback
# listener above, inside this container's own netns, which is # listener above, inside this container's own netns, which is
# what lets the scrape stay unauthenticated without the API # what lets the scrape stay unauthenticated without the API
# listener's client-cert requirement being touched. # listener's client-cert requirement being touched.
receivers.prometheus.config.scrape_configs = [ receivers.prometheus.config.scrape_configs = [
{ {
job_name = "bao"; job_name = "bao";
static_configs = [ { targets = [ metricsScrapeTarget ]; } ]; static_configs = [ { targets = [ metricsScrapeTarget ]; } ];
metrics_path = "/v1/sys/metrics"; metrics_path = "/v1/sys/metrics";
params.format = [ "prometheus" ]; params.format = [ "prometheus" ];
} }
]; ];
# Identity the hop above cannot supply: a host-side reader can # Identity the hop above cannot supply: a host-side reader can
# say which machine a line came from, and only a collector # say which machine a line came from, and only a collector
# inside this container can say it was the secret store's. No # inside this container can say it was the secret store's. No
# `hive` or `swarm` label — the swarm tier upserts `hive` from # `hive` or `swarm` label — the swarm tier upserts `hive` from
# the receiver that accepted the record, precisely so it comes # the receiver that accepted the record, precisely so it comes
# from something the sender cannot write. # from something the sender cannot write.
# #
# `collectorServiceName`, not `cfg.machine`: what the # `collectorServiceName`, not `cfg.machine`: what the
# dashboard selects on is a display identity, not the # dashboard selects on is a display identity, not the
# container/receiver name — see that binding's own comment. # container/receiver name — see that binding's own comment.
processors.resource.attributes = [ processors.resource.attributes = [
{ {
key = "service.name"; key = "service.name";
value = collectorServiceName; value = collectorServiceName;
action = "upsert"; action = "upsert";
} }
]; ];
# `otlphttp` unconditionally, unlike the agent forwarder's # `otlphttp` unconditionally, unlike the agent forwarder's
# protocol-derived exporter name: the swarm receiver speaks # protocol-derived exporter name: the swarm receiver speaks
# OTLP/HTTP protobuf whatever this tier's *upstream* # OTLP/HTTP protobuf whatever this tier's *upstream*
# protocol is, which is the same reason # protocol is, which is the same reason
# ./hive-c0re/environment.nix pins the protocol it hands out. # ./hive-c0re/environment.nix pins the protocol it hands out.
# `endpoint` is a BASE the exporter appends `/v1/logs` to. # `endpoint` is a BASE the exporter appends `/v1/logs` to.
exporters.otlphttp = { exporters.otlphttp = {
endpoint = otelFirstHop; endpoint = otelFirstHop;
# The receiver at that address accepts an OIDC token and # The receiver at that address accepts an OIDC token and
# nothing else (./swarm-otel.nix's `oidc/*` authenticators), # nothing else (./swarm-otel.nix's `oidc/*` authenticators),
# so this is what makes the hop deliver rather than collect # so this is what makes the hop deliver rather than collect
# 401s — unconditional, because a forwarder that exports # 401s — unconditional, because a forwarder that exports
# unauthenticated has no working mode to fall back to. # unauthenticated has no working mode to fall back to.
auth.authenticator = forwarderAuthName; auth.authenticator = forwarderAuthName;
}; };
# The push side, opposite in direction to the receiver's # The push side, opposite in direction to the receiver's
# `oidc/*`: those VALIDATE a token arriving, this OBTAINS one # `oidc/*`: those VALIDATE a token arriving, this OBTAINS one
# to send. Hence `oauth2client`, and hence a client id and # to send. Hence `oauth2client`, and hence a client id and
# secret rather than an issuer and an audience to check. # secret rather than an issuer and an audience to check.
extensions.${forwarderAuthName} = { extensions.${forwarderAuthName} = {
client_id = cfg.otel.clientId; client_id = cfg.otel.clientId;
# A path, never a value — and one systemd resolves at # A path, never a value — and one systemd resolves at
# runtime, so neither the secret nor the credentials # runtime, so neither the secret nor the credentials
# directory is rendered into the nix store. # directory is rendered into the nix store.
client_secret_file = "\${env:CREDENTIALS_DIRECTORY}/${forwarderCredentialId}"; client_secret_file = "\${env:CREDENTIALS_DIRECTORY}/${forwarderCredentialId}";
token_url = "${toString autheliaCfg.url}/api/oidc/token"; token_url = "${toString autheliaCfg.url}/api/oidc/token";
# ⚠️ THE AUDIENCE HAS TO BE REQUESTED, not merely granted. # ⚠️ THE AUDIENCE HAS TO BE REQUESTED, not merely granted.
# Registering it on the client only makes it permissible; a # Registering it on the client only makes it permissible; a
# token minted without asking carries `aud: []` and the # token minted without asking carries `aud: []` and the
# receiver refuses it — with a config that reads perfectly # receiver refuses it — with a config that reads perfectly
# at both ends. # at both ends.
endpoint_params.audience = cfg.otel.clientId; endpoint_params.audience = cfg.otel.clientId;
}; };
service.pipelines.logs = { service.pipelines.logs = {
receivers = [ "journald" ]; receivers = [ "journald" ];
processors = [ "resource" ]; processors = [ "resource" ];
exporters = [ "otlphttp" ]; exporters = [ "otlphttp" ];
}; };
# A SECOND pipeline, not a second receiver on the logs one: a # A SECOND pipeline, not a second receiver on the logs one: a
# pipeline is typed, and a prometheus receiver in `logs` is a # pipeline is typed, and a prometheus receiver in `logs` is a
# config the collector rejects at startup rather than one that # config the collector rejects at startup rather than one that
# quietly ships metrics as log records. # quietly ships metrics as log records.
# #
# Same `resource` processor and same `otlphttp` exporter, so # Same `resource` processor and same `otlphttp` exporter, so
# the store's metrics carry the same `service.name` its logs # the store's metrics carry the same `service.name` its logs
# do and leave by the same hop — no otel bypass, one exit from # do and leave by the same hop — no otel bypass, one exit from
# this container. The exporter treats `endpoint` as a base and # this container. The exporter treats `endpoint` as a base and
# appends the signal's own path, so `/v1/metrics` here and # appends the signal's own path, so `/v1/metrics` here and
# `/v1/logs` above come from the one address. # `/v1/logs` above come from the one address.
service.pipelines.metrics = { service.pipelines.metrics = {
receivers = [ "prometheus" ]; receivers = [ "prometheus" ];
processors = [ "resource" ]; processors = [ "resource" ];
exporters = [ "otlphttp" ]; exporters = [ "otlphttp" ];
}; };
# An extension configured but not listed here is INERT — the # Moves this collector's self-metrics off the built-in
# receiver's `storage: file_storage` above would name a # default of `localhost:8888`, which the hive tier holds —
# component the collector never starts, and the exporter's # this container shares the host's netns, so leaving the
# authenticator would push unauthenticated while the # default in place is the hive collector failing to bind.
# collector reports healthy. #
service.extensions = [ # ⚠️ `metrics.address` is the spelling that looks right and
"file_storage" # is REJECTED by this collector version:
forwarderAuthName # `'migration.MetricsConfigV030' has invalid keys:
]; # address`. `readers` is the schema it accepts, and the
} # difference is a startup failure rather than a warning.
# Off the built-in default of `localhost:8888`, which the hive tier service.telemetry.metrics.readers = [
# holds: this container shares the host's netns. {
(import ../collector-self-scrape.nix { pull.exporter.prometheus = {
inherit lib; host = "127.0.0.1";
port = cfg.otel.telemetryPort; port = cfg.otel.telemetryPort;
exporter = "otlphttp"; };
attributes.container = collectorServiceName; }
}) ];
];
# An extension configured but not listed here is INERT — the
# receiver's `storage: file_storage` above would name a
# component the collector never starts, and the exporter's
# authenticator would push unauthenticated while the
# collector reports healthy.
service.extensions = [
"file_storage"
forwarderAuthName
];
};
}; };
# How the secret crosses the sandbox. The host unit # How the secret crosses the sandbox. The host unit

View file

@ -1,7 +1,7 @@
{ {
"title": "hyperhive \u00b7 collector (self-telemetry)", "title": "hyperhive \u00b7 collector (self-telemetry)",
"uid": "hyperhive-swarm-collector", "uid": "hyperhive-swarm-collector",
"description": "OTEL collectors: accepted, refused, exported, queued. 8888 hive, 8889 swarm, others containers.", "description": "What the OTEL collectors accept, refuse, export and queue. Port 8888 is the hive, 8889 swarm-otel.",
"editable": true, "editable": true,
"refresh": "1m", "refresh": "1m",
"schemaVersion": 39, "schemaVersion": 39,

View file

@ -21,7 +21,6 @@ let
agent agent
carriesJournaldSeverity carriesJournaldSeverity
runGroup runGroup
selfScrapes
; ;
# The log path's three hops, one fixture each. Nothing carries a journal # The log path's three hops, one fixture each. Nothing carries a journal
@ -56,20 +55,6 @@ let
agentSettings = machine: machine.services.opentelemetry-collector.settings; agentSettings = machine: machine.services.opentelemetry-collector.settings;
cases = [ cases = [
{
# The collector's own counters are the only signal that it is dropping
# telemetry. 8904 keeps agents off 8888, which the collector dashboard
# reads as the hive collector; the exporter follows the wire protocol.
name = "an agent forwarder scrapes its own telemetry endpoint on loopback, off the hive collector's port";
ok =
selfScrapes (agentSettings agentOtel) 8904 { agent = agentOtel.services.hyperhive.agent.user.name; }
&& selfScrapes (agentSettings agentOtelGrpc) 8904 {
agent = agentOtelGrpc.services.hyperhive.agent.user.name;
}
# Control: with the switch off the lookup finds no collector config.
&& !(agentNoOtel.services.opentelemetry-collector.settings ? receivers)
&& (agentSettings agentOtelGrpc).service.pipelines."metrics/self".exporters == [ "otlp" ];
}
{ {
# The journald receiver's own default directory is the RUNTIME # The journald receiver's own default directory is the RUNTIME
# journal, and a container that stores persistently leaves that # journal, and a container that stores persistently leaves that

View file

@ -22,7 +22,6 @@ let
runGroup runGroup
baoSettings baoSettings
forceServiceConfigs forceServiceConfigs
selfScrapes
; ;
baoTwoAddresses = hive { baoTwoAddresses = hive {
@ -186,20 +185,6 @@ let
&& m.exporters == [ "otlphttp" ] && m.exporters == [ "otlphttp" ]
&& m.exporters == s.service.pipelines.logs.exporters; && m.exporters == s.service.pipelines.logs.exporters;
} }
{
# The store's collector reports its own health through a pipeline of its
# own: the store pipeline above upserts `service.name`, and sharing it
# would relabel these series away from the collector dashboard.
name = "the store's collector scrapes its own telemetry endpoint on loopback";
ok =
let
s = (baoForwarder baoWithCollector).settings;
port = baoWithCollector.services.hyperhive.swarm.bao.otel.telemetryPort;
in
selfScrapes s port { container = "bao"; }
&& s.service.pipelines.metrics.receivers == [ "prometheus" ]
&& s.service.pipelines.metrics.processors == [ "resource" ];
}
{ {
# Presence control for the case above: both fields are omitted rather # Presence control for the case above: both fields are omitted rather
# than defaulted, so a target declared as bare `host:port` renders what # than defaulted, so a target declared as bare `host:port` renders what

View file

@ -25,7 +25,6 @@ let
}) })
hive hive
runGroup runGroup
selfScrapes
; ;
inherit (shared) bare; inherit (shared) bare;
@ -559,10 +558,11 @@ let
ok = ok =
let let
c = ciAlone.containers.swarm-ci-runner.config; c = ciAlone.containers.swarm-ci-runner.config;
s = c.services.opentelemetry-collector.settings; reader = lib.head c.services.opentelemetry-collector.settings.service.telemetry.metrics.readers;
in in
c.services.hyperhive.swarmJournalForwarder.telemetryPort == 8895 c.services.hyperhive.swarmJournalForwarder.telemetryPort == 8895
&& selfScrapes s 8895 { container = "ci-runner"; }; && reader.pull.exporter.prometheus.host == "127.0.0.1"
&& reader.pull.exporter.prometheus.port == 8895;
} }
]; ];
in in

View file

@ -208,37 +208,6 @@ let
# the two tier cases differ only in which receiver they hand it. # the two tier cases differ only in which receiver they hand it.
carriesJournaldSeverity = receiver: lib.elem journaldSeverityParser (receiver.operators or [ ]); carriesJournaldSeverity = receiver: lib.elem journaldSeverityParser (receiver.operators or [ ]);
# Whether a container collector's settings carry the self-scrape of
# ../collector-self-scrape.nix: the telemetry endpoint on loopback at `port`,
# a job scraping exactly that address, and a metrics pipeline that reads the
# job and leaves through an exporter the settings define. `attrs` are the
# resource attributes the pipeline stamps.
selfScrapes =
s: port: attrs:
let
job = lib.findFirst (j: j.job_name == "collector") null (
s.receivers."prometheus/self".config.scrape_configs or [ ]
);
pipeline = s.service.pipelines."metrics/self" or null;
reader = lib.head (s.service.telemetry.metrics.readers or [ { } ]);
stamped = lib.mapAttrsToList (key: value: {
inherit key value;
action = "upsert";
}) attrs;
in
job != null
&& job.static_configs == [ { targets = [ "127.0.0.1:${toString port}" ]; } ]
&&
(reader.pull.exporter.prometheus or null) == {
host = "127.0.0.1";
inherit port;
}
&& pipeline != null
&& pipeline.receivers == [ "prometheus/self" ]
&& (s.processors."resource/self".attributes or null) == stamped
&& pipeline.exporters != [ ]
&& lib.all (e: s.exporters ? ${e}) pipeline.exporters;
# The enables that switch owns. The hive tier's collector is not among them: it # The enables that switch owns. The hive tier's collector is not among them: it
# follows `deploy.hive-controller.enable`, and a services host without a hive # follows `deploy.hive-controller.enable`, and a services host without a hive
# runs none. # runs none.
@ -314,7 +283,6 @@ in
inherit agentHarness; inherit agentHarness;
inherit journaldSeverityParser; inherit journaldSeverityParser;
inherit carriesJournaldSeverity; inherit carriesJournaldSeverity;
inherit selfScrapes;
inherit swarmServiceEnables; inherit swarmServiceEnables;
inherit runGroup; inherit runGroup;
inherit forceServiceConfigs; inherit forceServiceConfigs;

View file

@ -20,7 +20,6 @@ let
}) })
hive hive
runGroup runGroup
selfScrapes
; ;
# No store identity, which the first case below relies on. # No store identity, which the first case below relies on.
@ -158,10 +157,12 @@ let
h.services.hyperhive.swarm.nats.otel.telemetryPort h.services.hyperhive.swarm.nats.otel.telemetryPort
] ]
++ map (o: o.port) (lib.filter (o: o.machine != svc.machine) forwarderServices); ++ map (o: o.port) (lib.filter (o: o.machine != svc.machine) forwarderServices);
reader = lib.head s.service.telemetry.metrics.readers;
in in
c.services.hyperhive.swarmJournalForwarder.telemetryPort == svc.port c.services.hyperhive.swarmJournalForwarder.telemetryPort == svc.port
&& !(lib.elem svc.port taken) && !(lib.elem svc.port taken)
&& selfScrapes s svc.port { container = svc.name; }; && reader.pull.exporter.prometheus.host == "127.0.0.1"
&& reader.pull.exporter.prometheus.port == svc.port;
} }
] ]
) forwarderServices; ) forwarderServices;
@ -268,10 +269,12 @@ let
h.services.hyperhive.swarm.nats.otel.telemetryPort h.services.hyperhive.swarm.nats.otel.telemetryPort
] ]
++ map (o: o.port) (lib.filter (o: o.machine != svc.machine) hostNetnsForwarders); ++ map (o: o.port) (lib.filter (o: o.machine != svc.machine) hostNetnsForwarders);
reader = lib.head s.service.telemetry.metrics.readers;
in in
c.services.hyperhive.swarmJournalForwarder.telemetryPort == svc.port c.services.hyperhive.swarmJournalForwarder.telemetryPort == svc.port
&& !(lib.elem svc.port taken) && !(lib.elem svc.port taken)
&& selfScrapes s svc.port { container = svc.name; }; && reader.pull.exporter.prometheus.host == "127.0.0.1"
&& reader.pull.exporter.prometheus.port == svc.port;
} }
] ]
) hostNetnsForwarders; ) hostNetnsForwarders;
@ -737,7 +740,6 @@ let
&& s.extensions."oauth2client/swarm".endpoint_params.audience == clientId && s.extensions."oauth2client/swarm".endpoint_params.audience == clientId
&& lib.elem "oauth2client/swarm" s.service.extensions && lib.elem "oauth2client/swarm" s.service.extensions
&& s.service.pipelines.logs.receivers == [ "journald" ] && s.service.pipelines.logs.receivers == [ "journald" ]
&& selfScrapes s h.services.hyperhive.swarm.nats.otel.telemetryPort { container = "nats"; }
&& lib.elem "oidc-client-secret:/run/swarm-journal-forwarder-oidc/client-secret" c.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential && lib.elem "oidc-client-secret:/run/swarm-journal-forwarder-oidc/client-secret" c.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential
&& fetch.environment.BAO_CLIENT_CERT == "/var/lib/swarm-nats-auth/bao-client.pem" && fetch.environment.BAO_CLIENT_CERT == "/var/lib/swarm-nats-auth/bao-client.pem"
&& lib.hasInfix "bao kv get -field=value secret/swarm/services/${clientId}/oidc/client" fetch.script && lib.hasInfix "bao kv get -field=value secret/swarm/services/${clientId}/oidc/client" fetch.script
@ -995,7 +997,7 @@ let
registered = lib.findFirst ( registered = lib.findFirst (
x: x.id == clientId x: x.id == clientId
) null h.services.hyperhive.swarm.authelia.oidc.clients; ) null h.services.hyperhive.swarm.authelia.oidc.clients;
port = c.services.hyperhive.swarmJournalForwarder.telemetryPort; reader = lib.head s.service.telemetry.metrics.readers;
taken = [ taken = [
8888 8888
8890 8890
@ -1020,8 +1022,8 @@ let
&& registered.audience == [ clientId ] && registered.audience == [ clientId ]
&& !(lib.elem "--link-journal=host" container.extraFlags) && !(lib.elem "--link-journal=host" container.extraFlags)
&& lib.any (lib.hasPrefix "--load-credential=publisher-bao-cert:") container.extraFlags && lib.any (lib.hasPrefix "--load-credential=publisher-bao-cert:") container.extraFlags
&& selfScrapes s port { container = "authelia"; } && reader.pull.exporter.prometheus.host == "127.0.0.1"
&& !(lib.elem port taken) && !(lib.elem reader.pull.exporter.prometheus.port taken)
&& !autheliaColocated.containers.swarm-authelia.config.services.opentelemetry-collector.enable && !autheliaColocated.containers.swarm-authelia.config.services.opentelemetry-collector.enable
&& !( && !(
autheliaColocated.containers.swarm-authelia.config.systemd.services ? swarm-journal-forwarder-oidc autheliaColocated.containers.swarm-authelia.config.systemd.services ? swarm-journal-forwarder-oidc