vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

merge the self-scrape into each collector's settings instead of defining them twice

Browse source
This commit is contained in:
atlas 2026-10-28 22:20:24 +01:00 • committed by mara
commit a00bcb8c05
2 changed files with 243 additions and 241 deletions
Download patch file Download diff file Expand all files Collapse all files

177
nix/agent-modules/otel.nix
View file

@ -215,100 +215,101 @@ in
# and the collector then dies at startup. Same caveat as both other # and the collector then dies at startup. Same caveat as both other
# tiers; see nix/host-modules/otel.nix for the measurement. # tiers; see nix/host-modules/otel.nix for the measurement.
validateConfigFile = true; validateConfigFile = true;
settings = { settings = lib.mkMerge [
# `journalctl --follow --lines=0` (what this receiver runs under {
# the hood) ships only what's written *after* it starts — every # `journalctl --follow --lines=0` (what this receiver runs under
# collector start has a silent gap at the front. `storage:` below # the hood) ships only what's written *after* it starts — every
# persists the read cursor, so a restart resumes from where it # collector start has a silent gap at the front. `storage:` below
# left off instead of re-opening that gap; `start_at` stays `end` # persists the read cursor, so a restart resumes from where it
# (its default) because a cursor already covers every run after # left off instead of re-opening that gap; `start_at` stays `end`
# the first, and `beginning` without one would re-ship the whole # (its default) because a cursor already covers every run after
# journal on every restart. The cursor lives in the unit's own # the first, and `beginning` without one would re-ship the whole
# `StateDirectory` (nixpkgs' module already sets one, `%S`, below # journal on every restart. The cursor lives in the unit's own
# as `WorkingDirectory` too) — container-lifetime, not the # `StateDirectory` (nixpkgs' module already sets one, `%S`, below
# host-persisted state bind mount: it indexes /var/log/journal, # as `WorkingDirectory` too) — container-lifetime, not the
# itself fresh on every container start, so the two must share # host-persisted state bind mount: it indexes /var/log/journal,
# a lifetime or the cursor outlives the journal it points into. # itself fresh on every container start, so the two must share
# `create_directory` matters here for a build-time-vs-runtime # a lifetime or the cursor outlives the journal it points into.
# reason, not a convenience one: `validateConfigFile` above runs # `create_directory` matters here for a build-time-vs-runtime
# `otelcol validate` in the nix build sandbox, before systemd's # reason, not a convenience one: `validateConfigFile` above runs
# `StateDirectory=` has ever created this path. Without it the # `otelcol validate` in the nix build sandbox, before systemd's
# validator refuses with "directory must exist" and the build fails. # `StateDirectory=` has ever created this path. Without it the
extensions.file_storage = { # validator refuses with "directory must exist" and the build fails.
directory = "/var/lib/opentelemetry-collector"; extensions.file_storage = {
create_directory = true; directory = "/var/lib/opentelemetry-collector";
}; create_directory = true;
};
receivers.journald = { receivers.journald = {
# ⚠️ STATED, and it must stay stated: the receiver's own default # ⚠️ STATED, and it must stay stated: the receiver's own default
# is the RUNTIME journal (`/run/log/journal`), which in an agent # is the RUNTIME journal (`/run/log/journal`), which in an agent
# container is empty — journald stores persistently here, so # container is empty — journald stores persistently here, so
# every entry is under /var/log/journal. Dropping this line # every entry is under /var/log/journal. Dropping this line
# leaves a collector that validates, starts, reports healthy and # leaves a collector that validates, starts, reports healthy and
# forwards nothing. # forwards nothing.
directory = "/var/log/journal"; directory = "/var/log/journal";
storage = "file_storage"; storage = "file_storage";
# No `units` allowlist, and no user-session filter like the swarm # No `units` allowlist, and no user-session filter like the swarm
# tier's: a container's journal is the harness and what the # tier's: a container's journal is the harness and what the
# harness spawns, so there is no foreign traffic to filter out and # harness spawns, so there is no foreign traffic to filter out and
# an allowlist would only be a list to forget to update. # an allowlist would only be a list to forget to update.
# journald's PRIORITY carries the level every line already has; # journald's PRIORITY carries the level every line already has;
# without this the record reaches VictoriaLogs with # without this the record reaches VictoriaLogs with
# `severity_text: Unspecified` and the store cannot tell an error # `severity_text: Unspecified` and the store cannot tell an error
# from a debug line. The table — and the inverted direction that # from a debug line. The table — and the inverted direction that
# makes it worth a file of its own — lives in # makes it worth a file of its own — lives in
# ../journald-severity.nix, shared with the swarm tier's receiver. # ../journald-severity.nix, shared with the swarm tier's receiver.
operators = import ../journald-severity.nix; operators = import ../journald-severity.nix;
}; };
# Identity the hop above cannot supply. A host-side reader can say # Identity the hop above cannot supply. A host-side reader can say
# which machine a line came from; only a collector running as the # which machine a line came from; only a collector running as the
# agent can say which agent. `hive` and `swarm` are deliberately # agent can say which agent. `hive` and `swarm` are deliberately
# NOT stamped here: the swarm tier upserts both from the receiver # NOT stamped here: the swarm tier upserts both from the receiver
# that accepted the sample, precisely so the labels come from # that accepted the sample, precisely so the labels come from
# something the sender cannot write. # something the sender cannot write.
processors.resource.attributes = [ processors.resource.attributes = [
{ {
key = "service.name"; key = "service.name";
value = "hyperhive-agent"; value = "hyperhive-agent";
action = "upsert"; action = "upsert";
}
{
key = "agent";
value = userName;
action = "upsert";
}
];
# The hive's collector, at the same base address every in-process
# producer already exports to. `endpoint` is a BASE the exporter
# appends `/v1/logs` to — the path an OTLP/HTTP receiver serves.
exporters.${logExporter} = {
endpoint = cfg.endpoint;
} }
{ // lib.optionalAttrs (cfg.protocol == "http/json") { encoding = "json"; };
key = "agent";
value = userName;
action = "upsert";
}
];
# The hive's collector, at the same base address every in-process service.pipelines.logs = {
# producer already exports to. `endpoint` is a BASE the exporter receivers = [ "journald" ];
# appends `/v1/logs` to — the path an OTLP/HTTP receiver serves. processors = [ "resource" ];
exporters.${logExporter} = { exporters = [ logExporter ];
endpoint = cfg.endpoint; };
# An extension configured but not listed here is INERT — the
# journald receiver's `storage: file_storage` above would name a
# component the collector never starts.
service.extensions = [ "file_storage" ];
} }
// lib.optionalAttrs (cfg.protocol == "http/json") { encoding = "json"; }; # Every agent container has a network namespace of its own, so they can all
# use the same port.
service.pipelines.logs = { (import ../collector-self-scrape.nix {
receivers = [ "journald" ]; inherit lib;
processors = [ "resource" ]; port = selfTelemetryPort;
exporters = [ logExporter ]; exporter = logExporter;
}; attributes.agent = userName;
})
# An extension configured but not listed here is INERT — the ];
# journald receiver's `storage: file_storage` above would name a
# component the collector never starts.
service.extensions = [ "file_storage" ];
};
};
# Every agent container has a network namespace of its own, so they can all
# use the same port.
services.opentelemetry-collector.settings = import ../collector-self-scrape.nix {
inherit lib;
port = selfTelemetryPort;
exporter = logExporter;
attributes.agent = userName;
}; };
}; };
} }

297
nix/host-modules/swarm-bao.nix
View file

@ -4438,162 +4438,163 @@ in
# caveat as every other tier; see nix/host-modules/otel.nix for # caveat as every other tier; see nix/host-modules/otel.nix for
# the measurement. # the measurement.
validateConfigFile = true; validateConfigFile = true;
settings = { settings = lib.mkMerge [
# `journalctl --follow --lines=0` (what the receiver runs under {
# the hood) ships only what is written *after* it starts, so # `journalctl --follow --lines=0` (what the receiver runs under
# every collector start has a silent gap at the front. This # the hood) ships only what is written *after* it starts, so
# persists the read cursor, so a restart resumes instead of # every collector start has a silent gap at the front. This
# re-opening that gap; `start_at` stays `end` (its default), # persists the read cursor, so a restart resumes instead of
# since a cursor covers every run after the first and # re-opening that gap; `start_at` stays `end` (its default),
# `beginning` without one would re-ship the whole journal on # since a cursor covers every run after the first and
# every restart. `create_directory` is required for a # `beginning` without one would re-ship the whole journal on
# build-time reason rather than a convenience one: # every restart. `create_directory` is required for a
# `validateConfigFile` above runs `otelcol validate` in the nix # build-time reason rather than a convenience one:
# sandbox, before systemd's `StateDirectory=` has created the # `validateConfigFile` above runs `otelcol validate` in the nix
# path, and the validator refuses with "directory must exist". # sandbox, before systemd's `StateDirectory=` has created the
extensions.file_storage = { # path, and the validator refuses with "directory must exist".
directory = "/var/lib/opentelemetry-collector"; extensions.file_storage = {
create_directory = true; directory = "/var/lib/opentelemetry-collector";
}; create_directory = true;
};
receivers.journald = { receivers.journald = {
# ⚠️ STATED, and it must stay stated: the receiver's own # ⚠️ STATED, and it must stay stated: the receiver's own
# default is the RUNTIME journal (`/run/log/journal`), which # default is the RUNTIME journal (`/run/log/journal`), which
# in this container is empty — journald stores persistently # in this container is empty — journald stores persistently
# here, so every entry is under /var/log/journal. Dropping # here, so every entry is under /var/log/journal. Dropping
# this line leaves a collector that validates, starts, # this line leaves a collector that validates, starts,
# reports healthy and forwards nothing. # reports healthy and forwards nothing.
directory = "/var/log/journal"; directory = "/var/log/journal";
# ⚠️ `--merge` is a guard: an unqualified # ⚠️ `--merge` is a guard: an unqualified
# `journalctl --follow` over a journal with no entry for the # `journalctl --follow` over a journal with no entry for the
# current boot (a host-linked one) crash-loops the receiver # current boot (a host-linked one) crash-loops the receiver
# every ~2s with "No journal boot entry found for the # every ~2s with "No journal boot entry found for the
# specified boot (+0)". This journal is container-local, so # specified boot (+0)". This journal is container-local, so
# the current boot has an entry and the flag changes nothing. # the current boot has an entry and the flag changes nothing.
merge = true; merge = true;
storage = "file_storage"; storage = "file_storage";
}; };
# The store's scrape, run by this container's own collector. # The store's scrape, run by this container's own collector.
# #
# ⚠️ `metrics_path` and `params` are STATED, and both have to # ⚠️ `metrics_path` and `params` are STATED, and both have to
# be: openbao serves no `/metrics` at all, and # be: openbao serves no `/metrics` at all, and
# `/v1/sys/metrics` answers JSON unless the format is asked # `/v1/sys/metrics` answers JSON unless the format is asked
# for. A scrape of the default path 404s and a scrape without # for. A scrape of the default path 404s and a scrape without
# the query parses as nothing — each reads as a dead exporter # the query parses as nothing — each reads as a dead exporter
# rather than as a wrong request. The spelling matches # rather than as a wrong request. The spelling matches
# ./swarm-otel.nix's `loopbackScrapeConfig`, the renderer for # ./swarm-otel.nix's `loopbackScrapeConfig`, the renderer for
# every other service's target. # every other service's target.
# #
# No credential, unlike the swarm collector's # No credential, unlike the swarm collector's
# `publishedScrapeTargets` jobs: the target is the loopback # `publishedScrapeTargets` jobs: the target is the loopback
# listener above, inside this container's own netns, which is # listener above, inside this container's own netns, which is
# what lets the scrape stay unauthenticated without the API # what lets the scrape stay unauthenticated without the API
# listener's client-cert requirement being touched. # listener's client-cert requirement being touched.
receivers.prometheus.config.scrape_configs = [ receivers.prometheus.config.scrape_configs = [
{ {
job_name = "bao"; job_name = "bao";
static_configs = [ { targets = [ metricsScrapeTarget ]; } ]; static_configs = [ { targets = [ metricsScrapeTarget ]; } ];
metrics_path = "/v1/sys/metrics"; metrics_path = "/v1/sys/metrics";
params.format = [ "prometheus" ]; params.format = [ "prometheus" ];
} }
]; ];
# Identity the hop above cannot supply: a host-side reader can # Identity the hop above cannot supply: a host-side reader can
# say which machine a line came from, and only a collector # say which machine a line came from, and only a collector
# inside this container can say it was the secret store's. No # inside this container can say it was the secret store's. No
# `hive` or `swarm` label — the swarm tier upserts `hive` from # `hive` or `swarm` label — the swarm tier upserts `hive` from
# the receiver that accepted the record, precisely so it comes # the receiver that accepted the record, precisely so it comes
# from something the sender cannot write. # from something the sender cannot write.
# #
# `collectorServiceName`, not `cfg.machine`: what the # `collectorServiceName`, not `cfg.machine`: what the
# dashboard selects on is a display identity, not the # dashboard selects on is a display identity, not the
# container/receiver name — see that binding's own comment. # container/receiver name — see that binding's own comment.
processors.resource.attributes = [ processors.resource.attributes = [
{ {
key = "service.name"; key = "service.name";
value = collectorServiceName; value = collectorServiceName;
action = "upsert"; action = "upsert";
} }
]; ];
# `otlphttp` unconditionally, unlike the agent forwarder's # `otlphttp` unconditionally, unlike the agent forwarder's
# protocol-derived exporter name: the swarm receiver speaks # protocol-derived exporter name: the swarm receiver speaks
# OTLP/HTTP protobuf whatever this tier's *upstream* # OTLP/HTTP protobuf whatever this tier's *upstream*
# protocol is, which is the same reason # protocol is, which is the same reason
# ./hive-c0re/environment.nix pins the protocol it hands out. # ./hive-c0re/environment.nix pins the protocol it hands out.
# `endpoint` is a BASE the exporter appends `/v1/logs` to. # `endpoint` is a BASE the exporter appends `/v1/logs` to.
exporters.otlphttp = { exporters.otlphttp = {
endpoint = otelFirstHop; endpoint = otelFirstHop;
# The receiver at that address accepts an OIDC token and # The receiver at that address accepts an OIDC token and
# nothing else (./swarm-otel.nix's `oidc/*` authenticators), # nothing else (./swarm-otel.nix's `oidc/*` authenticators),
# so this is what makes the hop deliver rather than collect # so this is what makes the hop deliver rather than collect
# 401s — unconditional, because a forwarder that exports # 401s — unconditional, because a forwarder that exports
# unauthenticated has no working mode to fall back to. # unauthenticated has no working mode to fall back to.
auth.authenticator = forwarderAuthName; auth.authenticator = forwarderAuthName;
}; };
# The push side, opposite in direction to the receiver's # The push side, opposite in direction to the receiver's
# `oidc/*`: those VALIDATE a token arriving, this OBTAINS one # `oidc/*`: those VALIDATE a token arriving, this OBTAINS one
# to send. Hence `oauth2client`, and hence a client id and # to send. Hence `oauth2client`, and hence a client id and
# secret rather than an issuer and an audience to check. # secret rather than an issuer and an audience to check.
extensions.${forwarderAuthName} = { extensions.${forwarderAuthName} = {
client_id = cfg.otel.clientId; client_id = cfg.otel.clientId;
# A path, never a value — and one systemd resolves at # A path, never a value — and one systemd resolves at
# runtime, so neither the secret nor the credentials # runtime, so neither the secret nor the credentials
# directory is rendered into the nix store. # directory is rendered into the nix store.
client_secret_file = "\${env:CREDENTIALS_DIRECTORY}/${forwarderCredentialId}"; client_secret_file = "\${env:CREDENTIALS_DIRECTORY}/${forwarderCredentialId}";
token_url = "${toString autheliaCfg.url}/api/oidc/token"; token_url = "${toString autheliaCfg.url}/api/oidc/token";
# ⚠️ THE AUDIENCE HAS TO BE REQUESTED, not merely granted. # ⚠️ THE AUDIENCE HAS TO BE REQUESTED, not merely granted.
# Registering it on the client only makes it permissible; a # Registering it on the client only makes it permissible; a
# token minted without asking carries `aud: []` and the # token minted without asking carries `aud: []` and the
# receiver refuses it — with a config that reads perfectly # receiver refuses it — with a config that reads perfectly
# at both ends. # at both ends.
endpoint_params.audience = cfg.otel.clientId; endpoint_params.audience = cfg.otel.clientId;
}; };
service.pipelines.logs = { service.pipelines.logs = {
receivers = [ "journald" ]; receivers = [ "journald" ];
processors = [ "resource" ]; processors = [ "resource" ];
exporters = [ "otlphttp" ]; exporters = [ "otlphttp" ];
}; };
# A SECOND pipeline, not a second receiver on the logs one: a # A SECOND pipeline, not a second receiver on the logs one: a
# pipeline is typed, and a prometheus receiver in `logs` is a # pipeline is typed, and a prometheus receiver in `logs` is a
# config the collector rejects at startup rather than one that # config the collector rejects at startup rather than one that
# quietly ships metrics as log records. # quietly ships metrics as log records.
# #
# Same `resource` processor and same `otlphttp` exporter, so # Same `resource` processor and same `otlphttp` exporter, so
# the store's metrics carry the same `service.name` its logs # the store's metrics carry the same `service.name` its logs
# do and leave by the same hop — no otel bypass, one exit from # do and leave by the same hop — no otel bypass, one exit from
# this container. The exporter treats `endpoint` as a base and # this container. The exporter treats `endpoint` as a base and
# appends the signal's own path, so `/v1/metrics` here and # appends the signal's own path, so `/v1/metrics` here and
# `/v1/logs` above come from the one address. # `/v1/logs` above come from the one address.
service.pipelines.metrics = { service.pipelines.metrics = {
receivers = [ "prometheus" ]; receivers = [ "prometheus" ];
processors = [ "resource" ]; processors = [ "resource" ];
exporters = [ "otlphttp" ]; exporters = [ "otlphttp" ];
}; };
# An extension configured but not listed here is INERT — the # An extension configured but not listed here is INERT — the
# receiver's `storage: file_storage` above would name a # receiver's `storage: file_storage` above would name a
# component the collector never starts, and the exporter's # component the collector never starts, and the exporter's
# authenticator would push unauthenticated while the # authenticator would push unauthenticated while the
# collector reports healthy. # collector reports healthy.
service.extensions = [ service.extensions = [
"file_storage" "file_storage"
forwarderAuthName forwarderAuthName
]; ];
}; }
}; # Off the built-in default of `localhost:8888`, which the hive tier
# holds: this container shares the host's netns.
# Off the built-in default of `localhost:8888`, which the hive tier (import ../collector-self-scrape.nix {
# holds: this container shares the host's netns. inherit lib;
services.opentelemetry-collector.settings = import ../collector-self-scrape.nix { port = cfg.otel.telemetryPort;
inherit lib; exporter = "otlphttp";
port = cfg.otel.telemetryPort; attributes.container = collectorServiceName;
exporter = "otlphttp"; })
attributes.container = collectorServiceName; ];
}; };
# How the secret crosses the sandbox. The host unit # How the secret crosses the sandbox. The host unit