diff --git a/docs/swarm/services.md b/docs/swarm/services.md index 406461a59..7523f31fd 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -219,10 +219,12 @@ and failure series on `collector` appear once something fails, and the collector and of the swarm's collector, which both carry the collector binary's default `service.name`, `otelcol-contrib`. The timeseries panels split by `server.port`: 8888 is the hive collector, 8889 the swarm -collector. The per-container collectors (the secret store, NATS, Grafana, -Authelia, the Victoria stores and the agents' forwarders) don't report -their own metrics yet, so they have no lines here; a separate issue on -the forge covers that. `scripts/check-dashboard-service-names.sh` runs in CI +collector. Every per-container collector (the secret store, the other service +containers and the agents' forwarders) also scrapes its own endpoint on +loopback and pushes the result up its normal hop, so each has a line of its +own, on its own port: agents all report on 8904, service containers on +8890 to 8903. The series carry `container` (services) or `agent` (agents) +to tell them apart. `scripts/check-dashboard-service-names.sh` runs in CI and in `scripts/pre-push`. It fails when a dashboard query selects a `service.name` that nothing emits. diff --git a/nix/agent-modules/otel.nix b/nix/agent-modules/otel.nix index 5a622b81e..f913ffce5 100644 --- a/nix/agent-modules/otel.nix +++ b/nix/agent-modules/otel.nix @@ -72,6 +72,11 @@ let OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE = "cumulative"; }; + # Where the collector serves its own metrics, inside the container's own + # netns. Not the binary's default 8888, which the collector dashboard + # reads as the hive collector's. + selfTelemetryPort = 8904; + # The exporter's NAME picks the wire protocol, so `protocol` has to be # read here as well as handed to the SDKs above — same expression as the # swarm tier's upstream exporter in nix/host-modules/swarm-otel.nix. @@ -210,91 +215,101 @@ in # and the collector then dies at startup. Same caveat as both other # tiers; see nix/host-modules/otel.nix for the measurement. validateConfigFile = true; - settings = { - # `journalctl --follow --lines=0` (what this receiver runs under - # the hood) ships only what's written *after* it starts — every - # collector start has a silent gap at the front. `storage:` below - # persists the read cursor, so a restart resumes from where it - # left off instead of re-opening that gap; `start_at` stays `end` - # (its default) because a cursor already covers every run after - # the first, and `beginning` without one would re-ship the whole - # journal on every restart. The cursor lives in the unit's own - # `StateDirectory` (nixpkgs' module already sets one, `%S`, below - # as `WorkingDirectory` too) — container-lifetime, not the - # host-persisted state bind mount: it indexes /var/log/journal, - # itself fresh on every container start, so the two must share - # a lifetime or the cursor outlives the journal it points into. - # `create_directory` matters here for a build-time-vs-runtime - # reason, not a convenience one: `validateConfigFile` above runs - # `otelcol validate` in the nix build sandbox, before systemd's - # `StateDirectory=` has ever created this path. Without it the - # validator refuses with "directory must exist" and the build fails. - extensions.file_storage = { - directory = "/var/lib/opentelemetry-collector"; - create_directory = true; - }; + settings = lib.mkMerge [ + { + # `journalctl --follow --lines=0` (what this receiver runs under + # the hood) ships only what's written *after* it starts — every + # collector start has a silent gap at the front. `storage:` below + # persists the read cursor, so a restart resumes from where it + # left off instead of re-opening that gap; `start_at` stays `end` + # (its default) because a cursor already covers every run after + # the first, and `beginning` without one would re-ship the whole + # journal on every restart. The cursor lives in the unit's own + # `StateDirectory` (nixpkgs' module already sets one, `%S`, below + # as `WorkingDirectory` too) — container-lifetime, not the + # host-persisted state bind mount: it indexes /var/log/journal, + # itself fresh on every container start, so the two must share + # a lifetime or the cursor outlives the journal it points into. + # `create_directory` matters here for a build-time-vs-runtime + # reason, not a convenience one: `validateConfigFile` above runs + # `otelcol validate` in the nix build sandbox, before systemd's + # `StateDirectory=` has ever created this path. Without it the + # validator refuses with "directory must exist" and the build fails. + extensions.file_storage = { + directory = "/var/lib/opentelemetry-collector"; + create_directory = true; + }; - receivers.journald = { - # ⚠️ STATED, and it must stay stated: the receiver's own default - # is the RUNTIME journal (`/run/log/journal`), which in an agent - # container is empty — journald stores persistently here, so - # every entry is under /var/log/journal. Dropping this line - # leaves a collector that validates, starts, reports healthy and - # forwards nothing. - directory = "/var/log/journal"; - storage = "file_storage"; - # No `units` allowlist, and no user-session filter like the swarm - # tier's: a container's journal is the harness and what the - # harness spawns, so there is no foreign traffic to filter out and - # an allowlist would only be a list to forget to update. + receivers.journald = { + # ⚠️ STATED, and it must stay stated: the receiver's own default + # is the RUNTIME journal (`/run/log/journal`), which in an agent + # container is empty — journald stores persistently here, so + # every entry is under /var/log/journal. Dropping this line + # leaves a collector that validates, starts, reports healthy and + # forwards nothing. + directory = "/var/log/journal"; + storage = "file_storage"; + # No `units` allowlist, and no user-session filter like the swarm + # tier's: a container's journal is the harness and what the + # harness spawns, so there is no foreign traffic to filter out and + # an allowlist would only be a list to forget to update. - # journald's PRIORITY carries the level every line already has; - # without this the record reaches VictoriaLogs with - # `severity_text: Unspecified` and the store cannot tell an error - # from a debug line. The table — and the inverted direction that - # makes it worth a file of its own — lives in - # ../journald-severity.nix, shared with the swarm tier's receiver. - operators = import ../journald-severity.nix; - }; + # journald's PRIORITY carries the level every line already has; + # without this the record reaches VictoriaLogs with + # `severity_text: Unspecified` and the store cannot tell an error + # from a debug line. The table — and the inverted direction that + # makes it worth a file of its own — lives in + # ../journald-severity.nix, shared with the swarm tier's receiver. + operators = import ../journald-severity.nix; + }; - # Identity the hop above cannot supply. A host-side reader can say - # which machine a line came from; only a collector running as the - # agent can say which agent. `hive` and `swarm` are deliberately - # NOT stamped here: the swarm tier upserts both from the receiver - # that accepted the sample, precisely so the labels come from - # something the sender cannot write. - processors.resource.attributes = [ - { - key = "service.name"; - value = "hyperhive-agent"; - action = "upsert"; + # Identity the hop above cannot supply. A host-side reader can say + # which machine a line came from; only a collector running as the + # agent can say which agent. `hive` and `swarm` are deliberately + # NOT stamped here: the swarm tier upserts both from the receiver + # that accepted the sample, precisely so the labels come from + # something the sender cannot write. + processors.resource.attributes = [ + { + key = "service.name"; + value = "hyperhive-agent"; + action = "upsert"; + } + { + key = "agent"; + value = userName; + action = "upsert"; + } + ]; + + # The hive's collector, at the same base address every in-process + # producer already exports to. `endpoint` is a BASE the exporter + # appends `/v1/logs` to — the path an OTLP/HTTP receiver serves. + exporters.${logExporter} = { + endpoint = cfg.endpoint; } - { - key = "agent"; - value = userName; - action = "upsert"; - } - ]; + // lib.optionalAttrs (cfg.protocol == "http/json") { encoding = "json"; }; - # The hive's collector, at the same base address every in-process - # producer already exports to. `endpoint` is a BASE the exporter - # appends `/v1/logs` to — the path an OTLP/HTTP receiver serves. - exporters.${logExporter} = { - endpoint = cfg.endpoint; + service.pipelines.logs = { + receivers = [ "journald" ]; + processors = [ "resource" ]; + exporters = [ logExporter ]; + }; + + # An extension configured but not listed here is INERT — the + # journald receiver's `storage: file_storage` above would name a + # component the collector never starts. + service.extensions = [ "file_storage" ]; } - // lib.optionalAttrs (cfg.protocol == "http/json") { encoding = "json"; }; - - service.pipelines.logs = { - receivers = [ "journald" ]; - processors = [ "resource" ]; - exporters = [ logExporter ]; - }; - - # An extension configured but not listed here is INERT — the - # journald receiver's `storage: file_storage` above would name a - # component the collector never starts. - service.extensions = [ "file_storage" ]; - }; + # Every agent container has a network namespace of its own, so they can all + # use the same port. + (import ../collector-self-scrape.nix { + inherit lib; + port = selfTelemetryPort; + exporter = logExporter; + attributes.agent = userName; + }) + ]; }; }; } diff --git a/nix/collector-self-scrape.nix b/nix/collector-self-scrape.nix new file mode 100644 index 000000000..12cf796c4 --- /dev/null +++ b/nix/collector-self-scrape.nix @@ -0,0 +1,52 @@ +# Collector settings that make a container's OpenTelemetry collector report its +# own health (refused and failed records, exporter queue, memory), imported by +# every container collector: the agent forwarder, the service journal +# forwarder and the secret store's collector. The two host-side tiers scrape +# themselves through their `scrapeTargets` instead. +# +# One function renders both halves, so the endpoint and the job that reads it +# cannot name different ports. The endpoint binds loopback: the collector +# scrapes itself, and nothing outside the container has a reason to. +# +# The job runs in its own pipeline. A container's other metrics pipeline may +# carry a `resource` processor that upserts `service.name`, which would +# relabel these series away from the `otelcol-contrib` the collector dashboard +# selects on. The attributes stamped here tell the container's collectors +# apart. +# +# `readers`, not `metrics.address`: this collector version rejects `address` +# at startup. +{ + lib, + port, + exporter, + attributes, +}: +{ + service.telemetry.metrics.readers = [ + { + pull.exporter.prometheus = { + host = "127.0.0.1"; + inherit port; + }; + } + ]; + + receivers."prometheus/self".config.scrape_configs = [ + { + job_name = "collector"; + static_configs = [ { targets = [ "127.0.0.1:${toString port}" ]; } ]; + } + ]; + + processors."resource/self".attributes = lib.mapAttrsToList (key: value: { + inherit key value; + action = "upsert"; + }) attributes; + + service.pipelines."metrics/self" = { + receivers = [ "prometheus/self" ]; + processors = [ "resource/self" ]; + exporters = [ exporter ]; + }; +} diff --git a/nix/container-modules/swarm-journal-forwarder.nix b/nix/container-modules/swarm-journal-forwarder.nix index 952d9df56..1348d00d4 100644 --- a/nix/container-modules/swarm-journal-forwarder.nix +++ b/nix/container-modules/swarm-journal-forwarder.nix @@ -96,8 +96,9 @@ in type = lib.types.port; description = '' Loopback port for the collector's own metrics (refused and dropped - records, exporter failures). Required because the container may share - the host netns, where the binary's default 8888 is the hive tier's. + records, exporter failures), which it scrapes and pushes to the swarm + collector. Required because the container may share the host netns, + where the binary's default 8888 is the hive tier's. ''; }; @@ -242,17 +243,6 @@ in exporters = [ "otlphttp" ]; }; - # ⚠️ `readers`, not `metrics.address`: this collector version rejects - # `address` at startup. - service.telemetry.metrics.readers = [ - { - pull.exporter.prometheus = { - host = "127.0.0.1"; - port = cfg.telemetryPort; - }; - } - ]; - # An extension not listed here is inert: the receiver's storage would # name a component that never starts, and the exporter would push # unauthenticated. @@ -261,6 +251,12 @@ in authName ]; } + (import ../collector-self-scrape.nix { + inherit lib; + port = cfg.telemetryPort; + exporter = "otlphttp"; + attributes.container = cfg.serviceName; + }) # A separate set the module system merges in: joining it to the first with # `//` would replace the first set's whole `service` attrset. (lib.mkIf (cfg.otlpMetricsPort != null) { diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 7a79b2e4d..dd5b1bab7 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -4438,172 +4438,163 @@ in # caveat as every other tier; see nix/host-modules/otel.nix for # the measurement. validateConfigFile = true; - settings = { - # `journalctl --follow --lines=0` (what the receiver runs under - # the hood) ships only what is written *after* it starts, so - # every collector start has a silent gap at the front. This - # persists the read cursor, so a restart resumes instead of - # re-opening that gap; `start_at` stays `end` (its default), - # since a cursor covers every run after the first and - # `beginning` without one would re-ship the whole journal on - # every restart. `create_directory` is required for a - # build-time reason rather than a convenience one: - # `validateConfigFile` above runs `otelcol validate` in the nix - # sandbox, before systemd's `StateDirectory=` has created the - # path, and the validator refuses with "directory must exist". - extensions.file_storage = { - directory = "/var/lib/opentelemetry-collector"; - create_directory = true; - }; + settings = lib.mkMerge [ + { + # `journalctl --follow --lines=0` (what the receiver runs under + # the hood) ships only what is written *after* it starts, so + # every collector start has a silent gap at the front. This + # persists the read cursor, so a restart resumes instead of + # re-opening that gap; `start_at` stays `end` (its default), + # since a cursor covers every run after the first and + # `beginning` without one would re-ship the whole journal on + # every restart. `create_directory` is required for a + # build-time reason rather than a convenience one: + # `validateConfigFile` above runs `otelcol validate` in the nix + # sandbox, before systemd's `StateDirectory=` has created the + # path, and the validator refuses with "directory must exist". + extensions.file_storage = { + directory = "/var/lib/opentelemetry-collector"; + create_directory = true; + }; - receivers.journald = { - # ⚠️ STATED, and it must stay stated: the receiver's own - # default is the RUNTIME journal (`/run/log/journal`), which - # in this container is empty — journald stores persistently - # here, so every entry is under /var/log/journal. Dropping - # this line leaves a collector that validates, starts, - # reports healthy and forwards nothing. - directory = "/var/log/journal"; - # ⚠️ `--merge` is a guard: an unqualified - # `journalctl --follow` over a journal with no entry for the - # current boot (a host-linked one) crash-loops the receiver - # every ~2s with "No journal boot entry found for the - # specified boot (+0)". This journal is container-local, so - # the current boot has an entry and the flag changes nothing. - merge = true; - storage = "file_storage"; - }; + receivers.journald = { + # ⚠️ STATED, and it must stay stated: the receiver's own + # default is the RUNTIME journal (`/run/log/journal`), which + # in this container is empty — journald stores persistently + # here, so every entry is under /var/log/journal. Dropping + # this line leaves a collector that validates, starts, + # reports healthy and forwards nothing. + directory = "/var/log/journal"; + # ⚠️ `--merge` is a guard: an unqualified + # `journalctl --follow` over a journal with no entry for the + # current boot (a host-linked one) crash-loops the receiver + # every ~2s with "No journal boot entry found for the + # specified boot (+0)". This journal is container-local, so + # the current boot has an entry and the flag changes nothing. + merge = true; + storage = "file_storage"; + }; - # The store's scrape, run by this container's own collector. - # - # ⚠️ `metrics_path` and `params` are STATED, and both have to - # be: openbao serves no `/metrics` at all, and - # `/v1/sys/metrics` answers JSON unless the format is asked - # for. A scrape of the default path 404s and a scrape without - # the query parses as nothing — each reads as a dead exporter - # rather than as a wrong request. The spelling matches - # ./swarm-otel.nix's `loopbackScrapeConfig`, the renderer for - # every other service's target. - # - # No credential, unlike the swarm collector's - # `publishedScrapeTargets` jobs: the target is the loopback - # listener above, inside this container's own netns, which is - # what lets the scrape stay unauthenticated without the API - # listener's client-cert requirement being touched. - receivers.prometheus.config.scrape_configs = [ - { - job_name = "bao"; - static_configs = [ { targets = [ metricsScrapeTarget ]; } ]; - metrics_path = "/v1/sys/metrics"; - params.format = [ "prometheus" ]; - } - ]; + # The store's scrape, run by this container's own collector. + # + # ⚠️ `metrics_path` and `params` are STATED, and both have to + # be: openbao serves no `/metrics` at all, and + # `/v1/sys/metrics` answers JSON unless the format is asked + # for. A scrape of the default path 404s and a scrape without + # the query parses as nothing — each reads as a dead exporter + # rather than as a wrong request. The spelling matches + # ./swarm-otel.nix's `loopbackScrapeConfig`, the renderer for + # every other service's target. + # + # No credential, unlike the swarm collector's + # `publishedScrapeTargets` jobs: the target is the loopback + # listener above, inside this container's own netns, which is + # what lets the scrape stay unauthenticated without the API + # listener's client-cert requirement being touched. + receivers.prometheus.config.scrape_configs = [ + { + job_name = "bao"; + static_configs = [ { targets = [ metricsScrapeTarget ]; } ]; + metrics_path = "/v1/sys/metrics"; + params.format = [ "prometheus" ]; + } + ]; - # Identity the hop above cannot supply: a host-side reader can - # say which machine a line came from, and only a collector - # inside this container can say it was the secret store's. No - # `hive` or `swarm` label — the swarm tier upserts `hive` from - # the receiver that accepted the record, precisely so it comes - # from something the sender cannot write. - # - # `collectorServiceName`, not `cfg.machine`: what the - # dashboard selects on is a display identity, not the - # container/receiver name — see that binding's own comment. - processors.resource.attributes = [ - { - key = "service.name"; - value = collectorServiceName; - action = "upsert"; - } - ]; + # Identity the hop above cannot supply: a host-side reader can + # say which machine a line came from, and only a collector + # inside this container can say it was the secret store's. No + # `hive` or `swarm` label — the swarm tier upserts `hive` from + # the receiver that accepted the record, precisely so it comes + # from something the sender cannot write. + # + # `collectorServiceName`, not `cfg.machine`: what the + # dashboard selects on is a display identity, not the + # container/receiver name — see that binding's own comment. + processors.resource.attributes = [ + { + key = "service.name"; + value = collectorServiceName; + action = "upsert"; + } + ]; - # `otlphttp` unconditionally, unlike the agent forwarder's - # protocol-derived exporter name: the swarm receiver speaks - # OTLP/HTTP protobuf whatever this tier's *upstream* - # protocol is, which is the same reason - # ./hive-c0re/environment.nix pins the protocol it hands out. - # `endpoint` is a BASE the exporter appends `/v1/logs` to. - exporters.otlphttp = { - endpoint = otelFirstHop; - # The receiver at that address accepts an OIDC token and - # nothing else (./swarm-otel.nix's `oidc/*` authenticators), - # so this is what makes the hop deliver rather than collect - # 401s — unconditional, because a forwarder that exports - # unauthenticated has no working mode to fall back to. - auth.authenticator = forwarderAuthName; - }; + # `otlphttp` unconditionally, unlike the agent forwarder's + # protocol-derived exporter name: the swarm receiver speaks + # OTLP/HTTP protobuf whatever this tier's *upstream* + # protocol is, which is the same reason + # ./hive-c0re/environment.nix pins the protocol it hands out. + # `endpoint` is a BASE the exporter appends `/v1/logs` to. + exporters.otlphttp = { + endpoint = otelFirstHop; + # The receiver at that address accepts an OIDC token and + # nothing else (./swarm-otel.nix's `oidc/*` authenticators), + # so this is what makes the hop deliver rather than collect + # 401s — unconditional, because a forwarder that exports + # unauthenticated has no working mode to fall back to. + auth.authenticator = forwarderAuthName; + }; - # The push side, opposite in direction to the receiver's - # `oidc/*`: those VALIDATE a token arriving, this OBTAINS one - # to send. Hence `oauth2client`, and hence a client id and - # secret rather than an issuer and an audience to check. - extensions.${forwarderAuthName} = { - client_id = cfg.otel.clientId; - # A path, never a value — and one systemd resolves at - # runtime, so neither the secret nor the credentials - # directory is rendered into the nix store. - client_secret_file = "\${env:CREDENTIALS_DIRECTORY}/${forwarderCredentialId}"; - token_url = "${toString autheliaCfg.url}/api/oidc/token"; - # ⚠️ THE AUDIENCE HAS TO BE REQUESTED, not merely granted. - # Registering it on the client only makes it permissible; a - # token minted without asking carries `aud: []` and the - # receiver refuses it — with a config that reads perfectly - # at both ends. - endpoint_params.audience = cfg.otel.clientId; - }; + # The push side, opposite in direction to the receiver's + # `oidc/*`: those VALIDATE a token arriving, this OBTAINS one + # to send. Hence `oauth2client`, and hence a client id and + # secret rather than an issuer and an audience to check. + extensions.${forwarderAuthName} = { + client_id = cfg.otel.clientId; + # A path, never a value — and one systemd resolves at + # runtime, so neither the secret nor the credentials + # directory is rendered into the nix store. + client_secret_file = "\${env:CREDENTIALS_DIRECTORY}/${forwarderCredentialId}"; + token_url = "${toString autheliaCfg.url}/api/oidc/token"; + # ⚠️ THE AUDIENCE HAS TO BE REQUESTED, not merely granted. + # Registering it on the client only makes it permissible; a + # token minted without asking carries `aud: []` and the + # receiver refuses it — with a config that reads perfectly + # at both ends. + endpoint_params.audience = cfg.otel.clientId; + }; - service.pipelines.logs = { - receivers = [ "journald" ]; - processors = [ "resource" ]; - exporters = [ "otlphttp" ]; - }; + service.pipelines.logs = { + receivers = [ "journald" ]; + processors = [ "resource" ]; + exporters = [ "otlphttp" ]; + }; - # A SECOND pipeline, not a second receiver on the logs one: a - # pipeline is typed, and a prometheus receiver in `logs` is a - # config the collector rejects at startup rather than one that - # quietly ships metrics as log records. - # - # Same `resource` processor and same `otlphttp` exporter, so - # the store's metrics carry the same `service.name` its logs - # do and leave by the same hop — no otel bypass, one exit from - # this container. The exporter treats `endpoint` as a base and - # appends the signal's own path, so `/v1/metrics` here and - # `/v1/logs` above come from the one address. - service.pipelines.metrics = { - receivers = [ "prometheus" ]; - processors = [ "resource" ]; - exporters = [ "otlphttp" ]; - }; + # A SECOND pipeline, not a second receiver on the logs one: a + # pipeline is typed, and a prometheus receiver in `logs` is a + # config the collector rejects at startup rather than one that + # quietly ships metrics as log records. + # + # Same `resource` processor and same `otlphttp` exporter, so + # the store's metrics carry the same `service.name` its logs + # do and leave by the same hop — no otel bypass, one exit from + # this container. The exporter treats `endpoint` as a base and + # appends the signal's own path, so `/v1/metrics` here and + # `/v1/logs` above come from the one address. + service.pipelines.metrics = { + receivers = [ "prometheus" ]; + processors = [ "resource" ]; + exporters = [ "otlphttp" ]; + }; - # Moves this collector's self-metrics off the built-in - # default of `localhost:8888`, which the hive tier holds — - # this container shares the host's netns, so leaving the - # default in place is the hive collector failing to bind. - # - # ⚠️ `metrics.address` is the spelling that looks right and - # is REJECTED by this collector version: - # `'migration.MetricsConfigV030' has invalid keys: - # address`. `readers` is the schema it accepts, and the - # difference is a startup failure rather than a warning. - service.telemetry.metrics.readers = [ - { - pull.exporter.prometheus = { - host = "127.0.0.1"; - port = cfg.otel.telemetryPort; - }; - } - ]; - - # An extension configured but not listed here is INERT — the - # receiver's `storage: file_storage` above would name a - # component the collector never starts, and the exporter's - # authenticator would push unauthenticated while the - # collector reports healthy. - service.extensions = [ - "file_storage" - forwarderAuthName - ]; - }; + # An extension configured but not listed here is INERT — the + # receiver's `storage: file_storage` above would name a + # component the collector never starts, and the exporter's + # authenticator would push unauthenticated while the + # collector reports healthy. + service.extensions = [ + "file_storage" + forwarderAuthName + ]; + } + # Off the built-in default of `localhost:8888`, which the hive tier + # holds: this container shares the host's netns. + (import ../collector-self-scrape.nix { + inherit lib; + port = cfg.otel.telemetryPort; + exporter = "otlphttp"; + attributes.container = collectorServiceName; + }) + ]; }; # How the secret crosses the sandbox. The host unit diff --git a/nix/host-modules/swarm-grafana/dashboards/collector.json b/nix/host-modules/swarm-grafana/dashboards/collector.json index 8f1facb57..d8d2f3c38 100644 --- a/nix/host-modules/swarm-grafana/dashboards/collector.json +++ b/nix/host-modules/swarm-grafana/dashboards/collector.json @@ -1,7 +1,7 @@ { "title": "hyperhive \u00b7 collector (self-telemetry)", "uid": "hyperhive-swarm-collector", - "description": "What the OTEL collectors accept, refuse, export and queue. Port 8888 is the hive, 8889 swarm-otel.", + "description": "OTEL collectors: accepted, refused, exported, queued. 8888 hive, 8889 swarm, others containers.", "editable": true, "refresh": "1m", "schemaVersion": 39, diff --git a/nix/module-eval/agent-otel.nix b/nix/module-eval/agent-otel.nix index d92614a17..c7f3b0955 100644 --- a/nix/module-eval/agent-otel.nix +++ b/nix/module-eval/agent-otel.nix @@ -21,6 +21,7 @@ let agent carriesJournaldSeverity runGroup + selfScrapes ; # The log path's three hops, one fixture each. Nothing carries a journal @@ -55,6 +56,20 @@ let agentSettings = machine: machine.services.opentelemetry-collector.settings; cases = [ + { + # The collector's own counters are the only signal that it is dropping + # telemetry. 8904 keeps agents off 8888, which the collector dashboard + # reads as the hive collector; the exporter follows the wire protocol. + name = "an agent forwarder scrapes its own telemetry endpoint on loopback, off the hive collector's port"; + ok = + selfScrapes (agentSettings agentOtel) 8904 { agent = agentOtel.services.hyperhive.agent.user.name; } + && selfScrapes (agentSettings agentOtelGrpc) 8904 { + agent = agentOtelGrpc.services.hyperhive.agent.user.name; + } + # Control: with the switch off the lookup finds no collector config. + && !(agentNoOtel.services.opentelemetry-collector.settings ? receivers) + && (agentSettings agentOtelGrpc).service.pipelines."metrics/self".exporters == [ "otlp" ]; + } { # The journald receiver's own default directory is the RUNTIME # journal, and a container that stores persistently leaves that diff --git a/nix/module-eval/bao-otel-collector.nix b/nix/module-eval/bao-otel-collector.nix index 565645853..2e84c632c 100644 --- a/nix/module-eval/bao-otel-collector.nix +++ b/nix/module-eval/bao-otel-collector.nix @@ -22,6 +22,7 @@ let runGroup baoSettings forceServiceConfigs + selfScrapes ; baoTwoAddresses = hive { @@ -185,6 +186,20 @@ let && m.exporters == [ "otlphttp" ] && m.exporters == s.service.pipelines.logs.exporters; } + { + # The store's collector reports its own health through a pipeline of its + # own: the store pipeline above upserts `service.name`, and sharing it + # would relabel these series away from the collector dashboard. + name = "the store's collector scrapes its own telemetry endpoint on loopback"; + ok = + let + s = (baoForwarder baoWithCollector).settings; + port = baoWithCollector.services.hyperhive.swarm.bao.otel.telemetryPort; + in + selfScrapes s port { container = "bao"; } + && s.service.pipelines.metrics.receivers == [ "prometheus" ] + && s.service.pipelines.metrics.processors == [ "resource" ]; + } { # Presence control for the case above: both fields are omitted rather # than defaulted, so a target declared as bare `host:port` renders what diff --git a/nix/module-eval/forge-placement.nix b/nix/module-eval/forge-placement.nix index c07f3c16f..da8b5c1e6 100644 --- a/nix/module-eval/forge-placement.nix +++ b/nix/module-eval/forge-placement.nix @@ -25,6 +25,7 @@ let }) hive runGroup + selfScrapes ; inherit (shared) bare; @@ -558,11 +559,10 @@ let ok = let c = ciAlone.containers.swarm-ci-runner.config; - reader = lib.head c.services.opentelemetry-collector.settings.service.telemetry.metrics.readers; + s = c.services.opentelemetry-collector.settings; in c.services.hyperhive.swarmJournalForwarder.telemetryPort == 8895 - && reader.pull.exporter.prometheus.host == "127.0.0.1" - && reader.pull.exporter.prometheus.port == 8895; + && selfScrapes s 8895 { container = "ci-runner"; }; } ]; in diff --git a/nix/module-eval/lib.nix b/nix/module-eval/lib.nix index 5205e2f88..b9f84ac54 100644 --- a/nix/module-eval/lib.nix +++ b/nix/module-eval/lib.nix @@ -208,6 +208,37 @@ let # the two tier cases differ only in which receiver they hand it. carriesJournaldSeverity = receiver: lib.elem journaldSeverityParser (receiver.operators or [ ]); + # Whether a container collector's settings carry the self-scrape of + # ../collector-self-scrape.nix: the telemetry endpoint on loopback at `port`, + # a job scraping exactly that address, and a metrics pipeline that reads the + # job and leaves through an exporter the settings define. `attrs` are the + # resource attributes the pipeline stamps. + selfScrapes = + s: port: attrs: + let + job = lib.findFirst (j: j.job_name == "collector") null ( + s.receivers."prometheus/self".config.scrape_configs or [ ] + ); + pipeline = s.service.pipelines."metrics/self" or null; + reader = lib.head (s.service.telemetry.metrics.readers or [ { } ]); + stamped = lib.mapAttrsToList (key: value: { + inherit key value; + action = "upsert"; + }) attrs; + in + job != null + && job.static_configs == [ { targets = [ "127.0.0.1:${toString port}" ]; } ] + && + (reader.pull.exporter.prometheus or null) == { + host = "127.0.0.1"; + inherit port; + } + && pipeline != null + && pipeline.receivers == [ "prometheus/self" ] + && (s.processors."resource/self".attributes or null) == stamped + && pipeline.exporters != [ ] + && lib.all (e: s.exporters ? ${e}) pipeline.exporters; + # The enables that switch owns. The hive tier's collector is not among them: it # follows `deploy.hive-controller.enable`, and a services host without a hive # runs none. @@ -283,6 +314,7 @@ in inherit agentHarness; inherit journaldSeverityParser; inherit carriesJournaldSeverity; + inherit selfScrapes; inherit swarmServiceEnables; inherit runGroup; inherit forceServiceConfigs; diff --git a/nix/module-eval/nats-authelia.nix b/nix/module-eval/nats-authelia.nix index d21bc6466..56baa5ffa 100644 --- a/nix/module-eval/nats-authelia.nix +++ b/nix/module-eval/nats-authelia.nix @@ -20,6 +20,7 @@ let }) hive runGroup + selfScrapes ; # No store identity, which the first case below relies on. @@ -157,12 +158,10 @@ let h.services.hyperhive.swarm.nats.otel.telemetryPort ] ++ map (o: o.port) (lib.filter (o: o.machine != svc.machine) forwarderServices); - reader = lib.head s.service.telemetry.metrics.readers; in c.services.hyperhive.swarmJournalForwarder.telemetryPort == svc.port && !(lib.elem svc.port taken) - && reader.pull.exporter.prometheus.host == "127.0.0.1" - && reader.pull.exporter.prometheus.port == svc.port; + && selfScrapes s svc.port { container = svc.name; }; } ] ) forwarderServices; @@ -269,12 +268,10 @@ let h.services.hyperhive.swarm.nats.otel.telemetryPort ] ++ map (o: o.port) (lib.filter (o: o.machine != svc.machine) hostNetnsForwarders); - reader = lib.head s.service.telemetry.metrics.readers; in c.services.hyperhive.swarmJournalForwarder.telemetryPort == svc.port && !(lib.elem svc.port taken) - && reader.pull.exporter.prometheus.host == "127.0.0.1" - && reader.pull.exporter.prometheus.port == svc.port; + && selfScrapes s svc.port { container = svc.name; }; } ] ) hostNetnsForwarders; @@ -740,6 +737,7 @@ let && s.extensions."oauth2client/swarm".endpoint_params.audience == clientId && lib.elem "oauth2client/swarm" s.service.extensions && s.service.pipelines.logs.receivers == [ "journald" ] + && selfScrapes s h.services.hyperhive.swarm.nats.otel.telemetryPort { container = "nats"; } && lib.elem "oidc-client-secret:/run/swarm-journal-forwarder-oidc/client-secret" c.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential && fetch.environment.BAO_CLIENT_CERT == "/var/lib/swarm-nats-auth/bao-client.pem" && lib.hasInfix "bao kv get -field=value secret/swarm/services/${clientId}/oidc/client" fetch.script @@ -997,7 +995,7 @@ let registered = lib.findFirst ( x: x.id == clientId ) null h.services.hyperhive.swarm.authelia.oidc.clients; - reader = lib.head s.service.telemetry.metrics.readers; + port = c.services.hyperhive.swarmJournalForwarder.telemetryPort; taken = [ 8888 8890 @@ -1022,8 +1020,8 @@ let && registered.audience == [ clientId ] && !(lib.elem "--link-journal=host" container.extraFlags) && lib.any (lib.hasPrefix "--load-credential=publisher-bao-cert:") container.extraFlags - && reader.pull.exporter.prometheus.host == "127.0.0.1" - && !(lib.elem reader.pull.exporter.prometheus.port taken) + && selfScrapes s port { container = "authelia"; } + && !(lib.elem port taken) && !autheliaColocated.containers.swarm-authelia.config.services.opentelemetry-collector.enable && !( autheliaColocated.containers.swarm-authelia.config.systemd.services ? swarm-journal-forwarder-oidc