diff --git a/docs/swarm/services.md b/docs/swarm/services.md index 7523f31fd..406461a59 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -219,12 +219,10 @@ and failure series on `collector` appear once something fails, and the collector and of the swarm's collector, which both carry the collector binary's default `service.name`, `otelcol-contrib`. The timeseries panels split by `server.port`: 8888 is the hive collector, 8889 the swarm -collector. Every per-container collector (the secret store, the other service -containers and the agents' forwarders) also scrapes its own endpoint on -loopback and pushes the result up its normal hop, so each has a line of its -own, on its own port: agents all report on 8904, service containers on -8890 to 8903. The series carry `container` (services) or `agent` (agents) -to tell them apart. `scripts/check-dashboard-service-names.sh` runs in CI +collector. The per-container collectors (the secret store, NATS, Grafana, +Authelia, the Victoria stores and the agents' forwarders) don't report +their own metrics yet, so they have no lines here; a separate issue on +the forge covers that. `scripts/check-dashboard-service-names.sh` runs in CI and in `scripts/pre-push`. It fails when a dashboard query selects a `service.name` that nothing emits. diff --git a/nix/agent-modules/otel.nix b/nix/agent-modules/otel.nix index f913ffce5..5a622b81e 100644 --- a/nix/agent-modules/otel.nix +++ b/nix/agent-modules/otel.nix @@ -72,11 +72,6 @@ let OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE = "cumulative"; }; - # Where the collector serves its own metrics, inside the container's own - # netns. Not the binary's default 8888, which the collector dashboard - # reads as the hive collector's. - selfTelemetryPort = 8904; - # The exporter's NAME picks the wire protocol, so `protocol` has to be # read here as well as handed to the SDKs above — same expression as the # swarm tier's upstream exporter in nix/host-modules/swarm-otel.nix. @@ -215,101 +210,91 @@ in # and the collector then dies at startup. Same caveat as both other # tiers; see nix/host-modules/otel.nix for the measurement. validateConfigFile = true; - settings = lib.mkMerge [ - { - # `journalctl --follow --lines=0` (what this receiver runs under - # the hood) ships only what's written *after* it starts — every - # collector start has a silent gap at the front. `storage:` below - # persists the read cursor, so a restart resumes from where it - # left off instead of re-opening that gap; `start_at` stays `end` - # (its default) because a cursor already covers every run after - # the first, and `beginning` without one would re-ship the whole - # journal on every restart. The cursor lives in the unit's own - # `StateDirectory` (nixpkgs' module already sets one, `%S`, below - # as `WorkingDirectory` too) — container-lifetime, not the - # host-persisted state bind mount: it indexes /var/log/journal, - # itself fresh on every container start, so the two must share - # a lifetime or the cursor outlives the journal it points into. - # `create_directory` matters here for a build-time-vs-runtime - # reason, not a convenience one: `validateConfigFile` above runs - # `otelcol validate` in the nix build sandbox, before systemd's - # `StateDirectory=` has ever created this path. Without it the - # validator refuses with "directory must exist" and the build fails. - extensions.file_storage = { - directory = "/var/lib/opentelemetry-collector"; - create_directory = true; - }; + settings = { + # `journalctl --follow --lines=0` (what this receiver runs under + # the hood) ships only what's written *after* it starts — every + # collector start has a silent gap at the front. `storage:` below + # persists the read cursor, so a restart resumes from where it + # left off instead of re-opening that gap; `start_at` stays `end` + # (its default) because a cursor already covers every run after + # the first, and `beginning` without one would re-ship the whole + # journal on every restart. The cursor lives in the unit's own + # `StateDirectory` (nixpkgs' module already sets one, `%S`, below + # as `WorkingDirectory` too) — container-lifetime, not the + # host-persisted state bind mount: it indexes /var/log/journal, + # itself fresh on every container start, so the two must share + # a lifetime or the cursor outlives the journal it points into. + # `create_directory` matters here for a build-time-vs-runtime + # reason, not a convenience one: `validateConfigFile` above runs + # `otelcol validate` in the nix build sandbox, before systemd's + # `StateDirectory=` has ever created this path. Without it the + # validator refuses with "directory must exist" and the build fails. + extensions.file_storage = { + directory = "/var/lib/opentelemetry-collector"; + create_directory = true; + }; - receivers.journald = { - # ⚠️ STATED, and it must stay stated: the receiver's own default - # is the RUNTIME journal (`/run/log/journal`), which in an agent - # container is empty — journald stores persistently here, so - # every entry is under /var/log/journal. Dropping this line - # leaves a collector that validates, starts, reports healthy and - # forwards nothing. - directory = "/var/log/journal"; - storage = "file_storage"; - # No `units` allowlist, and no user-session filter like the swarm - # tier's: a container's journal is the harness and what the - # harness spawns, so there is no foreign traffic to filter out and - # an allowlist would only be a list to forget to update. + receivers.journald = { + # ⚠️ STATED, and it must stay stated: the receiver's own default + # is the RUNTIME journal (`/run/log/journal`), which in an agent + # container is empty — journald stores persistently here, so + # every entry is under /var/log/journal. Dropping this line + # leaves a collector that validates, starts, reports healthy and + # forwards nothing. + directory = "/var/log/journal"; + storage = "file_storage"; + # No `units` allowlist, and no user-session filter like the swarm + # tier's: a container's journal is the harness and what the + # harness spawns, so there is no foreign traffic to filter out and + # an allowlist would only be a list to forget to update. - # journald's PRIORITY carries the level every line already has; - # without this the record reaches VictoriaLogs with - # `severity_text: Unspecified` and the store cannot tell an error - # from a debug line. The table — and the inverted direction that - # makes it worth a file of its own — lives in - # ../journald-severity.nix, shared with the swarm tier's receiver. - operators = import ../journald-severity.nix; - }; + # journald's PRIORITY carries the level every line already has; + # without this the record reaches VictoriaLogs with + # `severity_text: Unspecified` and the store cannot tell an error + # from a debug line. The table — and the inverted direction that + # makes it worth a file of its own — lives in + # ../journald-severity.nix, shared with the swarm tier's receiver. + operators = import ../journald-severity.nix; + }; - # Identity the hop above cannot supply. A host-side reader can say - # which machine a line came from; only a collector running as the - # agent can say which agent. `hive` and `swarm` are deliberately - # NOT stamped here: the swarm tier upserts both from the receiver - # that accepted the sample, precisely so the labels come from - # something the sender cannot write. - processors.resource.attributes = [ - { - key = "service.name"; - value = "hyperhive-agent"; - action = "upsert"; - } - { - key = "agent"; - value = userName; - action = "upsert"; - } - ]; - - # The hive's collector, at the same base address every in-process - # producer already exports to. `endpoint` is a BASE the exporter - # appends `/v1/logs` to — the path an OTLP/HTTP receiver serves. - exporters.${logExporter} = { - endpoint = cfg.endpoint; + # Identity the hop above cannot supply. A host-side reader can say + # which machine a line came from; only a collector running as the + # agent can say which agent. `hive` and `swarm` are deliberately + # NOT stamped here: the swarm tier upserts both from the receiver + # that accepted the sample, precisely so the labels come from + # something the sender cannot write. + processors.resource.attributes = [ + { + key = "service.name"; + value = "hyperhive-agent"; + action = "upsert"; } - // lib.optionalAttrs (cfg.protocol == "http/json") { encoding = "json"; }; + { + key = "agent"; + value = userName; + action = "upsert"; + } + ]; - service.pipelines.logs = { - receivers = [ "journald" ]; - processors = [ "resource" ]; - exporters = [ logExporter ]; - }; - - # An extension configured but not listed here is INERT — the - # journald receiver's `storage: file_storage` above would name a - # component the collector never starts. - service.extensions = [ "file_storage" ]; + # The hive's collector, at the same base address every in-process + # producer already exports to. `endpoint` is a BASE the exporter + # appends `/v1/logs` to — the path an OTLP/HTTP receiver serves. + exporters.${logExporter} = { + endpoint = cfg.endpoint; } - # Every agent container has a network namespace of its own, so they can all - # use the same port. - (import ../collector-self-scrape.nix { - inherit lib; - port = selfTelemetryPort; - exporter = logExporter; - attributes.agent = userName; - }) - ]; + // lib.optionalAttrs (cfg.protocol == "http/json") { encoding = "json"; }; + + service.pipelines.logs = { + receivers = [ "journald" ]; + processors = [ "resource" ]; + exporters = [ logExporter ]; + }; + + # An extension configured but not listed here is INERT — the + # journald receiver's `storage: file_storage` above would name a + # component the collector never starts. + service.extensions = [ "file_storage" ]; + }; }; }; } diff --git a/nix/collector-self-scrape.nix b/nix/collector-self-scrape.nix deleted file mode 100644 index 12cf796c4..000000000 --- a/nix/collector-self-scrape.nix +++ /dev/null @@ -1,52 +0,0 @@ -# Collector settings that make a container's OpenTelemetry collector report its -# own health (refused and failed records, exporter queue, memory), imported by -# every container collector: the agent forwarder, the service journal -# forwarder and the secret store's collector. The two host-side tiers scrape -# themselves through their `scrapeTargets` instead. -# -# One function renders both halves, so the endpoint and the job that reads it -# cannot name different ports. The endpoint binds loopback: the collector -# scrapes itself, and nothing outside the container has a reason to. -# -# The job runs in its own pipeline. A container's other metrics pipeline may -# carry a `resource` processor that upserts `service.name`, which would -# relabel these series away from the `otelcol-contrib` the collector dashboard -# selects on. The attributes stamped here tell the container's collectors -# apart. -# -# `readers`, not `metrics.address`: this collector version rejects `address` -# at startup. -{ - lib, - port, - exporter, - attributes, -}: -{ - service.telemetry.metrics.readers = [ - { - pull.exporter.prometheus = { - host = "127.0.0.1"; - inherit port; - }; - } - ]; - - receivers."prometheus/self".config.scrape_configs = [ - { - job_name = "collector"; - static_configs = [ { targets = [ "127.0.0.1:${toString port}" ]; } ]; - } - ]; - - processors."resource/self".attributes = lib.mapAttrsToList (key: value: { - inherit key value; - action = "upsert"; - }) attributes; - - service.pipelines."metrics/self" = { - receivers = [ "prometheus/self" ]; - processors = [ "resource/self" ]; - exporters = [ exporter ]; - }; -} diff --git a/nix/container-modules/swarm-journal-forwarder.nix b/nix/container-modules/swarm-journal-forwarder.nix index 1348d00d4..952d9df56 100644 --- a/nix/container-modules/swarm-journal-forwarder.nix +++ b/nix/container-modules/swarm-journal-forwarder.nix @@ -96,9 +96,8 @@ in type = lib.types.port; description = '' Loopback port for the collector's own metrics (refused and dropped - records, exporter failures), which it scrapes and pushes to the swarm - collector. Required because the container may share the host netns, - where the binary's default 8888 is the hive tier's. + records, exporter failures). Required because the container may share + the host netns, where the binary's default 8888 is the hive tier's. ''; }; @@ -243,6 +242,17 @@ in exporters = [ "otlphttp" ]; }; + # ⚠️ `readers`, not `metrics.address`: this collector version rejects + # `address` at startup. + service.telemetry.metrics.readers = [ + { + pull.exporter.prometheus = { + host = "127.0.0.1"; + port = cfg.telemetryPort; + }; + } + ]; + # An extension not listed here is inert: the receiver's storage would # name a component that never starts, and the exporter would push # unauthenticated. @@ -251,12 +261,6 @@ in authName ]; } - (import ../collector-self-scrape.nix { - inherit lib; - port = cfg.telemetryPort; - exporter = "otlphttp"; - attributes.container = cfg.serviceName; - }) # A separate set the module system merges in: joining it to the first with # `//` would replace the first set's whole `service` attrset. (lib.mkIf (cfg.otlpMetricsPort != null) { diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index dd5b1bab7..7a79b2e4d 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -4438,163 +4438,172 @@ in # caveat as every other tier; see nix/host-modules/otel.nix for # the measurement. validateConfigFile = true; - settings = lib.mkMerge [ - { - # `journalctl --follow --lines=0` (what the receiver runs under - # the hood) ships only what is written *after* it starts, so - # every collector start has a silent gap at the front. This - # persists the read cursor, so a restart resumes instead of - # re-opening that gap; `start_at` stays `end` (its default), - # since a cursor covers every run after the first and - # `beginning` without one would re-ship the whole journal on - # every restart. `create_directory` is required for a - # build-time reason rather than a convenience one: - # `validateConfigFile` above runs `otelcol validate` in the nix - # sandbox, before systemd's `StateDirectory=` has created the - # path, and the validator refuses with "directory must exist". - extensions.file_storage = { - directory = "/var/lib/opentelemetry-collector"; - create_directory = true; - }; + settings = { + # `journalctl --follow --lines=0` (what the receiver runs under + # the hood) ships only what is written *after* it starts, so + # every collector start has a silent gap at the front. This + # persists the read cursor, so a restart resumes instead of + # re-opening that gap; `start_at` stays `end` (its default), + # since a cursor covers every run after the first and + # `beginning` without one would re-ship the whole journal on + # every restart. `create_directory` is required for a + # build-time reason rather than a convenience one: + # `validateConfigFile` above runs `otelcol validate` in the nix + # sandbox, before systemd's `StateDirectory=` has created the + # path, and the validator refuses with "directory must exist". + extensions.file_storage = { + directory = "/var/lib/opentelemetry-collector"; + create_directory = true; + }; - receivers.journald = { - # ⚠️ STATED, and it must stay stated: the receiver's own - # default is the RUNTIME journal (`/run/log/journal`), which - # in this container is empty — journald stores persistently - # here, so every entry is under /var/log/journal. Dropping - # this line leaves a collector that validates, starts, - # reports healthy and forwards nothing. - directory = "/var/log/journal"; - # ⚠️ `--merge` is a guard: an unqualified - # `journalctl --follow` over a journal with no entry for the - # current boot (a host-linked one) crash-loops the receiver - # every ~2s with "No journal boot entry found for the - # specified boot (+0)". This journal is container-local, so - # the current boot has an entry and the flag changes nothing. - merge = true; - storage = "file_storage"; - }; + receivers.journald = { + # ⚠️ STATED, and it must stay stated: the receiver's own + # default is the RUNTIME journal (`/run/log/journal`), which + # in this container is empty — journald stores persistently + # here, so every entry is under /var/log/journal. Dropping + # this line leaves a collector that validates, starts, + # reports healthy and forwards nothing. + directory = "/var/log/journal"; + # ⚠️ `--merge` is a guard: an unqualified + # `journalctl --follow` over a journal with no entry for the + # current boot (a host-linked one) crash-loops the receiver + # every ~2s with "No journal boot entry found for the + # specified boot (+0)". This journal is container-local, so + # the current boot has an entry and the flag changes nothing. + merge = true; + storage = "file_storage"; + }; - # The store's scrape, run by this container's own collector. - # - # ⚠️ `metrics_path` and `params` are STATED, and both have to - # be: openbao serves no `/metrics` at all, and - # `/v1/sys/metrics` answers JSON unless the format is asked - # for. A scrape of the default path 404s and a scrape without - # the query parses as nothing — each reads as a dead exporter - # rather than as a wrong request. The spelling matches - # ./swarm-otel.nix's `loopbackScrapeConfig`, the renderer for - # every other service's target. - # - # No credential, unlike the swarm collector's - # `publishedScrapeTargets` jobs: the target is the loopback - # listener above, inside this container's own netns, which is - # what lets the scrape stay unauthenticated without the API - # listener's client-cert requirement being touched. - receivers.prometheus.config.scrape_configs = [ - { - job_name = "bao"; - static_configs = [ { targets = [ metricsScrapeTarget ]; } ]; - metrics_path = "/v1/sys/metrics"; - params.format = [ "prometheus" ]; - } - ]; + # The store's scrape, run by this container's own collector. + # + # ⚠️ `metrics_path` and `params` are STATED, and both have to + # be: openbao serves no `/metrics` at all, and + # `/v1/sys/metrics` answers JSON unless the format is asked + # for. A scrape of the default path 404s and a scrape without + # the query parses as nothing — each reads as a dead exporter + # rather than as a wrong request. The spelling matches + # ./swarm-otel.nix's `loopbackScrapeConfig`, the renderer for + # every other service's target. + # + # No credential, unlike the swarm collector's + # `publishedScrapeTargets` jobs: the target is the loopback + # listener above, inside this container's own netns, which is + # what lets the scrape stay unauthenticated without the API + # listener's client-cert requirement being touched. + receivers.prometheus.config.scrape_configs = [ + { + job_name = "bao"; + static_configs = [ { targets = [ metricsScrapeTarget ]; } ]; + metrics_path = "/v1/sys/metrics"; + params.format = [ "prometheus" ]; + } + ]; - # Identity the hop above cannot supply: a host-side reader can - # say which machine a line came from, and only a collector - # inside this container can say it was the secret store's. No - # `hive` or `swarm` label — the swarm tier upserts `hive` from - # the receiver that accepted the record, precisely so it comes - # from something the sender cannot write. - # - # `collectorServiceName`, not `cfg.machine`: what the - # dashboard selects on is a display identity, not the - # container/receiver name — see that binding's own comment. - processors.resource.attributes = [ - { - key = "service.name"; - value = collectorServiceName; - action = "upsert"; - } - ]; + # Identity the hop above cannot supply: a host-side reader can + # say which machine a line came from, and only a collector + # inside this container can say it was the secret store's. No + # `hive` or `swarm` label — the swarm tier upserts `hive` from + # the receiver that accepted the record, precisely so it comes + # from something the sender cannot write. + # + # `collectorServiceName`, not `cfg.machine`: what the + # dashboard selects on is a display identity, not the + # container/receiver name — see that binding's own comment. + processors.resource.attributes = [ + { + key = "service.name"; + value = collectorServiceName; + action = "upsert"; + } + ]; - # `otlphttp` unconditionally, unlike the agent forwarder's - # protocol-derived exporter name: the swarm receiver speaks - # OTLP/HTTP protobuf whatever this tier's *upstream* - # protocol is, which is the same reason - # ./hive-c0re/environment.nix pins the protocol it hands out. - # `endpoint` is a BASE the exporter appends `/v1/logs` to. - exporters.otlphttp = { - endpoint = otelFirstHop; - # The receiver at that address accepts an OIDC token and - # nothing else (./swarm-otel.nix's `oidc/*` authenticators), - # so this is what makes the hop deliver rather than collect - # 401s — unconditional, because a forwarder that exports - # unauthenticated has no working mode to fall back to. - auth.authenticator = forwarderAuthName; - }; + # `otlphttp` unconditionally, unlike the agent forwarder's + # protocol-derived exporter name: the swarm receiver speaks + # OTLP/HTTP protobuf whatever this tier's *upstream* + # protocol is, which is the same reason + # ./hive-c0re/environment.nix pins the protocol it hands out. + # `endpoint` is a BASE the exporter appends `/v1/logs` to. + exporters.otlphttp = { + endpoint = otelFirstHop; + # The receiver at that address accepts an OIDC token and + # nothing else (./swarm-otel.nix's `oidc/*` authenticators), + # so this is what makes the hop deliver rather than collect + # 401s — unconditional, because a forwarder that exports + # unauthenticated has no working mode to fall back to. + auth.authenticator = forwarderAuthName; + }; - # The push side, opposite in direction to the receiver's - # `oidc/*`: those VALIDATE a token arriving, this OBTAINS one - # to send. Hence `oauth2client`, and hence a client id and - # secret rather than an issuer and an audience to check. - extensions.${forwarderAuthName} = { - client_id = cfg.otel.clientId; - # A path, never a value — and one systemd resolves at - # runtime, so neither the secret nor the credentials - # directory is rendered into the nix store. - client_secret_file = "\${env:CREDENTIALS_DIRECTORY}/${forwarderCredentialId}"; - token_url = "${toString autheliaCfg.url}/api/oidc/token"; - # ⚠️ THE AUDIENCE HAS TO BE REQUESTED, not merely granted. - # Registering it on the client only makes it permissible; a - # token minted without asking carries `aud: []` and the - # receiver refuses it — with a config that reads perfectly - # at both ends. - endpoint_params.audience = cfg.otel.clientId; - }; + # The push side, opposite in direction to the receiver's + # `oidc/*`: those VALIDATE a token arriving, this OBTAINS one + # to send. Hence `oauth2client`, and hence a client id and + # secret rather than an issuer and an audience to check. + extensions.${forwarderAuthName} = { + client_id = cfg.otel.clientId; + # A path, never a value — and one systemd resolves at + # runtime, so neither the secret nor the credentials + # directory is rendered into the nix store. + client_secret_file = "\${env:CREDENTIALS_DIRECTORY}/${forwarderCredentialId}"; + token_url = "${toString autheliaCfg.url}/api/oidc/token"; + # ⚠️ THE AUDIENCE HAS TO BE REQUESTED, not merely granted. + # Registering it on the client only makes it permissible; a + # token minted without asking carries `aud: []` and the + # receiver refuses it — with a config that reads perfectly + # at both ends. + endpoint_params.audience = cfg.otel.clientId; + }; - service.pipelines.logs = { - receivers = [ "journald" ]; - processors = [ "resource" ]; - exporters = [ "otlphttp" ]; - }; + service.pipelines.logs = { + receivers = [ "journald" ]; + processors = [ "resource" ]; + exporters = [ "otlphttp" ]; + }; - # A SECOND pipeline, not a second receiver on the logs one: a - # pipeline is typed, and a prometheus receiver in `logs` is a - # config the collector rejects at startup rather than one that - # quietly ships metrics as log records. - # - # Same `resource` processor and same `otlphttp` exporter, so - # the store's metrics carry the same `service.name` its logs - # do and leave by the same hop — no otel bypass, one exit from - # this container. The exporter treats `endpoint` as a base and - # appends the signal's own path, so `/v1/metrics` here and - # `/v1/logs` above come from the one address. - service.pipelines.metrics = { - receivers = [ "prometheus" ]; - processors = [ "resource" ]; - exporters = [ "otlphttp" ]; - }; + # A SECOND pipeline, not a second receiver on the logs one: a + # pipeline is typed, and a prometheus receiver in `logs` is a + # config the collector rejects at startup rather than one that + # quietly ships metrics as log records. + # + # Same `resource` processor and same `otlphttp` exporter, so + # the store's metrics carry the same `service.name` its logs + # do and leave by the same hop — no otel bypass, one exit from + # this container. The exporter treats `endpoint` as a base and + # appends the signal's own path, so `/v1/metrics` here and + # `/v1/logs` above come from the one address. + service.pipelines.metrics = { + receivers = [ "prometheus" ]; + processors = [ "resource" ]; + exporters = [ "otlphttp" ]; + }; - # An extension configured but not listed here is INERT — the - # receiver's `storage: file_storage` above would name a - # component the collector never starts, and the exporter's - # authenticator would push unauthenticated while the - # collector reports healthy. - service.extensions = [ - "file_storage" - forwarderAuthName - ]; - } - # Off the built-in default of `localhost:8888`, which the hive tier - # holds: this container shares the host's netns. - (import ../collector-self-scrape.nix { - inherit lib; - port = cfg.otel.telemetryPort; - exporter = "otlphttp"; - attributes.container = collectorServiceName; - }) - ]; + # Moves this collector's self-metrics off the built-in + # default of `localhost:8888`, which the hive tier holds — + # this container shares the host's netns, so leaving the + # default in place is the hive collector failing to bind. + # + # ⚠️ `metrics.address` is the spelling that looks right and + # is REJECTED by this collector version: + # `'migration.MetricsConfigV030' has invalid keys: + # address`. `readers` is the schema it accepts, and the + # difference is a startup failure rather than a warning. + service.telemetry.metrics.readers = [ + { + pull.exporter.prometheus = { + host = "127.0.0.1"; + port = cfg.otel.telemetryPort; + }; + } + ]; + + # An extension configured but not listed here is INERT — the + # receiver's `storage: file_storage` above would name a + # component the collector never starts, and the exporter's + # authenticator would push unauthenticated while the + # collector reports healthy. + service.extensions = [ + "file_storage" + forwarderAuthName + ]; + }; }; # How the secret crosses the sandbox. The host unit diff --git a/nix/host-modules/swarm-grafana/dashboards/collector.json b/nix/host-modules/swarm-grafana/dashboards/collector.json index d8d2f3c38..8f1facb57 100644 --- a/nix/host-modules/swarm-grafana/dashboards/collector.json +++ b/nix/host-modules/swarm-grafana/dashboards/collector.json @@ -1,7 +1,7 @@ { "title": "hyperhive \u00b7 collector (self-telemetry)", "uid": "hyperhive-swarm-collector", - "description": "OTEL collectors: accepted, refused, exported, queued. 8888 hive, 8889 swarm, others containers.", + "description": "What the OTEL collectors accept, refuse, export and queue. Port 8888 is the hive, 8889 swarm-otel.", "editable": true, "refresh": "1m", "schemaVersion": 39, diff --git a/nix/module-eval/agent-otel.nix b/nix/module-eval/agent-otel.nix index c7f3b0955..d92614a17 100644 --- a/nix/module-eval/agent-otel.nix +++ b/nix/module-eval/agent-otel.nix @@ -21,7 +21,6 @@ let agent carriesJournaldSeverity runGroup - selfScrapes ; # The log path's three hops, one fixture each. Nothing carries a journal @@ -56,20 +55,6 @@ let agentSettings = machine: machine.services.opentelemetry-collector.settings; cases = [ - { - # The collector's own counters are the only signal that it is dropping - # telemetry. 8904 keeps agents off 8888, which the collector dashboard - # reads as the hive collector; the exporter follows the wire protocol. - name = "an agent forwarder scrapes its own telemetry endpoint on loopback, off the hive collector's port"; - ok = - selfScrapes (agentSettings agentOtel) 8904 { agent = agentOtel.services.hyperhive.agent.user.name; } - && selfScrapes (agentSettings agentOtelGrpc) 8904 { - agent = agentOtelGrpc.services.hyperhive.agent.user.name; - } - # Control: with the switch off the lookup finds no collector config. - && !(agentNoOtel.services.opentelemetry-collector.settings ? receivers) - && (agentSettings agentOtelGrpc).service.pipelines."metrics/self".exporters == [ "otlp" ]; - } { # The journald receiver's own default directory is the RUNTIME # journal, and a container that stores persistently leaves that diff --git a/nix/module-eval/bao-otel-collector.nix b/nix/module-eval/bao-otel-collector.nix index 2e84c632c..565645853 100644 --- a/nix/module-eval/bao-otel-collector.nix +++ b/nix/module-eval/bao-otel-collector.nix @@ -22,7 +22,6 @@ let runGroup baoSettings forceServiceConfigs - selfScrapes ; baoTwoAddresses = hive { @@ -186,20 +185,6 @@ let && m.exporters == [ "otlphttp" ] && m.exporters == s.service.pipelines.logs.exporters; } - { - # The store's collector reports its own health through a pipeline of its - # own: the store pipeline above upserts `service.name`, and sharing it - # would relabel these series away from the collector dashboard. - name = "the store's collector scrapes its own telemetry endpoint on loopback"; - ok = - let - s = (baoForwarder baoWithCollector).settings; - port = baoWithCollector.services.hyperhive.swarm.bao.otel.telemetryPort; - in - selfScrapes s port { container = "bao"; } - && s.service.pipelines.metrics.receivers == [ "prometheus" ] - && s.service.pipelines.metrics.processors == [ "resource" ]; - } { # Presence control for the case above: both fields are omitted rather # than defaulted, so a target declared as bare `host:port` renders what diff --git a/nix/module-eval/forge-placement.nix b/nix/module-eval/forge-placement.nix index da8b5c1e6..c07f3c16f 100644 --- a/nix/module-eval/forge-placement.nix +++ b/nix/module-eval/forge-placement.nix @@ -25,7 +25,6 @@ let }) hive runGroup - selfScrapes ; inherit (shared) bare; @@ -559,10 +558,11 @@ let ok = let c = ciAlone.containers.swarm-ci-runner.config; - s = c.services.opentelemetry-collector.settings; + reader = lib.head c.services.opentelemetry-collector.settings.service.telemetry.metrics.readers; in c.services.hyperhive.swarmJournalForwarder.telemetryPort == 8895 - && selfScrapes s 8895 { container = "ci-runner"; }; + && reader.pull.exporter.prometheus.host == "127.0.0.1" + && reader.pull.exporter.prometheus.port == 8895; } ]; in diff --git a/nix/module-eval/lib.nix b/nix/module-eval/lib.nix index b9f84ac54..5205e2f88 100644 --- a/nix/module-eval/lib.nix +++ b/nix/module-eval/lib.nix @@ -208,37 +208,6 @@ let # the two tier cases differ only in which receiver they hand it. carriesJournaldSeverity = receiver: lib.elem journaldSeverityParser (receiver.operators or [ ]); - # Whether a container collector's settings carry the self-scrape of - # ../collector-self-scrape.nix: the telemetry endpoint on loopback at `port`, - # a job scraping exactly that address, and a metrics pipeline that reads the - # job and leaves through an exporter the settings define. `attrs` are the - # resource attributes the pipeline stamps. - selfScrapes = - s: port: attrs: - let - job = lib.findFirst (j: j.job_name == "collector") null ( - s.receivers."prometheus/self".config.scrape_configs or [ ] - ); - pipeline = s.service.pipelines."metrics/self" or null; - reader = lib.head (s.service.telemetry.metrics.readers or [ { } ]); - stamped = lib.mapAttrsToList (key: value: { - inherit key value; - action = "upsert"; - }) attrs; - in - job != null - && job.static_configs == [ { targets = [ "127.0.0.1:${toString port}" ]; } ] - && - (reader.pull.exporter.prometheus or null) == { - host = "127.0.0.1"; - inherit port; - } - && pipeline != null - && pipeline.receivers == [ "prometheus/self" ] - && (s.processors."resource/self".attributes or null) == stamped - && pipeline.exporters != [ ] - && lib.all (e: s.exporters ? ${e}) pipeline.exporters; - # The enables that switch owns. The hive tier's collector is not among them: it # follows `deploy.hive-controller.enable`, and a services host without a hive # runs none. @@ -314,7 +283,6 @@ in inherit agentHarness; inherit journaldSeverityParser; inherit carriesJournaldSeverity; - inherit selfScrapes; inherit swarmServiceEnables; inherit runGroup; inherit forceServiceConfigs; diff --git a/nix/module-eval/nats-authelia.nix b/nix/module-eval/nats-authelia.nix index 56baa5ffa..d21bc6466 100644 --- a/nix/module-eval/nats-authelia.nix +++ b/nix/module-eval/nats-authelia.nix @@ -20,7 +20,6 @@ let }) hive runGroup - selfScrapes ; # No store identity, which the first case below relies on. @@ -158,10 +157,12 @@ let h.services.hyperhive.swarm.nats.otel.telemetryPort ] ++ map (o: o.port) (lib.filter (o: o.machine != svc.machine) forwarderServices); + reader = lib.head s.service.telemetry.metrics.readers; in c.services.hyperhive.swarmJournalForwarder.telemetryPort == svc.port && !(lib.elem svc.port taken) - && selfScrapes s svc.port { container = svc.name; }; + && reader.pull.exporter.prometheus.host == "127.0.0.1" + && reader.pull.exporter.prometheus.port == svc.port; } ] ) forwarderServices; @@ -268,10 +269,12 @@ let h.services.hyperhive.swarm.nats.otel.telemetryPort ] ++ map (o: o.port) (lib.filter (o: o.machine != svc.machine) hostNetnsForwarders); + reader = lib.head s.service.telemetry.metrics.readers; in c.services.hyperhive.swarmJournalForwarder.telemetryPort == svc.port && !(lib.elem svc.port taken) - && selfScrapes s svc.port { container = svc.name; }; + && reader.pull.exporter.prometheus.host == "127.0.0.1" + && reader.pull.exporter.prometheus.port == svc.port; } ] ) hostNetnsForwarders; @@ -737,7 +740,6 @@ let && s.extensions."oauth2client/swarm".endpoint_params.audience == clientId && lib.elem "oauth2client/swarm" s.service.extensions && s.service.pipelines.logs.receivers == [ "journald" ] - && selfScrapes s h.services.hyperhive.swarm.nats.otel.telemetryPort { container = "nats"; } && lib.elem "oidc-client-secret:/run/swarm-journal-forwarder-oidc/client-secret" c.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential && fetch.environment.BAO_CLIENT_CERT == "/var/lib/swarm-nats-auth/bao-client.pem" && lib.hasInfix "bao kv get -field=value secret/swarm/services/${clientId}/oidc/client" fetch.script @@ -995,7 +997,7 @@ let registered = lib.findFirst ( x: x.id == clientId ) null h.services.hyperhive.swarm.authelia.oidc.clients; - port = c.services.hyperhive.swarmJournalForwarder.telemetryPort; + reader = lib.head s.service.telemetry.metrics.readers; taken = [ 8888 8890 @@ -1020,8 +1022,8 @@ let && registered.audience == [ clientId ] && !(lib.elem "--link-journal=host" container.extraFlags) && lib.any (lib.hasPrefix "--load-credential=publisher-bao-cert:") container.extraFlags - && selfScrapes s port { container = "authelia"; } - && !(lib.elem port taken) + && reader.pull.exporter.prometheus.host == "127.0.0.1" + && !(lib.elem reader.pull.exporter.prometheus.port taken) && !autheliaColocated.containers.swarm-authelia.config.services.opentelemetry-collector.enable && !( autheliaColocated.containers.swarm-authelia.config.systemd.services ? swarm-journal-forwarder-oidc