diff --git a/docs/swarm/services.md b/docs/swarm/services.md index 406461a5..7523f31f 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -219,10 +219,12 @@ and failure series on `collector` appear once something fails, and the collector and of the swarm's collector, which both carry the collector binary's default `service.name`, `otelcol-contrib`. The timeseries panels split by `server.port`: 8888 is the hive collector, 8889 the swarm -collector. The per-container collectors (the secret store, NATS, Grafana, -Authelia, the Victoria stores and the agents' forwarders) don't report -their own metrics yet, so they have no lines here; a separate issue on -the forge covers that. `scripts/check-dashboard-service-names.sh` runs in CI +collector. Every per-container collector (the secret store, the other service +containers and the agents' forwarders) also scrapes its own endpoint on +loopback and pushes the result up its normal hop, so each has a line of its +own, on its own port: agents all report on 8904, service containers on +8890 to 8903. The series carry `container` (services) or `agent` (agents) +to tell them apart. `scripts/check-dashboard-service-names.sh` runs in CI and in `scripts/pre-push`. It fails when a dashboard query selects a `service.name` that nothing emits. diff --git a/nix/agent-modules/otel.nix b/nix/agent-modules/otel.nix index 5a622b81..1feaa003 100644 --- a/nix/agent-modules/otel.nix +++ b/nix/agent-modules/otel.nix @@ -72,6 +72,11 @@ let OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE = "cumulative"; }; + # Where the collector serves its own metrics, inside the container's own + # netns. Not the binary's default 8888, which the collector dashboard + # reads as the hive collector's. + selfTelemetryPort = 8904; + # The exporter's NAME picks the wire protocol, so `protocol` has to be # read here as well as handed to the SDKs above — same expression as the # swarm tier's upstream exporter in nix/host-modules/swarm-otel.nix. @@ -296,5 +301,14 @@ in service.extensions = [ "file_storage" ]; }; }; + + # Every agent container has a network namespace of its own, so they can all + # use the same port. + services.opentelemetry-collector.settings = import ../collector-self-scrape.nix { + inherit lib; + port = selfTelemetryPort; + exporter = logExporter; + attributes.agent = userName; + }; }; } diff --git a/nix/collector-self-scrape.nix b/nix/collector-self-scrape.nix new file mode 100644 index 00000000..12cf796c --- /dev/null +++ b/nix/collector-self-scrape.nix @@ -0,0 +1,52 @@ +# Collector settings that make a container's OpenTelemetry collector report its +# own health (refused and failed records, exporter queue, memory), imported by +# every container collector: the agent forwarder, the service journal +# forwarder and the secret store's collector. The two host-side tiers scrape +# themselves through their `scrapeTargets` instead. +# +# One function renders both halves, so the endpoint and the job that reads it +# cannot name different ports. The endpoint binds loopback: the collector +# scrapes itself, and nothing outside the container has a reason to. +# +# The job runs in its own pipeline. A container's other metrics pipeline may +# carry a `resource` processor that upserts `service.name`, which would +# relabel these series away from the `otelcol-contrib` the collector dashboard +# selects on. The attributes stamped here tell the container's collectors +# apart. +# +# `readers`, not `metrics.address`: this collector version rejects `address` +# at startup. +{ + lib, + port, + exporter, + attributes, +}: +{ + service.telemetry.metrics.readers = [ + { + pull.exporter.prometheus = { + host = "127.0.0.1"; + inherit port; + }; + } + ]; + + receivers."prometheus/self".config.scrape_configs = [ + { + job_name = "collector"; + static_configs = [ { targets = [ "127.0.0.1:${toString port}" ]; } ]; + } + ]; + + processors."resource/self".attributes = lib.mapAttrsToList (key: value: { + inherit key value; + action = "upsert"; + }) attributes; + + service.pipelines."metrics/self" = { + receivers = [ "prometheus/self" ]; + processors = [ "resource/self" ]; + exporters = [ exporter ]; + }; +} diff --git a/nix/container-modules/swarm-journal-forwarder.nix b/nix/container-modules/swarm-journal-forwarder.nix index 952d9df5..1348d00d 100644 --- a/nix/container-modules/swarm-journal-forwarder.nix +++ b/nix/container-modules/swarm-journal-forwarder.nix @@ -96,8 +96,9 @@ in type = lib.types.port; description = '' Loopback port for the collector's own metrics (refused and dropped - records, exporter failures). Required because the container may share - the host netns, where the binary's default 8888 is the hive tier's. + records, exporter failures), which it scrapes and pushes to the swarm + collector. Required because the container may share the host netns, + where the binary's default 8888 is the hive tier's. ''; }; @@ -242,17 +243,6 @@ in exporters = [ "otlphttp" ]; }; - # ⚠️ `readers`, not `metrics.address`: this collector version rejects - # `address` at startup. - service.telemetry.metrics.readers = [ - { - pull.exporter.prometheus = { - host = "127.0.0.1"; - port = cfg.telemetryPort; - }; - } - ]; - # An extension not listed here is inert: the receiver's storage would # name a component that never starts, and the exporter would push # unauthenticated. @@ -261,6 +251,12 @@ in authName ]; } + (import ../collector-self-scrape.nix { + inherit lib; + port = cfg.telemetryPort; + exporter = "otlphttp"; + attributes.container = cfg.serviceName; + }) # A separate set the module system merges in: joining it to the first with # `//` would replace the first set's whole `service` attrset. (lib.mkIf (cfg.otlpMetricsPort != null) { diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 7a79b2e4..4307c39e 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -4575,25 +4575,6 @@ in exporters = [ "otlphttp" ]; }; - # Moves this collector's self-metrics off the built-in - # default of `localhost:8888`, which the hive tier holds — - # this container shares the host's netns, so leaving the - # default in place is the hive collector failing to bind. - # - # ⚠️ `metrics.address` is the spelling that looks right and - # is REJECTED by this collector version: - # `'migration.MetricsConfigV030' has invalid keys: - # address`. `readers` is the schema it accepts, and the - # difference is a startup failure rather than a warning. - service.telemetry.metrics.readers = [ - { - pull.exporter.prometheus = { - host = "127.0.0.1"; - port = cfg.otel.telemetryPort; - }; - } - ]; - # An extension configured but not listed here is INERT — the # receiver's `storage: file_storage` above would name a # component the collector never starts, and the exporter's @@ -4606,6 +4587,15 @@ in }; }; + # Off the built-in default of `localhost:8888`, which the hive tier + # holds: this container shares the host's netns. + services.opentelemetry-collector.settings = import ../collector-self-scrape.nix { + inherit lib; + port = cfg.otel.telemetryPort; + exporter = "otlphttp"; + attributes.container = collectorServiceName; + }; + # How the secret crosses the sandbox. The host unit # `swarm-bao-forwarder-oidc` leaves a root-owned 0400 file at this # path on the host's /run, which the container bind-mounts diff --git a/nix/host-modules/swarm-grafana/dashboards/collector.json b/nix/host-modules/swarm-grafana/dashboards/collector.json index 8f1facb5..d8d2f3c3 100644 --- a/nix/host-modules/swarm-grafana/dashboards/collector.json +++ b/nix/host-modules/swarm-grafana/dashboards/collector.json @@ -1,7 +1,7 @@ { "title": "hyperhive \u00b7 collector (self-telemetry)", "uid": "hyperhive-swarm-collector", - "description": "What the OTEL collectors accept, refuse, export and queue. Port 8888 is the hive, 8889 swarm-otel.", + "description": "OTEL collectors: accepted, refused, exported, queued. 8888 hive, 8889 swarm, others containers.", "editable": true, "refresh": "1m", "schemaVersion": 39, diff --git a/nix/module-eval/agent-otel.nix b/nix/module-eval/agent-otel.nix index d92614a1..c7f3b095 100644 --- a/nix/module-eval/agent-otel.nix +++ b/nix/module-eval/agent-otel.nix @@ -21,6 +21,7 @@ let agent carriesJournaldSeverity runGroup + selfScrapes ; # The log path's three hops, one fixture each. Nothing carries a journal @@ -55,6 +56,20 @@ let agentSettings = machine: machine.services.opentelemetry-collector.settings; cases = [ + { + # The collector's own counters are the only signal that it is dropping + # telemetry. 8904 keeps agents off 8888, which the collector dashboard + # reads as the hive collector; the exporter follows the wire protocol. + name = "an agent forwarder scrapes its own telemetry endpoint on loopback, off the hive collector's port"; + ok = + selfScrapes (agentSettings agentOtel) 8904 { agent = agentOtel.services.hyperhive.agent.user.name; } + && selfScrapes (agentSettings agentOtelGrpc) 8904 { + agent = agentOtelGrpc.services.hyperhive.agent.user.name; + } + # Control: with the switch off the lookup finds no collector config. + && !(agentNoOtel.services.opentelemetry-collector.settings ? receivers) + && (agentSettings agentOtelGrpc).service.pipelines."metrics/self".exporters == [ "otlp" ]; + } { # The journald receiver's own default directory is the RUNTIME # journal, and a container that stores persistently leaves that diff --git a/nix/module-eval/bao-otel-collector.nix b/nix/module-eval/bao-otel-collector.nix index 56564585..2e84c632 100644 --- a/nix/module-eval/bao-otel-collector.nix +++ b/nix/module-eval/bao-otel-collector.nix @@ -22,6 +22,7 @@ let runGroup baoSettings forceServiceConfigs + selfScrapes ; baoTwoAddresses = hive { @@ -185,6 +186,20 @@ let && m.exporters == [ "otlphttp" ] && m.exporters == s.service.pipelines.logs.exporters; } + { + # The store's collector reports its own health through a pipeline of its + # own: the store pipeline above upserts `service.name`, and sharing it + # would relabel these series away from the collector dashboard. + name = "the store's collector scrapes its own telemetry endpoint on loopback"; + ok = + let + s = (baoForwarder baoWithCollector).settings; + port = baoWithCollector.services.hyperhive.swarm.bao.otel.telemetryPort; + in + selfScrapes s port { container = "bao"; } + && s.service.pipelines.metrics.receivers == [ "prometheus" ] + && s.service.pipelines.metrics.processors == [ "resource" ]; + } { # Presence control for the case above: both fields are omitted rather # than defaulted, so a target declared as bare `host:port` renders what diff --git a/nix/module-eval/forge-placement.nix b/nix/module-eval/forge-placement.nix index c07f3c16..da8b5c1e 100644 --- a/nix/module-eval/forge-placement.nix +++ b/nix/module-eval/forge-placement.nix @@ -25,6 +25,7 @@ let }) hive runGroup + selfScrapes ; inherit (shared) bare; @@ -558,11 +559,10 @@ let ok = let c = ciAlone.containers.swarm-ci-runner.config; - reader = lib.head c.services.opentelemetry-collector.settings.service.telemetry.metrics.readers; + s = c.services.opentelemetry-collector.settings; in c.services.hyperhive.swarmJournalForwarder.telemetryPort == 8895 - && reader.pull.exporter.prometheus.host == "127.0.0.1" - && reader.pull.exporter.prometheus.port == 8895; + && selfScrapes s 8895 { container = "ci-runner"; }; } ]; in diff --git a/nix/module-eval/lib.nix b/nix/module-eval/lib.nix index 5205e2f8..b9f84ac5 100644 --- a/nix/module-eval/lib.nix +++ b/nix/module-eval/lib.nix @@ -208,6 +208,37 @@ let # the two tier cases differ only in which receiver they hand it. carriesJournaldSeverity = receiver: lib.elem journaldSeverityParser (receiver.operators or [ ]); + # Whether a container collector's settings carry the self-scrape of + # ../collector-self-scrape.nix: the telemetry endpoint on loopback at `port`, + # a job scraping exactly that address, and a metrics pipeline that reads the + # job and leaves through an exporter the settings define. `attrs` are the + # resource attributes the pipeline stamps. + selfScrapes = + s: port: attrs: + let + job = lib.findFirst (j: j.job_name == "collector") null ( + s.receivers."prometheus/self".config.scrape_configs or [ ] + ); + pipeline = s.service.pipelines."metrics/self" or null; + reader = lib.head (s.service.telemetry.metrics.readers or [ { } ]); + stamped = lib.mapAttrsToList (key: value: { + inherit key value; + action = "upsert"; + }) attrs; + in + job != null + && job.static_configs == [ { targets = [ "127.0.0.1:${toString port}" ]; } ] + && + (reader.pull.exporter.prometheus or null) == { + host = "127.0.0.1"; + inherit port; + } + && pipeline != null + && pipeline.receivers == [ "prometheus/self" ] + && (s.processors."resource/self".attributes or null) == stamped + && pipeline.exporters != [ ] + && lib.all (e: s.exporters ? ${e}) pipeline.exporters; + # The enables that switch owns. The hive tier's collector is not among them: it # follows `deploy.hive-controller.enable`, and a services host without a hive # runs none. @@ -283,6 +314,7 @@ in inherit agentHarness; inherit journaldSeverityParser; inherit carriesJournaldSeverity; + inherit selfScrapes; inherit swarmServiceEnables; inherit runGroup; inherit forceServiceConfigs; diff --git a/nix/module-eval/nats-authelia.nix b/nix/module-eval/nats-authelia.nix index d21bc646..56baa5ff 100644 --- a/nix/module-eval/nats-authelia.nix +++ b/nix/module-eval/nats-authelia.nix @@ -20,6 +20,7 @@ let }) hive runGroup + selfScrapes ; # No store identity, which the first case below relies on. @@ -157,12 +158,10 @@ let h.services.hyperhive.swarm.nats.otel.telemetryPort ] ++ map (o: o.port) (lib.filter (o: o.machine != svc.machine) forwarderServices); - reader = lib.head s.service.telemetry.metrics.readers; in c.services.hyperhive.swarmJournalForwarder.telemetryPort == svc.port && !(lib.elem svc.port taken) - && reader.pull.exporter.prometheus.host == "127.0.0.1" - && reader.pull.exporter.prometheus.port == svc.port; + && selfScrapes s svc.port { container = svc.name; }; } ] ) forwarderServices; @@ -269,12 +268,10 @@ let h.services.hyperhive.swarm.nats.otel.telemetryPort ] ++ map (o: o.port) (lib.filter (o: o.machine != svc.machine) hostNetnsForwarders); - reader = lib.head s.service.telemetry.metrics.readers; in c.services.hyperhive.swarmJournalForwarder.telemetryPort == svc.port && !(lib.elem svc.port taken) - && reader.pull.exporter.prometheus.host == "127.0.0.1" - && reader.pull.exporter.prometheus.port == svc.port; + && selfScrapes s svc.port { container = svc.name; }; } ] ) hostNetnsForwarders; @@ -740,6 +737,7 @@ let && s.extensions."oauth2client/swarm".endpoint_params.audience == clientId && lib.elem "oauth2client/swarm" s.service.extensions && s.service.pipelines.logs.receivers == [ "journald" ] + && selfScrapes s h.services.hyperhive.swarm.nats.otel.telemetryPort { container = "nats"; } && lib.elem "oidc-client-secret:/run/swarm-journal-forwarder-oidc/client-secret" c.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential && fetch.environment.BAO_CLIENT_CERT == "/var/lib/swarm-nats-auth/bao-client.pem" && lib.hasInfix "bao kv get -field=value secret/swarm/services/${clientId}/oidc/client" fetch.script @@ -997,7 +995,7 @@ let registered = lib.findFirst ( x: x.id == clientId ) null h.services.hyperhive.swarm.authelia.oidc.clients; - reader = lib.head s.service.telemetry.metrics.readers; + port = c.services.hyperhive.swarmJournalForwarder.telemetryPort; taken = [ 8888 8890 @@ -1022,8 +1020,8 @@ let && registered.audience == [ clientId ] && !(lib.elem "--link-journal=host" container.extraFlags) && lib.any (lib.hasPrefix "--load-credential=publisher-bao-cert:") container.extraFlags - && reader.pull.exporter.prometheus.host == "127.0.0.1" - && !(lib.elem reader.pull.exporter.prometheus.port taken) + && selfScrapes s port { container = "authelia"; } + && !(lib.elem port taken) && !autheliaColocated.containers.swarm-authelia.config.services.opentelemetry-collector.enable && !( autheliaColocated.containers.swarm-authelia.config.systemd.services ? swarm-journal-forwarder-oidc