container collectors scrape their own telemetry and push it up their hop

docs/swarm/services.md

@@ -219,10 +219,12 @@ and failure series on `collector` appear once something fails, and the
 collector and of the swarm's collector, which both carry the collector
 binary's default `service.name`, `otelcol-contrib`. The timeseries panels
 split by `server.port`: 8888 is the hive collector, 8889 the swarm
-collector. The per-container collectors (the secret store, NATS, Grafana,
+collector. Every per-container collector (the secret store, the other service
-Authelia, the Victoria stores and the agents' forwarders) don't report
+containers and the agents' forwarders) also scrapes its own endpoint on
-their own metrics yet, so they have no lines here; a separate issue on
+loopback and pushes the result up its normal hop, so each has a line of its
-the forge covers that. `scripts/check-dashboard-service-names.sh` runs in CI
+own, on its own port: agents all report on 8904, service containers on
+8890 to 8903. The series carry `container` (services) or `agent` (agents)
+to tell them apart. `scripts/check-dashboard-service-names.sh` runs in CI
 and in `scripts/pre-push`. It fails when a dashboard query selects a
 `service.name` that nothing emits.
 

nix/agent-modules/otel.nix

@@ -72,6 +72,11 @@ let
     OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE = "cumulative";
   };
 
+  # Where the collector serves its own metrics, inside the container's own
+  # netns. Not the binary's default 8888, which the collector dashboard
+  # reads as the hive collector's.
+  selfTelemetryPort = 8904;
+
   # The exporter's NAME picks the wire protocol, so `protocol` has to be
   # read here as well as handed to the SDKs above — same expression as the
   # swarm tier's upstream exporter in nix/host-modules/swarm-otel.nix.
@@ -296,5 +301,14 @@ in
         service.extensions = [ "file_storage" ];
       };
     };
+
+    # Every agent container has a network namespace of its own, so they can all
+    # use the same port.
+    services.opentelemetry-collector.settings = import ../collector-self-scrape.nix {
+      inherit lib;
+      port = selfTelemetryPort;
+      exporter = logExporter;
+      attributes.agent = userName;
+    };
   };
 }

nix/collector-self-scrape.nix

@@ -0,0 +1,52 @@
+# Collector settings that make a container's OpenTelemetry collector report its
+# own health (refused and failed records, exporter queue, memory), imported by
+# every container collector: the agent forwarder, the service journal
+# forwarder and the secret store's collector. The two host-side tiers scrape
+# themselves through their `scrapeTargets` instead.
+#
+# One function renders both halves, so the endpoint and the job that reads it
+# cannot name different ports. The endpoint binds loopback: the collector
+# scrapes itself, and nothing outside the container has a reason to.
+#
+# The job runs in its own pipeline. A container's other metrics pipeline may
+# carry a `resource` processor that upserts `service.name`, which would
+# relabel these series away from the `otelcol-contrib` the collector dashboard
+# selects on. The attributes stamped here tell the container's collectors
+# apart.
+#
+# `readers`, not `metrics.address`: this collector version rejects `address`
+# at startup.
+{
+  lib,
+  port,
+  exporter,
+  attributes,
+}:
+{
+  service.telemetry.metrics.readers = [
+    {
+      pull.exporter.prometheus = {
+        host = "127.0.0.1";
+        inherit port;
+      };
+    }
+  ];
+
+  receivers."prometheus/self".config.scrape_configs = [
+    {
+      job_name = "collector";
+      static_configs = [ { targets = [ "127.0.0.1:${toString port}" ]; } ];
+    }
+  ];
+
+  processors."resource/self".attributes = lib.mapAttrsToList (key: value: {
+    inherit key value;
+    action = "upsert";
+  }) attributes;
+
+  service.pipelines."metrics/self" = {
+    receivers = [ "prometheus/self" ];
+    processors = [ "resource/self" ];
+    exporters = [ exporter ];
+  };
+}

nix/container-modules/swarm-journal-forwarder.nix

@@ -96,8 +96,9 @@ in
       type = lib.types.port;
       description = ''
         Loopback port for the collector's own metrics (refused and dropped
-        records, exporter failures). Required because the container may share
+        records, exporter failures), which it scrapes and pushes to the swarm
-        the host netns, where the binary's default 8888 is the hive tier's.
+        collector. Required because the container may share the host netns,
+        where the binary's default 8888 is the hive tier's.
       '';
     };
 
@@ -242,17 +243,6 @@ in
             exporters = [ "otlphttp" ];
           };
 
-          # ⚠️ `readers`, not `metrics.address`: this collector version rejects
-          # `address` at startup.
-          service.telemetry.metrics.readers = [
-            {
-              pull.exporter.prometheus = {
-                host = "127.0.0.1";
-                port = cfg.telemetryPort;
-              };
-            }
-          ];
-
           # An extension not listed here is inert: the receiver's storage would
           # name a component that never starts, and the exporter would push
           # unauthenticated.
@@ -261,6 +251,12 @@ in
             authName
           ];
         }
+        (import ../collector-self-scrape.nix {
+          inherit lib;
+          port = cfg.telemetryPort;
+          exporter = "otlphttp";
+          attributes.container = cfg.serviceName;
+        })
         # A separate set the module system merges in: joining it to the first with
         # `//` would replace the first set's whole `service` attrset.
         (lib.mkIf (cfg.otlpMetricsPort != null) {

nix/host-modules/swarm-bao.nix

@@ -4575,25 +4575,6 @@ in
                   exporters = [ "otlphttp" ];
                 };
 
-                # Moves this collector's self-metrics off the built-in
-                # default of `localhost:8888`, which the hive tier holds —
-                # this container shares the host's netns, so leaving the
-                # default in place is the hive collector failing to bind.
-                #
-                # ⚠️ `metrics.address` is the spelling that looks right and
-                # is REJECTED by this collector version:
-                # `'migration.MetricsConfigV030' has invalid keys:
-                # address`. `readers` is the schema it accepts, and the
-                # difference is a startup failure rather than a warning.
-                service.telemetry.metrics.readers = [
-                  {
-                    pull.exporter.prometheus = {
-                      host = "127.0.0.1";
-                      port = cfg.otel.telemetryPort;
-                    };
-                  }
-                ];
-
                 # An extension configured but not listed here is INERT — the
                 # receiver's `storage: file_storage` above would name a
                 # component the collector never starts, and the exporter's
@@ -4606,6 +4587,15 @@ in
               };
             };
 
+            # Off the built-in default of `localhost:8888`, which the hive tier
+            # holds: this container shares the host's netns.
+            services.opentelemetry-collector.settings = import ../collector-self-scrape.nix {
+              inherit lib;
+              port = cfg.otel.telemetryPort;
+              exporter = "otlphttp";
+              attributes.container = collectorServiceName;
+            };
+
             # How the secret crosses the sandbox. The host unit
             # `swarm-bao-forwarder-oidc` leaves a root-owned 0400 file at this
             # path on the host's /run, which the container bind-mounts

nix/host-modules/swarm-grafana/dashboards/collector.json

@@ -1,7 +1,7 @@
 {
   "title": "hyperhive \u00b7 collector (self-telemetry)",
   "uid": "hyperhive-swarm-collector",
-  "description": "What the OTEL collectors accept, refuse, export and queue. Port 8888 is the hive, 8889 swarm-otel.",
+  "description": "OTEL collectors: accepted, refused, exported, queued. 8888 hive, 8889 swarm, others containers.",
   "editable": true,
   "refresh": "1m",
   "schemaVersion": 39,

nix/module-eval/agent-otel.nix

@@ -21,6 +21,7 @@ let
     agent
     carriesJournaldSeverity
     runGroup
+    selfScrapes
     ;
 
   # The log path's three hops, one fixture each. Nothing carries a journal
@@ -55,6 +56,20 @@ let
 
   agentSettings = machine: machine.services.opentelemetry-collector.settings;
   cases = [
+    {
+      # The collector's own counters are the only signal that it is dropping
+      # telemetry. 8904 keeps agents off 8888, which the collector dashboard
+      # reads as the hive collector; the exporter follows the wire protocol.
+      name = "an agent forwarder scrapes its own telemetry endpoint on loopback, off the hive collector's port";
+      ok =
+        selfScrapes (agentSettings agentOtel) 8904 { agent = agentOtel.services.hyperhive.agent.user.name; }
+        && selfScrapes (agentSettings agentOtelGrpc) 8904 {
+          agent = agentOtelGrpc.services.hyperhive.agent.user.name;
+        }
+        # Control: with the switch off the lookup finds no collector config.
+        && !(agentNoOtel.services.opentelemetry-collector.settings ? receivers)
+        && (agentSettings agentOtelGrpc).service.pipelines."metrics/self".exporters == [ "otlp" ];
+    }
     {
       # The journald receiver's own default directory is the RUNTIME
       # journal, and a container that stores persistently leaves that

nix/module-eval/bao-otel-collector.nix

@@ -22,6 +22,7 @@ let
     runGroup
     baoSettings
     forceServiceConfigs
+    selfScrapes
     ;
 
   baoTwoAddresses = hive {
@@ -185,6 +186,20 @@ let
         && m.exporters == [ "otlphttp" ]
         && m.exporters == s.service.pipelines.logs.exporters;
     }
+    {
+      # The store's collector reports its own health through a pipeline of its
+      # own: the store pipeline above upserts `service.name`, and sharing it
+      # would relabel these series away from the collector dashboard.
+      name = "the store's collector scrapes its own telemetry endpoint on loopback";
+      ok =
+        let
+          s = (baoForwarder baoWithCollector).settings;
+          port = baoWithCollector.services.hyperhive.swarm.bao.otel.telemetryPort;
+        in
+        selfScrapes s port { container = "bao"; }
+        && s.service.pipelines.metrics.receivers == [ "prometheus" ]
+        && s.service.pipelines.metrics.processors == [ "resource" ];
+    }
     {
       # Presence control for the case above: both fields are omitted rather
       # than defaulted, so a target declared as bare `host:port` renders what

nix/module-eval/forge-placement.nix

@@ -25,6 +25,7 @@ let
     })
     hive
     runGroup
+    selfScrapes
     ;
 
   inherit (shared) bare;
@@ -558,11 +559,10 @@ let
       ok =
         let
           c = ciAlone.containers.swarm-ci-runner.config;
-          reader = lib.head c.services.opentelemetry-collector.settings.service.telemetry.metrics.readers;
+          s = c.services.opentelemetry-collector.settings;
         in
         c.services.hyperhive.swarmJournalForwarder.telemetryPort == 8895
-        && reader.pull.exporter.prometheus.host == "127.0.0.1"
+        && selfScrapes s 8895 { container = "ci-runner"; };
-        && reader.pull.exporter.prometheus.port == 8895;
     }
   ];
 in

nix/module-eval/lib.nix

@@ -208,6 +208,37 @@ let
   # the two tier cases differ only in which receiver they hand it.
   carriesJournaldSeverity = receiver: lib.elem journaldSeverityParser (receiver.operators or [ ]);
 
+  # Whether a container collector's settings carry the self-scrape of
+  # ../collector-self-scrape.nix: the telemetry endpoint on loopback at `port`,
+  # a job scraping exactly that address, and a metrics pipeline that reads the
+  # job and leaves through an exporter the settings define. `attrs` are the
+  # resource attributes the pipeline stamps.
+  selfScrapes =
+    s: port: attrs:
+    let
+      job = lib.findFirst (j: j.job_name == "collector") null (
+        s.receivers."prometheus/self".config.scrape_configs or [ ]
+      );
+      pipeline = s.service.pipelines."metrics/self" or null;
+      reader = lib.head (s.service.telemetry.metrics.readers or [ { } ]);
+      stamped = lib.mapAttrsToList (key: value: {
+        inherit key value;
+        action = "upsert";
+      }) attrs;
+    in
+    job != null
+    && job.static_configs == [ { targets = [ "127.0.0.1:${toString port}" ]; } ]
+    &&
+      (reader.pull.exporter.prometheus or null) == {
+        host = "127.0.0.1";
+        inherit port;
+      }
+    && pipeline != null
+    && pipeline.receivers == [ "prometheus/self" ]
+    && (s.processors."resource/self".attributes or null) == stamped
+    && pipeline.exporters != [ ]
+    && lib.all (e: s.exporters ? ${e}) pipeline.exporters;
+
   # The enables that switch owns. The hive tier's collector is not among them: it
   # follows `deploy.hive-controller.enable`, and a services host without a hive
   # runs none.
@@ -283,6 +314,7 @@ in
   inherit agentHarness;
   inherit journaldSeverityParser;
   inherit carriesJournaldSeverity;
+  inherit selfScrapes;
   inherit swarmServiceEnables;
   inherit runGroup;
   inherit forceServiceConfigs;

nix/module-eval/nats-authelia.nix

@@ -20,6 +20,7 @@ let
     })
     hive
     runGroup
+    selfScrapes
     ;
 
   # No store identity, which the first case below relies on.
@@ -157,12 +158,10 @@ let
               h.services.hyperhive.swarm.nats.otel.telemetryPort
             ]
             ++ map (o: o.port) (lib.filter (o: o.machine != svc.machine) forwarderServices);
-            reader = lib.head s.service.telemetry.metrics.readers;
           in
           c.services.hyperhive.swarmJournalForwarder.telemetryPort == svc.port
           && !(lib.elem svc.port taken)
-          && reader.pull.exporter.prometheus.host == "127.0.0.1"
+          && selfScrapes s svc.port { container = svc.name; };
-          && reader.pull.exporter.prometheus.port == svc.port;
       }
     ]
   ) forwarderServices;
@@ -269,12 +268,10 @@ let
               h.services.hyperhive.swarm.nats.otel.telemetryPort
             ]
             ++ map (o: o.port) (lib.filter (o: o.machine != svc.machine) hostNetnsForwarders);
-            reader = lib.head s.service.telemetry.metrics.readers;
           in
           c.services.hyperhive.swarmJournalForwarder.telemetryPort == svc.port
           && !(lib.elem svc.port taken)
-          && reader.pull.exporter.prometheus.host == "127.0.0.1"
+          && selfScrapes s svc.port { container = svc.name; };
-          && reader.pull.exporter.prometheus.port == svc.port;
       }
     ]
   ) hostNetnsForwarders;
@@ -740,6 +737,7 @@ let
             && s.extensions."oauth2client/swarm".endpoint_params.audience == clientId
             && lib.elem "oauth2client/swarm" s.service.extensions
             && s.service.pipelines.logs.receivers == [ "journald" ]
+            && selfScrapes s h.services.hyperhive.swarm.nats.otel.telemetryPort { container = "nats"; }
             && lib.elem "oidc-client-secret:/run/swarm-journal-forwarder-oidc/client-secret" c.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential
             && fetch.environment.BAO_CLIENT_CERT == "/var/lib/swarm-nats-auth/bao-client.pem"
             && lib.hasInfix "bao kv get -field=value secret/swarm/services/${clientId}/oidc/client" fetch.script
@@ -997,7 +995,7 @@ let
           registered = lib.findFirst (
             x: x.id == clientId
           ) null h.services.hyperhive.swarm.authelia.oidc.clients;
-          reader = lib.head s.service.telemetry.metrics.readers;
+          port = c.services.hyperhive.swarmJournalForwarder.telemetryPort;
           taken = [
             8888
             8890
@@ -1022,8 +1020,8 @@ let
         && registered.audience == [ clientId ]
         && !(lib.elem "--link-journal=host" container.extraFlags)
         && lib.any (lib.hasPrefix "--load-credential=publisher-bao-cert:") container.extraFlags
-        && reader.pull.exporter.prometheus.host == "127.0.0.1"
+        && selfScrapes s port { container = "authelia"; }
-        && !(lib.elem reader.pull.exporter.prometheus.port taken)
+        && !(lib.elem port taken)
         && !autheliaColocated.containers.swarm-authelia.config.services.opentelemetry-collector.enable
         && !(
           autheliaColocated.containers.swarm-authelia.config.systemd.services ? swarm-journal-forwarder-oidc