hive collector runs only on hive hosts; review classifier pushes metrics through its forwarder

docs/networking/network.md

@@ -47,7 +47,7 @@ that touches it.
 | `hive-forge`              | host (shared)           | host addresses | forgejo `:3000` http, `:2222` git-ssh; fronted by the `forge.<swarm-domain>` vhost              |
 | `hive-matrix`             | private, veth on bridge | DHCP pool      | tuwunel on UDS `/run/hive-matrix/tuwunel.sock` → the matrix vhost; out to store, SSO, peers     |
 | `swarm-ci-runner`         | private, veth on bridge | DHCP pool      | outbound only (runner → forge); no inbound surface                                              |
-| `swarm-review-classifier` | private, veth on bridge | DHCP pool      | outbound only (classifier → queue, secret store, hive collector); no inbound surface            |
+| `swarm-review-classifier` | private, veth on bridge | DHCP pool      | outbound only (classifier → queue, secret store, swarm collector); no inbound surface           |
 | `h-<agent>`               | private, veth on bridge | DHCP pool      | web UI via UDS `/run/hive-agent/<name>` → nginx sub-path; in-container UI port hashed 8100–8999 |
 
 The flows, end to end:

docs/scheduler/observability.md

@@ -20,7 +20,11 @@ the host is:
 | **swarm tier** — `deploy.swarm-otel` | once per swarm | every hive's collector            | writes the swarm's stores and exports upstream  |
 | **hive tier** — always on            | every hive     | that hive's agents, on the bridge | forwards to the swarm tier. Holds no credential |
 
-An all-local host runs both; the hop between them configures itself.
+An all-local host runs both; the hop between them configures itself. A host that
+runs the swarm's services without a hive runs only the swarm tier: the hive tier
+labels a hive's telemetry outside its agents' control, and that host has no hive.
+The swarm services there push to the swarm collector directly, each as its own
+OAuth2 client.
 
 ```nix
 services.hyperhive.deploy.swarm-otel.upstream = {

docs/swarm/review-classifier.md

@@ -655,14 +655,15 @@ Add a filter such as `F_OUTCOME:="post-failed"` before the `|` to narrow it.
 The swarm UI shows no token counts, cache counts or cost, on any page. On the
 claude runtimes, every classifier run starts its `claude` process with
 `OTEL_RESOURCE_ATTRIBUTES` naming the repo and the PR, so the cost reads from
-the metrics store by repo and PR. The process exports to the hive-tier collector on
+the metrics store by repo and PR. The process exports to the OTLP receiver of the
-the classifier's host, where that host runs one. On a subscription, cost is notional.
+container's own forwarder on loopback, which pushes to the swarm collector as the
+classifier's OAuth2 client, on a host with a hive or without one. On a subscription, cost is notional.
 
 On the ACP runtime, the service computes each classification's cost itself,
 from the settings' `acp.prices` and the token counts the agent reports for the
 prompt, and adds it to the counter `hyperhive.review_classifier.cost.usage`,
 labelled `repo`, `pr`, `model` (the provider's) and `currency` (`USD`). It
-exports to the same collector `claude` does. A classification stopped at its
+exports to the same receiver `claude` does. A classification stopped at its
 deadline, or whose agent reports no token counts, adds nothing. opencode
 reports no cost of its own for a provider without prices in its config, which
 is why the prices are a setting.

nix/container-modules/swarm-journal-forwarder.nix

@@ -101,6 +101,20 @@ in
       '';
     };
 
+    otlpMetricsPort = lib.mkOption {
+      type = lib.types.nullOr lib.types.port;
+      default = null;
+      example = 4318;
+      description = ''
+        When set, the collector also takes OTLP/HTTP metrics on
+        `127.0.0.1:<port>` inside this container and pushes them to
+        {option}`services.hyperhive.swarmJournalForwarder.endpoint` under the
+        same OIDC client as the logs. For a process in the container that
+        speaks OTLP but cannot fetch a token. Null: the collector reads the
+        journal and nothing else.
+      '';
+    };
+
     hostSecretFile = lib.mkOption {
       type = lib.types.nullOr lib.types.str;
       default = null;
@@ -176,76 +190,89 @@ in
       # A parser, not a wiring check: it accepts an extension that is never
       # started.
       validateConfigFile = true;
-      settings = {
+      settings = lib.mkMerge [
-        # The receiver's read cursor, so a restart resumes rather than skipping
+        {
-        # what was written while it was down. `create_directory`, because
+          # The receiver's read cursor, so a restart resumes rather than skipping
-        # `otelcol validate` runs in the build sandbox before systemd's
+          # what was written while it was down. `create_directory`, because
-        # `StateDirectory=` exists.
+          # `otelcol validate` runs in the build sandbox before systemd's
-        extensions.file_storage = {
+          # `StateDirectory=` exists.
-          directory = "/var/lib/opentelemetry-collector";
+          extensions.file_storage = {
-          create_directory = true;
+            directory = "/var/lib/opentelemetry-collector";
-        };
+            create_directory = true;
+          };
 
-        receivers.journald = {
+          receivers.journald = {
-          # ⚠️ Stated: the receiver's default is the runtime journal under
+            # ⚠️ Stated: the receiver's default is the runtime journal under
-          # /run/log/journal, which is empty when journald stores persistently.
+            # /run/log/journal, which is empty when journald stores persistently.
-          directory = "/var/log/journal";
+            directory = "/var/log/journal";
-          # Without `--merge`, `journalctl --follow` over a journal with no
+            # Without `--merge`, `journalctl --follow` over a journal with no
-          # entry for the current boot crash-loops the receiver.
+            # entry for the current boot crash-loops the receiver.
-          merge = true;
+            merge = true;
-          storage = "file_storage";
+            storage = "file_storage";
-        };
+          };
 
-        processors.resource.attributes = [
+          processors.resource.attributes = [
-          {
+            {
-            key = "service.name";
+              key = "service.name";
-            value = cfg.serviceName;
+              value = cfg.serviceName;
-            action = "upsert";
+              action = "upsert";
-          }
+            }
-        ];
+          ];
 
-        exporters.otlphttp = {
+          exporters.otlphttp = {
-          endpoint = cfg.endpoint;
+            endpoint = cfg.endpoint;
-          # The receiver accepts an OIDC token and nothing else.
+            # The receiver accepts an OIDC token and nothing else.
-          auth.authenticator = authName;
+            auth.authenticator = authName;
-        };
+          };
 
-        extensions.${authName} = {
+          extensions.${authName} = {
-          client_id = cfg.clientId;
+            client_id = cfg.clientId;
-          # A path systemd resolves at runtime, so neither the secret nor the
+            # A path systemd resolves at runtime, so neither the secret nor the
-          # credentials directory is rendered into the nix store.
+            # credentials directory is rendered into the nix store.
-          client_secret_file = "\${env:CREDENTIALS_DIRECTORY}/${credentialId}";
+            client_secret_file = "\${env:CREDENTIALS_DIRECTORY}/${credentialId}";
-          token_url = cfg.tokenUrl;
+            token_url = cfg.tokenUrl;
-          # ⚠️ Requested, not merely registered: a token minted without asking
+            # ⚠️ Requested, not merely registered: a token minted without asking
-          # carries `aud: []` and the receiver refuses it.
+            # carries `aud: []` and the receiver refuses it.
-          endpoint_params.audience = cfg.clientId;
+            endpoint_params.audience = cfg.clientId;
-        };
+          };
 
-        service.pipelines.logs = {
+          service.pipelines.logs = {
-          receivers = [ "journald" ];
+            receivers = [ "journald" ];
-          processors = [ "resource" ];
+            processors = [ "resource" ];
-          exporters = [ "otlphttp" ];
+            exporters = [ "otlphttp" ];
-        };
+          };
 
-        # ⚠️ `readers`, not `metrics.address`: this collector version rejects
+          # ⚠️ `readers`, not `metrics.address`: this collector version rejects
-        # `address` at startup.
+          # `address` at startup.
-        service.telemetry.metrics.readers = [
+          service.telemetry.metrics.readers = [
-          {
+            {
-            pull.exporter.prometheus = {
+              pull.exporter.prometheus = {
-              host = "127.0.0.1";
+                host = "127.0.0.1";
-              port = cfg.telemetryPort;
+                port = cfg.telemetryPort;
-            };
+              };
-          }
+            }
-        ];
+          ];
 
-        # An extension not listed here is inert: the receiver's storage would
+          # An extension not listed here is inert: the receiver's storage would
-        # name a component that never starts, and the exporter would push
+          # name a component that never starts, and the exporter would push
-        # unauthenticated.
+          # unauthenticated.
-        service.extensions = [
+          service.extensions = [
-          "file_storage"
+            "file_storage"
-          authName
+            authName
-        ];
+          ];
-      };
+        }
+        # A separate set the module system merges in: joining it to the first with
+        # `//` would replace the first set's whole `service` attrset.
+        (lib.mkIf (cfg.otlpMetricsPort != null) {
+          receivers.otlp.protocols.http.endpoint = "127.0.0.1:${toString cfg.otlpMetricsPort}";
+          # No `resource` processor: it upserts the journal's `service.name`, and
+          # the sender names itself.
+          service.pipelines.metrics = {
+            receivers = [ "otlp" ];
+            exporters = [ "otlphttp" ];
+          };
+        })
+      ];
     };
 
     # `Restart` is nixpkgs' `"always"`; a second definition conflicts. Only

nix/host-modules/otel.nix

@@ -91,17 +91,16 @@ in
     collector.enable = lib.mkOption {
       type = lib.types.bool;
       readOnly = true;
-      default =
+      default = config.services.hyperhive.deploy.hive-controller.enable;
-        config.services.hyperhive.deploy.hive-controller.enable
+      defaultText = lib.literalExpression "deploy.hive-controller.enable";
-        || config.services.hyperhive.deploy.allSwarmServices;
-      defaultText = lib.literalExpression "deploy.hive-controller.enable || deploy.allSwarmServices";
       description = ''
         Whether this host runs the hive tier's collector. Not a switch:
         every host that runs a hive
         ({option}`services.hyperhive.deploy.hive-controller.enable`) runs
-        one, and so does a host carrying all the swarm's shared services
+        one, and no other host does. It exists to stamp the `hive` label on
-        ({option}`services.hyperhive.deploy.allSwarmServices`), whose own
+        a hive's telemetry outside its agents' control, so a host without a
-        containers export to it.
+        hive has nothing for it to label: the swarm services there export to
+        the swarm collector directly, each as its own client.
 
         Agents export to it unauthenticated at
         {option}`services.hyperhive.otel.collector.host`, a name this

nix/host-modules/swarm-review-classifier.nix

@@ -39,7 +39,6 @@ let
   autheliaCfg = hyperhiveCfg.swarm.authelia;
   gatewayCfg = hyperhiveCfg.gateway;
   networkCfg = hyperhiveCfg.network;
-  otelCfg = hyperhiveCfg.otel;
   tlsCfg = hyperhiveCfg.deploy.hive-controller.tls;
 
   machine = "swarm-review-classifier";
@@ -55,6 +54,8 @@ let
   # Loopback in the container's own netns, and distinct from the other service
   # containers' forwarders all the same.
   forwarderTelemetryPort = 8894;
+  # The standard OTLP/HTTP port, on the container's loopback.
+  forwarderOtlpPort = 4318;
 
   # The daemon reaches authelia, the store and the queue by name over TLS, and
   # in self-signed mode their leaves chain to the hive CA.
@@ -89,14 +90,15 @@ let
   credential = id: "/run/credentials/${unit}.service/${id}";
   forwarderCredential = id: "/run/credentials/${forwarderUnit}.service/${id}";
 
-  # What `claude` exports its metrics with, and the daemon its ACP runs' cost,
+  # What `claude` exports its metrics with, and the daemon its ACP runs' cost:
-  # where this host runs a hive-tier collector on the bridge. The daemon adds
+  # the container's own forwarder, which pushes them to the swarm collector as
-  # the repo and the PR to each `claude` run's resource attributes, and to
+  # this service's client, on every host. The daemon adds the repo and the PR
-  # each cost record's attributes.
+  # to each `claude` run's resource attributes, and to each cost record's
-  otelEnv = lib.optionalAttrs otelCfg.collector.enable {
+  # attributes.
+  otelEnv = {
     CLAUDE_CODE_ENABLE_TELEMETRY = "1";
     OTEL_METRICS_EXPORTER = "otlp";
-    OTEL_EXPORTER_OTLP_ENDPOINT = "http://${networkCfg.bridgeIp}:${toString hyperhiveCfg.swarm.hiveOtel.collectorPort}";
+    OTEL_EXPORTER_OTLP_ENDPOINT = "http://127.0.0.1:${toString forwarderOtlpPort}";
     OTEL_EXPORTER_OTLP_PROTOCOL = "http/protobuf";
     # Claude Code defaults to delta, which the metrics store drops.
     OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE = "cumulative";
@@ -218,6 +220,7 @@ in
             inherit clientId;
             tokenUrl = "${autheliaCfg.url}/api/oidc/token";
             telemetryPort = forwarderTelemetryPort;
+            otlpMetricsPort = forwarderOtlpPort;
             store = {
               package = baoDeploy.package;
               addr = "https://${baoCfg.domain}:${toString baoCfg.port}";

nix/module-eval/core-toggle.nix

@@ -85,6 +85,13 @@ let
     deploy.review-classifier.enable = true;
   };
 
+  # The same classifier on a host that runs a hive: its metrics path is the
+  # same as without one.
+  withClassifierOnHive = hiveWithController {
+    deploy.bao.enable = true;
+    deploy.review-classifier.enable = true;
+  };
+
   # The host fetches that restart their consumer when the store's value
   # changes (../host-modules/lib/refresh-consumer.nix), each on a fixture
   # that renders it.
@@ -142,12 +149,11 @@ let
         let
           es = swarmServiceEnables bare;
         in
-        # `otel` is the hive tier, which every hive runs and so `bare` does
+        # The hosting switch's host is the control that the lookup finds what
-        # too: it is the control that the lookup finds what is there, and the
+        # is there.
-        # other eight are the swarm's shared services.
+        lib.length (lib.attrNames es) == 8
-        lib.length (lib.attrNames es) == 9
+        && !lib.any lib.id (lib.attrValues es)
-        && es.otel
+        && lib.all lib.id (lib.attrValues (swarmServiceEnables shared.servicesHere));
-        && !lib.any lib.id (lib.attrValues (removeAttrs es [ "otel" ]));
     }
     {
       name = "the all-local mode turns the swarm controller on";
@@ -716,6 +722,64 @@ let
           lib.attrValues withClassifier.systemd.services
         ));
     }
+    {
+      # The collector the classifier's metrics reach is the container's own
+      # forwarder, which holds the service's client, so the path is the same
+      # with or without a hive on the host. The logs pipeline is the control
+      # that the lookup finds what is there.
+      name = "the review classifier pushes its metrics to the swarm collector through its own forwarder, with or without a hive";
+      ok =
+        let
+          check =
+            h:
+            let
+              c = h.containers.swarm-review-classifier.config;
+              s = c.services.opentelemetry-collector.settings;
+              otel = h.services.hyperhive.swarm.otel;
+            in
+            c.systemd.services.swarm-review-classifier.environment.OTEL_EXPORTER_OTLP_ENDPOINT
+            == "http://127.0.0.1:4318"
+            && s.receivers.otlp.protocols.http.endpoint == "127.0.0.1:4318"
+            && s.service.pipelines.metrics.receivers == [ "otlp" ]
+            && s.service.pipelines.metrics.exporters == [ "otlphttp" ]
+            && s.exporters.otlphttp.endpoint == "https://${otel.domain}/swarm-review-classifier"
+            && s.exporters.otlphttp.auth.authenticator == "oauth2client/swarm"
+            && s.service.pipelines.logs.receivers == [ "journald" ];
+        in
+        check withClassifier && check withClassifierOnHive;
+    }
+    {
+      # Only the classifier asks the forwarder for a metrics receiver; the
+      # others carry a journal. The classifier is the control.
+      name = "a service forwarder without an OTLP port has no metrics pipeline";
+      ok =
+        let
+          s = withCi.containers.swarm-ci-runner.config.services.opentelemetry-collector.settings;
+          c =
+            withClassifier.containers.swarm-review-classifier.config.services.opentelemetry-collector.settings;
+        in
+        c.service.pipelines ? metrics && !(s.service.pipelines ? metrics) && !(s.receivers ? otlp);
+    }
+    {
+      # The hive tier is the labelling point for a hive's telemetry, so it runs
+      # where a hive does and nowhere else. A services host with no hive keeps
+      # its swarm collector and loses only the host-level one.
+      name = "the hive collector runs on a hive host and not on a swarm-services host without one";
+      ok =
+        let
+          bridge = bare.services.hyperhive.network.bridgeIp;
+        in
+        bare.services.hyperhive.otel.collector.enable
+        && bare.services.opentelemetry-collector.enable
+        &&
+          bare.services.opentelemetry-collector.settings.receivers.otlp.protocols.http.endpoint
+          == "${bridge}:4318"
+        && shared.swarmServicesWithController.services.opentelemetry-collector.enable
+        && !shared.servicesHere.services.hyperhive.otel.collector.enable
+        && !shared.servicesHere.services.opentelemetry-collector.enable
+        && lib.elem "hive-collector.hive" bare.services.hyperhive.gateway.localNames
+        && !(lib.elem "hive-collector.hive" shared.servicesHere.services.hyperhive.gateway.localNames);
+    }
     {
       # See the same case in ./builder.nix.
       name = "the review classifier's container runs the leaf-minting unit before every start";

nix/module-eval/lib.nix

@@ -208,12 +208,9 @@ let
   # the two tier cases differ only in which receiver they hand it.
   carriesJournaldSeverity = receiver: lib.elem journaldSeverityParser (receiver.operators or [ ]);
 
-  # The enables that switch owns. ⚠️ `otel` is the per-hive collector's own
+  # The enables that switch owns. The hive tier's collector is not among them: it
-  # option and is NOT under `deploy` — spelled at the wrong path it would be
+  # follows `deploy.hive-controller.enable`, and a services host without a hive
-  # undeclared rather than false, and a roster that quietly loses a member is
+  # runs none.
-  # what the count guard in the cases below exists to catch. Its membership
-  # here is deliberate and was the fix for a gap, not an oversight: the hive
-  # tier lands wherever the swarm services do.
   swarmServiceEnables =
     machine:
     let
@@ -221,7 +218,6 @@ let
     in
     {
       matrix = h.deploy.matrix.enable;
-      otel = h.otel.collector.enable;
       authelia = h.deploy.authelia.enable;
       nats = h.deploy.nats.enable;
       swarm-otel = h.deploy.swarm-otel.enable;

nix/module-eval/swarm-services-switch.nix

@@ -207,7 +207,7 @@ let
         let
           es = swarmServiceEnables swarmServicesHere;
         in
-        lib.length (lib.attrNames es) == 9 && lib.all lib.id (lib.attrValues es);
+        lib.length (lib.attrNames es) == 8 && lib.all lib.id (lib.attrValues es);
     }
     {
       # The container is the control that the toggle reaches the module, not
@@ -1211,7 +1211,6 @@ let
           readers = [
             "swarm-bao-forge-oidc"
             "swarm-bao-matrix-oidc"
-            "swarm-bao-hive-otel-oidc"
             "swarm-bao-grafana-oidc"
             "swarm-bao-grafana-admin"
             "swarm-bao-grafana-metrics-bearer"
@@ -1240,7 +1239,7 @@ let
             !(lib.any (n: lib.elem n (s.after ++ s.wants ++ s.requires)) storeUnits);
         in
         lib.all waits readers
-        && lib.length readersHere == 10
+        && lib.length readersHere == 9
         && lib.elem "swarm-bao-grafana-admin" readersHere
         && lib.length readersElsewhere == 7
         && lib.all namesNoStoreUnit readersElsewhere