diff --git a/docs/networking/network.md b/docs/networking/network.md index 8475a758..0025ad18 100644 --- a/docs/networking/network.md +++ b/docs/networking/network.md @@ -47,7 +47,7 @@ that touches it. | `hive-forge` | host (shared) | host addresses | forgejo `:3000` http, `:2222` git-ssh; fronted by the `forge.` vhost | | `hive-matrix` | private, veth on bridge | DHCP pool | tuwunel on UDS `/run/hive-matrix/tuwunel.sock` → the matrix vhost; out to store, SSO, peers | | `swarm-ci-runner` | private, veth on bridge | DHCP pool | outbound only (runner → forge); no inbound surface | -| `swarm-review-classifier` | private, veth on bridge | DHCP pool | outbound only (classifier → queue, secret store, hive collector); no inbound surface | +| `swarm-review-classifier` | private, veth on bridge | DHCP pool | outbound only (classifier → queue, secret store, swarm collector); no inbound surface | | `h-` | private, veth on bridge | DHCP pool | web UI via UDS `/run/hive-agent/` → nginx sub-path; in-container UI port hashed 8100–8999 | The flows, end to end: diff --git a/docs/scheduler/observability.md b/docs/scheduler/observability.md index 1fde60f1..0585e9c0 100644 --- a/docs/scheduler/observability.md +++ b/docs/scheduler/observability.md @@ -20,7 +20,11 @@ the host is: | **swarm tier** — `deploy.swarm-otel` | once per swarm | every hive's collector | writes the swarm's stores and exports upstream | | **hive tier** — always on | every hive | that hive's agents, on the bridge | forwards to the swarm tier. Holds no credential | -An all-local host runs both; the hop between them configures itself. +An all-local host runs both; the hop between them configures itself. A host that +runs the swarm's services without a hive runs only the swarm tier: the hive tier +labels a hive's telemetry outside its agents' control, and that host has no hive. +The swarm services there push to the swarm collector directly, each as its own +OAuth2 client. ```nix services.hyperhive.deploy.swarm-otel.upstream = { diff --git a/docs/swarm/review-classifier.md b/docs/swarm/review-classifier.md index 1c76bf9a..86d8d381 100644 --- a/docs/swarm/review-classifier.md +++ b/docs/swarm/review-classifier.md @@ -655,14 +655,15 @@ Add a filter such as `F_OUTCOME:="post-failed"` before the `|` to narrow it. The swarm UI shows no token counts, cache counts or cost, on any page. On the claude runtimes, every classifier run starts its `claude` process with `OTEL_RESOURCE_ATTRIBUTES` naming the repo and the PR, so the cost reads from -the metrics store by repo and PR. The process exports to the hive-tier collector on -the classifier's host, where that host runs one. On a subscription, cost is notional. +the metrics store by repo and PR. The process exports to the OTLP receiver of the +container's own forwarder on loopback, which pushes to the swarm collector as the +classifier's OAuth2 client, on a host with a hive or without one. On a subscription, cost is notional. On the ACP runtime, the service computes each classification's cost itself, from the settings' `acp.prices` and the token counts the agent reports for the prompt, and adds it to the counter `hyperhive.review_classifier.cost.usage`, labelled `repo`, `pr`, `model` (the provider's) and `currency` (`USD`). It -exports to the same collector `claude` does. A classification stopped at its +exports to the same receiver `claude` does. A classification stopped at its deadline, or whose agent reports no token counts, adds nothing. opencode reports no cost of its own for a provider without prices in its config, which is why the prices are a setting. diff --git a/nix/container-modules/swarm-journal-forwarder.nix b/nix/container-modules/swarm-journal-forwarder.nix index f6419777..952d9df5 100644 --- a/nix/container-modules/swarm-journal-forwarder.nix +++ b/nix/container-modules/swarm-journal-forwarder.nix @@ -101,6 +101,20 @@ in ''; }; + otlpMetricsPort = lib.mkOption { + type = lib.types.nullOr lib.types.port; + default = null; + example = 4318; + description = '' + When set, the collector also takes OTLP/HTTP metrics on + `127.0.0.1:` inside this container and pushes them to + {option}`services.hyperhive.swarmJournalForwarder.endpoint` under the + same OIDC client as the logs. For a process in the container that + speaks OTLP but cannot fetch a token. Null: the collector reads the + journal and nothing else. + ''; + }; + hostSecretFile = lib.mkOption { type = lib.types.nullOr lib.types.str; default = null; @@ -176,76 +190,89 @@ in # A parser, not a wiring check: it accepts an extension that is never # started. validateConfigFile = true; - settings = { - # The receiver's read cursor, so a restart resumes rather than skipping - # what was written while it was down. `create_directory`, because - # `otelcol validate` runs in the build sandbox before systemd's - # `StateDirectory=` exists. - extensions.file_storage = { - directory = "/var/lib/opentelemetry-collector"; - create_directory = true; - }; + settings = lib.mkMerge [ + { + # The receiver's read cursor, so a restart resumes rather than skipping + # what was written while it was down. `create_directory`, because + # `otelcol validate` runs in the build sandbox before systemd's + # `StateDirectory=` exists. + extensions.file_storage = { + directory = "/var/lib/opentelemetry-collector"; + create_directory = true; + }; - receivers.journald = { - # ⚠️ Stated: the receiver's default is the runtime journal under - # /run/log/journal, which is empty when journald stores persistently. - directory = "/var/log/journal"; - # Without `--merge`, `journalctl --follow` over a journal with no - # entry for the current boot crash-loops the receiver. - merge = true; - storage = "file_storage"; - }; + receivers.journald = { + # ⚠️ Stated: the receiver's default is the runtime journal under + # /run/log/journal, which is empty when journald stores persistently. + directory = "/var/log/journal"; + # Without `--merge`, `journalctl --follow` over a journal with no + # entry for the current boot crash-loops the receiver. + merge = true; + storage = "file_storage"; + }; - processors.resource.attributes = [ - { - key = "service.name"; - value = cfg.serviceName; - action = "upsert"; - } - ]; + processors.resource.attributes = [ + { + key = "service.name"; + value = cfg.serviceName; + action = "upsert"; + } + ]; - exporters.otlphttp = { - endpoint = cfg.endpoint; - # The receiver accepts an OIDC token and nothing else. - auth.authenticator = authName; - }; + exporters.otlphttp = { + endpoint = cfg.endpoint; + # The receiver accepts an OIDC token and nothing else. + auth.authenticator = authName; + }; - extensions.${authName} = { - client_id = cfg.clientId; - # A path systemd resolves at runtime, so neither the secret nor the - # credentials directory is rendered into the nix store. - client_secret_file = "\${env:CREDENTIALS_DIRECTORY}/${credentialId}"; - token_url = cfg.tokenUrl; - # ⚠️ Requested, not merely registered: a token minted without asking - # carries `aud: []` and the receiver refuses it. - endpoint_params.audience = cfg.clientId; - }; + extensions.${authName} = { + client_id = cfg.clientId; + # A path systemd resolves at runtime, so neither the secret nor the + # credentials directory is rendered into the nix store. + client_secret_file = "\${env:CREDENTIALS_DIRECTORY}/${credentialId}"; + token_url = cfg.tokenUrl; + # ⚠️ Requested, not merely registered: a token minted without asking + # carries `aud: []` and the receiver refuses it. + endpoint_params.audience = cfg.clientId; + }; - service.pipelines.logs = { - receivers = [ "journald" ]; - processors = [ "resource" ]; - exporters = [ "otlphttp" ]; - }; + service.pipelines.logs = { + receivers = [ "journald" ]; + processors = [ "resource" ]; + exporters = [ "otlphttp" ]; + }; - # ⚠️ `readers`, not `metrics.address`: this collector version rejects - # `address` at startup. - service.telemetry.metrics.readers = [ - { - pull.exporter.prometheus = { - host = "127.0.0.1"; - port = cfg.telemetryPort; - }; - } - ]; + # ⚠️ `readers`, not `metrics.address`: this collector version rejects + # `address` at startup. + service.telemetry.metrics.readers = [ + { + pull.exporter.prometheus = { + host = "127.0.0.1"; + port = cfg.telemetryPort; + }; + } + ]; - # An extension not listed here is inert: the receiver's storage would - # name a component that never starts, and the exporter would push - # unauthenticated. - service.extensions = [ - "file_storage" - authName - ]; - }; + # An extension not listed here is inert: the receiver's storage would + # name a component that never starts, and the exporter would push + # unauthenticated. + service.extensions = [ + "file_storage" + authName + ]; + } + # A separate set the module system merges in: joining it to the first with + # `//` would replace the first set's whole `service` attrset. + (lib.mkIf (cfg.otlpMetricsPort != null) { + receivers.otlp.protocols.http.endpoint = "127.0.0.1:${toString cfg.otlpMetricsPort}"; + # No `resource` processor: it upserts the journal's `service.name`, and + # the sender names itself. + service.pipelines.metrics = { + receivers = [ "otlp" ]; + exporters = [ "otlphttp" ]; + }; + }) + ]; }; # `Restart` is nixpkgs' `"always"`; a second definition conflicts. Only diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index a3dfc6e7..3e8c4c90 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -91,17 +91,16 @@ in collector.enable = lib.mkOption { type = lib.types.bool; readOnly = true; - default = - config.services.hyperhive.deploy.hive-controller.enable - || config.services.hyperhive.deploy.allSwarmServices; - defaultText = lib.literalExpression "deploy.hive-controller.enable || deploy.allSwarmServices"; + default = config.services.hyperhive.deploy.hive-controller.enable; + defaultText = lib.literalExpression "deploy.hive-controller.enable"; description = '' Whether this host runs the hive tier's collector. Not a switch: every host that runs a hive ({option}`services.hyperhive.deploy.hive-controller.enable`) runs - one, and so does a host carrying all the swarm's shared services - ({option}`services.hyperhive.deploy.allSwarmServices`), whose own - containers export to it. + one, and no other host does. It exists to stamp the `hive` label on + a hive's telemetry outside its agents' control, so a host without a + hive has nothing for it to label: the swarm services there export to + the swarm collector directly, each as its own client. Agents export to it unauthenticated at {option}`services.hyperhive.otel.collector.host`, a name this diff --git a/nix/host-modules/swarm-review-classifier.nix b/nix/host-modules/swarm-review-classifier.nix index 3393b82e..f6fd2d4a 100644 --- a/nix/host-modules/swarm-review-classifier.nix +++ b/nix/host-modules/swarm-review-classifier.nix @@ -39,7 +39,6 @@ let autheliaCfg = hyperhiveCfg.swarm.authelia; gatewayCfg = hyperhiveCfg.gateway; networkCfg = hyperhiveCfg.network; - otelCfg = hyperhiveCfg.otel; tlsCfg = hyperhiveCfg.deploy.hive-controller.tls; machine = "swarm-review-classifier"; @@ -55,6 +54,8 @@ let # Loopback in the container's own netns, and distinct from the other service # containers' forwarders all the same. forwarderTelemetryPort = 8894; + # The standard OTLP/HTTP port, on the container's loopback. + forwarderOtlpPort = 4318; # The daemon reaches authelia, the store and the queue by name over TLS, and # in self-signed mode their leaves chain to the hive CA. @@ -89,14 +90,15 @@ let credential = id: "/run/credentials/${unit}.service/${id}"; forwarderCredential = id: "/run/credentials/${forwarderUnit}.service/${id}"; - # What `claude` exports its metrics with, and the daemon its ACP runs' cost, - # where this host runs a hive-tier collector on the bridge. The daemon adds - # the repo and the PR to each `claude` run's resource attributes, and to - # each cost record's attributes. - otelEnv = lib.optionalAttrs otelCfg.collector.enable { + # What `claude` exports its metrics with, and the daemon its ACP runs' cost: + # the container's own forwarder, which pushes them to the swarm collector as + # this service's client, on every host. The daemon adds the repo and the PR + # to each `claude` run's resource attributes, and to each cost record's + # attributes. + otelEnv = { CLAUDE_CODE_ENABLE_TELEMETRY = "1"; OTEL_METRICS_EXPORTER = "otlp"; - OTEL_EXPORTER_OTLP_ENDPOINT = "http://${networkCfg.bridgeIp}:${toString hyperhiveCfg.swarm.hiveOtel.collectorPort}"; + OTEL_EXPORTER_OTLP_ENDPOINT = "http://127.0.0.1:${toString forwarderOtlpPort}"; OTEL_EXPORTER_OTLP_PROTOCOL = "http/protobuf"; # Claude Code defaults to delta, which the metrics store drops. OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE = "cumulative"; @@ -218,6 +220,7 @@ in inherit clientId; tokenUrl = "${autheliaCfg.url}/api/oidc/token"; telemetryPort = forwarderTelemetryPort; + otlpMetricsPort = forwarderOtlpPort; store = { package = baoDeploy.package; addr = "https://${baoCfg.domain}:${toString baoCfg.port}"; diff --git a/nix/module-eval/core-toggle.nix b/nix/module-eval/core-toggle.nix index 166aa5a9..c13dab80 100644 --- a/nix/module-eval/core-toggle.nix +++ b/nix/module-eval/core-toggle.nix @@ -85,6 +85,13 @@ let deploy.review-classifier.enable = true; }; + # The same classifier on a host that runs a hive: its metrics path is the + # same as without one. + withClassifierOnHive = hiveWithController { + deploy.bao.enable = true; + deploy.review-classifier.enable = true; + }; + # The host fetches that restart their consumer when the store's value # changes (../host-modules/lib/refresh-consumer.nix), each on a fixture # that renders it. @@ -142,12 +149,11 @@ let let es = swarmServiceEnables bare; in - # `otel` is the hive tier, which every hive runs and so `bare` does - # too: it is the control that the lookup finds what is there, and the - # other eight are the swarm's shared services. - lib.length (lib.attrNames es) == 9 - && es.otel - && !lib.any lib.id (lib.attrValues (removeAttrs es [ "otel" ])); + # The hosting switch's host is the control that the lookup finds what + # is there. + lib.length (lib.attrNames es) == 8 + && !lib.any lib.id (lib.attrValues es) + && lib.all lib.id (lib.attrValues (swarmServiceEnables shared.servicesHere)); } { name = "the all-local mode turns the swarm controller on"; @@ -716,6 +722,64 @@ let lib.attrValues withClassifier.systemd.services )); } + { + # The collector the classifier's metrics reach is the container's own + # forwarder, which holds the service's client, so the path is the same + # with or without a hive on the host. The logs pipeline is the control + # that the lookup finds what is there. + name = "the review classifier pushes its metrics to the swarm collector through its own forwarder, with or without a hive"; + ok = + let + check = + h: + let + c = h.containers.swarm-review-classifier.config; + s = c.services.opentelemetry-collector.settings; + otel = h.services.hyperhive.swarm.otel; + in + c.systemd.services.swarm-review-classifier.environment.OTEL_EXPORTER_OTLP_ENDPOINT + == "http://127.0.0.1:4318" + && s.receivers.otlp.protocols.http.endpoint == "127.0.0.1:4318" + && s.service.pipelines.metrics.receivers == [ "otlp" ] + && s.service.pipelines.metrics.exporters == [ "otlphttp" ] + && s.exporters.otlphttp.endpoint == "https://${otel.domain}/swarm-review-classifier" + && s.exporters.otlphttp.auth.authenticator == "oauth2client/swarm" + && s.service.pipelines.logs.receivers == [ "journald" ]; + in + check withClassifier && check withClassifierOnHive; + } + { + # Only the classifier asks the forwarder for a metrics receiver; the + # others carry a journal. The classifier is the control. + name = "a service forwarder without an OTLP port has no metrics pipeline"; + ok = + let + s = withCi.containers.swarm-ci-runner.config.services.opentelemetry-collector.settings; + c = + withClassifier.containers.swarm-review-classifier.config.services.opentelemetry-collector.settings; + in + c.service.pipelines ? metrics && !(s.service.pipelines ? metrics) && !(s.receivers ? otlp); + } + { + # The hive tier is the labelling point for a hive's telemetry, so it runs + # where a hive does and nowhere else. A services host with no hive keeps + # its swarm collector and loses only the host-level one. + name = "the hive collector runs on a hive host and not on a swarm-services host without one"; + ok = + let + bridge = bare.services.hyperhive.network.bridgeIp; + in + bare.services.hyperhive.otel.collector.enable + && bare.services.opentelemetry-collector.enable + && + bare.services.opentelemetry-collector.settings.receivers.otlp.protocols.http.endpoint + == "${bridge}:4318" + && shared.swarmServicesWithController.services.opentelemetry-collector.enable + && !shared.servicesHere.services.hyperhive.otel.collector.enable + && !shared.servicesHere.services.opentelemetry-collector.enable + && lib.elem "hive-collector.hive" bare.services.hyperhive.gateway.localNames + && !(lib.elem "hive-collector.hive" shared.servicesHere.services.hyperhive.gateway.localNames); + } { # See the same case in ./builder.nix. name = "the review classifier's container runs the leaf-minting unit before every start"; diff --git a/nix/module-eval/lib.nix b/nix/module-eval/lib.nix index ad0a8a32..be481dca 100644 --- a/nix/module-eval/lib.nix +++ b/nix/module-eval/lib.nix @@ -208,12 +208,9 @@ let # the two tier cases differ only in which receiver they hand it. carriesJournaldSeverity = receiver: lib.elem journaldSeverityParser (receiver.operators or [ ]); - # The enables that switch owns. ⚠️ `otel` is the per-hive collector's own - # option and is NOT under `deploy` — spelled at the wrong path it would be - # undeclared rather than false, and a roster that quietly loses a member is - # what the count guard in the cases below exists to catch. Its membership - # here is deliberate and was the fix for a gap, not an oversight: the hive - # tier lands wherever the swarm services do. + # The enables that switch owns. The hive tier's collector is not among them: it + # follows `deploy.hive-controller.enable`, and a services host without a hive + # runs none. swarmServiceEnables = machine: let @@ -221,7 +218,6 @@ let in { matrix = h.deploy.matrix.enable; - otel = h.otel.collector.enable; authelia = h.deploy.authelia.enable; nats = h.deploy.nats.enable; swarm-otel = h.deploy.swarm-otel.enable; diff --git a/nix/module-eval/swarm-services-switch.nix b/nix/module-eval/swarm-services-switch.nix index 27128cff..ecb3d04e 100644 --- a/nix/module-eval/swarm-services-switch.nix +++ b/nix/module-eval/swarm-services-switch.nix @@ -207,7 +207,7 @@ let let es = swarmServiceEnables swarmServicesHere; in - lib.length (lib.attrNames es) == 9 && lib.all lib.id (lib.attrValues es); + lib.length (lib.attrNames es) == 8 && lib.all lib.id (lib.attrValues es); } { # The container is the control that the toggle reaches the module, not @@ -1211,7 +1211,6 @@ let readers = [ "swarm-bao-forge-oidc" "swarm-bao-matrix-oidc" - "swarm-bao-hive-otel-oidc" "swarm-bao-grafana-oidc" "swarm-bao-grafana-admin" "swarm-bao-grafana-metrics-bearer" @@ -1240,7 +1239,7 @@ let !(lib.any (n: lib.elem n (s.after ++ s.wants ++ s.requires)) storeUnits); in lib.all waits readers - && lib.length readersHere == 10 + && lib.length readersHere == 9 && lib.elem "swarm-bao-grafana-admin" readersHere && lib.length readersElsewhere == 7 && lib.all namesNoStoreUnit readersElsewhere