vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

hive collector runs only on hive hosts; review classifier pushes metrics through its forwarder

Browse source
This commit is contained in:
atlas 2026-10-26 15:58:44 +01:00
commit 0cbfa806d3
9 changed files with 191 additions and 98 deletions
Download patch file Download diff file Expand all files Collapse all files

153
nix/container-modules/swarm-journal-forwarder.nix
View file

@ -101,6 +101,20 @@ in
'';
};
otlpMetricsPort = lib.mkOption {
type = lib.types.nullOr lib.types.port;
default = null;
example = 4318;
description = ''
When set, the collector also takes OTLP/HTTP metrics on
`127.0.0.1:<port>` inside this container and pushes them to
{option}`services.hyperhive.swarmJournalForwarder.endpoint` under the
same OIDC client as the logs. For a process in the container that
speaks OTLP but cannot fetch a token. Null: the collector reads the
journal and nothing else.
'';
};
hostSecretFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
@ -176,76 +190,89 @@ in
# A parser, not a wiring check: it accepts an extension that is never
# started.
validateConfigFile = true;
settings = {
# The receiver's read cursor, so a restart resumes rather than skipping
# what was written while it was down. `create_directory`, because
# `otelcol validate` runs in the build sandbox before systemd's
# `StateDirectory=` exists.
extensions.file_storage = {
directory = "/var/lib/opentelemetry-collector";
create_directory = true;
};
settings = lib.mkMerge [
{
# The receiver's read cursor, so a restart resumes rather than skipping
# what was written while it was down. `create_directory`, because
# `otelcol validate` runs in the build sandbox before systemd's
# `StateDirectory=` exists.
extensions.file_storage = {
directory = "/var/lib/opentelemetry-collector";
create_directory = true;
};
receivers.journald = {
# ⚠️ Stated: the receiver's default is the runtime journal under
# /run/log/journal, which is empty when journald stores persistently.
directory = "/var/log/journal";
# Without `--merge`, `journalctl --follow` over a journal with no
# entry for the current boot crash-loops the receiver.
merge = true;
storage = "file_storage";
};
receivers.journald = {
# ⚠️ Stated: the receiver's default is the runtime journal under
# /run/log/journal, which is empty when journald stores persistently.
directory = "/var/log/journal";
# Without `--merge`, `journalctl --follow` over a journal with no
# entry for the current boot crash-loops the receiver.
merge = true;
storage = "file_storage";
};
processors.resource.attributes = [
{
key = "service.name";
value = cfg.serviceName;
action = "upsert";
}
];
processors.resource.attributes = [
{
key = "service.name";
value = cfg.serviceName;
action = "upsert";
}
];
exporters.otlphttp = {
endpoint = cfg.endpoint;
# The receiver accepts an OIDC token and nothing else.
auth.authenticator = authName;
};
exporters.otlphttp = {
endpoint = cfg.endpoint;
# The receiver accepts an OIDC token and nothing else.
auth.authenticator = authName;
};
extensions.${authName} = {
client_id = cfg.clientId;
# A path systemd resolves at runtime, so neither the secret nor the
# credentials directory is rendered into the nix store.
client_secret_file = "\${env:CREDENTIALS_DIRECTORY}/${credentialId}";
token_url = cfg.tokenUrl;
# ⚠️ Requested, not merely registered: a token minted without asking
# carries `aud: []` and the receiver refuses it.
endpoint_params.audience = cfg.clientId;
};
extensions.${authName} = {
client_id = cfg.clientId;
# A path systemd resolves at runtime, so neither the secret nor the
# credentials directory is rendered into the nix store.
client_secret_file = "\${env:CREDENTIALS_DIRECTORY}/${credentialId}";
token_url = cfg.tokenUrl;
# ⚠️ Requested, not merely registered: a token minted without asking
# carries `aud: []` and the receiver refuses it.
endpoint_params.audience = cfg.clientId;
};
service.pipelines.logs = {
receivers = [ "journald" ];
processors = [ "resource" ];
exporters = [ "otlphttp" ];
};
service.pipelines.logs = {
receivers = [ "journald" ];
processors = [ "resource" ];
exporters = [ "otlphttp" ];
};
# ⚠️ `readers`, not `metrics.address`: this collector version rejects
# `address` at startup.
service.telemetry.metrics.readers = [
{
pull.exporter.prometheus = {
host = "127.0.0.1";
port = cfg.telemetryPort;
};
}
];
# ⚠️ `readers`, not `metrics.address`: this collector version rejects
# `address` at startup.
service.telemetry.metrics.readers = [
{
pull.exporter.prometheus = {
host = "127.0.0.1";
port = cfg.telemetryPort;
};
}
];
# An extension not listed here is inert: the receiver's storage would
# name a component that never starts, and the exporter would push
# unauthenticated.
service.extensions = [
"file_storage"
authName
];
};
# An extension not listed here is inert: the receiver's storage would
# name a component that never starts, and the exporter would push
# unauthenticated.
service.extensions = [
"file_storage"
authName
];
}
# A separate set the module system merges in: joining it to the first with
# `//` would replace the first set's whole `service` attrset.
(lib.mkIf (cfg.otlpMetricsPort != null) {
receivers.otlp.protocols.http.endpoint = "127.0.0.1:${toString cfg.otlpMetricsPort}";
# No `resource` processor: it upserts the journal's `service.name`, and
# the sender names itself.
service.pipelines.metrics = {
receivers = [ "otlp" ];
exporters = [ "otlphttp" ];
};
})
];
};
# `Restart` is nixpkgs' `"always"`; a second definition conflicts. Only

13
nix/host-modules/otel.nix
View file

@ -91,17 +91,16 @@ in
collector.enable = lib.mkOption {
type = lib.types.bool;
readOnly = true;
default =
config.services.hyperhive.deploy.hive-controller.enable
|| config.services.hyperhive.deploy.allSwarmServices;
defaultText = lib.literalExpression "deploy.hive-controller.enable || deploy.allSwarmServices";
default = config.services.hyperhive.deploy.hive-controller.enable;
defaultText = lib.literalExpression "deploy.hive-controller.enable";
description = ''
Whether this host runs the hive tier's collector. Not a switch:
every host that runs a hive
({option}`services.hyperhive.deploy.hive-controller.enable`) runs
one, and so does a host carrying all the swarm's shared services
({option}`services.hyperhive.deploy.allSwarmServices`), whose own
containers export to it.
one, and no other host does. It exists to stamp the `hive` label on
a hive's telemetry outside its agents' control, so a host without a
hive has nothing for it to label: the swarm services there export to
the swarm collector directly, each as its own client.
Agents export to it unauthenticated at
{option}`services.hyperhive.otel.collector.host`, a name this

17
nix/host-modules/swarm-review-classifier.nix
View file

@ -39,7 +39,6 @@ let
autheliaCfg = hyperhiveCfg.swarm.authelia;
gatewayCfg = hyperhiveCfg.gateway;
networkCfg = hyperhiveCfg.network;
otelCfg = hyperhiveCfg.otel;
tlsCfg = hyperhiveCfg.deploy.hive-controller.tls;
machine = "swarm-review-classifier";
@ -55,6 +54,8 @@ let
# Loopback in the container's own netns, and distinct from the other service
# containers' forwarders all the same.
forwarderTelemetryPort = 8894;
# The standard OTLP/HTTP port, on the container's loopback.
forwarderOtlpPort = 4318;
# The daemon reaches authelia, the store and the queue by name over TLS, and
# in self-signed mode their leaves chain to the hive CA.
@ -89,14 +90,15 @@ let
credential = id: "/run/credentials/${unit}.service/${id}";
forwarderCredential = id: "/run/credentials/${forwarderUnit}.service/${id}";
# What `claude` exports its metrics with, and the daemon its ACP runs' cost,
# where this host runs a hive-tier collector on the bridge. The daemon adds
# the repo and the PR to each `claude` run's resource attributes, and to
# each cost record's attributes.
otelEnv = lib.optionalAttrs otelCfg.collector.enable {
# What `claude` exports its metrics with, and the daemon its ACP runs' cost:
# the container's own forwarder, which pushes them to the swarm collector as
# this service's client, on every host. The daemon adds the repo and the PR
# to each `claude` run's resource attributes, and to each cost record's
# attributes.
otelEnv = {
CLAUDE_CODE_ENABLE_TELEMETRY = "1";
OTEL_METRICS_EXPORTER = "otlp";
OTEL_EXPORTER_OTLP_ENDPOINT = "http://${networkCfg.bridgeIp}:${toString hyperhiveCfg.swarm.hiveOtel.collectorPort}";
OTEL_EXPORTER_OTLP_ENDPOINT = "http://127.0.0.1:${toString forwarderOtlpPort}";
OTEL_EXPORTER_OTLP_PROTOCOL = "http/protobuf";
# Claude Code defaults to delta, which the metrics store drops.
OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE = "cumulative";
@ -218,6 +220,7 @@ in
inherit clientId;
tokenUrl = "${autheliaCfg.url}/api/oidc/token";
telemetryPort = forwarderTelemetryPort;
otlpMetricsPort = forwarderOtlpPort;
store = {
package = baoDeploy.package;
addr = "https://${baoCfg.domain}:${toString baoCfg.port}";

76
nix/module-eval/core-toggle.nix
View file

@ -85,6 +85,13 @@ let
deploy.review-classifier.enable = true;
};
# The same classifier on a host that runs a hive: its metrics path is the
# same as without one.
withClassifierOnHive = hiveWithController {
deploy.bao.enable = true;
deploy.review-classifier.enable = true;
};
# The host fetches that restart their consumer when the store's value
# changes (../host-modules/lib/refresh-consumer.nix), each on a fixture
# that renders it.
@ -142,12 +149,11 @@ let
let
es = swarmServiceEnables bare;
in
# `otel` is the hive tier, which every hive runs and so `bare` does
# too: it is the control that the lookup finds what is there, and the
# other eight are the swarm's shared services.
lib.length (lib.attrNames es) == 9
&& es.otel
&& !lib.any lib.id (lib.attrValues (removeAttrs es [ "otel" ]));
# The hosting switch's host is the control that the lookup finds what
# is there.
lib.length (lib.attrNames es) == 8
&& !lib.any lib.id (lib.attrValues es)
&& lib.all lib.id (lib.attrValues (swarmServiceEnables shared.servicesHere));
}
{
name = "the all-local mode turns the swarm controller on";
@ -716,6 +722,64 @@ let
lib.attrValues withClassifier.systemd.services
));
}
{
# The collector the classifier's metrics reach is the container's own
# forwarder, which holds the service's client, so the path is the same
# with or without a hive on the host. The logs pipeline is the control
# that the lookup finds what is there.
name = "the review classifier pushes its metrics to the swarm collector through its own forwarder, with or without a hive";
ok =
let
check =
h:
let
c = h.containers.swarm-review-classifier.config;
s = c.services.opentelemetry-collector.settings;
otel = h.services.hyperhive.swarm.otel;
in
c.systemd.services.swarm-review-classifier.environment.OTEL_EXPORTER_OTLP_ENDPOINT
== "http://127.0.0.1:4318"
&& s.receivers.otlp.protocols.http.endpoint == "127.0.0.1:4318"
&& s.service.pipelines.metrics.receivers == [ "otlp" ]
&& s.service.pipelines.metrics.exporters == [ "otlphttp" ]
&& s.exporters.otlphttp.endpoint == "https://${otel.domain}/swarm-review-classifier"
&& s.exporters.otlphttp.auth.authenticator == "oauth2client/swarm"
&& s.service.pipelines.logs.receivers == [ "journald" ];
in
check withClassifier && check withClassifierOnHive;
}
{
# Only the classifier asks the forwarder for a metrics receiver; the
# others carry a journal. The classifier is the control.
name = "a service forwarder without an OTLP port has no metrics pipeline";
ok =
let
s = withCi.containers.swarm-ci-runner.config.services.opentelemetry-collector.settings;
c =
withClassifier.containers.swarm-review-classifier.config.services.opentelemetry-collector.settings;
in
c.service.pipelines ? metrics && !(s.service.pipelines ? metrics) && !(s.receivers ? otlp);
}
{
# The hive tier is the labelling point for a hive's telemetry, so it runs
# where a hive does and nowhere else. A services host with no hive keeps
# its swarm collector and loses only the host-level one.
name = "the hive collector runs on a hive host and not on a swarm-services host without one";
ok =
let
bridge = bare.services.hyperhive.network.bridgeIp;
in
bare.services.hyperhive.otel.collector.enable
&& bare.services.opentelemetry-collector.enable
&&
bare.services.opentelemetry-collector.settings.receivers.otlp.protocols.http.endpoint
== "${bridge}:4318"
&& shared.swarmServicesWithController.services.opentelemetry-collector.enable
&& !shared.servicesHere.services.hyperhive.otel.collector.enable
&& !shared.servicesHere.services.opentelemetry-collector.enable
&& lib.elem "hive-collector.hive" bare.services.hyperhive.gateway.localNames
&& !(lib.elem "hive-collector.hive" shared.servicesHere.services.hyperhive.gateway.localNames);
}
{
# See the same case in ./builder.nix.
name = "the review classifier's container runs the leaf-minting unit before every start";

10
nix/module-eval/lib.nix
View file

@ -208,12 +208,9 @@ let
# the two tier cases differ only in which receiver they hand it.
carriesJournaldSeverity = receiver: lib.elem journaldSeverityParser (receiver.operators or [ ]);
# The enables that switch owns. ⚠️ `otel` is the per-hive collector's own
# option and is NOT under `deploy` — spelled at the wrong path it would be
# undeclared rather than false, and a roster that quietly loses a member is
# what the count guard in the cases below exists to catch. Its membership
# here is deliberate and was the fix for a gap, not an oversight: the hive
# tier lands wherever the swarm services do.
# The enables that switch owns. The hive tier's collector is not among them: it
# follows `deploy.hive-controller.enable`, and a services host without a hive
# runs none.
swarmServiceEnables =
machine:
let
@ -221,7 +218,6 @@ let
in
{
matrix = h.deploy.matrix.enable;
otel = h.otel.collector.enable;
authelia = h.deploy.authelia.enable;
nats = h.deploy.nats.enable;
swarm-otel = h.deploy.swarm-otel.enable;

5
nix/module-eval/swarm-services-switch.nix
View file

@ -207,7 +207,7 @@ let
let
es = swarmServiceEnables swarmServicesHere;
in
lib.length (lib.attrNames es) == 9 && lib.all lib.id (lib.attrValues es);
lib.length (lib.attrNames es) == 8 && lib.all lib.id (lib.attrValues es);
}
{
# The container is the control that the toggle reaches the module, not
@ -1211,7 +1211,6 @@ let
readers = [
"swarm-bao-forge-oidc"
"swarm-bao-matrix-oidc"
"swarm-bao-hive-otel-oidc"
"swarm-bao-grafana-oidc"
"swarm-bao-grafana-admin"
"swarm-bao-grafana-metrics-bearer"
@ -1240,7 +1239,7 @@ let
!(lib.any (n: lib.elem n (s.after ++ s.wants ++ s.requires)) storeUnits);
in
lib.all waits readers
&& lib.length readersHere == 10
&& lib.length readersHere == 9
&& lib.elem "swarm-bao-grafana-admin" readersHere
&& lib.length readersElsewhere == 7
&& lib.all namesNoStoreUnit readersElsewhere