mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

ci runner: export its built-in metrics through the container's forwarder

Browse source
This commit is contained in:
atlas 2026-11-04 22:26:54 +01:00
commit 0bc350a8a1
4 changed files with 78 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

13
docs/swarm/services.md
View file

@ -430,6 +430,19 @@ On a host without the store, that leaf is the one file to copy. Registration,
the container and the threat model are in
[`../scheduler/ci.md`](../scheduler/ci.md).
The runner's built-in Prometheus endpoint is on, bound to `127.0.0.1:9101` in
the container's own network namespace. The container's journal forwarder
scrapes it (`services.hyperhive.swarmJournalForwarder.extraScrapeTargets`) and
pushes the series to the swarm collector under the job `gitea-runner`, so
nothing listens on the bridge. Series, all prefixed `gitea_runner_`:
`job_running`, `capacity`, `job_capacity_utilization_ratio`, `uptime_seconds`,
`info` (labels `version` and `name`), `job_total{status}`,
`job_duration_seconds`, `poll_fetch_total{result}`,
`poll_fetch_duration_seconds`, `poll_backoff_seconds`, `report_log_*`,
`report_state_*` and `client_errors_total`. A pull-request job runs in the same
namespace and can read the endpoint, which carries counts, durations, the
runner's name and its version, and no secrets.
### Builder
The swarm builder, in a `swarm-builder` container on the host that sets

26
nix/container-modules/swarm-journal-forwarder.nix
View file

@ -116,6 +116,22 @@ in
'';
};
extraScrapeTargets = lib.mkOption {
type = lib.types.attrsOf (lib.types.strMatching "127\\.0\\.0\\.1:[0-9]+");
default = { };
example = {
gitea-runner = "127.0.0.1:9101";
};
description = ''
Prometheus endpoints of processes in this container, scraped by the
collector and pushed to the swarm collector like its journal. The
attribute name is the scrape job, which becomes the series'
`service.name`; the value is a loopback `127.0.0.1:<port>` address, so
nothing listens outside the container. Empty: no scrape and no extra
pipeline.
'';
};
hostSecretFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
@ -257,6 +273,16 @@ in
exporter = "otlphttp";
attributes.container = cfg.serviceName;
})
(lib.mkIf (cfg.extraScrapeTargets != { }) {
receivers."prometheus/extra".config.scrape_configs = lib.mapAttrsToList (job: target: {
job_name = job;
static_configs = [ { targets = [ target ]; } ];
}) cfg.extraScrapeTargets;
service.pipelines."metrics/extra" = {
receivers = [ "prometheus/extra" ];
exporters = [ "otlphttp" ];
};
})
# A separate set the module system merges in: joining it to the first with
# `//` would replace the first set's whole `service` attrset.
(lib.mkIf (cfg.otlpMetricsPort != null) {

9
nix/host-modules/swarm-ci-runner.nix
View file

@ -75,6 +75,10 @@ let
# containers' forwarders all the same.
forwarderTelemetryPort = 8895;
# The runner's Prometheus endpoint. Loopback in the container's own netns: the
# forwarder scrapes it, and PR jobs in the same netns can read it.
runnerMetricsAddr = "127.0.0.1:9101";
# The runner's store identity, handed into the container as system
# credentials (credential id → host source) for the forwarder's fetch unit.
storeFiles = lib.filterAttrs (_: source: source != null) {
@ -414,6 +418,7 @@ in
clientId = forwarderClientId;
tokenUrl = forwarderTokenUrl;
telemetryPort = forwarderTelemetryPort;
extraScrapeTargets.gitea-runner = runnerMetricsAddr;
store = {
package = baoDeploy.package;
addr = "https://${baoCfg.domain}:${toString baoCfg.port}";
@ -482,6 +487,10 @@ in
settings = {
runner.capacity = cfg.concurrency;
runner.timeout = cfg.jobTimeout;
metrics = {
enabled = true;
addr = runnerMetricsAddr;
};
};
};

30
nix/module-eval/core-toggle.nix
View file

@ -842,6 +842,36 @@ let
in
check withClassifier && check withClassifierOnHive;
}
{
# The runner serves its Prometheus endpoint on loopback in its own
# netns, and the container's forwarder scrapes exactly that address into
# a pipeline that leaves through the swarm exporter. The classifier's
# forwarder, which sets no extra targets, is the control: no extra
# receiver, no extra pipeline.
name = "the CI runner's built-in metrics are scraped on loopback by its forwarder";
ok =
let
cc = withCi.containers.swarm-ci-runner.config;
s = cc.services.opentelemetry-collector.settings;
c =
withClassifier.containers.swarm-review-classifier.config.services.opentelemetry-collector.settings;
runner = cc.services.gitea-actions-runner.instances.swarm.settings;
in
runner.metrics.enabled == true
&& runner.metrics.addr == "127.0.0.1:9101"
&&
s.receivers."prometheus/extra".config.scrape_configs == [
{
job_name = "gitea-runner";
static_configs = [ { targets = [ "127.0.0.1:9101" ]; } ];
}
]
&& s.service.pipelines."metrics/extra".receivers == [ "prometheus/extra" ]
&& s.service.pipelines."metrics/extra".exporters == [ "otlphttp" ]
&& s.exporters ? otlphttp
&& !(c.receivers ? "prometheus/extra")
&& !(c.service.pipelines ? "metrics/extra");
}
{
# Only the classifier asks the forwarder for a metrics receiver; the
# others carry a journal. The classifier is the control.