From 0bc350a8a1502255e80f55042d1779c951af2886 Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 4 Nov 2026 22:26:54 +0100 Subject: [PATCH] ci runner: export its built-in metrics through the container's forwarder --- docs/swarm/services.md | 13 ++++++++ .../swarm-journal-forwarder.nix | 26 ++++++++++++++++ nix/host-modules/swarm-ci-runner.nix | 9 ++++++ nix/module-eval/core-toggle.nix | 30 +++++++++++++++++++ 4 files changed, 78 insertions(+) diff --git a/docs/swarm/services.md b/docs/swarm/services.md index 4eebef0b..a05f8c6a 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -430,6 +430,19 @@ On a host without the store, that leaf is the one file to copy. Registration, the container and the threat model are in [`../scheduler/ci.md`](../scheduler/ci.md). +The runner's built-in Prometheus endpoint is on, bound to `127.0.0.1:9101` in +the container's own network namespace. The container's journal forwarder +scrapes it (`services.hyperhive.swarmJournalForwarder.extraScrapeTargets`) and +pushes the series to the swarm collector under the job `gitea-runner`, so +nothing listens on the bridge. Series, all prefixed `gitea_runner_`: +`job_running`, `capacity`, `job_capacity_utilization_ratio`, `uptime_seconds`, +`info` (labels `version` and `name`), `job_total{status}`, +`job_duration_seconds`, `poll_fetch_total{result}`, +`poll_fetch_duration_seconds`, `poll_backoff_seconds`, `report_log_*`, +`report_state_*` and `client_errors_total`. A pull-request job runs in the same +namespace and can read the endpoint, which carries counts, durations, the +runner's name and its version, and no secrets. + ### Builder The swarm builder, in a `swarm-builder` container on the host that sets diff --git a/nix/container-modules/swarm-journal-forwarder.nix b/nix/container-modules/swarm-journal-forwarder.nix index f16312f3..6aa2f4cd 100644 --- a/nix/container-modules/swarm-journal-forwarder.nix +++ b/nix/container-modules/swarm-journal-forwarder.nix @@ -116,6 +116,22 @@ in ''; }; + extraScrapeTargets = lib.mkOption { + type = lib.types.attrsOf (lib.types.strMatching "127\\.0\\.0\\.1:[0-9]+"); + default = { }; + example = { + gitea-runner = "127.0.0.1:9101"; + }; + description = '' + Prometheus endpoints of processes in this container, scraped by the + collector and pushed to the swarm collector like its journal. The + attribute name is the scrape job, which becomes the series' + `service.name`; the value is a loopback `127.0.0.1:` address, so + nothing listens outside the container. Empty: no scrape and no extra + pipeline. + ''; + }; + hostSecretFile = lib.mkOption { type = lib.types.nullOr lib.types.str; default = null; @@ -257,6 +273,16 @@ in exporter = "otlphttp"; attributes.container = cfg.serviceName; }) + (lib.mkIf (cfg.extraScrapeTargets != { }) { + receivers."prometheus/extra".config.scrape_configs = lib.mapAttrsToList (job: target: { + job_name = job; + static_configs = [ { targets = [ target ]; } ]; + }) cfg.extraScrapeTargets; + service.pipelines."metrics/extra" = { + receivers = [ "prometheus/extra" ]; + exporters = [ "otlphttp" ]; + }; + }) # A separate set the module system merges in: joining it to the first with # `//` would replace the first set's whole `service` attrset. (lib.mkIf (cfg.otlpMetricsPort != null) { diff --git a/nix/host-modules/swarm-ci-runner.nix b/nix/host-modules/swarm-ci-runner.nix index 50ea4264..7bbd639a 100644 --- a/nix/host-modules/swarm-ci-runner.nix +++ b/nix/host-modules/swarm-ci-runner.nix @@ -75,6 +75,10 @@ let # containers' forwarders all the same. forwarderTelemetryPort = 8895; + # The runner's Prometheus endpoint. Loopback in the container's own netns: the + # forwarder scrapes it, and PR jobs in the same netns can read it. + runnerMetricsAddr = "127.0.0.1:9101"; + # The runner's store identity, handed into the container as system # credentials (credential id → host source) for the forwarder's fetch unit. storeFiles = lib.filterAttrs (_: source: source != null) { @@ -414,6 +418,7 @@ in clientId = forwarderClientId; tokenUrl = forwarderTokenUrl; telemetryPort = forwarderTelemetryPort; + extraScrapeTargets.gitea-runner = runnerMetricsAddr; store = { package = baoDeploy.package; addr = "https://${baoCfg.domain}:${toString baoCfg.port}"; @@ -482,6 +487,10 @@ in settings = { runner.capacity = cfg.concurrency; runner.timeout = cfg.jobTimeout; + metrics = { + enabled = true; + addr = runnerMetricsAddr; + }; }; }; diff --git a/nix/module-eval/core-toggle.nix b/nix/module-eval/core-toggle.nix index 2a68b560..67398f34 100644 --- a/nix/module-eval/core-toggle.nix +++ b/nix/module-eval/core-toggle.nix @@ -842,6 +842,36 @@ let in check withClassifier && check withClassifierOnHive; } + { + # The runner serves its Prometheus endpoint on loopback in its own + # netns, and the container's forwarder scrapes exactly that address into + # a pipeline that leaves through the swarm exporter. The classifier's + # forwarder, which sets no extra targets, is the control: no extra + # receiver, no extra pipeline. + name = "the CI runner's built-in metrics are scraped on loopback by its forwarder"; + ok = + let + cc = withCi.containers.swarm-ci-runner.config; + s = cc.services.opentelemetry-collector.settings; + c = + withClassifier.containers.swarm-review-classifier.config.services.opentelemetry-collector.settings; + runner = cc.services.gitea-actions-runner.instances.swarm.settings; + in + runner.metrics.enabled == true + && runner.metrics.addr == "127.0.0.1:9101" + && + s.receivers."prometheus/extra".config.scrape_configs == [ + { + job_name = "gitea-runner"; + static_configs = [ { targets = [ "127.0.0.1:9101" ]; } ]; + } + ] + && s.service.pipelines."metrics/extra".receivers == [ "prometheus/extra" ] + && s.service.pipelines."metrics/extra".exporters == [ "otlphttp" ] + && s.exporters ? otlphttp + && !(c.receivers ? "prometheus/extra") + && !(c.service.pipelines ? "metrics/extra"); + } { # Only the classifier asks the forwarder for a metrics receiver; the # others carry a journal. The classifier is the control.