From b072fbf4d69643faadff1d81c518b561db000bb4 Mon Sep 17 00:00:00 2001 From: atlas Date: Fri, 16 Oct 2026 18:23:59 +0200 Subject: [PATCH] push ci run metrics to victoriametrics, with a grafana board A swarm-ci-metrics host unit on a 5-minute timer lists every forge repo's Actions runs as the new read-only ci-metrics account and pushes each finished run's duration and queue wait over OTLP, stamped at the run's stop time. The first pass backfills from run 1. swarm-controller mints the account's token into bao and keeps a ci-metrics team with read on the Actions unit in every organisation. Closes #5376 --- Cargo.lock | 19 + Cargo.toml | 1 + docs/README.md | 2 + docs/swarm/ci-metrics.md | 96 ++++ docs/swarm/credentials.md | 55 +- docs/swarm/secrets.md | 26 +- docs/swarm/services.md | 15 + flake.nix | 3 + nix/checks.nix | 5 + nix/host-modules/default.nix | 2 + nix/host-modules/glue-bao-tls.nix | 9 + .../glue-ci-metrics-otel-oidc-client.nix | 36 ++ nix/host-modules/swarm-bao.nix | 70 ++- nix/host-modules/swarm-ci-metrics.nix | 166 ++++++ nix/host-modules/swarm-grafana.nix | 1 + .../swarm-grafana/dashboards/ci.json | 410 +++++++++++++++ nix/host-modules/swarm-otel-service.nix | 6 + nix/host-modules/swarm-required-services.nix | 7 + nix/host-modules/swarm-secret-publisher.nix | 2 + nix/host-modules/swarm.nix | 1 + nix/module-eval/bao-grants.nix | 99 +++- nix/module-eval/ci-metrics.nix | 98 ++++ nix/module-eval/grafana.nix | 29 ++ nix/packages/default.nix | 1 + nix/reserved-names.nix | 6 +- swarm-ci-metrics/Cargo.toml | 35 ++ swarm-ci-metrics/src/cursor.rs | 157 ++++++ swarm-ci-metrics/src/main.rs | 271 ++++++++++ swarm-ci-metrics/src/otlp.rs | 138 +++++ swarm-ci-metrics/src/pass.rs | 258 ++++++++++ swarm-ci-metrics/src/sample.rs | 274 ++++++++++ swarm-controller/src/forge.rs | 1 + .../src/forge/ci_metrics_account.rs | 476 ++++++++++++++++++ swarm-controller/src/main.rs | 62 ++- swarm-secret-client/src/ci_metrics.rs | 64 +++ swarm-secret-client/src/lib.rs | 4 +- 36 files changed, 2849 insertions(+), 56 deletions(-) create mode 100644 docs/swarm/ci-metrics.md create mode 100644 nix/host-modules/glue-ci-metrics-otel-oidc-client.nix create mode 100644 nix/host-modules/swarm-ci-metrics.nix create mode 100644 nix/host-modules/swarm-grafana/dashboards/ci.json create mode 100644 nix/module-eval/ci-metrics.nix create mode 100644 swarm-ci-metrics/Cargo.toml create mode 100644 swarm-ci-metrics/src/cursor.rs create mode 100644 swarm-ci-metrics/src/main.rs create mode 100644 swarm-ci-metrics/src/otlp.rs create mode 100644 swarm-ci-metrics/src/pass.rs create mode 100644 swarm-ci-metrics/src/sample.rs create mode 100644 swarm-controller/src/forge/ci_metrics_account.rs create mode 100644 swarm-secret-client/src/ci_metrics.rs diff --git a/Cargo.lock b/Cargo.lock index d6aaf983..c01aff5f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4807,6 +4807,25 @@ dependencies = [ "tracing", ] +[[package]] +name = "swarm-ci-metrics" +version = "0.1.0" +dependencies = [ + "anyhow", + "forgejo-api", + "hive-log", + "reqwest", + "serde", + "serde_json", + "swarm-queue-client", + "swarm-secret-client", + "tempfile", + "time", + "tokio", + "tracing", + "url", +] + [[package]] name = "swarm-claude-login" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 39dc1a02..f643c836 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,7 @@ members = [ "swarm-agent-verify", "swarm-authelia-bridge", "swarm-builder", + "swarm-ci-metrics", "swarm-claude-login", "swarm-authelia-bridge-sock", "swarm-controller", diff --git a/docs/README.md b/docs/README.md index da5f2cd2..47015297 100644 --- a/docs/README.md +++ b/docs/README.md @@ -91,6 +91,8 @@ declarations. (planned, not implemented yet). - **Does an agent config PR build before it's merged?** → [`swarm/builder.md`](swarm/builder.md). +- **How long do CI runs take, per repo and PR, and where does the board get + it?** → [`swarm/ci-metrics.md`](swarm/ci-metrics.md). - **How do I log an agent in to claude from the swarm UI, and why doesn't a stale stored copy log it out again?** → [`swarm/claude-login.md`](swarm/claude-login.md). diff --git a/docs/swarm/ci-metrics.md b/docs/swarm/ci-metrics.md new file mode 100644 index 00000000..f37b6b4d --- /dev/null +++ b/docs/swarm/ci-metrics.md @@ -0,0 +1,96 @@ +# CI metrics + +`swarm-ci-metrics` puts the forge's Actions run times into the swarm's metrics +store, one sample per finished run, so Grafana's **hyperhive · CI runs** board +can show run time and queue wait per repo, workflow and PR, over any range back +to a repo's first run. + +## Where it runs + +A oneshot `swarm-ci-metrics.service` on a 5-minute timer, on the host that sets +`deploy.ci-metrics.enable`. `allSwarmServices` defaults it on wherever the host +has the poller's store identity, which the host that runs the store has. Run it +on one host per swarm: each poller keeps its own cursor, so two of them push +every run twice. + +It reaches the forge, authelia, the store and the collector by name, so it needs +none of them on its host. On a host without the store, its leaf +(`deploy.bao.ciMetricsClientCertFile` and `ciMetricsClientKeyFile`) is the one +file to copy, as for the CI runner. + +## What it reads, and as whom + +| what | where | identity | +| ---------------------- | --------------------------------------------------------------------------------------- | -------------------------------------------------------- | +| its forge token | `swarm/services/ci-metrics/forge/token` in the secret store, once per pass | its store leaf, subject `deploy.bao.ciMetricsCommonName` | +| its OIDC client secret | `swarm/services/swarm-ci-metrics/oidc/client` in the secret store, once per pass | the same leaf | +| the repos | the forge's repo search: every repo with Actions on, neither archived nor a mirror | the forge account `ci-metrics`, with that token | +| each repo's runs | the forge's runs list, newest first, down to the repo's cursor, at most a page a second | the same account | + +swarm-controller creates the `ci-metrics` account and mints its token with the +scope `read:repository`, the way it does for the builder. It also keeps a +`ci-metrics` team in every organisation, with read on the Actions unit of every +repo in it, and the account as its one member. The poller therefore covers: + +- every public repo, which needs no grant; +- every private repo an organisation owns, through that team; +- **no** private repo a user owns. The forge grants a user's repo only to a + collaborator, and a collaborator reads every unit, code included. + +## What it pushes + +Two gauges, pushed over OTLP to `https:///swarm-ci-metrics` +as its own OIDC client (`swarm.ci-metrics.clientId`). The swarm collector +forwards them to the metrics store. Each sample carries the run's `stopped` +time as its timestamp, which the store keeps, so a run from last month lands +last month. + +| metric | value, in seconds | +| ----------------------------- | ------------------- | +| `hyperhive.ci.run.duration` | `stopped − started` | +| `hyperhive.ci.run.queue_wait` | `started − created` | + +Both carry the labels `repo` (`owner/name`), `run` (the run number), `workflow` +(the workflow file), `pr` (the PR number, or the ref for a run outside a PR), +`sha` (12 characters), `outcome` (`success` or `failure`) and `event`, plus +`service.name="swarm-ci-metrics"` and the collector's `swarm`. With `run` among +them, every run is a series of its own, and the board reads each with +`last_over_time`. + +A run becomes no sample when its status is `cancelled` or `skipped`, it has no +start (or the epoch as its start, which the forge writes for a run cancelled +before it started), or it stops no later than it starts. A run still going a day +after its `created` time counts as abandoned. The journal line for each repo counts +these by reason. + +## The cursor and the backfill + +The poller keeps one file, `cursor.json`, in its state directory +(`/var/lib/swarm-ci-metrics`). Per repo, it holds a run number below which the +poller has pushed or skipped every run, and the run numbers above it already +handled. A run still going holds that number, so a run that finishes after a +newer one still gets pushed. + +With no file, the cursor of each repo is at run 0, so the first pass is the +backfill: it walks every run back to the first, a page of 50 a second. For a +repo with 5000 runs that takes a few minutes. + +The poller writes the file only after the collector accepted a push, never +before. A crash between the two pushes those runs again on the next pass. The +store then holds the same point twice in the same series, which changes no +panel. Deleting the file re-runs the backfill the same way. + +A failed repo leaves its cursor where it was, the pass goes on with the next +repo, and the unit fails, so `systemctl status swarm-ci-metrics` shows it. The +timer runs it again 5 minutes later. + +## The board + +`nix/host-modules/swarm-grafana/dashboards/ci.json`, with a repo and a workflow +variable: + +- mean and 90th percentile run time of successful runs, over time; +- queue wait over time, mean and longest; +- finished runs per day, by outcome; +- a table per PR over the time range: mean and longest successful run time, + runs, failed runs. diff --git a/docs/swarm/credentials.md b/docs/swarm/credentials.md index 02c7dc54..89bdfcf0 100644 --- a/docs/swarm/credentials.md +++ b/docs/swarm/credentials.md @@ -55,33 +55,34 @@ of the cell says how. -| store path | minter | reader — pulls at runtime, holds in memory | automatic re-mint | automatic re-pull | -| ----------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `swarm/agents//matrix/main` | `swarm-controller`, with the swarm's appservice token, at agent creation and in a five-minute pass | the agent container itself, under the certificate its hive passed in | ✅ the pass re-mints when the stored token is missing, unknown to the homeserver, or someone else's | ✅ `hive-matrix-daemon` exits when the homeserver rejects its token, and a five-minute timer restarts it, which reads the store again | -| `swarm/agents//matrix/` | `swarm-controller` | the agent container itself, under the certificate its hive passed in | must be stated | ✅ a secondary that didn't come up at start, or whose token the homeserver rejected, is read from the store again two minutes later, then at doubling waits up to sixteen minutes, without restarting the daemon or its other accounts; one that's up and then rejected ends its sync loop, the daemon exits and its timer restarts it | -| `swarm/agents//matrix-recovery/` | `swarm-controller`, a random passphrase for every account listed under the agent's `matrix/`, in a five-minute pass | the agent container itself, under the certificate its hive passed in, when `hive-matrix-daemon` brings the account up | ❌ never replaced: the account's server-side key backup opens only with the passphrase it was enabled with, so the pass writes only to an empty path | ❌ read when the account comes up; an account whose passphrase lands later turns backup on at the next daemon start | -| `swarm/controller/swarm-controller/matrix/appservice-token` | `swarm-matrix-ctl`, inside the `hive-matrix` container, once | `swarm-controller`, under its own certificate | ❌ `swarm-matrix-ctl` mints it once; the container keeps its copy and republishes it when the store's differs | ✅ the controller reads it on every five-minute matrix pass | -| `swarm/controller/swarm-controller/oidc/client` | authelia, at its first boot; `swarm-secret-publish` copies it in | `swarm-controller`, under its own certificate, at start and on refusal | ❌ authelia mints it once. A re-mint is republished by `swarm-secret-publish`'s path unit | ✅ read at start and held in memory; read again when authelia refuses it, at most once per backoff of 5 s doubling to 5 min while the store holds the refused value | -| `swarm/controller/swarm-controller/forge/token` | the forge's `forgejo-swarm-controller-account`, in the `hive-forge` container, once, under the forge publisher's certificate with a write-only grant; it then revokes its earlier tokens | `swarm-controller`, under its own certificate, at start and on refusal | ❌ the forge mints it once and keeps only the token's name in the container, at `/var/lib/forgejo/swarm-controller-token-name`; deleting that file mints a new token on the unit's next start | ✅ the controller reads it at start, and in the background with backoff until it first finds it. When the forge answers 401 it reads it again and sends the request once more with the new token, at most once per backoff of 5 s doubling to 5 min while the store holds the refused value | -| `swarm/controller/swarm-controller/grafana/token` | `swarm-bao-grafana-controller-token` on Grafana's host, under Grafana's certificate: a token for the `swarm-controller` Viewer service account, minted with Grafana's admin password | `swarm-controller`, under its own certificate, on the first `/api/alerts` request and again when Grafana refuses it | ✅ a timer starts the unit again 10 minutes after each run; it keeps a stored token Grafana accepts, and mints a new one when Grafana answers 401 or the store holds none | ✅ the controller holds the token in memory; when Grafana answers 401 it reads the store once more and retries the request | -| `swarm/agents//bao-mtls` | the store's agent PKI mount (`deploy.bao.agentPkiMountPath`), which generates the key, at `swarm-controller`'s request at agent creation | `hive-c0re`, under the hive's own certificate, when it writes the agent's container config; then `hive-agent-bao-identity` in the agent container, under the leaf it holds | ✅ `swarm-controller`'s five-minute pass re-issues a live agent's leaf once it's past half its validity (45 of 90 days, read from the certificate itself) | ✅ `hive-agent-bao-identity` re-reads it every 10 minutes and swaps a new leaf into `/run/hive-agent-bao-identity/`, which every store login in the container presents. It reads under the leaf it holds, so it renews only while that leaf is valid | -| `swarm/agents//queue` | `swarm-controller`, at agent creation | `hive-agent` in the agent container, under the agent's own certificate, held in memory — the identity it presents to the swarm queue, naming that one agent rather than its hive; `swarm-logs` in the agent container, under the same certificate, on every query, as the bearer it presents at the log store; `swarm-nats-auth` and `swarm-logs-auth`, each under its own certificate, to check a presented secret | ✅ `swarm-controller`'s five-minute pass re-mints a live agent's secret once it's 45 days old by `minted_at` on the stored object; a secret with no `minted_at` gets one stamped, value unchanged. The pass skips agents declared `Destroyed` — declaring an agent destroyed deletes every version of the path instead, the undo of the mint rather than another one | ✅ `hive-agent` reads the path before its first connect and again on every reconnect attempt, so a reconnect after a re-mint presents the new secret. An open connection keeps the secret it connected with; after a revocation the agent keeps retrying under the queue client's backoff. `swarm-logs` reads it on every query and once more after a 401. `swarm-logs-auth` keeps granting a secret it verified for `deploy.victorialogs.authVerdictTtl` (60 s by default) without reading the path, so a replaced or deleted secret still works at the log store for up to that long | -| `swarm/agents//forge-token` | `swarm-controller`, at agent creation and in a pass every 5 minutes over every agent with a store identity | the agent container itself, under its own certificate, fetched to `/run/hive-agent-forge-token/token` | ✅ the controller re-mints when the stored token is missing or no longer matches the forge (last eight characters and scopes) | ✅ the agent re-fetches on a 10-minute timer | -| `swarm/agents//forge/