push ci run metrics to victoriametrics, with a grafana board
A swarm-ci-metrics host unit on a 5-minute timer lists every forge repo's Actions runs as the new read-only ci-metrics account and pushes each finished run's duration and queue wait over OTLP, stamped at the run's stop time. The first pass backfills from run 1. swarm-controller mints the account's token into bao and keeps a ci-metrics team with read on the Actions unit in every organisation. Closes #5376
This commit is contained in:
parent
8b7a4e7955
commit
b072fbf4d6
36 changed files with 2849 additions and 56 deletions
|
|
@ -47,6 +47,7 @@
|
|||
./glue-nats-wireguard.nix
|
||||
./glue-queue-agent-credential.nix
|
||||
./glue-builder-oidc-client.nix
|
||||
./glue-ci-metrics-otel-oidc-client.nix
|
||||
./glue-ci-runner-otel-oidc-client.nix
|
||||
./glue-claude-login-oidc-client.nix
|
||||
./glue-forge-otel-oidc-client.nix
|
||||
|
|
@ -65,6 +66,7 @@
|
|||
./swarm-bao.nix
|
||||
./swarm-builder.nix
|
||||
./swarm-claude-login.nix
|
||||
./swarm-ci-metrics.nix
|
||||
./swarm-ci-runner.nix
|
||||
./swarm-review-classifier.nix
|
||||
./swarm-secret-publisher.nix
|
||||
|
|
|
|||
|
|
@ -117,6 +117,8 @@ in
|
|||
reviewClassifierClientKeyFile = lib.mkDefault "${pkiDir}/review-classifier-key.pem";
|
||||
builderClientCertFile = lib.mkDefault "${pkiDir}/builder.pem";
|
||||
builderClientKeyFile = lib.mkDefault "${pkiDir}/builder-key.pem";
|
||||
ciMetricsClientCertFile = lib.mkDefault "${pkiDir}/ci-metrics.pem";
|
||||
ciMetricsClientKeyFile = lib.mkDefault "${pkiDir}/ci-metrics-key.pem";
|
||||
claudeLoginClientCertFile = lib.mkDefault "${pkiDir}/claude-login.pem";
|
||||
claudeLoginClientKeyFile = lib.mkDefault "${pkiDir}/claude-login-key.pem";
|
||||
forgePublisherClientCertFile = lib.mkDefault "${pkiDir}/forge-publisher.pem";
|
||||
|
|
@ -310,6 +312,13 @@ in
|
|||
[ -s ${pkiDir}/builder.pem ] || ${signLeaf} ${pkiDir} builder \
|
||||
${lib.escapeShellArg deployCfg.bao.builderCommonName} "" clientAuth
|
||||
|
||||
# The CI run poller's, which reads its forge token and OIDC client
|
||||
# secret. Minted here for the matrix-ctl leaf's reason: the poller is a
|
||||
# swarm singleton, so on a poller host without the store this is the
|
||||
# file an operator copies.
|
||||
[ -s ${pkiDir}/ci-metrics.pem ] || ${signLeaf} ${pkiDir} ci-metrics \
|
||||
${lib.escapeShellArg deployCfg.bao.ciMetricsCommonName} "" clientAuth
|
||||
|
||||
# The claude login minter's, which writes agents' claude credentials
|
||||
# and reads its queue client secret. Minted here for the matrix-ctl
|
||||
# leaf's reason: the minter is a swarm service, so on a minter host
|
||||
|
|
|
|||
36
nix/host-modules/glue-ci-metrics-otel-oidc-client.nix
Normal file
36
nix/host-modules/glue-ci-metrics-otel-oidc-client.nix
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
# Glue: register the CI run poller as an OIDC client wherever authelia runs.
|
||||
#
|
||||
# ONE PAIRING PER FILE: that poller ← authelia, and nothing else.
|
||||
# Without it authelia mints no secret, the publisher has nothing to copy into
|
||||
# the store, and the poller's push is a 401.
|
||||
#
|
||||
# Gated on authelia being HERE, not on this host running the poller: a
|
||||
# client is a row in THIS host's provider config, the split
|
||||
# ./glue-ci-runner-otel-oidc-client.nix makes for the same reason.
|
||||
{
|
||||
lib,
|
||||
config,
|
||||
...
|
||||
}:
|
||||
let
|
||||
hyperhiveCfg = config.services.hyperhive;
|
||||
clientId = hyperhiveCfg.swarm.ci-metrics.clientId;
|
||||
in
|
||||
{
|
||||
config = lib.mkIf hyperhiveCfg.deploy.authelia.enable {
|
||||
services.hyperhive.swarm.authelia.oidc.clients = [
|
||||
{
|
||||
id = clientId;
|
||||
description = "HyperHive CI run poller";
|
||||
kind = "machine";
|
||||
redirectUris = [ ];
|
||||
# Its own id as its own permitted audience, the one the swarm
|
||||
# collector's receiver for the poller checks.
|
||||
audience = [ clientId ];
|
||||
# The receiver verifies the token offline against `/jwks.json`;
|
||||
# authelia's default opaque handle fails there.
|
||||
accessTokenSignedResponseAlg = "RS256";
|
||||
}
|
||||
];
|
||||
};
|
||||
}
|
||||
|
|
@ -449,6 +449,10 @@ let
|
|||
capabilities = ["create", "read", "update"]
|
||||
}
|
||||
|
||||
path "${credentialMountPath}/data/${ciMetricsForgeTokenLeaf}" {
|
||||
capabilities = ["create", "read", "update"]
|
||||
}
|
||||
|
||||
path "${credentialMountPath}/data/${swarmAppserviceTokenLeaf}" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
|
@ -504,8 +508,8 @@ let
|
|||
# The review classifier's forge token is denied too: the controller posts
|
||||
# the classifier's reviews with whatever token is there, so a write to it
|
||||
# picks the account those reviews come from.
|
||||
# So is the builder's: a write to it picks the account the builder fetches
|
||||
# config repos as.
|
||||
# So are the builder's and the CI run poller's: a write to either picks the
|
||||
# account that service reads the forge as.
|
||||
secretPublisherPolicyText = ''
|
||||
path "${credentialMountPath}/data/swarm/hives/*" {
|
||||
capabilities = ["create", "update"]
|
||||
|
|
@ -539,6 +543,10 @@ let
|
|||
capabilities = ["deny"]
|
||||
}
|
||||
|
||||
path "${credentialMountPath}/data/${ciMetricsForgeTokenLeaf}" {
|
||||
capabilities = ["deny"]
|
||||
}
|
||||
|
||||
path "${credentialMountPath}/data/${controllerClientLeaf}" {
|
||||
capabilities = ["create", "update"]
|
||||
}
|
||||
|
|
@ -620,6 +628,12 @@ let
|
|||
# to fetch config repos.
|
||||
builderForgeTokenLeaf = "swarm/services/builder/forge/token";
|
||||
|
||||
# The CI run poller's forge token, the nix half of
|
||||
# `swarm_secret_client::ci_metrics::forge_token_path`. The controller mints it
|
||||
# and reads it back before deciding whether to re-mint; the poller reads it
|
||||
# to list runs.
|
||||
ciMetricsForgeTokenLeaf = "swarm/services/ci-metrics/forge/token";
|
||||
|
||||
# The review classifier's ACP provider key, the nix half of
|
||||
# `swarm_secret_client::review_classifier::acp_provider_key_path`. Written
|
||||
# and read like the API key above.
|
||||
|
|
@ -1095,6 +1109,18 @@ let
|
|||
}
|
||||
];
|
||||
|
||||
# The CI run poller: its forge token and its own OIDC client secret, both
|
||||
# read once per pass by the host unit.
|
||||
ciMetricsReaders = [
|
||||
{
|
||||
name = "swarm-ci-metrics";
|
||||
cn = baoDeploy.ciMetricsCommonName;
|
||||
policyText =
|
||||
readStanza "${credentialMountPath}/data/${ciMetricsForgeTokenLeaf}"
|
||||
+ readStanza "${credentialMountPath}/data/swarm/services/${hyperhiveCfg.swarm.ci-metrics.clientId}/oidc/client";
|
||||
}
|
||||
];
|
||||
|
||||
# The claude login minter: every agent's `claude-credentials` leaf,
|
||||
# create and update so a relogin replaces the last, and nothing else under
|
||||
# an agent; and its own queue client secret, read at start.
|
||||
|
|
@ -2003,6 +2029,19 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
ciMetricsCommonName = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "swarm-ci-metrics";
|
||||
description = ''
|
||||
Subject the store's `swarm-ci-metrics` cert-auth role accepts: the
|
||||
identity the CI run poller presents from its host unit. Its grant is
|
||||
two paths, read only: `swarm/services/ci-metrics/forge/token` and its
|
||||
own OIDC client secret.
|
||||
|
||||
⚠️ Reserved as a hive name by ./swarm.nix, like its siblings.
|
||||
'';
|
||||
};
|
||||
|
||||
claudeLoginCommonName = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "swarm-claude-login";
|
||||
|
|
@ -2318,6 +2357,31 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
ciMetricsClientCertFile = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
example = "/var/lib/swarm-bao-pki/ci-metrics.pem";
|
||||
description = ''
|
||||
Certificate the CI run poller presents to the store. Its subject must
|
||||
be {option}`services.hyperhive.deploy.bao.ciMetricsCommonName`.
|
||||
|
||||
./glue-bao-tls.nix points it at the leaf it mints on the store's host.
|
||||
On a poller host without the store, it is a file the operator copies
|
||||
from the store's host. A path, never a value.
|
||||
'';
|
||||
};
|
||||
|
||||
ciMetricsClientKeyFile = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
example = "/var/lib/swarm-bao-pki/ci-metrics-key.pem";
|
||||
description = ''
|
||||
Private key for
|
||||
{option}`services.hyperhive.deploy.bao.ciMetricsClientCertFile`.
|
||||
A path, never a value.
|
||||
'';
|
||||
};
|
||||
|
||||
claudeLoginClientCertFile = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
|
|
@ -3435,6 +3499,8 @@ in
|
|||
|
||||
systemd.services.swarm-bao-builder-policy = readerPolicyUnit "write the swarm builder's forge-token-reader bao policy and cert-auth role" builderReaders;
|
||||
|
||||
systemd.services.swarm-bao-ci-metrics-policy = readerPolicyUnit "write the CI run poller's forge-token-reader bao policy and cert-auth role" ciMetricsReaders;
|
||||
|
||||
systemd.services.swarm-bao-claude-login-policy = readerPolicyUnit "write the claude login minter's credentials-writer bao policy and cert-auth role" claudeLoginWriters;
|
||||
|
||||
systemd.services.swarm-bao-forge-publisher-policy = readerPolicyUnit "write the forge's controller-token-publisher bao policy and cert-auth role" forgePublisherWriters;
|
||||
|
|
|
|||
166
nix/host-modules/swarm-ci-metrics.nix
Normal file
166
nix/host-modules/swarm-ci-metrics.nix
Normal file
|
|
@ -0,0 +1,166 @@
|
|||
# The swarm's CI run poller: a oneshot `swarm-ci-metrics` on a 5-minute timer,
|
||||
# on the host that sets `deploy.ci-metrics.enable`, which
|
||||
# `deploy.allSwarmServices` defaults on where the host has the poller's store
|
||||
# identity (./swarm-required-services.nix).
|
||||
#
|
||||
# Each pass lists the Actions runs of every repo the `ci-metrics` forge account
|
||||
# sees and pushes each newly finished run's duration and queue wait to the
|
||||
# swarm collector, stamped at the run's end. The first pass, with no cursor in
|
||||
# the state directory, walks every run back to the first. docs/swarm/ci-metrics.md
|
||||
# has the design.
|
||||
#
|
||||
# It reaches the forge, authelia, the store and the collector by name, so
|
||||
# nothing here asks which host runs any of them. Its one credential on disk is
|
||||
# its store leaf, handed in by `LoadCredential=`; the forge token and its OIDC
|
||||
# client secret it reads from the store under that leaf on every pass.
|
||||
{
|
||||
lib,
|
||||
config,
|
||||
...
|
||||
}:
|
||||
let
|
||||
hyperhiveCfg = config.services.hyperhive;
|
||||
cfg = hyperhiveCfg.deploy.ci-metrics;
|
||||
baoCfg = hyperhiveCfg.swarm.bao;
|
||||
baoDeploy = hyperhiveCfg.deploy.bao;
|
||||
gatewayCfg = hyperhiveCfg.gateway;
|
||||
swarmOtelCfg = hyperhiveCfg.swarm.otel;
|
||||
|
||||
unit = "swarm-ci-metrics";
|
||||
clientId = hyperhiveCfg.swarm.ci-metrics.clientId;
|
||||
|
||||
# The route `serviceProducers.swarm-ci-metrics` in ./swarm-otel-service.nix
|
||||
# declares, by the collector's gateway name. The port suffix is elided on
|
||||
# 443, as in ./swarm-builder.nix.
|
||||
gatewayPortSuffix = lib.optionalString (
|
||||
gatewayCfg.httpsPort != 443
|
||||
) ":${toString gatewayCfg.httpsPort}";
|
||||
otlpEndpoint = "https://${swarmOtelCfg.domain}${gatewayPortSuffix}/${unit}";
|
||||
|
||||
haveClientIdentity =
|
||||
baoDeploy.ciMetricsClientCertFile != null && baoDeploy.ciMetricsClientKeyFile != null;
|
||||
|
||||
credential = id: "/run/credentials/${unit}.service/${id}";
|
||||
# Credential id → host path of the poller's store identity.
|
||||
storeFiles = {
|
||||
"bao-client.pem" = baoDeploy.ciMetricsClientCertFile;
|
||||
"bao-client-key.pem" = baoDeploy.ciMetricsClientKeyFile;
|
||||
}
|
||||
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
|
||||
"bao-ca.pem" = baoDeploy.serverCaFile;
|
||||
};
|
||||
in
|
||||
{
|
||||
options.services.hyperhive.swarm.ci-metrics.clientId = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
readOnly = true;
|
||||
default = "swarm-ci-metrics";
|
||||
description = ''
|
||||
OAuth2 client id the CI run poller authenticates to the swarm
|
||||
collector as, and the audience it asks its token for.
|
||||
|
||||
Read-only because three hosts have to agree on it: authelia registers
|
||||
the client (`glue-ci-metrics-otel-oidc-client.nix`), the swarm collector
|
||||
admits it on the poller's receiver (`swarm.otel.serviceProducers`), and
|
||||
the secret publisher carries its secret to the store.
|
||||
'';
|
||||
};
|
||||
|
||||
options.services.hyperhive.deploy.ci-metrics = {
|
||||
enable = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
description = ''
|
||||
Run the CI run poller on this host: every 5 minutes it pushes each
|
||||
newly finished forge Actions run's duration and queue wait to the
|
||||
swarm collector, for the Grafana CI dashboard. Its first run pushes
|
||||
every run since the first.
|
||||
|
||||
Requires the poller's store identity,
|
||||
{option}`services.hyperhive.deploy.bao.ciMetricsClientCertFile` and
|
||||
`ciMetricsClientKeyFile`. Run it on one host per swarm: two pollers
|
||||
keep two cursors and push every run twice.
|
||||
'';
|
||||
};
|
||||
|
||||
package = lib.mkOption {
|
||||
type = lib.types.package;
|
||||
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-ci-metrics";
|
||||
description = "The CI run poller package.";
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
assertions = [
|
||||
{
|
||||
assertion = haveClientIdentity;
|
||||
message = ''
|
||||
services.hyperhive.deploy.ci-metrics.enable requires the poller's
|
||||
swarm-secret-store client identity: set both
|
||||
|
||||
services.hyperhive.deploy.bao.ciMetricsClientCertFile
|
||||
services.hyperhive.deploy.bao.ciMetricsClientKeyFile
|
||||
|
||||
swarm-ci-metrics.service reads its forge token and its OIDC client
|
||||
secret with it.
|
||||
|
||||
Both default to the pair glue-bao-tls.nix mints when
|
||||
services.hyperhive.deploy.bao.enable is true (the host that runs
|
||||
the store). Otherwise copy ci-metrics.pem and ci-metrics-key.pem
|
||||
from the store host's /var/lib/swarm-bao-pki/ and name them in
|
||||
those two options.
|
||||
'';
|
||||
}
|
||||
];
|
||||
|
||||
systemd.services.${unit} = {
|
||||
description = "push finished forge CI runs to the swarm metrics store";
|
||||
after = [ "network-online.target" ];
|
||||
wants = [ "network-online.target" ];
|
||||
# No `RemainAfterExit`: the timer starts this again, and starting an
|
||||
# active unit is a no-op.
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
ExecStart = "${cfg.package}/bin/swarm-ci-metrics";
|
||||
# The first pass reads every run the forge has, a page a second.
|
||||
TimeoutStartSec = "30min";
|
||||
# Paths only: `argv` is world-readable via /proc/<pid>/cmdline.
|
||||
LoadCredential = lib.mapAttrsToList (id: path: "${id}:${path}") storeFiles;
|
||||
DynamicUser = true;
|
||||
# The cursor. Losing it re-pushes every run.
|
||||
StateDirectory = unit;
|
||||
StateDirectoryMode = "0700";
|
||||
SyslogIdentifier = unit;
|
||||
ProtectSystem = "strict";
|
||||
ProtectHome = true;
|
||||
PrivateTmp = true;
|
||||
PrivateDevices = true;
|
||||
NoNewPrivileges = true;
|
||||
RestrictSUIDSGID = true;
|
||||
RemoveIPC = true;
|
||||
};
|
||||
environment = {
|
||||
SWARM_CI_METRICS_FORGE_URL = "https://${hyperhiveCfg.swarm.forge.domain}";
|
||||
SWARM_CI_METRICS_OTLP_ENDPOINT = otlpEndpoint;
|
||||
SWARM_CI_METRICS_OIDC_TOKEN_ENDPOINT = "${hyperhiveCfg.swarm.authelia.url}/api/oidc/token";
|
||||
SWARM_CI_METRICS_OIDC_CLIENT_ID = clientId;
|
||||
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||||
BAO_CLIENT_CERT = credential "bao-client.pem";
|
||||
BAO_CLIENT_KEY = credential "bao-client-key.pem";
|
||||
}
|
||||
// lib.optionalAttrs (storeFiles ? "bao-ca.pem") {
|
||||
BAO_CACERT = credential "bao-ca.pem";
|
||||
};
|
||||
};
|
||||
|
||||
systemd.timers.${unit} = {
|
||||
description = "run the CI run poller every 5 minutes";
|
||||
wantedBy = [ "timers.target" ];
|
||||
timerConfig = {
|
||||
OnBootSec = "2min";
|
||||
OnUnitInactiveSec = "5min";
|
||||
RandomizedDelaySec = "30s";
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
@ -197,6 +197,7 @@ let
|
|||
"agents.json"
|
||||
"authelia.json"
|
||||
"bao.json"
|
||||
"ci.json"
|
||||
"claude-usage.json"
|
||||
"collector.json"
|
||||
"controller.json"
|
||||
|
|
|
|||
410
nix/host-modules/swarm-grafana/dashboards/ci.json
Normal file
410
nix/host-modules/swarm-grafana/dashboards/ci.json
Normal file
|
|
@ -0,0 +1,410 @@
|
|||
{
|
||||
"uid": "hyperhive-swarm-ci",
|
||||
"title": "hyperhive · CI runs",
|
||||
"description": "Forge Actions run time and queue wait, one sample per finished run, per repo and workflow.",
|
||||
"tags": ["hyperhive", "ci"],
|
||||
"timezone": "utc",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"editable": true,
|
||||
"refresh": "5m",
|
||||
"time": {
|
||||
"from": "now-30d",
|
||||
"to": "now"
|
||||
},
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"name": "repo",
|
||||
"label": "Repo",
|
||||
"type": "query",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"query": "label_values({__name__=\"hyperhive.ci.run.duration\"}, repo)",
|
||||
"refresh": 2,
|
||||
"includeAll": true,
|
||||
"allValue": ".*",
|
||||
"multi": true,
|
||||
"current": {
|
||||
"text": "All",
|
||||
"value": "$__all"
|
||||
},
|
||||
"sort": 1
|
||||
},
|
||||
{
|
||||
"name": "workflow",
|
||||
"label": "Workflow",
|
||||
"type": "query",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"query": "label_values({__name__=\"hyperhive.ci.run.duration\", repo=~\"$repo\"}, workflow)",
|
||||
"refresh": 2,
|
||||
"includeAll": true,
|
||||
"allValue": ".*",
|
||||
"multi": true,
|
||||
"current": {
|
||||
"text": "All",
|
||||
"value": "$__all"
|
||||
},
|
||||
"sort": 1
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "timeseries",
|
||||
"title": "Run duration, successful runs",
|
||||
"description": "Mean and 90th percentile of stopped minus started, over the runs that ended in each interval.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"interval": "1h",
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "avg(last_over_time({__name__=\"hyperhive.ci.run.duration\", repo=~\"$repo\", workflow=~\"$workflow\", outcome=\"success\"}[$__interval]))",
|
||||
"legendFormat": "mean"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "quantile(0.9, last_over_time({__name__=\"hyperhive.ci.run.duration\", repo=~\"$repo\", workflow=~\"$workflow\", outcome=\"success\"}[$__interval]))",
|
||||
"legendFormat": "p90"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s",
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineWidth": 2,
|
||||
"fillOpacity": 0,
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"axisSoftMin": 0
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true,
|
||||
"calcs": []
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "timeseries",
|
||||
"title": "Queue wait",
|
||||
"description": "Started minus created: how long each run waited for a runner, mean and longest per interval.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"interval": "1h",
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 0
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "avg(last_over_time({__name__=\"hyperhive.ci.run.queue_wait\", repo=~\"$repo\", workflow=~\"$workflow\"}[$__interval]))",
|
||||
"legendFormat": "mean"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "max(last_over_time({__name__=\"hyperhive.ci.run.queue_wait\", repo=~\"$repo\", workflow=~\"$workflow\"}[$__interval]))",
|
||||
"legendFormat": "longest"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s",
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineWidth": 2,
|
||||
"fillOpacity": 0,
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"axisSoftMin": 0
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true,
|
||||
"calcs": []
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "timeseries",
|
||||
"title": "Runs per day by outcome",
|
||||
"description": "Finished runs per UTC day. Cancelled runs are not counted.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"interval": "1d",
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 9
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "count by (outcome) (last_over_time({__name__=\"hyperhive.ci.run.duration\", repo=~\"$repo\", workflow=~\"$workflow\"}[$__interval]))",
|
||||
"legendFormat": "{{outcome}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"decimals": 0,
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"drawStyle": "bars",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 80,
|
||||
"showPoints": "never",
|
||||
"stacking": {
|
||||
"mode": "normal",
|
||||
"group": "A"
|
||||
},
|
||||
"axisSoftMin": 0
|
||||
}
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "success"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"mode": "fixed",
|
||||
"fixedColor": "green"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "failure"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"mode": "fixed",
|
||||
"fixedColor": "red"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true,
|
||||
"calcs": ["sum"]
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "table",
|
||||
"title": "Duration per PR",
|
||||
"description": "Per PR (or ref, for runs not on a PR) over the time range: run time of its successful runs.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 9
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "avg by (pr) (last_over_time({__name__=\"hyperhive.ci.run.duration\", repo=~\"$repo\", workflow=~\"$workflow\", outcome=\"success\"}[$__range]))",
|
||||
"format": "table",
|
||||
"instant": true
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "max by (pr) (last_over_time({__name__=\"hyperhive.ci.run.duration\", repo=~\"$repo\", workflow=~\"$workflow\", outcome=\"success\"}[$__range]))",
|
||||
"format": "table",
|
||||
"instant": true
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "count by (pr) (last_over_time({__name__=\"hyperhive.ci.run.duration\", repo=~\"$repo\", workflow=~\"$workflow\"}[$__range]))",
|
||||
"format": "table",
|
||||
"instant": true
|
||||
},
|
||||
{
|
||||
"refId": "D",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "count by (pr) (last_over_time({__name__=\"hyperhive.ci.run.duration\", repo=~\"$repo\", workflow=~\"$workflow\", outcome=\"failure\"}[$__range]))",
|
||||
"format": "table",
|
||||
"instant": true
|
||||
}
|
||||
],
|
||||
"transformations": [
|
||||
{
|
||||
"id": "merge",
|
||||
"options": {}
|
||||
},
|
||||
{
|
||||
"id": "organize",
|
||||
"options": {
|
||||
"excludeByName": {
|
||||
"Time": true
|
||||
},
|
||||
"renameByName": {
|
||||
"pr": "PR",
|
||||
"Value #A": "Mean (success)",
|
||||
"Value #B": "Longest (success)",
|
||||
"Value #C": "Runs",
|
||||
"Value #D": "Failed runs"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "sortBy",
|
||||
"options": {
|
||||
"sort": [
|
||||
{
|
||||
"field": "PR",
|
||||
"desc": true
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"align": "auto"
|
||||
}
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byRegexp",
|
||||
"options": "^(Mean|Longest).*"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "unit",
|
||||
"value": "s"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byRegexp",
|
||||
"options": "^(Runs|Failed runs)$"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "decimals",
|
||||
"value": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"showHeader": true,
|
||||
"cellHeight": "sm"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -445,6 +445,12 @@ in
|
|||
clientId = hyperhiveCfg.swarm.victoriametrics.otel.clientId;
|
||||
port = lib.mkDefault 4401;
|
||||
};
|
||||
# The CI run poller in ./swarm-ci-metrics.nix, a host unit that pushes
|
||||
# its run samples here directly.
|
||||
swarm-ci-metrics = {
|
||||
clientId = hyperhiveCfg.swarm.ci-metrics.clientId;
|
||||
port = lib.mkDefault 4403;
|
||||
};
|
||||
};
|
||||
|
||||
# Not under any `enable`: see `publishedScrapeTargets` for why every host
|
||||
|
|
|
|||
|
|
@ -127,6 +127,13 @@ in
|
|||
&& deployCfg.bao.builderClientKeyFile != null
|
||||
);
|
||||
|
||||
# The CI run poller, by the same rule: its module asserts its store identity.
|
||||
config.services.hyperhive.deploy.ci-metrics.enable = lib.mkDefault (
|
||||
deployCfg.allSwarmServices
|
||||
&& deployCfg.bao.ciMetricsClientCertFile != null
|
||||
&& deployCfg.bao.ciMetricsClientKeyFile != null
|
||||
);
|
||||
|
||||
# The claude login minter, defaulted on the same terms as the review
|
||||
# classifier.
|
||||
config.services.hyperhive.deploy.claude-login.enable = lib.mkDefault (
|
||||
|
|
|
|||
|
|
@ -108,6 +108,8 @@ let
|
|||
hyperhiveCfg.swarm.matrix.otel.clientId
|
||||
# The metrics store container's journal forwarder, whose host reads it.
|
||||
hyperhiveCfg.swarm.victoriametrics.otel.clientId
|
||||
# The CI run poller, which reads it itself on its host.
|
||||
hyperhiveCfg.swarm.ci-metrics.clientId
|
||||
];
|
||||
|
||||
# The swarm controller's OIDC client, which it reads back from the store
|
||||
|
|
|
|||
|
|
@ -59,6 +59,7 @@ let
|
|||
deployCfg.bao.metricsAuthCommonName
|
||||
deployCfg.bao.reviewClassifierCommonName
|
||||
deployCfg.bao.builderCommonName
|
||||
deployCfg.bao.ciMetricsCommonName
|
||||
deployCfg.bao.claudeLoginCommonName
|
||||
deployCfg.bao.forgePublisherCommonName
|
||||
]
|
||||
|
|
|
|||
Loading…
Reference in a new issue