push ci run metrics to victoriametrics, with a grafana board
A swarm-ci-metrics host unit on a 5-minute timer lists every forge repo's Actions runs as the new read-only ci-metrics account and pushes each finished run's duration and queue wait over OTLP, stamped at the run's stop time. The first pass backfills from run 1. swarm-controller mints the account's token into bao and keeps a ci-metrics team with read on the Actions unit in every organisation. Closes #5376
This commit is contained in:
atlas
mara
parent
8b7a4e7955
commit
b072fbf4d6
36 changed files with 2849 additions and 56 deletions
166
nix/host-modules/swarm-ci-metrics.nix
Normal file
166
nix/host-modules/swarm-ci-metrics.nix
Normal file
|
|
@ -0,0 +1,166 @@
|
|||
# The swarm's CI run poller: a oneshot `swarm-ci-metrics` on a 5-minute timer,
|
||||
# on the host that sets `deploy.ci-metrics.enable`, which
|
||||
# `deploy.allSwarmServices` defaults on where the host has the poller's store
|
||||
# identity (./swarm-required-services.nix).
|
||||
#
|
||||
# Each pass lists the Actions runs of every repo the `ci-metrics` forge account
|
||||
# sees and pushes each newly finished run's duration and queue wait to the
|
||||
# swarm collector, stamped at the run's end. The first pass, with no cursor in
|
||||
# the state directory, walks every run back to the first. docs/swarm/ci-metrics.md
|
||||
# has the design.
|
||||
#
|
||||
# It reaches the forge, authelia, the store and the collector by name, so
|
||||
# nothing here asks which host runs any of them. Its one credential on disk is
|
||||
# its store leaf, handed in by `LoadCredential=`; the forge token and its OIDC
|
||||
# client secret it reads from the store under that leaf on every pass.
|
||||
{
|
||||
lib,
|
||||
config,
|
||||
...
|
||||
}:
|
||||
let
|
||||
hyperhiveCfg = config.services.hyperhive;
|
||||
cfg = hyperhiveCfg.deploy.ci-metrics;
|
||||
baoCfg = hyperhiveCfg.swarm.bao;
|
||||
baoDeploy = hyperhiveCfg.deploy.bao;
|
||||
gatewayCfg = hyperhiveCfg.gateway;
|
||||
swarmOtelCfg = hyperhiveCfg.swarm.otel;
|
||||
|
||||
unit = "swarm-ci-metrics";
|
||||
clientId = hyperhiveCfg.swarm.ci-metrics.clientId;
|
||||
|
||||
# The route `serviceProducers.swarm-ci-metrics` in ./swarm-otel-service.nix
|
||||
# declares, by the collector's gateway name. The port suffix is elided on
|
||||
# 443, as in ./swarm-builder.nix.
|
||||
gatewayPortSuffix = lib.optionalString (
|
||||
gatewayCfg.httpsPort != 443
|
||||
) ":${toString gatewayCfg.httpsPort}";
|
||||
otlpEndpoint = "https://${swarmOtelCfg.domain}${gatewayPortSuffix}/${unit}";
|
||||
|
||||
haveClientIdentity =
|
||||
baoDeploy.ciMetricsClientCertFile != null && baoDeploy.ciMetricsClientKeyFile != null;
|
||||
|
||||
credential = id: "/run/credentials/${unit}.service/${id}";
|
||||
# Credential id → host path of the poller's store identity.
|
||||
storeFiles = {
|
||||
"bao-client.pem" = baoDeploy.ciMetricsClientCertFile;
|
||||
"bao-client-key.pem" = baoDeploy.ciMetricsClientKeyFile;
|
||||
}
|
||||
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
|
||||
"bao-ca.pem" = baoDeploy.serverCaFile;
|
||||
};
|
||||
in
|
||||
{
|
||||
options.services.hyperhive.swarm.ci-metrics.clientId = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
readOnly = true;
|
||||
default = "swarm-ci-metrics";
|
||||
description = ''
|
||||
OAuth2 client id the CI run poller authenticates to the swarm
|
||||
collector as, and the audience it asks its token for.
|
||||
|
||||
Read-only because three hosts have to agree on it: authelia registers
|
||||
the client (`glue-ci-metrics-otel-oidc-client.nix`), the swarm collector
|
||||
admits it on the poller's receiver (`swarm.otel.serviceProducers`), and
|
||||
the secret publisher carries its secret to the store.
|
||||
'';
|
||||
};
|
||||
|
||||
options.services.hyperhive.deploy.ci-metrics = {
|
||||
enable = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
description = ''
|
||||
Run the CI run poller on this host: every 5 minutes it pushes each
|
||||
newly finished forge Actions run's duration and queue wait to the
|
||||
swarm collector, for the Grafana CI dashboard. Its first run pushes
|
||||
every run since the first.
|
||||
|
||||
Requires the poller's store identity,
|
||||
{option}`services.hyperhive.deploy.bao.ciMetricsClientCertFile` and
|
||||
`ciMetricsClientKeyFile`. Run it on one host per swarm: two pollers
|
||||
keep two cursors and push every run twice.
|
||||
'';
|
||||
};
|
||||
|
||||
package = lib.mkOption {
|
||||
type = lib.types.package;
|
||||
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-ci-metrics";
|
||||
description = "The CI run poller package.";
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
assertions = [
|
||||
{
|
||||
assertion = haveClientIdentity;
|
||||
message = ''
|
||||
services.hyperhive.deploy.ci-metrics.enable requires the poller's
|
||||
swarm-secret-store client identity: set both
|
||||
|
||||
services.hyperhive.deploy.bao.ciMetricsClientCertFile
|
||||
services.hyperhive.deploy.bao.ciMetricsClientKeyFile
|
||||
|
||||
swarm-ci-metrics.service reads its forge token and its OIDC client
|
||||
secret with it.
|
||||
|
||||
Both default to the pair glue-bao-tls.nix mints when
|
||||
services.hyperhive.deploy.bao.enable is true (the host that runs
|
||||
the store). Otherwise copy ci-metrics.pem and ci-metrics-key.pem
|
||||
from the store host's /var/lib/swarm-bao-pki/ and name them in
|
||||
those two options.
|
||||
'';
|
||||
}
|
||||
];
|
||||
|
||||
systemd.services.${unit} = {
|
||||
description = "push finished forge CI runs to the swarm metrics store";
|
||||
after = [ "network-online.target" ];
|
||||
wants = [ "network-online.target" ];
|
||||
# No `RemainAfterExit`: the timer starts this again, and starting an
|
||||
# active unit is a no-op.
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
ExecStart = "${cfg.package}/bin/swarm-ci-metrics";
|
||||
# The first pass reads every run the forge has, a page a second.
|
||||
TimeoutStartSec = "30min";
|
||||
# Paths only: `argv` is world-readable via /proc/<pid>/cmdline.
|
||||
LoadCredential = lib.mapAttrsToList (id: path: "${id}:${path}") storeFiles;
|
||||
DynamicUser = true;
|
||||
# The cursor. Losing it re-pushes every run.
|
||||
StateDirectory = unit;
|
||||
StateDirectoryMode = "0700";
|
||||
SyslogIdentifier = unit;
|
||||
ProtectSystem = "strict";
|
||||
ProtectHome = true;
|
||||
PrivateTmp = true;
|
||||
PrivateDevices = true;
|
||||
NoNewPrivileges = true;
|
||||
RestrictSUIDSGID = true;
|
||||
RemoveIPC = true;
|
||||
};
|
||||
environment = {
|
||||
SWARM_CI_METRICS_FORGE_URL = "https://${hyperhiveCfg.swarm.forge.domain}";
|
||||
SWARM_CI_METRICS_OTLP_ENDPOINT = otlpEndpoint;
|
||||
SWARM_CI_METRICS_OIDC_TOKEN_ENDPOINT = "${hyperhiveCfg.swarm.authelia.url}/api/oidc/token";
|
||||
SWARM_CI_METRICS_OIDC_CLIENT_ID = clientId;
|
||||
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||||
BAO_CLIENT_CERT = credential "bao-client.pem";
|
||||
BAO_CLIENT_KEY = credential "bao-client-key.pem";
|
||||
}
|
||||
// lib.optionalAttrs (storeFiles ? "bao-ca.pem") {
|
||||
BAO_CACERT = credential "bao-ca.pem";
|
||||
};
|
||||
};
|
||||
|
||||
systemd.timers.${unit} = {
|
||||
description = "run the CI run poller every 5 minutes";
|
||||
wantedBy = [ "timers.target" ];
|
||||
timerConfig = {
|
||||
OnBootSec = "2min";
|
||||
OnUnitInactiveSec = "5min";
|
||||
RandomizedDelaySec = "30s";
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
Loading…
Reference in a new issue