vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

push ci run metrics to victoriametrics, with a grafana board

Browse source 
A swarm-ci-metrics host unit on a 5-minute timer lists every forge
repo's Actions runs as the new read-only ci-metrics account and pushes
each finished run's duration and queue wait over OTLP, stamped at the
run's stop time. The first pass backfills from run 1. swarm-controller
mints the account's token into bao and keeps a ci-metrics team with
read on the Actions unit in every organisation.

Closes #5376
This commit is contained in:
atlas 2026-10-16 18:23:59 +02:00 • committed by mara
commit b072fbf4d6
36 changed files with 2849 additions and 56 deletions
Download patch file Download diff file Expand all files Collapse all files

5
nix/checks.nix
View file

@ -49,6 +49,11 @@ in
inherit (pkgs) lib;
shared = moduleEvalShared;
};
module-eval-ci-metrics = import ./module-eval/ci-metrics.nix {
inherit pkgs self nixosSystem;
inherit (pkgs) lib;
shared = moduleEvalShared;
};
module-eval-nats-authelia = import ./module-eval/nats-authelia.nix {
inherit pkgs self nixosSystem;
inherit (pkgs) lib;

2
nix/host-modules/default.nix
View file

@ -47,6 +47,7 @@
./glue-nats-wireguard.nix
./glue-queue-agent-credential.nix
./glue-builder-oidc-client.nix
./glue-ci-metrics-otel-oidc-client.nix
./glue-ci-runner-otel-oidc-client.nix
./glue-claude-login-oidc-client.nix
./glue-forge-otel-oidc-client.nix
@ -65,6 +66,7 @@
./swarm-bao.nix
./swarm-builder.nix
./swarm-claude-login.nix
./swarm-ci-metrics.nix
./swarm-ci-runner.nix
./swarm-review-classifier.nix
./swarm-secret-publisher.nix

9
nix/host-modules/glue-bao-tls.nix
View file

@ -117,6 +117,8 @@ in
reviewClassifierClientKeyFile = lib.mkDefault "${pkiDir}/review-classifier-key.pem";
builderClientCertFile = lib.mkDefault "${pkiDir}/builder.pem";
builderClientKeyFile = lib.mkDefault "${pkiDir}/builder-key.pem";
ciMetricsClientCertFile = lib.mkDefault "${pkiDir}/ci-metrics.pem";
ciMetricsClientKeyFile = lib.mkDefault "${pkiDir}/ci-metrics-key.pem";
claudeLoginClientCertFile = lib.mkDefault "${pkiDir}/claude-login.pem";
claudeLoginClientKeyFile = lib.mkDefault "${pkiDir}/claude-login-key.pem";
forgePublisherClientCertFile = lib.mkDefault "${pkiDir}/forge-publisher.pem";
@ -310,6 +312,13 @@ in
[ -s ${pkiDir}/builder.pem ] || ${signLeaf} ${pkiDir} builder \
${lib.escapeShellArg deployCfg.bao.builderCommonName} "" clientAuth
# The CI run poller's, which reads its forge token and OIDC client
# secret. Minted here for the matrix-ctl leaf's reason: the poller is a
# swarm singleton, so on a poller host without the store this is the
# file an operator copies.
[ -s ${pkiDir}/ci-metrics.pem ] || ${signLeaf} ${pkiDir} ci-metrics \
${lib.escapeShellArg deployCfg.bao.ciMetricsCommonName} "" clientAuth
# The claude login minter's, which writes agents' claude credentials
# and reads its queue client secret. Minted here for the matrix-ctl
# leaf's reason: the minter is a swarm service, so on a minter host

36
nix/host-modules/glue-ci-metrics-otel-oidc-client.nix Normal file
View file

@ -0,0 +1,36 @@
# Glue: register the CI run poller as an OIDC client wherever authelia runs.
#
# ONE PAIRING PER FILE: that poller ← authelia, and nothing else.
# Without it authelia mints no secret, the publisher has nothing to copy into
# the store, and the poller's push is a 401.
#
# Gated on authelia being HERE, not on this host running the poller: a
# client is a row in THIS host's provider config, the split
# ./glue-ci-runner-otel-oidc-client.nix makes for the same reason.
{
lib,
config,
...
}:
let
hyperhiveCfg = config.services.hyperhive;
clientId = hyperhiveCfg.swarm.ci-metrics.clientId;
in
{
config = lib.mkIf hyperhiveCfg.deploy.authelia.enable {
services.hyperhive.swarm.authelia.oidc.clients = [
{
id = clientId;
description = "HyperHive CI run poller";
kind = "machine";
redirectUris = [ ];
# Its own id as its own permitted audience, the one the swarm
# collector's receiver for the poller checks.
audience = [ clientId ];
# The receiver verifies the token offline against `/jwks.json`;
# authelia's default opaque handle fails there.
accessTokenSignedResponseAlg = "RS256";
}
];
};
}

70
nix/host-modules/swarm-bao.nix
View file

@ -449,6 +449,10 @@ let
capabilities = ["create", "read", "update"]
}
path "${credentialMountPath}/data/${ciMetricsForgeTokenLeaf}" {
capabilities = ["create", "read", "update"]
}
path "${credentialMountPath}/data/${swarmAppserviceTokenLeaf}" {
capabilities = ["read"]
}
@ -504,8 +508,8 @@ let
# The review classifier's forge token is denied too: the controller posts
# the classifier's reviews with whatever token is there, so a write to it
# picks the account those reviews come from.
# So is the builder's: a write to it picks the account the builder fetches
# config repos as.
# So are the builder's and the CI run poller's: a write to either picks the
# account that service reads the forge as.
secretPublisherPolicyText = ''
path "${credentialMountPath}/data/swarm/hives/*" {
capabilities = ["create", "update"]
@ -539,6 +543,10 @@ let
capabilities = ["deny"]
}
path "${credentialMountPath}/data/${ciMetricsForgeTokenLeaf}" {
capabilities = ["deny"]
}
path "${credentialMountPath}/data/${controllerClientLeaf}" {
capabilities = ["create", "update"]
}
@ -620,6 +628,12 @@ let
# to fetch config repos.
builderForgeTokenLeaf = "swarm/services/builder/forge/token";
# The CI run poller's forge token, the nix half of
# `swarm_secret_client::ci_metrics::forge_token_path`. The controller mints it
# and reads it back before deciding whether to re-mint; the poller reads it
# to list runs.
ciMetricsForgeTokenLeaf = "swarm/services/ci-metrics/forge/token";
# The review classifier's ACP provider key, the nix half of
# `swarm_secret_client::review_classifier::acp_provider_key_path`. Written
# and read like the API key above.
@ -1095,6 +1109,18 @@ let
}
];
# The CI run poller: its forge token and its own OIDC client secret, both
# read once per pass by the host unit.
ciMetricsReaders = [
{
name = "swarm-ci-metrics";
cn = baoDeploy.ciMetricsCommonName;
policyText =
readStanza "${credentialMountPath}/data/${ciMetricsForgeTokenLeaf}"
+ readStanza "${credentialMountPath}/data/swarm/services/${hyperhiveCfg.swarm.ci-metrics.clientId}/oidc/client";
}
];
# The claude login minter: every agent's `claude-credentials` leaf,
# create and update so a relogin replaces the last, and nothing else under
# an agent; and its own queue client secret, read at start.
@ -2003,6 +2029,19 @@ in
'';
};
ciMetricsCommonName = lib.mkOption {
type = lib.types.str;
default = "swarm-ci-metrics";
description = ''
Subject the store's `swarm-ci-metrics` cert-auth role accepts: the
identity the CI run poller presents from its host unit. Its grant is
two paths, read only: `swarm/services/ci-metrics/forge/token` and its
own OIDC client secret.
⚠️ Reserved as a hive name by ./swarm.nix, like its siblings.
'';
};
claudeLoginCommonName = lib.mkOption {
type = lib.types.str;
default = "swarm-claude-login";
@ -2318,6 +2357,31 @@ in
'';
};
ciMetricsClientCertFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "/var/lib/swarm-bao-pki/ci-metrics.pem";
description = ''
Certificate the CI run poller presents to the store. Its subject must
be {option}`services.hyperhive.deploy.bao.ciMetricsCommonName`.
./glue-bao-tls.nix points it at the leaf it mints on the store's host.
On a poller host without the store, it is a file the operator copies
from the store's host. A path, never a value.
'';
};
ciMetricsClientKeyFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "/var/lib/swarm-bao-pki/ci-metrics-key.pem";
description = ''
Private key for
{option}`services.hyperhive.deploy.bao.ciMetricsClientCertFile`.
A path, never a value.
'';
};
claudeLoginClientCertFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
@ -3435,6 +3499,8 @@ in
systemd.services.swarm-bao-builder-policy = readerPolicyUnit "write the swarm builder's forge-token-reader bao policy and cert-auth role" builderReaders;
systemd.services.swarm-bao-ci-metrics-policy = readerPolicyUnit "write the CI run poller's forge-token-reader bao policy and cert-auth role" ciMetricsReaders;
systemd.services.swarm-bao-claude-login-policy = readerPolicyUnit "write the claude login minter's credentials-writer bao policy and cert-auth role" claudeLoginWriters;
systemd.services.swarm-bao-forge-publisher-policy = readerPolicyUnit "write the forge's controller-token-publisher bao policy and cert-auth role" forgePublisherWriters;

166
nix/host-modules/swarm-ci-metrics.nix Normal file
View file

@ -0,0 +1,166 @@
# The swarm's CI run poller: a oneshot `swarm-ci-metrics` on a 5-minute timer,
# on the host that sets `deploy.ci-metrics.enable`, which
# `deploy.allSwarmServices` defaults on where the host has the poller's store
# identity (./swarm-required-services.nix).
#
# Each pass lists the Actions runs of every repo the `ci-metrics` forge account
# sees and pushes each newly finished run's duration and queue wait to the
# swarm collector, stamped at the run's end. The first pass, with no cursor in
# the state directory, walks every run back to the first. docs/swarm/ci-metrics.md
# has the design.
#
# It reaches the forge, authelia, the store and the collector by name, so
# nothing here asks which host runs any of them. Its one credential on disk is
# its store leaf, handed in by `LoadCredential=`; the forge token and its OIDC
# client secret it reads from the store under that leaf on every pass.
{
lib,
config,
...
}:
let
hyperhiveCfg = config.services.hyperhive;
cfg = hyperhiveCfg.deploy.ci-metrics;
baoCfg = hyperhiveCfg.swarm.bao;
baoDeploy = hyperhiveCfg.deploy.bao;
gatewayCfg = hyperhiveCfg.gateway;
swarmOtelCfg = hyperhiveCfg.swarm.otel;
unit = "swarm-ci-metrics";
clientId = hyperhiveCfg.swarm.ci-metrics.clientId;
# The route `serviceProducers.swarm-ci-metrics` in ./swarm-otel-service.nix
# declares, by the collector's gateway name. The port suffix is elided on
# 443, as in ./swarm-builder.nix.
gatewayPortSuffix = lib.optionalString (
gatewayCfg.httpsPort != 443
) ":${toString gatewayCfg.httpsPort}";
otlpEndpoint = "https://${swarmOtelCfg.domain}${gatewayPortSuffix}/${unit}";
haveClientIdentity =
baoDeploy.ciMetricsClientCertFile != null && baoDeploy.ciMetricsClientKeyFile != null;
credential = id: "/run/credentials/${unit}.service/${id}";
# Credential id → host path of the poller's store identity.
storeFiles = {
"bao-client.pem" = baoDeploy.ciMetricsClientCertFile;
"bao-client-key.pem" = baoDeploy.ciMetricsClientKeyFile;
}
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
"bao-ca.pem" = baoDeploy.serverCaFile;
};
in
{
options.services.hyperhive.swarm.ci-metrics.clientId = lib.mkOption {
type = lib.types.str;
readOnly = true;
default = "swarm-ci-metrics";
description = ''
OAuth2 client id the CI run poller authenticates to the swarm
collector as, and the audience it asks its token for.
Read-only because three hosts have to agree on it: authelia registers
the client (`glue-ci-metrics-otel-oidc-client.nix`), the swarm collector
admits it on the poller's receiver (`swarm.otel.serviceProducers`), and
the secret publisher carries its secret to the store.
'';
};
options.services.hyperhive.deploy.ci-metrics = {
enable = lib.mkOption {
type = lib.types.bool;
default = false;
description = ''
Run the CI run poller on this host: every 5 minutes it pushes each
newly finished forge Actions run's duration and queue wait to the
swarm collector, for the Grafana CI dashboard. Its first run pushes
every run since the first.
Requires the poller's store identity,
{option}`services.hyperhive.deploy.bao.ciMetricsClientCertFile` and
`ciMetricsClientKeyFile`. Run it on one host per swarm: two pollers
keep two cursors and push every run twice.
'';
};
package = lib.mkOption {
type = lib.types.package;
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-ci-metrics";
description = "The CI run poller package.";
};
};
config = lib.mkIf cfg.enable {
assertions = [
{
assertion = haveClientIdentity;
message = ''
services.hyperhive.deploy.ci-metrics.enable requires the poller's
swarm-secret-store client identity: set both
services.hyperhive.deploy.bao.ciMetricsClientCertFile
services.hyperhive.deploy.bao.ciMetricsClientKeyFile
swarm-ci-metrics.service reads its forge token and its OIDC client
secret with it.
Both default to the pair glue-bao-tls.nix mints when
services.hyperhive.deploy.bao.enable is true (the host that runs
the store). Otherwise copy ci-metrics.pem and ci-metrics-key.pem
from the store host's /var/lib/swarm-bao-pki/ and name them in
those two options.
'';
}
];
systemd.services.${unit} = {
description = "push finished forge CI runs to the swarm metrics store";
after = [ "network-online.target" ];
wants = [ "network-online.target" ];
# No `RemainAfterExit`: the timer starts this again, and starting an
# active unit is a no-op.
serviceConfig = {
Type = "oneshot";
ExecStart = "${cfg.package}/bin/swarm-ci-metrics";
# The first pass reads every run the forge has, a page a second.
TimeoutStartSec = "30min";
# Paths only: `argv` is world-readable via /proc/<pid>/cmdline.
LoadCredential = lib.mapAttrsToList (id: path: "${id}:${path}") storeFiles;
DynamicUser = true;
# The cursor. Losing it re-pushes every run.
StateDirectory = unit;
StateDirectoryMode = "0700";
SyslogIdentifier = unit;
ProtectSystem = "strict";
ProtectHome = true;
PrivateTmp = true;
PrivateDevices = true;
NoNewPrivileges = true;
RestrictSUIDSGID = true;
RemoveIPC = true;
};
environment = {
SWARM_CI_METRICS_FORGE_URL = "https://${hyperhiveCfg.swarm.forge.domain}";
SWARM_CI_METRICS_OTLP_ENDPOINT = otlpEndpoint;
SWARM_CI_METRICS_OIDC_TOKEN_ENDPOINT = "${hyperhiveCfg.swarm.authelia.url}/api/oidc/token";
SWARM_CI_METRICS_OIDC_CLIENT_ID = clientId;
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
BAO_CLIENT_CERT = credential "bao-client.pem";
BAO_CLIENT_KEY = credential "bao-client-key.pem";
}
// lib.optionalAttrs (storeFiles ? "bao-ca.pem") {
BAO_CACERT = credential "bao-ca.pem";
};
};
systemd.timers.${unit} = {
description = "run the CI run poller every 5 minutes";
wantedBy = [ "timers.target" ];
timerConfig = {
OnBootSec = "2min";
OnUnitInactiveSec = "5min";
RandomizedDelaySec = "30s";
};
};
};
}

1
nix/host-modules/swarm-grafana.nix
View file

@ -197,6 +197,7 @@ let
"agents.json"
"authelia.json"
"bao.json"
"ci.json"
"claude-usage.json"
"collector.json"
"controller.json"

410
nix/host-modules/swarm-grafana/dashboards/ci.json Normal file
View file

@ -0,0 +1,410 @@
{
"uid": "hyperhive-swarm-ci",
"title": "hyperhive · CI runs",
"description": "Forge Actions run time and queue wait, one sample per finished run, per repo and workflow.",
"tags": ["hyperhive", "ci"],
"timezone": "utc",
"schemaVersion": 39,
"version": 1,
"editable": true,
"refresh": "5m",
"time": {
"from": "now-30d",
"to": "now"
},
"templating": {
"list": [
{
"name": "repo",
"label": "Repo",
"type": "query",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"query": "label_values({__name__=\"hyperhive.ci.run.duration\"}, repo)",
"refresh": 2,
"includeAll": true,
"allValue": ".*",
"multi": true,
"current": {
"text": "All",
"value": "$__all"
},
"sort": 1
},
{
"name": "workflow",
"label": "Workflow",
"type": "query",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"query": "label_values({__name__=\"hyperhive.ci.run.duration\", repo=~\"$repo\"}, workflow)",
"refresh": 2,
"includeAll": true,
"allValue": ".*",
"multi": true,
"current": {
"text": "All",
"value": "$__all"
},
"sort": 1
}
]
},
"panels": [
{
"id": 1,
"type": "timeseries",
"title": "Run duration, successful runs",
"description": "Mean and 90th percentile of stopped minus started, over the runs that ended in each interval.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"interval": "1h",
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 0
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "avg(last_over_time({__name__=\"hyperhive.ci.run.duration\", repo=~\"$repo\", workflow=~\"$workflow\", outcome=\"success\"}[$__interval]))",
"legendFormat": "mean"
},
{
"refId": "B",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "quantile(0.9, last_over_time({__name__=\"hyperhive.ci.run.duration\", repo=~\"$repo\", workflow=~\"$workflow\", outcome=\"success\"}[$__interval]))",
"legendFormat": "p90"
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "line",
"lineWidth": 2,
"fillOpacity": 0,
"showPoints": "auto",
"spanNulls": false,
"axisSoftMin": 0
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true,
"calcs": []
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 2,
"type": "timeseries",
"title": "Queue wait",
"description": "Started minus created: how long each run waited for a runner, mean and longest per interval.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"interval": "1h",
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 0
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "avg(last_over_time({__name__=\"hyperhive.ci.run.queue_wait\", repo=~\"$repo\", workflow=~\"$workflow\"}[$__interval]))",
"legendFormat": "mean"
},
{
"refId": "B",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "max(last_over_time({__name__=\"hyperhive.ci.run.queue_wait\", repo=~\"$repo\", workflow=~\"$workflow\"}[$__interval]))",
"legendFormat": "longest"
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "line",
"lineWidth": 2,
"fillOpacity": 0,
"showPoints": "auto",
"spanNulls": false,
"axisSoftMin": 0
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true,
"calcs": []
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 3,
"type": "timeseries",
"title": "Runs per day by outcome",
"description": "Finished runs per UTC day. Cancelled runs are not counted.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"interval": "1d",
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 9
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "count by (outcome) (last_over_time({__name__=\"hyperhive.ci.run.duration\", repo=~\"$repo\", workflow=~\"$workflow\"}[$__interval]))",
"legendFormat": "{{outcome}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "bars",
"lineWidth": 1,
"fillOpacity": 80,
"showPoints": "never",
"stacking": {
"mode": "normal",
"group": "A"
},
"axisSoftMin": 0
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "success"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "green"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "failure"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "red"
}
}
]
}
]
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true,
"calcs": ["sum"]
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 4,
"type": "table",
"title": "Duration per PR",
"description": "Per PR (or ref, for runs not on a PR) over the time range: run time of its successful runs.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 9
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "avg by (pr) (last_over_time({__name__=\"hyperhive.ci.run.duration\", repo=~\"$repo\", workflow=~\"$workflow\", outcome=\"success\"}[$__range]))",
"format": "table",
"instant": true
},
{
"refId": "B",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "max by (pr) (last_over_time({__name__=\"hyperhive.ci.run.duration\", repo=~\"$repo\", workflow=~\"$workflow\", outcome=\"success\"}[$__range]))",
"format": "table",
"instant": true
},
{
"refId": "C",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "count by (pr) (last_over_time({__name__=\"hyperhive.ci.run.duration\", repo=~\"$repo\", workflow=~\"$workflow\"}[$__range]))",
"format": "table",
"instant": true
},
{
"refId": "D",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "count by (pr) (last_over_time({__name__=\"hyperhive.ci.run.duration\", repo=~\"$repo\", workflow=~\"$workflow\", outcome=\"failure\"}[$__range]))",
"format": "table",
"instant": true
}
],
"transformations": [
{
"id": "merge",
"options": {}
},
{
"id": "organize",
"options": {
"excludeByName": {
"Time": true
},
"renameByName": {
"pr": "PR",
"Value #A": "Mean (success)",
"Value #B": "Longest (success)",
"Value #C": "Runs",
"Value #D": "Failed runs"
}
}
},
{
"id": "sortBy",
"options": {
"sort": [
{
"field": "PR",
"desc": true
}
]
}
}
],
"fieldConfig": {
"defaults": {
"custom": {
"align": "auto"
}
},
"overrides": [
{
"matcher": {
"id": "byRegexp",
"options": "^(Mean|Longest).*"
},
"properties": [
{
"id": "unit",
"value": "s"
}
]
},
{
"matcher": {
"id": "byRegexp",
"options": "^(Runs|Failed runs)$"
},
"properties": [
{
"id": "decimals",
"value": 0
}
]
}
]
},
"options": {
"showHeader": true,
"cellHeight": "sm"
}
}
]
}

6
nix/host-modules/swarm-otel-service.nix
View file

@ -445,6 +445,12 @@ in
clientId = hyperhiveCfg.swarm.victoriametrics.otel.clientId;
port = lib.mkDefault 4401;
};
# The CI run poller in ./swarm-ci-metrics.nix, a host unit that pushes
# its run samples here directly.
swarm-ci-metrics = {
clientId = hyperhiveCfg.swarm.ci-metrics.clientId;
port = lib.mkDefault 4403;
};
};
# Not under any `enable`: see `publishedScrapeTargets` for why every host

7
nix/host-modules/swarm-required-services.nix
View file

@ -127,6 +127,13 @@ in
&& deployCfg.bao.builderClientKeyFile != null
);
# The CI run poller, by the same rule: its module asserts its store identity.
config.services.hyperhive.deploy.ci-metrics.enable = lib.mkDefault (
deployCfg.allSwarmServices
&& deployCfg.bao.ciMetricsClientCertFile != null
&& deployCfg.bao.ciMetricsClientKeyFile != null
);
# The claude login minter, defaulted on the same terms as the review
# classifier.
config.services.hyperhive.deploy.claude-login.enable = lib.mkDefault (

2
nix/host-modules/swarm-secret-publisher.nix
View file

@ -108,6 +108,8 @@ let
hyperhiveCfg.swarm.matrix.otel.clientId
# The metrics store container's journal forwarder, whose host reads it.
hyperhiveCfg.swarm.victoriametrics.otel.clientId
# The CI run poller, which reads it itself on its host.
hyperhiveCfg.swarm.ci-metrics.clientId
];
# The swarm controller's OIDC client, which it reads back from the store

1
nix/host-modules/swarm.nix
View file

@ -59,6 +59,7 @@ let
deployCfg.bao.metricsAuthCommonName
deployCfg.bao.reviewClassifierCommonName
deployCfg.bao.builderCommonName
deployCfg.bao.ciMetricsCommonName
deployCfg.bao.claudeLoginCommonName
deployCfg.bao.forgePublisherCommonName
]

99
nix/module-eval/bao-grants.nix
View file

@ -169,8 +169,8 @@ let
_: u: (u.environment.BAO_CLIENT_CERT or null) == granterCertFile
) baoGrantWithConsumers.systemd.services;
# The twenty units that write a `swarm-*` grant, by name, for the discovery
# control below.
# The twenty-one units that write a `swarm-*` grant, by name, for the
# discovery control below.
grantingUnitNames = [
"swarm-bao-controller-policy"
"swarm-bao-secret-publisher-policy"
@ -191,6 +191,7 @@ let
"swarm-bao-forge-publisher-policy"
"swarm-bao-review-classifier-policy"
"swarm-bao-builder-policy"
"swarm-bao-ci-metrics-policy"
"swarm-bao-claude-login-policy"
];
@ -1032,24 +1033,24 @@ let
}
{
# A store host without the granter's pair writes its grants some other
# way, so none of the twenty units may exist. Without this arm
# way, so none of the twenty-one units may exist. Without this arm
# `lib.mkIf haveGranter` could be dropped from any of them and every other
# case here would still pass.
name = "without the granter's pair none of the twenty granting units render";
name = "without the granter's pair none of the twenty-one granting units render";
ok =
let
s = baoGranterOptOut.systemd.services;
in
lib.all (unit: !(s ? ${unit})) (grantingUnitNames ++ [ "swarm-bao-granter-role" ])
# The control: the same store with the pair renders all twenty.
# The control: the same store with the pair renders all twenty-one.
&& lib.all (unit: baoGrantHere.systemd.services ? ${unit}) grantingUnitNames;
}
{
# 🩸 What replaced the silent skip. With no bootstrap token the twenty still
# render, and a refused granter fails them with the step that fixes it.
# 🩸 What replaced the silent skip. With no bootstrap token the twenty-one
# still render, and a refused granter fails them with the step that fixes it.
# A store host that never named a token is told to name one, since the
# unit that sets the granter up renders only where it has.
name = "a store host without a bootstrap token renders the twenty, each failing loudly with the one-time step";
name = "a store host without a bootstrap token renders the twenty-one, each failing loudly with the one-time step";
ok =
let
s = baoGranterNoToken.systemd.services;
@ -1670,6 +1671,84 @@ let
in
lib.hasInfix "/builder.pem ]" s && lib.hasInfix "swarm-builder \"\" clientAuth" s;
}
{
# The poller reads its forge token and its own OIDC client secret, each
# read-only and spelled to the leaf. The whole list, so the token's
# stanza is the presence control and any further path fails it.
name = "the CI run poller's grant is its forge token and its OIDC client secret, and nothing else";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-ci-metrics-policy.script;
in
grantsIn s == [
{
path = "secret/data/swarm/services/ci-metrics/forge/token";
caps = [ "read" ];
}
{
path = "secret/data/swarm/services/swarm-ci-metrics/oidc/client";
caps = [ "read" ];
}
]
&& lib.hasInfix "auth/cert/certs/swarm-ci-metrics" s
&& lib.hasInfix "allowed_common_names=swarm-ci-metrics" s
&& lib.hasInfix "token_policies=swarm-ci-metrics" s;
}
{
# Who may read the token is the units whose grant on it holds `read`:
# the controller, which mints it, and the poller. The publisher names it
# only to deny it.
name = "no granting unit but the controller's and the CI run poller's may read the poller's forge token";
ok =
let
path = "secret/data/swarm/services/ci-metrics/forge/token";
reading = lib.filter (
u:
lib.any (g: g.path == path && lib.elem "read" g.caps) (
grantsIn baoGrantHere.systemd.services.${u}.script
)
) grantingUnitNames;
in
lib.sort lib.lessThan reading == [
"swarm-bao-ci-metrics-policy"
"swarm-bao-controller-policy"
];
}
{
# `forge::ci_metrics_account` mints the token and reads it back to decide
# on a re-mint. Pinned as the whole stanza, and spelled to the leaf.
name = "the controller's grant on the CI run poller is its forge token alone";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script;
in
lib.hasInfix "path \"secret/data/swarm/services/ci-metrics/forge/token\" {\n capabilities = [\"create\", \"read\", \"update\"]\n}" s
&& !(lib.hasInfix "secret/data/swarm/services/ci-metrics/*" s);
}
{
# The token sits under the publisher's `services/*` write, and the
# poller lists runs with whatever is there. Its OIDC leaf is the
# control: still the prefix's.
name = "the publisher may not write the CI run poller's forge token";
ok =
let
grants = grantsIn baoGrantHere.systemd.services.swarm-bao-secret-publisher-policy.script;
in
(grantFor grants "secret/data/swarm/services/ci-metrics/forge/token").caps == [ "deny" ]
&&
(grantFor grants "secret/data/swarm/services/swarm-ci-metrics/oidc/client").caps == [
"create"
"update"
];
}
{
name = "the PKI unit signs the CI run poller's leaf under its own subject";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-pki.script;
in
lib.hasInfix "/ci-metrics.pem ]" s && lib.hasInfix "swarm-ci-metrics \"\" clientAuth" s;
}
{
# The minter writes one leaf under every agent, create and update so a
# relogin replaces the last, and reads its own queue client secret. No
@ -1974,8 +2053,8 @@ let
}
{
# What makes the case above mean something: discovery by the granter's
# certificate reaches all twenty units, and each yields calls.
name = "the granter-policy check sees all twenty granting units, and parses calls from each";
# certificate reaches all twenty-one units, and each yields calls.
name = "the granter-policy check sees all twenty-one granting units, and parses calls from each";
ok =
lib.sort lib.lessThan (lib.attrNames granterUnits) == lib.sort lib.lessThan grantingUnitNames
&& lib.all (u: baoCalls u.script != [ ]) (lib.attrValues granterUnits)

98
nix/module-eval/ci-metrics.nix Normal file
View file

@ -0,0 +1,98 @@
# `checks.module-eval-ci-metrics` — see ./lib.nix for the shared rationale
# (why this suite exists, naming convention, "evaluates not executes"). The CI
# run poller's unit and timer, its default, its collector route and its IdP
# registration; its store grants are in ./bao-grants.nix beside every other
# principal's, and its dashboard in ./grafana.nix.
#
# Every fixture is a shared one, so this cluster evaluates no host of its own.
{
pkgs,
lib,
self,
nixosSystem,
shared,
}:
let
inherit
(import ./lib.nix {
inherit
pkgs
lib
self
nixosSystem
;
})
runGroup
;
bare = shared.bareWithController;
# Every swarm service on one host: the store mints the poller's leaf, so the
# poller defaults on, beside authelia, the publisher and the collector.
allLocal = shared.allLocal;
svc = allLocal.systemd.services.swarm-ci-metrics;
env = svc.environment;
clientId = allLocal.services.hyperhive.swarm.ci-metrics.clientId;
otel = allLocal.services.hyperhive.swarm.otel;
cases = [
{
# The control is the same unit and timer on the all-local host: a
# misspelled attribute path would make the absence arm pass alone.
name = "the CI run poller's unit and timer exist only where its host enables it";
ok =
!(bare.systemd.services ? swarm-ci-metrics)
&& !(bare.systemd.timers ? swarm-ci-metrics)
&& allLocal.services.hyperhive.deploy.ci-metrics.enable
&& svc.serviceConfig.Type == "oneshot"
&& !(svc.serviceConfig.RemainAfterExit or false)
&& allLocal.systemd.timers.swarm-ci-metrics.timerConfig.OnUnitInactiveSec == "5min"
&& lib.elem "timers.target" allLocal.systemd.timers.swarm-ci-metrics.wantedBy;
}
{
# The leaf is handed in as a credential, never an argument; the forge
# token and the client secret are read from the store per pass.
name = "the poller loads only its store leaf, keeps its cursor in its state directory and carries no secret in its unit";
ok =
lib.elem "bao-client-key.pem:/var/lib/swarm-bao-pki/ci-metrics-key.pem" svc.serviceConfig.LoadCredential
&& lib.elem "bao-client.pem:/var/lib/swarm-bao-pki/ci-metrics.pem" svc.serviceConfig.LoadCredential
&& env.BAO_CLIENT_KEY == "/run/credentials/swarm-ci-metrics.service/bao-client-key.pem"
&& svc.serviceConfig.DynamicUser
&& svc.serviceConfig.StateDirectory == "swarm-ci-metrics"
&& !(lib.any (n: lib.hasSuffix "TOKEN" n || lib.hasSuffix "SECRET" n) (lib.attrNames env));
}
{
# The endpoint the poller pushes to is the route the collector admits
# its client on, and that route feeds the swarm tier's metrics pipeline.
name = "the poller pushes to its own collector route, which admits its client into the metrics pipeline";
ok =
let
h = shared.baoWithCollector;
s = h.containers.swarm-otel.config.services.opentelemetry-collector.settings;
in
env.SWARM_CI_METRICS_OTLP_ENDPOINT == "https://${otel.domain}/swarm-ci-metrics"
&& env.SWARM_CI_METRICS_OIDC_CLIENT_ID == clientId
&& otel.serviceProducers.swarm-ci-metrics.clientId == clientId
&& s.receivers."otlp/swarm-ci-metrics".protocols.http.auth.authenticator == "oidc/swarm-ci-metrics"
&& s.extensions."oidc/swarm-ci-metrics".audience == clientId
&& lib.elem "otlp/swarm-ci-metrics" s.service.pipelines."metrics/${otel.producerName}".receivers
&& h.services.nginx.virtualHosts.${otel.domain}.locations ? "/swarm-ci-metrics/";
}
{
name = "the poller's client is registered as a machine, and the publisher carries its secret";
ok =
let
c = lib.findFirst (
c: c.id == clientId
) null allLocal.services.hyperhive.swarm.authelia.oidc.clients;
in
c != null
&& c.kind == "machine"
&& c.audience == [ clientId ]
&& lib.hasInfix "secret/swarm/services/${clientId}/oidc/client" (
allLocal.systemd.services.swarm-secret-publish.script
);
}
];
in
runGroup "ci-metrics" cases

29
nix/module-eval/grafana.nix
View file

@ -354,6 +354,35 @@ let
grafanaRemoteAuthelia.services.hyperhive.swarm.statusPublish.tokenEndpoint
== "https://auth.example.invalid/api/oidc/token";
}
{
# The CI board is in the directory the provider scans (`bao.json` is the
# control that the entries are read at all), and every metric it queries
# is a name the poller pushes: a renamed metric on either side is a board
# of empty panels.
name = "the CI dashboard ships, and queries the metric names the CI run poller pushes";
ok =
let
provider = lib.head grafanaWithAuthelia.containers.swarm-grafana.config.services.grafana.provision.dashboards.settings.providers;
shipped = map (e: e.name) provider.options.path.entries;
board = builtins.fromJSON (builtins.readFile ../host-modules/swarm-grafana/dashboards/ci.json);
exprs = lib.concatMap (p: map (t: t.expr or "") (p.targets or [ ])) board.panels;
queried = lib.unique (
lib.concatMap (
e: map lib.head (lib.filter lib.isList (builtins.split ''"(hyperhive\.ci\.[a-z_.]+)"'' e))
) exprs
);
pushed = builtins.readFile ../../swarm-ci-metrics/src/sample.rs;
in
lib.elem "ci.json" shipped
&& lib.elem "bao.json" shipped
&&
lib.sort lib.lessThan queried == [
"hyperhive.ci.run.duration"
"hyperhive.ci.run.queue_wait"
]
&& lib.all (m: lib.hasInfix "\"${m}\"" pushed) queried
&& lib.any (v: v.name == "repo") board.templating.list;
}
{
# The standing check on the collector's PRIORITY mapping — whether a line
# ever reaches the log store carrying no severity. It is shipped config

1
nix/packages/default.nix
View file

@ -182,6 +182,7 @@ in
# reason: it runs inside its own container, not on the host.
swarm-review-classifier = mkBinPackage "swarm-review-classifier" "hyperhive review classifier try-out daemon";
swarm-builder = mkBinPackage "swarm-builder" "hyperhive swarm builder: agent config build checks";
swarm-ci-metrics = mkBinPackage "swarm-ci-metrics" "hyperhive swarm CI run poller";
swarm-claude-login = mkBinPackage "swarm-claude-login" "hyperhive claude login minter";
# The only process allowed to write swarm-authelia's users database —

6
nix/reserved-names.nix
View file

@ -93,10 +93,12 @@
# ---- forge accounts swarm-controller owns ----------------------------
# Service accounts it creates under these logins (`forge::builder_account`,
# `forge::classifier_account`). An agent of the same name would be refused
# one node into provisioning, and would keep the service from its token.
# `forge::classifier_account`, `forge::ci_metrics_account`). An agent of the
# same name would be refused one node into provisioning, and would keep the
# service from its token.
"builder"
"review-classifier"
"ci-metrics"
]
# Deliberately absent, and it is a load-bearing omission:
#