push ci run metrics to victoriametrics, with a grafana board
A swarm-ci-metrics host unit on a 5-minute timer lists every forge repo's Actions runs as the new read-only ci-metrics account and pushes each finished run's duration and queue wait over OTLP, stamped at the run's stop time. The first pass backfills from run 1. swarm-controller mints the account's token into bao and keeps a ci-metrics team with read on the Actions unit in every organisation. Closes #5376
This commit is contained in:
atlas
mara
parent
8b7a4e7955
commit
b072fbf4d6
36 changed files with 2849 additions and 56 deletions
|
|
@ -169,8 +169,8 @@ let
|
|||
_: u: (u.environment.BAO_CLIENT_CERT or null) == granterCertFile
|
||||
) baoGrantWithConsumers.systemd.services;
|
||||
|
||||
# The twenty units that write a `swarm-*` grant, by name, for the discovery
|
||||
# control below.
|
||||
# The twenty-one units that write a `swarm-*` grant, by name, for the
|
||||
# discovery control below.
|
||||
grantingUnitNames = [
|
||||
"swarm-bao-controller-policy"
|
||||
"swarm-bao-secret-publisher-policy"
|
||||
|
|
@ -191,6 +191,7 @@ let
|
|||
"swarm-bao-forge-publisher-policy"
|
||||
"swarm-bao-review-classifier-policy"
|
||||
"swarm-bao-builder-policy"
|
||||
"swarm-bao-ci-metrics-policy"
|
||||
"swarm-bao-claude-login-policy"
|
||||
];
|
||||
|
||||
|
|
@ -1032,24 +1033,24 @@ let
|
|||
}
|
||||
{
|
||||
# A store host without the granter's pair writes its grants some other
|
||||
# way, so none of the twenty units may exist. Without this arm
|
||||
# way, so none of the twenty-one units may exist. Without this arm
|
||||
# `lib.mkIf haveGranter` could be dropped from any of them and every other
|
||||
# case here would still pass.
|
||||
name = "without the granter's pair none of the twenty granting units render";
|
||||
name = "without the granter's pair none of the twenty-one granting units render";
|
||||
ok =
|
||||
let
|
||||
s = baoGranterOptOut.systemd.services;
|
||||
in
|
||||
lib.all (unit: !(s ? ${unit})) (grantingUnitNames ++ [ "swarm-bao-granter-role" ])
|
||||
# The control: the same store with the pair renders all twenty.
|
||||
# The control: the same store with the pair renders all twenty-one.
|
||||
&& lib.all (unit: baoGrantHere.systemd.services ? ${unit}) grantingUnitNames;
|
||||
}
|
||||
{
|
||||
# 🩸 What replaced the silent skip. With no bootstrap token the twenty still
|
||||
# render, and a refused granter fails them with the step that fixes it.
|
||||
# 🩸 What replaced the silent skip. With no bootstrap token the twenty-one
|
||||
# still render, and a refused granter fails them with the step that fixes it.
|
||||
# A store host that never named a token is told to name one, since the
|
||||
# unit that sets the granter up renders only where it has.
|
||||
name = "a store host without a bootstrap token renders the twenty, each failing loudly with the one-time step";
|
||||
name = "a store host without a bootstrap token renders the twenty-one, each failing loudly with the one-time step";
|
||||
ok =
|
||||
let
|
||||
s = baoGranterNoToken.systemd.services;
|
||||
|
|
@ -1670,6 +1671,84 @@ let
|
|||
in
|
||||
lib.hasInfix "/builder.pem ]" s && lib.hasInfix "swarm-builder \"\" clientAuth" s;
|
||||
}
|
||||
{
|
||||
# The poller reads its forge token and its own OIDC client secret, each
|
||||
# read-only and spelled to the leaf. The whole list, so the token's
|
||||
# stanza is the presence control and any further path fails it.
|
||||
name = "the CI run poller's grant is its forge token and its OIDC client secret, and nothing else";
|
||||
ok =
|
||||
let
|
||||
s = baoGrantHere.systemd.services.swarm-bao-ci-metrics-policy.script;
|
||||
in
|
||||
grantsIn s == [
|
||||
{
|
||||
path = "secret/data/swarm/services/ci-metrics/forge/token";
|
||||
caps = [ "read" ];
|
||||
}
|
||||
{
|
||||
path = "secret/data/swarm/services/swarm-ci-metrics/oidc/client";
|
||||
caps = [ "read" ];
|
||||
}
|
||||
]
|
||||
&& lib.hasInfix "auth/cert/certs/swarm-ci-metrics" s
|
||||
&& lib.hasInfix "allowed_common_names=swarm-ci-metrics" s
|
||||
&& lib.hasInfix "token_policies=swarm-ci-metrics" s;
|
||||
}
|
||||
{
|
||||
# Who may read the token is the units whose grant on it holds `read`:
|
||||
# the controller, which mints it, and the poller. The publisher names it
|
||||
# only to deny it.
|
||||
name = "no granting unit but the controller's and the CI run poller's may read the poller's forge token";
|
||||
ok =
|
||||
let
|
||||
path = "secret/data/swarm/services/ci-metrics/forge/token";
|
||||
reading = lib.filter (
|
||||
u:
|
||||
lib.any (g: g.path == path && lib.elem "read" g.caps) (
|
||||
grantsIn baoGrantHere.systemd.services.${u}.script
|
||||
)
|
||||
) grantingUnitNames;
|
||||
in
|
||||
lib.sort lib.lessThan reading == [
|
||||
"swarm-bao-ci-metrics-policy"
|
||||
"swarm-bao-controller-policy"
|
||||
];
|
||||
}
|
||||
{
|
||||
# `forge::ci_metrics_account` mints the token and reads it back to decide
|
||||
# on a re-mint. Pinned as the whole stanza, and spelled to the leaf.
|
||||
name = "the controller's grant on the CI run poller is its forge token alone";
|
||||
ok =
|
||||
let
|
||||
s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script;
|
||||
in
|
||||
lib.hasInfix "path \"secret/data/swarm/services/ci-metrics/forge/token\" {\n capabilities = [\"create\", \"read\", \"update\"]\n}" s
|
||||
&& !(lib.hasInfix "secret/data/swarm/services/ci-metrics/*" s);
|
||||
}
|
||||
{
|
||||
# The token sits under the publisher's `services/*` write, and the
|
||||
# poller lists runs with whatever is there. Its OIDC leaf is the
|
||||
# control: still the prefix's.
|
||||
name = "the publisher may not write the CI run poller's forge token";
|
||||
ok =
|
||||
let
|
||||
grants = grantsIn baoGrantHere.systemd.services.swarm-bao-secret-publisher-policy.script;
|
||||
in
|
||||
(grantFor grants "secret/data/swarm/services/ci-metrics/forge/token").caps == [ "deny" ]
|
||||
&&
|
||||
(grantFor grants "secret/data/swarm/services/swarm-ci-metrics/oidc/client").caps == [
|
||||
"create"
|
||||
"update"
|
||||
];
|
||||
}
|
||||
{
|
||||
name = "the PKI unit signs the CI run poller's leaf under its own subject";
|
||||
ok =
|
||||
let
|
||||
s = baoGrantHere.systemd.services.swarm-bao-pki.script;
|
||||
in
|
||||
lib.hasInfix "/ci-metrics.pem ]" s && lib.hasInfix "swarm-ci-metrics \"\" clientAuth" s;
|
||||
}
|
||||
{
|
||||
# The minter writes one leaf under every agent, create and update so a
|
||||
# relogin replaces the last, and reads its own queue client secret. No
|
||||
|
|
@ -1974,8 +2053,8 @@ let
|
|||
}
|
||||
{
|
||||
# What makes the case above mean something: discovery by the granter's
|
||||
# certificate reaches all twenty units, and each yields calls.
|
||||
name = "the granter-policy check sees all twenty granting units, and parses calls from each";
|
||||
# certificate reaches all twenty-one units, and each yields calls.
|
||||
name = "the granter-policy check sees all twenty-one granting units, and parses calls from each";
|
||||
ok =
|
||||
lib.sort lib.lessThan (lib.attrNames granterUnits) == lib.sort lib.lessThan grantingUnitNames
|
||||
&& lib.all (u: baoCalls u.script != [ ]) (lib.attrValues granterUnits)
|
||||
|
|
|
|||
98
nix/module-eval/ci-metrics.nix
Normal file
98
nix/module-eval/ci-metrics.nix
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
# `checks.module-eval-ci-metrics` — see ./lib.nix for the shared rationale
|
||||
# (why this suite exists, naming convention, "evaluates not executes"). The CI
|
||||
# run poller's unit and timer, its default, its collector route and its IdP
|
||||
# registration; its store grants are in ./bao-grants.nix beside every other
|
||||
# principal's, and its dashboard in ./grafana.nix.
|
||||
#
|
||||
# Every fixture is a shared one, so this cluster evaluates no host of its own.
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
self,
|
||||
nixosSystem,
|
||||
shared,
|
||||
}:
|
||||
let
|
||||
inherit
|
||||
(import ./lib.nix {
|
||||
inherit
|
||||
pkgs
|
||||
lib
|
||||
self
|
||||
nixosSystem
|
||||
;
|
||||
})
|
||||
runGroup
|
||||
;
|
||||
|
||||
bare = shared.bareWithController;
|
||||
# Every swarm service on one host: the store mints the poller's leaf, so the
|
||||
# poller defaults on, beside authelia, the publisher and the collector.
|
||||
allLocal = shared.allLocal;
|
||||
|
||||
svc = allLocal.systemd.services.swarm-ci-metrics;
|
||||
env = svc.environment;
|
||||
clientId = allLocal.services.hyperhive.swarm.ci-metrics.clientId;
|
||||
otel = allLocal.services.hyperhive.swarm.otel;
|
||||
|
||||
cases = [
|
||||
{
|
||||
# The control is the same unit and timer on the all-local host: a
|
||||
# misspelled attribute path would make the absence arm pass alone.
|
||||
name = "the CI run poller's unit and timer exist only where its host enables it";
|
||||
ok =
|
||||
!(bare.systemd.services ? swarm-ci-metrics)
|
||||
&& !(bare.systemd.timers ? swarm-ci-metrics)
|
||||
&& allLocal.services.hyperhive.deploy.ci-metrics.enable
|
||||
&& svc.serviceConfig.Type == "oneshot"
|
||||
&& !(svc.serviceConfig.RemainAfterExit or false)
|
||||
&& allLocal.systemd.timers.swarm-ci-metrics.timerConfig.OnUnitInactiveSec == "5min"
|
||||
&& lib.elem "timers.target" allLocal.systemd.timers.swarm-ci-metrics.wantedBy;
|
||||
}
|
||||
{
|
||||
# The leaf is handed in as a credential, never an argument; the forge
|
||||
# token and the client secret are read from the store per pass.
|
||||
name = "the poller loads only its store leaf, keeps its cursor in its state directory and carries no secret in its unit";
|
||||
ok =
|
||||
lib.elem "bao-client-key.pem:/var/lib/swarm-bao-pki/ci-metrics-key.pem" svc.serviceConfig.LoadCredential
|
||||
&& lib.elem "bao-client.pem:/var/lib/swarm-bao-pki/ci-metrics.pem" svc.serviceConfig.LoadCredential
|
||||
&& env.BAO_CLIENT_KEY == "/run/credentials/swarm-ci-metrics.service/bao-client-key.pem"
|
||||
&& svc.serviceConfig.DynamicUser
|
||||
&& svc.serviceConfig.StateDirectory == "swarm-ci-metrics"
|
||||
&& !(lib.any (n: lib.hasSuffix "TOKEN" n || lib.hasSuffix "SECRET" n) (lib.attrNames env));
|
||||
}
|
||||
{
|
||||
# The endpoint the poller pushes to is the route the collector admits
|
||||
# its client on, and that route feeds the swarm tier's metrics pipeline.
|
||||
name = "the poller pushes to its own collector route, which admits its client into the metrics pipeline";
|
||||
ok =
|
||||
let
|
||||
h = shared.baoWithCollector;
|
||||
s = h.containers.swarm-otel.config.services.opentelemetry-collector.settings;
|
||||
in
|
||||
env.SWARM_CI_METRICS_OTLP_ENDPOINT == "https://${otel.domain}/swarm-ci-metrics"
|
||||
&& env.SWARM_CI_METRICS_OIDC_CLIENT_ID == clientId
|
||||
&& otel.serviceProducers.swarm-ci-metrics.clientId == clientId
|
||||
&& s.receivers."otlp/swarm-ci-metrics".protocols.http.auth.authenticator == "oidc/swarm-ci-metrics"
|
||||
&& s.extensions."oidc/swarm-ci-metrics".audience == clientId
|
||||
&& lib.elem "otlp/swarm-ci-metrics" s.service.pipelines."metrics/${otel.producerName}".receivers
|
||||
&& h.services.nginx.virtualHosts.${otel.domain}.locations ? "/swarm-ci-metrics/";
|
||||
}
|
||||
{
|
||||
name = "the poller's client is registered as a machine, and the publisher carries its secret";
|
||||
ok =
|
||||
let
|
||||
c = lib.findFirst (
|
||||
c: c.id == clientId
|
||||
) null allLocal.services.hyperhive.swarm.authelia.oidc.clients;
|
||||
in
|
||||
c != null
|
||||
&& c.kind == "machine"
|
||||
&& c.audience == [ clientId ]
|
||||
&& lib.hasInfix "secret/swarm/services/${clientId}/oidc/client" (
|
||||
allLocal.systemd.services.swarm-secret-publish.script
|
||||
);
|
||||
}
|
||||
];
|
||||
in
|
||||
runGroup "ci-metrics" cases
|
||||
|
|
@ -354,6 +354,35 @@ let
|
|||
grafanaRemoteAuthelia.services.hyperhive.swarm.statusPublish.tokenEndpoint
|
||||
== "https://auth.example.invalid/api/oidc/token";
|
||||
}
|
||||
{
|
||||
# The CI board is in the directory the provider scans (`bao.json` is the
|
||||
# control that the entries are read at all), and every metric it queries
|
||||
# is a name the poller pushes: a renamed metric on either side is a board
|
||||
# of empty panels.
|
||||
name = "the CI dashboard ships, and queries the metric names the CI run poller pushes";
|
||||
ok =
|
||||
let
|
||||
provider = lib.head grafanaWithAuthelia.containers.swarm-grafana.config.services.grafana.provision.dashboards.settings.providers;
|
||||
shipped = map (e: e.name) provider.options.path.entries;
|
||||
board = builtins.fromJSON (builtins.readFile ../host-modules/swarm-grafana/dashboards/ci.json);
|
||||
exprs = lib.concatMap (p: map (t: t.expr or "") (p.targets or [ ])) board.panels;
|
||||
queried = lib.unique (
|
||||
lib.concatMap (
|
||||
e: map lib.head (lib.filter lib.isList (builtins.split ''"(hyperhive\.ci\.[a-z_.]+)"'' e))
|
||||
) exprs
|
||||
);
|
||||
pushed = builtins.readFile ../../swarm-ci-metrics/src/sample.rs;
|
||||
in
|
||||
lib.elem "ci.json" shipped
|
||||
&& lib.elem "bao.json" shipped
|
||||
&&
|
||||
lib.sort lib.lessThan queried == [
|
||||
"hyperhive.ci.run.duration"
|
||||
"hyperhive.ci.run.queue_wait"
|
||||
]
|
||||
&& lib.all (m: lib.hasInfix "\"${m}\"" pushed) queried
|
||||
&& lib.any (v: v.name == "repo") board.templating.list;
|
||||
}
|
||||
{
|
||||
# The standing check on the collector's PRIORITY mapping — whether a line
|
||||
# ever reaches the log store carrying no severity. It is shipped config
|
||||
|
|
|
|||
Loading…
Reference in a new issue