push ci run metrics to victoriametrics, with a grafana board
A swarm-ci-metrics host unit on a 5-minute timer lists every forge repo's Actions runs as the new read-only ci-metrics account and pushes each finished run's duration and queue wait over OTLP, stamped at the run's stop time. The first pass backfills from run 1. swarm-controller mints the account's token into bao and keeps a ci-metrics team with read on the Actions unit in every organisation. Closes #5376
This commit is contained in:
atlas
mara
parent
8b7a4e7955
commit
b072fbf4d6
36 changed files with 2849 additions and 56 deletions
|
|
@ -169,8 +169,8 @@ let
|
|||
_: u: (u.environment.BAO_CLIENT_CERT or null) == granterCertFile
|
||||
) baoGrantWithConsumers.systemd.services;
|
||||
|
||||
# The twenty units that write a `swarm-*` grant, by name, for the discovery
|
||||
# control below.
|
||||
# The twenty-one units that write a `swarm-*` grant, by name, for the
|
||||
# discovery control below.
|
||||
grantingUnitNames = [
|
||||
"swarm-bao-controller-policy"
|
||||
"swarm-bao-secret-publisher-policy"
|
||||
|
|
@ -191,6 +191,7 @@ let
|
|||
"swarm-bao-forge-publisher-policy"
|
||||
"swarm-bao-review-classifier-policy"
|
||||
"swarm-bao-builder-policy"
|
||||
"swarm-bao-ci-metrics-policy"
|
||||
"swarm-bao-claude-login-policy"
|
||||
];
|
||||
|
||||
|
|
@ -1032,24 +1033,24 @@ let
|
|||
}
|
||||
{
|
||||
# A store host without the granter's pair writes its grants some other
|
||||
# way, so none of the twenty units may exist. Without this arm
|
||||
# way, so none of the twenty-one units may exist. Without this arm
|
||||
# `lib.mkIf haveGranter` could be dropped from any of them and every other
|
||||
# case here would still pass.
|
||||
name = "without the granter's pair none of the twenty granting units render";
|
||||
name = "without the granter's pair none of the twenty-one granting units render";
|
||||
ok =
|
||||
let
|
||||
s = baoGranterOptOut.systemd.services;
|
||||
in
|
||||
lib.all (unit: !(s ? ${unit})) (grantingUnitNames ++ [ "swarm-bao-granter-role" ])
|
||||
# The control: the same store with the pair renders all twenty.
|
||||
# The control: the same store with the pair renders all twenty-one.
|
||||
&& lib.all (unit: baoGrantHere.systemd.services ? ${unit}) grantingUnitNames;
|
||||
}
|
||||
{
|
||||
# 🩸 What replaced the silent skip. With no bootstrap token the twenty still
|
||||
# render, and a refused granter fails them with the step that fixes it.
|
||||
# 🩸 What replaced the silent skip. With no bootstrap token the twenty-one
|
||||
# still render, and a refused granter fails them with the step that fixes it.
|
||||
# A store host that never named a token is told to name one, since the
|
||||
# unit that sets the granter up renders only where it has.
|
||||
name = "a store host without a bootstrap token renders the twenty, each failing loudly with the one-time step";
|
||||
name = "a store host without a bootstrap token renders the twenty-one, each failing loudly with the one-time step";
|
||||
ok =
|
||||
let
|
||||
s = baoGranterNoToken.systemd.services;
|
||||
|
|
@ -1670,6 +1671,84 @@ let
|
|||
in
|
||||
lib.hasInfix "/builder.pem ]" s && lib.hasInfix "swarm-builder \"\" clientAuth" s;
|
||||
}
|
||||
{
|
||||
# The poller reads its forge token and its own OIDC client secret, each
|
||||
# read-only and spelled to the leaf. The whole list, so the token's
|
||||
# stanza is the presence control and any further path fails it.
|
||||
name = "the CI run poller's grant is its forge token and its OIDC client secret, and nothing else";
|
||||
ok =
|
||||
let
|
||||
s = baoGrantHere.systemd.services.swarm-bao-ci-metrics-policy.script;
|
||||
in
|
||||
grantsIn s == [
|
||||
{
|
||||
path = "secret/data/swarm/services/ci-metrics/forge/token";
|
||||
caps = [ "read" ];
|
||||
}
|
||||
{
|
||||
path = "secret/data/swarm/services/swarm-ci-metrics/oidc/client";
|
||||
caps = [ "read" ];
|
||||
}
|
||||
]
|
||||
&& lib.hasInfix "auth/cert/certs/swarm-ci-metrics" s
|
||||
&& lib.hasInfix "allowed_common_names=swarm-ci-metrics" s
|
||||
&& lib.hasInfix "token_policies=swarm-ci-metrics" s;
|
||||
}
|
||||
{
|
||||
# Who may read the token is the units whose grant on it holds `read`:
|
||||
# the controller, which mints it, and the poller. The publisher names it
|
||||
# only to deny it.
|
||||
name = "no granting unit but the controller's and the CI run poller's may read the poller's forge token";
|
||||
ok =
|
||||
let
|
||||
path = "secret/data/swarm/services/ci-metrics/forge/token";
|
||||
reading = lib.filter (
|
||||
u:
|
||||
lib.any (g: g.path == path && lib.elem "read" g.caps) (
|
||||
grantsIn baoGrantHere.systemd.services.${u}.script
|
||||
)
|
||||
) grantingUnitNames;
|
||||
in
|
||||
lib.sort lib.lessThan reading == [
|
||||
"swarm-bao-ci-metrics-policy"
|
||||
"swarm-bao-controller-policy"
|
||||
];
|
||||
}
|
||||
{
|
||||
# `forge::ci_metrics_account` mints the token and reads it back to decide
|
||||
# on a re-mint. Pinned as the whole stanza, and spelled to the leaf.
|
||||
name = "the controller's grant on the CI run poller is its forge token alone";
|
||||
ok =
|
||||
let
|
||||
s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script;
|
||||
in
|
||||
lib.hasInfix "path \"secret/data/swarm/services/ci-metrics/forge/token\" {\n capabilities = [\"create\", \"read\", \"update\"]\n}" s
|
||||
&& !(lib.hasInfix "secret/data/swarm/services/ci-metrics/*" s);
|
||||
}
|
||||
{
|
||||
# The token sits under the publisher's `services/*` write, and the
|
||||
# poller lists runs with whatever is there. Its OIDC leaf is the
|
||||
# control: still the prefix's.
|
||||
name = "the publisher may not write the CI run poller's forge token";
|
||||
ok =
|
||||
let
|
||||
grants = grantsIn baoGrantHere.systemd.services.swarm-bao-secret-publisher-policy.script;
|
||||
in
|
||||
(grantFor grants "secret/data/swarm/services/ci-metrics/forge/token").caps == [ "deny" ]
|
||||
&&
|
||||
(grantFor grants "secret/data/swarm/services/swarm-ci-metrics/oidc/client").caps == [
|
||||
"create"
|
||||
"update"
|
||||
];
|
||||
}
|
||||
{
|
||||
name = "the PKI unit signs the CI run poller's leaf under its own subject";
|
||||
ok =
|
||||
let
|
||||
s = baoGrantHere.systemd.services.swarm-bao-pki.script;
|
||||
in
|
||||
lib.hasInfix "/ci-metrics.pem ]" s && lib.hasInfix "swarm-ci-metrics \"\" clientAuth" s;
|
||||
}
|
||||
{
|
||||
# The minter writes one leaf under every agent, create and update so a
|
||||
# relogin replaces the last, and reads its own queue client secret. No
|
||||
|
|
@ -1974,8 +2053,8 @@ let
|
|||
}
|
||||
{
|
||||
# What makes the case above mean something: discovery by the granter's
|
||||
# certificate reaches all twenty units, and each yields calls.
|
||||
name = "the granter-policy check sees all twenty granting units, and parses calls from each";
|
||||
# certificate reaches all twenty-one units, and each yields calls.
|
||||
name = "the granter-policy check sees all twenty-one granting units, and parses calls from each";
|
||||
ok =
|
||||
lib.sort lib.lessThan (lib.attrNames granterUnits) == lib.sort lib.lessThan grantingUnitNames
|
||||
&& lib.all (u: baoCalls u.script != [ ]) (lib.attrValues granterUnits)
|
||||
|
|
|
|||
Loading…
Reference in a new issue