d30cb7d122
The mint creates each client's secret in the swarm secret store and writes only the digest to disk. It mints only when the store answers and holds none. A new unit moves the plaintext files an earlier release left into the store and removes each once the store holds the same value. The bridge reads its secret from the store into /run before each start. The secret publisher unit and its path unit are gone.
187 lines
7 KiB
Nix
187 lines
7 KiB
Nix
# The swarm's CI run poller: a oneshot `swarm-ci-metrics` on a 5-minute timer,
|
|
# on the host that sets `deploy.ci-metrics.enable`, which
|
|
# `deploy.allSwarmServices` defaults on where the host has the poller's store
|
|
# identity (./swarm-required-services.nix).
|
|
#
|
|
# Each pass lists the Actions runs of every repo the `ci-metrics` forge account
|
|
# sees and pushes each newly finished run's duration and queue wait to the
|
|
# swarm collector, stamped at the run's end. The first pass, with no cursor in
|
|
# the state directory, walks every run back to the first. docs/swarm/ci-metrics.md
|
|
# has the design.
|
|
#
|
|
# It reaches the forge, authelia, the store and the collector by name, so
|
|
# nothing here asks which host runs any of them. Its one credential on disk is
|
|
# its store leaf, handed in by `LoadCredential=`; the forge token and its OIDC
|
|
# client secret it reads from the store under that leaf on every pass.
|
|
{
|
|
pkgs,
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
hyperhiveCfg = config.services.hyperhive;
|
|
cfg = hyperhiveCfg.deploy.ci-metrics;
|
|
baoCfg = hyperhiveCfg.swarm.bao;
|
|
baoDeploy = hyperhiveCfg.deploy.bao;
|
|
gatewayCfg = hyperhiveCfg.gateway;
|
|
swarmOtelCfg = hyperhiveCfg.swarm.otel;
|
|
|
|
unit = "swarm-ci-metrics";
|
|
clientId = hyperhiveCfg.swarm.ci-metrics.clientId;
|
|
|
|
# The route `serviceProducers.swarm-ci-metrics` in ./swarm-otel-service.nix
|
|
# declares, by the collector's gateway name.
|
|
otlpEndpoint = "https://${swarmOtelCfg.domain}/${unit}";
|
|
|
|
haveClientIdentity =
|
|
baoDeploy.ciMetricsClientCertFile != null && baoDeploy.ciMetricsClientKeyFile != null;
|
|
|
|
# The forge, authelia and the collector serve leaves issued by the swarm's
|
|
# own CA, which the system store lacks, and the poller's `reqwest`/`rustls`
|
|
# roots come from `SSL_CERT_FILE`, which replaces that store. `trustBundle`
|
|
# below sets it to the system CAs plus the hive anchor; without it every
|
|
# pass fails `invalid peer certificate: UnknownIssuer`.
|
|
caTrust = import ./lib/hive-ca-trust.nix {
|
|
inherit lib gatewayCfg;
|
|
tlsCfg = hyperhiveCfg.deploy.hive-controller.tls;
|
|
};
|
|
|
|
credential = id: "/run/credentials/${unit}.service/${id}";
|
|
# Credential id → host path of the poller's store identity.
|
|
storeFiles = {
|
|
"bao-client.pem" = baoDeploy.ciMetricsClientCertFile;
|
|
"bao-client-key.pem" = baoDeploy.ciMetricsClientKeyFile;
|
|
}
|
|
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
|
|
"bao-ca.pem" = baoDeploy.serverCaFile;
|
|
};
|
|
in
|
|
{
|
|
# `hostUnit`: a host service, so the bundle reads the host CA path and waits
|
|
# on `hive-tls-ca.service`. `enable`: `imports` is unconditional, so without
|
|
# it a host with the poller off gets the bundle unit and an `ExecStart`-less
|
|
# `swarm-ci-metrics` service.
|
|
imports = [
|
|
(caTrust.trustBundle {
|
|
inherit pkgs;
|
|
name = unit;
|
|
consumers = [ unit ];
|
|
hostUnit = true;
|
|
inherit (cfg) enable;
|
|
})
|
|
];
|
|
|
|
options.services.hyperhive.swarm.ci-metrics.clientId = lib.mkOption {
|
|
type = lib.types.str;
|
|
readOnly = true;
|
|
default = "swarm-ci-metrics";
|
|
description = ''
|
|
OAuth2 client id the CI run poller authenticates to the swarm
|
|
collector as, and the audience it asks its token for.
|
|
|
|
Read-only because three hosts have to agree on it: authelia registers
|
|
the client (`glue-ci-metrics-otel-oidc-client.nix`), the swarm collector
|
|
admits it on the poller's receiver (`swarm.otel.serviceProducers`), and
|
|
authelia's mint stores its secret in the store.
|
|
'';
|
|
};
|
|
|
|
options.services.hyperhive.deploy.ci-metrics = {
|
|
enable = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = false;
|
|
description = ''
|
|
Run the CI run poller on this host: every 5 minutes it pushes each
|
|
newly finished forge Actions run's duration and queue wait to the
|
|
swarm collector, for the Grafana CI dashboard. Its first run pushes
|
|
every run since the first.
|
|
|
|
Requires the poller's store identity,
|
|
{option}`services.hyperhive.deploy.bao.ciMetricsClientCertFile` and
|
|
`ciMetricsClientKeyFile`. Run it on one host per swarm: two pollers
|
|
keep two cursors and push every run twice.
|
|
'';
|
|
};
|
|
|
|
package = lib.mkOption {
|
|
type = lib.types.package;
|
|
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-ci-metrics";
|
|
description = "The CI run poller package.";
|
|
};
|
|
};
|
|
|
|
config = lib.mkIf cfg.enable {
|
|
assertions = [
|
|
{
|
|
assertion = haveClientIdentity;
|
|
message = ''
|
|
services.hyperhive.deploy.ci-metrics.enable requires the poller's
|
|
swarm-secret-store client identity: set both
|
|
|
|
services.hyperhive.deploy.bao.ciMetricsClientCertFile
|
|
services.hyperhive.deploy.bao.ciMetricsClientKeyFile
|
|
|
|
swarm-ci-metrics.service reads its forge token and its OIDC client
|
|
secret with it.
|
|
|
|
Both default to the pair glue-bao-tls.nix mints when
|
|
services.hyperhive.deploy.bao.enable is true (the host that runs
|
|
the store). Otherwise copy ci-metrics.pem and ci-metrics-key.pem
|
|
from the store host's /var/lib/swarm-bao-pki/ and name them in
|
|
those two options.
|
|
'';
|
|
}
|
|
];
|
|
|
|
systemd.services.${unit} = {
|
|
description = "push finished forge CI runs to the swarm metrics store";
|
|
after = [ "network-online.target" ];
|
|
wants = [ "network-online.target" ];
|
|
# No `RemainAfterExit`: the timer starts this again, and starting an
|
|
# active unit is a no-op.
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
ExecStart = "${cfg.package}/bin/swarm-ci-metrics";
|
|
# The first pass reads every run the forge has, a page a second.
|
|
TimeoutStartSec = "30min";
|
|
# Paths only: `argv` is world-readable via /proc/<pid>/cmdline.
|
|
LoadCredential = lib.mapAttrsToList (id: path: "${id}:${path}") storeFiles;
|
|
DynamicUser = true;
|
|
# The cursor. Losing it re-pushes every run.
|
|
StateDirectory = unit;
|
|
StateDirectoryMode = "0700";
|
|
SyslogIdentifier = unit;
|
|
ProtectSystem = "strict";
|
|
ProtectHome = true;
|
|
PrivateTmp = true;
|
|
PrivateDevices = true;
|
|
NoNewPrivileges = true;
|
|
RestrictSUIDSGID = true;
|
|
RemoveIPC = true;
|
|
};
|
|
environment = {
|
|
SWARM_CI_METRICS_FORGE_URL = "https://${hyperhiveCfg.swarm.forge.domain}";
|
|
SWARM_CI_METRICS_OTLP_ENDPOINT = otlpEndpoint;
|
|
SWARM_CI_METRICS_OIDC_TOKEN_ENDPOINT = "${hyperhiveCfg.swarm.authelia.url}/api/oidc/token";
|
|
SWARM_CI_METRICS_OIDC_CLIENT_ID = clientId;
|
|
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
|
BAO_CLIENT_CERT = credential "bao-client.pem";
|
|
BAO_CLIENT_KEY = credential "bao-client-key.pem";
|
|
}
|
|
// lib.optionalAttrs (storeFiles ? "bao-ca.pem") {
|
|
BAO_CACERT = credential "bao-ca.pem";
|
|
};
|
|
};
|
|
|
|
systemd.timers.${unit} = {
|
|
description = "run the CI run poller every 5 minutes";
|
|
wantedBy = [ "timers.target" ];
|
|
timerConfig = {
|
|
OnBootSec = "2min";
|
|
OnUnitInactiveSec = "5min";
|
|
RandomizedDelaySec = "30s";
|
|
};
|
|
};
|
|
};
|
|
}
|