swarm UI: an alerts page, read by the controller with a Viewer Grafana token from bao
swarm-bao-grafana-controller-token, on the host that runs Grafana, logs in to Grafana's API over its socket as the built-in admin (the password swarm-bao-grafana-admin minted, #5138) and keeps a `swarm-controller` service account that is an enabled Viewer. It mints a token for that account and writes it to bao at swarm/controller/swarm-controller/grafana/token under Grafana's leaf. A ten-minute timer runs it again. The unit keeps a stored token Grafana accepts. If Grafana refuses the stored token with 401, or bao holds none, the unit deletes every token of the account and mints a new one. Grafana's grant gains create/read/update on that leaf, and the controller's gains read. swarm-controller serves GET /api/alerts from Grafana's /api/prometheus/grafana/api/v1/alerts. It reads the token from bao, keeps it in memory, and re-reads bao once on a 401. The HTTP client has 5s connect and 15s request timeouts. Any failure is a 503 that names its cause. swarm-ui gets an /alerts page listing them, built the same way as the hives page. This is option (c) on #5129, the operator's choice.
This commit is contained in:
atlas
mara
parent
8cb6273782
commit
7d09776742
23 changed files with 888 additions and 37 deletions
|
|
@ -67,7 +67,8 @@ let
|
|||
];
|
||||
};
|
||||
# ./swarm-grafana.nix, under Grafana's leaf, which may create the admin
|
||||
# password and the metrics bearer only once that policy is written.
|
||||
# password, the metrics bearer and the controller's Grafana token only
|
||||
# once that policy is written.
|
||||
swarm-bao-grafana-admin = {
|
||||
here = deployCfg.grafana.enable;
|
||||
policies = [ "swarm-bao-grafana-oidc-policy" ];
|
||||
|
|
@ -76,6 +77,10 @@ let
|
|||
here = deployCfg.grafana.enable;
|
||||
policies = [ "swarm-bao-grafana-oidc-policy" ];
|
||||
};
|
||||
swarm-bao-grafana-controller-token = {
|
||||
here = deployCfg.grafana.enable;
|
||||
policies = [ "swarm-bao-grafana-oidc-policy" ];
|
||||
};
|
||||
# ./hive-matrix.nix, under matrix-ctl's leaf.
|
||||
swarm-bao-matrix-oidc = {
|
||||
here = deployCfg.matrix.enable;
|
||||
|
|
|
|||
|
|
@ -47,6 +47,7 @@ let
|
|||
swarm-bao-grafana-oidc = deployCfg.grafana.enable;
|
||||
swarm-bao-grafana-admin = deployCfg.grafana.enable;
|
||||
swarm-bao-grafana-metrics-bearer = deployCfg.grafana.enable;
|
||||
swarm-bao-grafana-controller-token = deployCfg.grafana.enable;
|
||||
# ./swarm-nats.nix
|
||||
swarm-bao-nats-callout-xkey = deployCfg.nats.enable;
|
||||
# ./swarm-ci-runner.nix
|
||||
|
|
|
|||
|
|
@ -369,9 +369,9 @@ let
|
|||
# Each hive's matrix sender token (`matrix_account::hive_sender`) grants `read`
|
||||
# for the same keep-if-live reason, and `+` for the same glob-only-as-last-segment reason.
|
||||
#
|
||||
# Three leaves it reads and never writes: the swarm appservice token
|
||||
# (matrix-ctl, `matrixCtlPolicyText` below), its own OIDC client secret (the
|
||||
# publisher) and its own forge token (the forge's host, `forgePublisherWriters`).
|
||||
# Four leaves it reads and never writes: the swarm appservice token (matrix-ctl),
|
||||
# its OIDC client secret (the publisher), its forge token (the forge's host)
|
||||
# and its Grafana token (Grafana's host). Each writer's grant is named below.
|
||||
#
|
||||
# The agent PKI grant is its only path on that mount: it can ask the one role
|
||||
# for a certificate, not write that role or reach the issuer.
|
||||
|
|
@ -457,6 +457,10 @@ let
|
|||
capabilities = ["read"]
|
||||
}
|
||||
|
||||
path "${credentialMountPath}/data/${controllerGrafanaTokenLeaf}" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
||||
path "${agentPkiMountPath}/issue/${agentPkiRoleName}" {
|
||||
capabilities = ["update"]
|
||||
}
|
||||
|
|
@ -582,6 +586,11 @@ let
|
|||
# it; the controller reads it and holds no other copy.
|
||||
controllerForgeTokenLeaf = "swarm/controller/swarm-controller/forge/token";
|
||||
|
||||
# The controller's Viewer token for Grafana's API, the nix half of
|
||||
# `swarm_secret_client::grafana::controller_token_path`. Grafana's host
|
||||
# writes it; the controller reads it and holds no other copy.
|
||||
controllerGrafanaTokenLeaf = "swarm/controller/swarm-controller/grafana/token";
|
||||
|
||||
# The forge's runner registration token, the nix half of
|
||||
# `swarm_secret_client::ci::registration_token_path`. The controller writes
|
||||
# it and cannot read it back (`forge::runner_token` keeps the value it last
|
||||
|
|
@ -911,6 +920,10 @@ let
|
|||
# in ./swarm-grafana.nix) and the bearer its metrics datasource presents
|
||||
# (`swarm-bao-grafana-metrics-bearer`). `create` without `update`, so a stored
|
||||
# value is never replaced.
|
||||
#
|
||||
# The controller's Grafana token is the one it may replace:
|
||||
# `swarm-bao-grafana-controller-token` mints a new one when Grafana refuses
|
||||
# the stored one, as it does after its database is recreated.
|
||||
grafanaOidcReaders = [
|
||||
{
|
||||
name = "swarm-grafana-oidc";
|
||||
|
|
@ -918,7 +931,12 @@ let
|
|||
policyText =
|
||||
readStanza "${credentialMountPath}/data/swarm/services/${hyperhiveCfg.swarm.grafana.oidc.clientId}/oidc/client"
|
||||
+ createStanza "${credentialMountPath}/data/${grafanaAdminLeaf}"
|
||||
+ createStanza "${credentialMountPath}/data/${grafanaMetricsBearerLeaf}";
|
||||
+ createStanza "${credentialMountPath}/data/${grafanaMetricsBearerLeaf}"
|
||||
+ ''
|
||||
path "${credentialMountPath}/data/${controllerGrafanaTokenLeaf}" {
|
||||
capabilities = ["create", "read", "update"]
|
||||
}
|
||||
'';
|
||||
}
|
||||
];
|
||||
|
||||
|
|
|
|||
|
|
@ -811,6 +811,11 @@ in
|
|||
# separated: a matrix user id cannot contain a space. Consumed by
|
||||
# swarm-controller/src/matrix_account/operator_admin.rs.
|
||||
SWARM_CONTROLLER_MATRIX_ADMIN_SYNC_IGNORE_USERS = lib.concatStringsSep " " matrixCfg.adminSync.ignoreUsers;
|
||||
# Where `GET /api/alerts` reads alerts (swarm-controller/src/alerts.rs).
|
||||
# Every swarm has one Grafana, reached by name wherever it runs. The
|
||||
# token is in the store, written by `swarm-bao-grafana-controller-token`
|
||||
# on Grafana's host.
|
||||
SWARM_CONTROLLER_GRAFANA_URL = "https://${swarmCfg.grafana.domain}";
|
||||
}
|
||||
// queueEnv
|
||||
// forgeEnv
|
||||
|
|
|
|||
|
|
@ -273,6 +273,14 @@ let
|
|||
metricsBearerPath = "/var/lib/grafana-metrics/bearer";
|
||||
hostMetricsBearerPath = "/var/lib/nixos-containers/${cfg.machine}${metricsBearerPath}";
|
||||
|
||||
# swarm-controller's token for Grafana's API, minted by
|
||||
# `swarm-bao-grafana-controller-token` below for a Viewer service account of
|
||||
# this name. `controllerGrafanaTokenLeaf` in ./swarm-bao.nix is the nix half
|
||||
# of the path, `swarm_secret_client::grafana::controller_token_path` the Rust
|
||||
# half.
|
||||
controllerAccountName = "swarm-controller";
|
||||
controllerTokenStorePath = "secret/swarm/controller/swarm-controller/grafana/token";
|
||||
|
||||
socketPath = "${deployCfg.grafana.socketDir}/grafana.sock";
|
||||
|
||||
# Both static NixOS ids, and both checked rather than assumed: nginx has
|
||||
|
|
@ -849,6 +857,148 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
# swarm-controller reads Grafana's alerts with a token of a Viewer service
|
||||
# account, created here through Grafana's API as the built-in admin and
|
||||
# handed over through the store.
|
||||
#
|
||||
# Re-run by the timer below. A stored token Grafana still accepts is kept;
|
||||
# one it refuses with 401, or none at all, is replaced: every token of the
|
||||
# account is deleted and one new one is minted and stored. The account's
|
||||
# role is set back to Viewer on every run.
|
||||
systemd.services.swarm-bao-grafana-controller-token = {
|
||||
description = "give swarm-controller a Viewer token for Grafana's API through the swarm secret store";
|
||||
# Ordered after the store where it runs here:
|
||||
# ./glue-bao-readers-store-order.nix.
|
||||
after = [
|
||||
"container@${cfg.machine}.service"
|
||||
"swarm-bao-grafana-admin.service"
|
||||
];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
path = [
|
||||
baoDeploy.package
|
||||
pkgs.coreutils
|
||||
pkgs.curl
|
||||
pkgs.jq
|
||||
];
|
||||
# ./lib/store-retry.nix. A Grafana still starting refuses the
|
||||
# connection, and that attempt fails into the same retry.
|
||||
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
|
||||
serviceConfig = storeRetry.serviceConfig // {
|
||||
# No `RemainAfterExit`: the timer below starts this again.
|
||||
Type = "oneshot";
|
||||
SyslogIdentifier = "swarm-bao-grafana-controller-token";
|
||||
# Grafana's answers, the new token among them, pass through a temp file.
|
||||
PrivateTmp = true;
|
||||
# Each call below is bounded by curl's own timeouts; a run makes up to
|
||||
# about ten of them.
|
||||
TimeoutStartSec = "3min";
|
||||
};
|
||||
environment = {
|
||||
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||||
BAO_CLIENT_CERT = baoDeploy.grafanaOidcClientCertFile;
|
||||
BAO_CLIENT_KEY = baoDeploy.grafanaOidcClientKeyFile;
|
||||
}
|
||||
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
|
||||
BAO_CACERT = baoDeploy.serverCaFile;
|
||||
};
|
||||
script = ''
|
||||
set -euo pipefail
|
||||
|
||||
err="$(mktemp)"
|
||||
body="$(mktemp)"
|
||||
trap 'rm -f "$err" "$body"' EXIT
|
||||
|
||||
# One call to Grafana's API through its socket. Prints the HTTP
|
||||
# status; the body lands in "$body". The credential reaches curl as
|
||||
# a header file, never as an argument.
|
||||
grafana() {
|
||||
local auth="$1" method="$2" path="$3"
|
||||
shift 3
|
||||
curl --silent --show-error --connect-timeout 5 --max-time 15 \
|
||||
--unix-socket ${lib.escapeShellArg socketPath} \
|
||||
-H @<(printf 'Authorization: %s\n' "$auth") \
|
||||
-H 'Content-Type: application/json' \
|
||||
-X "$method" -o "$body" -w '%{http_code}' "$@" \
|
||||
"http://localhost$path"
|
||||
}
|
||||
|
||||
# Fails the attempt unless the last call answered one of the statuses
|
||||
# given.
|
||||
expect() {
|
||||
local what="$1" status="$2"
|
||||
shift 2
|
||||
for ok in "$@"; do
|
||||
[ "$status" = "$ok" ] && return 0
|
||||
done
|
||||
echo "Grafana answered $status to $what:" >&2
|
||||
cat "$body" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
if [ ! -s ${lib.escapeShellArg hostAdminPasswordPath} ]; then
|
||||
echo "no Grafana admin password at ${hostAdminPasswordPath} yet; swarm-bao-grafana-admin writes it." >&2
|
||||
exit 1
|
||||
fi
|
||||
admin="Basic $(printf 'admin:%s' "$(< ${lib.escapeShellArg hostAdminPasswordPath})" | base64 -w0)"
|
||||
|
||||
status="$(grafana "$admin" GET '/api/serviceaccounts/search?perpage=1000&query=${controllerAccountName}')"
|
||||
expect "the service account search" "$status" 200
|
||||
account="$(jq -c --arg n ${controllerAccountName} '[.serviceAccounts[] | select(.name == $n)][0] // empty' "$body")"
|
||||
if [ -z "$account" ]; then
|
||||
status="$(grafana "$admin" POST /api/serviceaccounts \
|
||||
--data '{"name":"${controllerAccountName}","role":"Viewer","isDisabled":false}')"
|
||||
expect "creating the ${controllerAccountName} service account" "$status" 200 201
|
||||
account="$(jq -c . "$body")"
|
||||
echo "created the ${controllerAccountName} service account as a Viewer"
|
||||
fi
|
||||
id="$(jq -r .id <<<"$account")"
|
||||
if [ "$(jq -r '"\(.role) \(.isDisabled // false)"' <<<"$account")" != "Viewer false" ]; then
|
||||
status="$(grafana "$admin" PATCH "/api/serviceaccounts/$id" \
|
||||
--data '{"role":"Viewer","isDisabled":false}')"
|
||||
expect "setting the ${controllerAccountName} service account to an enabled Viewer" "$status" 200
|
||||
echo "set the ${controllerAccountName} service account back to an enabled Viewer"
|
||||
fi
|
||||
|
||||
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
|
||||
echo "could not log in to swarm-bao with Grafana's certificate; leaving swarm-controller's Grafana token as it is." >&2
|
||||
cat "$err" >&2
|
||||
exit 1
|
||||
fi
|
||||
export BAO_TOKEN
|
||||
|
||||
# Checked against the call the controller makes with it.
|
||||
if stored="$(bao kv get -field=value ${lib.escapeShellArg controllerTokenStorePath} 2>"$err")" && [ -n "$stored" ]; then
|
||||
status="$(grafana "Bearer $stored" GET /api/prometheus/grafana/api/v1/alerts)"
|
||||
case "$status" in
|
||||
200) exit 0 ;;
|
||||
401) echo "Grafana refuses the stored token for ${controllerAccountName}; replacing it." ;;
|
||||
*) expect "reading alerts with the stored token" "$status" 200 ;;
|
||||
esac
|
||||
fi
|
||||
|
||||
status="$(grafana "$admin" GET "/api/serviceaccounts/$id/tokens")"
|
||||
expect "listing the ${controllerAccountName} tokens" "$status" 200
|
||||
for token_id in $(jq -r '.[].id' "$body"); do
|
||||
status="$(grafana "$admin" DELETE "/api/serviceaccounts/$id/tokens/$token_id")"
|
||||
expect "deleting token $token_id of ${controllerAccountName}" "$status" 200
|
||||
done
|
||||
|
||||
status="$(grafana "$admin" POST "/api/serviceaccounts/$id/tokens" \
|
||||
--data "{\"name\":\"${controllerAccountName}-$(date +%s)\"}")"
|
||||
expect "minting a ${controllerAccountName} token" "$status" 200
|
||||
key="$(jq -r .key "$body")"
|
||||
: >"$body"
|
||||
if ! printf '%s' "$key" | bao kv put ${lib.escapeShellArg controllerTokenStorePath} value=- >/dev/null 2>"$err"; then
|
||||
echo "could not store the new ${controllerAccountName} token at ${controllerTokenStorePath}; the next run replaces it." >&2
|
||||
cat "$err" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "stored a new ${controllerAccountName} token at ${controllerTokenStorePath}"
|
||||
'';
|
||||
};
|
||||
|
||||
systemd.timers.swarm-bao-grafana-controller-token = storeRefreshTimer "check swarm-controller's Grafana token and replace it if Grafana refuses it";
|
||||
|
||||
containers.${cfg.machine} = {
|
||||
autoStart = true;
|
||||
ephemeral = false;
|
||||
|
|
|
|||
Loading…
Reference in a new issue