swarm UI: an alerts page, read by the controller with a Viewer Grafana token from bao
swarm-bao-grafana-controller-token, on the host that runs Grafana, logs in to Grafana's API over its socket as the built-in admin (the password swarm-bao-grafana-admin minted, #5138) and keeps a `swarm-controller` service account that is an enabled Viewer. It mints a token for that account and writes it to bao at swarm/controller/swarm-controller/grafana/token under Grafana's leaf. A ten-minute timer runs it again. The unit keeps a stored token Grafana accepts. If Grafana refuses the stored token with 401, or bao holds none, the unit deletes every token of the account and mints a new one. Grafana's grant gains create/read/update on that leaf, and the controller's gains read. swarm-controller serves GET /api/alerts from Grafana's /api/prometheus/grafana/api/v1/alerts. It reads the token from bao, keeps it in memory, and re-reads bao once on a 401. The HTTP client has 5s connect and 15s request timeouts. Any failure is a 503 that names its cause. swarm-ui gets an /alerts page listing them, built the same way as the hives page. This is option (c) on #5129, the operator's choice.
This commit is contained in:
atlas
mara
parent
8cb6273782
commit
7d09776742
23 changed files with 888 additions and 37 deletions
|
|
@ -273,6 +273,14 @@ let
|
|||
metricsBearerPath = "/var/lib/grafana-metrics/bearer";
|
||||
hostMetricsBearerPath = "/var/lib/nixos-containers/${cfg.machine}${metricsBearerPath}";
|
||||
|
||||
# swarm-controller's token for Grafana's API, minted by
|
||||
# `swarm-bao-grafana-controller-token` below for a Viewer service account of
|
||||
# this name. `controllerGrafanaTokenLeaf` in ./swarm-bao.nix is the nix half
|
||||
# of the path, `swarm_secret_client::grafana::controller_token_path` the Rust
|
||||
# half.
|
||||
controllerAccountName = "swarm-controller";
|
||||
controllerTokenStorePath = "secret/swarm/controller/swarm-controller/grafana/token";
|
||||
|
||||
socketPath = "${deployCfg.grafana.socketDir}/grafana.sock";
|
||||
|
||||
# Both static NixOS ids, and both checked rather than assumed: nginx has
|
||||
|
|
@ -849,6 +857,148 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
# swarm-controller reads Grafana's alerts with a token of a Viewer service
|
||||
# account, created here through Grafana's API as the built-in admin and
|
||||
# handed over through the store.
|
||||
#
|
||||
# Re-run by the timer below. A stored token Grafana still accepts is kept;
|
||||
# one it refuses with 401, or none at all, is replaced: every token of the
|
||||
# account is deleted and one new one is minted and stored. The account's
|
||||
# role is set back to Viewer on every run.
|
||||
systemd.services.swarm-bao-grafana-controller-token = {
|
||||
description = "give swarm-controller a Viewer token for Grafana's API through the swarm secret store";
|
||||
# Ordered after the store where it runs here:
|
||||
# ./glue-bao-readers-store-order.nix.
|
||||
after = [
|
||||
"container@${cfg.machine}.service"
|
||||
"swarm-bao-grafana-admin.service"
|
||||
];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
path = [
|
||||
baoDeploy.package
|
||||
pkgs.coreutils
|
||||
pkgs.curl
|
||||
pkgs.jq
|
||||
];
|
||||
# ./lib/store-retry.nix. A Grafana still starting refuses the
|
||||
# connection, and that attempt fails into the same retry.
|
||||
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
|
||||
serviceConfig = storeRetry.serviceConfig // {
|
||||
# No `RemainAfterExit`: the timer below starts this again.
|
||||
Type = "oneshot";
|
||||
SyslogIdentifier = "swarm-bao-grafana-controller-token";
|
||||
# Grafana's answers, the new token among them, pass through a temp file.
|
||||
PrivateTmp = true;
|
||||
# Each call below is bounded by curl's own timeouts; a run makes up to
|
||||
# about ten of them.
|
||||
TimeoutStartSec = "3min";
|
||||
};
|
||||
environment = {
|
||||
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||||
BAO_CLIENT_CERT = baoDeploy.grafanaOidcClientCertFile;
|
||||
BAO_CLIENT_KEY = baoDeploy.grafanaOidcClientKeyFile;
|
||||
}
|
||||
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
|
||||
BAO_CACERT = baoDeploy.serverCaFile;
|
||||
};
|
||||
script = ''
|
||||
set -euo pipefail
|
||||
|
||||
err="$(mktemp)"
|
||||
body="$(mktemp)"
|
||||
trap 'rm -f "$err" "$body"' EXIT
|
||||
|
||||
# One call to Grafana's API through its socket. Prints the HTTP
|
||||
# status; the body lands in "$body". The credential reaches curl as
|
||||
# a header file, never as an argument.
|
||||
grafana() {
|
||||
local auth="$1" method="$2" path="$3"
|
||||
shift 3
|
||||
curl --silent --show-error --connect-timeout 5 --max-time 15 \
|
||||
--unix-socket ${lib.escapeShellArg socketPath} \
|
||||
-H @<(printf 'Authorization: %s\n' "$auth") \
|
||||
-H 'Content-Type: application/json' \
|
||||
-X "$method" -o "$body" -w '%{http_code}' "$@" \
|
||||
"http://localhost$path"
|
||||
}
|
||||
|
||||
# Fails the attempt unless the last call answered one of the statuses
|
||||
# given.
|
||||
expect() {
|
||||
local what="$1" status="$2"
|
||||
shift 2
|
||||
for ok in "$@"; do
|
||||
[ "$status" = "$ok" ] && return 0
|
||||
done
|
||||
echo "Grafana answered $status to $what:" >&2
|
||||
cat "$body" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
if [ ! -s ${lib.escapeShellArg hostAdminPasswordPath} ]; then
|
||||
echo "no Grafana admin password at ${hostAdminPasswordPath} yet; swarm-bao-grafana-admin writes it." >&2
|
||||
exit 1
|
||||
fi
|
||||
admin="Basic $(printf 'admin:%s' "$(< ${lib.escapeShellArg hostAdminPasswordPath})" | base64 -w0)"
|
||||
|
||||
status="$(grafana "$admin" GET '/api/serviceaccounts/search?perpage=1000&query=${controllerAccountName}')"
|
||||
expect "the service account search" "$status" 200
|
||||
account="$(jq -c --arg n ${controllerAccountName} '[.serviceAccounts[] | select(.name == $n)][0] // empty' "$body")"
|
||||
if [ -z "$account" ]; then
|
||||
status="$(grafana "$admin" POST /api/serviceaccounts \
|
||||
--data '{"name":"${controllerAccountName}","role":"Viewer","isDisabled":false}')"
|
||||
expect "creating the ${controllerAccountName} service account" "$status" 200 201
|
||||
account="$(jq -c . "$body")"
|
||||
echo "created the ${controllerAccountName} service account as a Viewer"
|
||||
fi
|
||||
id="$(jq -r .id <<<"$account")"
|
||||
if [ "$(jq -r '"\(.role) \(.isDisabled // false)"' <<<"$account")" != "Viewer false" ]; then
|
||||
status="$(grafana "$admin" PATCH "/api/serviceaccounts/$id" \
|
||||
--data '{"role":"Viewer","isDisabled":false}')"
|
||||
expect "setting the ${controllerAccountName} service account to an enabled Viewer" "$status" 200
|
||||
echo "set the ${controllerAccountName} service account back to an enabled Viewer"
|
||||
fi
|
||||
|
||||
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
|
||||
echo "could not log in to swarm-bao with Grafana's certificate; leaving swarm-controller's Grafana token as it is." >&2
|
||||
cat "$err" >&2
|
||||
exit 1
|
||||
fi
|
||||
export BAO_TOKEN
|
||||
|
||||
# Checked against the call the controller makes with it.
|
||||
if stored="$(bao kv get -field=value ${lib.escapeShellArg controllerTokenStorePath} 2>"$err")" && [ -n "$stored" ]; then
|
||||
status="$(grafana "Bearer $stored" GET /api/prometheus/grafana/api/v1/alerts)"
|
||||
case "$status" in
|
||||
200) exit 0 ;;
|
||||
401) echo "Grafana refuses the stored token for ${controllerAccountName}; replacing it." ;;
|
||||
*) expect "reading alerts with the stored token" "$status" 200 ;;
|
||||
esac
|
||||
fi
|
||||
|
||||
status="$(grafana "$admin" GET "/api/serviceaccounts/$id/tokens")"
|
||||
expect "listing the ${controllerAccountName} tokens" "$status" 200
|
||||
for token_id in $(jq -r '.[].id' "$body"); do
|
||||
status="$(grafana "$admin" DELETE "/api/serviceaccounts/$id/tokens/$token_id")"
|
||||
expect "deleting token $token_id of ${controllerAccountName}" "$status" 200
|
||||
done
|
||||
|
||||
status="$(grafana "$admin" POST "/api/serviceaccounts/$id/tokens" \
|
||||
--data "{\"name\":\"${controllerAccountName}-$(date +%s)\"}")"
|
||||
expect "minting a ${controllerAccountName} token" "$status" 200
|
||||
key="$(jq -r .key "$body")"
|
||||
: >"$body"
|
||||
if ! printf '%s' "$key" | bao kv put ${lib.escapeShellArg controllerTokenStorePath} value=- >/dev/null 2>"$err"; then
|
||||
echo "could not store the new ${controllerAccountName} token at ${controllerTokenStorePath}; the next run replaces it." >&2
|
||||
cat "$err" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "stored a new ${controllerAccountName} token at ${controllerTokenStorePath}"
|
||||
'';
|
||||
};
|
||||
|
||||
systemd.timers.swarm-bao-grafana-controller-token = storeRefreshTimer "check swarm-controller's Grafana token and replace it if Grafana refuses it";
|
||||
|
||||
containers.${cfg.machine} = {
|
||||
autoStart = true;
|
||||
ephemeral = false;
|
||||
|
|
|
|||
Loading…
Reference in a new issue