swarm UI: an alerts page, read by the controller with a Viewer Grafana token from bao
swarm-bao-grafana-controller-token, on the host that runs Grafana, logs in to Grafana's API over its socket as the built-in admin (the password swarm-bao-grafana-admin minted, #5138) and keeps a `swarm-controller` service account that is an enabled Viewer. It mints a token for that account and writes it to bao at swarm/controller/swarm-controller/grafana/token under Grafana's leaf. A ten-minute timer runs it again. The unit keeps a stored token Grafana accepts. If Grafana refuses the stored token with 401, or bao holds none, the unit deletes every token of the account and mints a new one. Grafana's grant gains create/read/update on that leaf, and the controller's gains read. swarm-controller serves GET /api/alerts from Grafana's /api/prometheus/grafana/api/v1/alerts. It reads the token from bao, keeps it in memory, and re-reads bao once on a 401. The HTTP client has 5s connect and 15s request timeouts. Any failure is a 503 that names its cause. swarm-ui gets an /alerts page listing them, built the same way as the hives page. This is option (c) on #5129, the operator's choice.
This commit is contained in:
atlas
mara
parent
8cb6273782
commit
7d09776742
23 changed files with 888 additions and 37 deletions
|
|
@ -221,10 +221,10 @@ let
|
|||
&& builtins.elem "bao-client-key.pem:/var/lib/swarm-bao-pki/controller-key.pem" s.swarm-controller.serviceConfig.LoadCredential;
|
||||
}
|
||||
{
|
||||
# The forge token comes from the store, so the unit is handed no file for
|
||||
# it, and neither fixture runs the forge: a controller reaches it by name
|
||||
# wherever it runs.
|
||||
name = "the controller is handed no forge token file, and is pointed at the forge away from it";
|
||||
# The forge and Grafana tokens come from the store, so the unit is handed
|
||||
# no file for either, and neither fixture runs the forge or Grafana: a
|
||||
# controller reaches both by name wherever they run.
|
||||
name = "the controller is handed no forge token file, and is pointed at the forge and Grafana away from it";
|
||||
ok =
|
||||
lib.all
|
||||
(
|
||||
|
|
@ -235,6 +235,7 @@ let
|
|||
!(lib.any (c: lib.hasPrefix "forge-token:" c) u.serviceConfig.LoadCredential)
|
||||
&& !(u.environment ? SWARM_CONTROLLER_FORGE_TOKEN_FILE)
|
||||
&& u.environment.SWARM_CONTROLLER_FORGE_URL == "https://forge.t.local"
|
||||
&& u.environment.SWARM_CONTROLLER_GRAFANA_URL == "https://grafana.t.local"
|
||||
)
|
||||
[
|
||||
baoControllerHere
|
||||
|
|
|
|||
|
|
@ -597,8 +597,10 @@ let
|
|||
#
|
||||
# The admin password and the metrics bearer are the leaves it may create,
|
||||
# and without `update`: a grant that could replace one would let a second
|
||||
# run rotate a value Grafana or the metrics store already holds.
|
||||
name = "the Grafana reader reads its own client secret and creates its admin password and metrics bearer, and nothing else";
|
||||
# run rotate a value Grafana or the metrics store already holds. The
|
||||
# controller's Grafana token is the one leaf it may replace, which it
|
||||
# does when Grafana refuses the stored token.
|
||||
name = "the Grafana reader reads its own client secret, creates its admin password and metrics bearer, and writes the controller's Grafana token, and nothing else";
|
||||
ok =
|
||||
let
|
||||
s = baoGrantHere.systemd.services.swarm-bao-grafana-oidc-policy.script;
|
||||
|
|
@ -622,6 +624,14 @@ let
|
|||
"read"
|
||||
];
|
||||
}
|
||||
{
|
||||
path = "secret/data/swarm/controller/swarm-controller/grafana/token";
|
||||
caps = [
|
||||
"create"
|
||||
"read"
|
||||
"update"
|
||||
];
|
||||
}
|
||||
]
|
||||
&& !(lib.hasInfix "secret/data/swarm/agents" s)
|
||||
&& !(lib.hasInfix "secret/data/swarm/hives" s)
|
||||
|
|
@ -938,18 +948,27 @@ let
|
|||
&& !(noReaders ? swarm-bao-nats-callout-xkey);
|
||||
}
|
||||
{
|
||||
# The admin password is created under Grafana's leaf, so its unit waits
|
||||
# for the unit writing that leaf's role and policy; before it, the create
|
||||
# is refused. The second arm is the absence control.
|
||||
name = "grafana's admin password unit is ordered after grafana's policy unit";
|
||||
# The admin password and the controller's Grafana token are written under
|
||||
# Grafana's leaf, so their units wait for the unit writing that leaf's
|
||||
# role and policy; before it, the write is refused. The last arm is the
|
||||
# absence control.
|
||||
name = "grafana's admin password and controller token units are ordered after grafana's policy unit";
|
||||
ok =
|
||||
let
|
||||
u = baoGrantWithConsumers.systemd.services.swarm-bao-grafana-admin;
|
||||
in
|
||||
lib.elem "swarm-bao-grafana-oidc-policy.service" u.after
|
||||
&& lib.elem "swarm-bao-grafana-oidc-policy.service" u.wants
|
||||
&& !(lib.elem "swarm-bao-grafana-oidc-policy.service" u.requires)
|
||||
&& !(baoGrantNoReaders.systemd.services ? swarm-bao-grafana-admin);
|
||||
lib.all
|
||||
(
|
||||
unit:
|
||||
let
|
||||
u = baoGrantWithConsumers.systemd.services.${unit};
|
||||
in
|
||||
lib.elem "swarm-bao-grafana-oidc-policy.service" u.after
|
||||
&& lib.elem "swarm-bao-grafana-oidc-policy.service" u.wants
|
||||
&& !(lib.elem "swarm-bao-grafana-oidc-policy.service" u.requires)
|
||||
&& !(baoGrantNoReaders.systemd.services ? ${unit})
|
||||
)
|
||||
[
|
||||
"swarm-bao-grafana-admin"
|
||||
"swarm-bao-grafana-controller-token"
|
||||
];
|
||||
}
|
||||
{
|
||||
# The ordering is set apart from each reader's own definition, so it can
|
||||
|
|
@ -1100,6 +1119,7 @@ let
|
|||
"swarm-bao-forwarder-oidc"
|
||||
"swarm-bao-grafana-admin"
|
||||
"swarm-bao-grafana-metrics-bearer"
|
||||
"swarm-bao-grafana-controller-token"
|
||||
]
|
||||
++ policyReaders
|
||||
);
|
||||
|
|
@ -1715,6 +1735,25 @@ let
|
|||
"swarm-bao-forge-publisher-policy"
|
||||
];
|
||||
}
|
||||
{
|
||||
# The controller's Grafana token: read on the one leaf, pinned as the
|
||||
# whole stanza so an added write fails, and named by no policy but the
|
||||
# controller's and Grafana's, which writes it.
|
||||
name = "the controller reads its Grafana token, and only Grafana's grant writes it";
|
||||
ok =
|
||||
let
|
||||
s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script;
|
||||
naming = lib.filter (
|
||||
u: lib.hasInfix "swarm-controller/grafana/token" baoGrantHere.systemd.services.${u}.script
|
||||
) grantingUnitNames;
|
||||
in
|
||||
lib.hasInfix "path \"secret/data/swarm/controller/swarm-controller/grafana/token\" {\n capabilities = [\"read\"]\n}" s
|
||||
&&
|
||||
lib.sort lib.lessThan naming == [
|
||||
"swarm-bao-controller-policy"
|
||||
"swarm-bao-grafana-oidc-policy"
|
||||
];
|
||||
}
|
||||
{
|
||||
# The forge's OIDC client secret by name: the forge publisher reads it,
|
||||
# and the secret publisher writes it under its `swarm/services/*` grant,
|
||||
|
|
|
|||
|
|
@ -249,6 +249,33 @@ let
|
|||
&& lib.hasInfix "admin reset-admin-password --password-from-stdin < /var/lib/grafana-admin/password" service.preStart
|
||||
&& lib.hasInfix "-config ${resetConfig}" service.script;
|
||||
}
|
||||
{
|
||||
# swarm-controller reads alerts with this token, so the account it
|
||||
# belongs to is created and kept a Viewer, the token lands at the path
|
||||
# the controller's grant reads, and an attempt against a Grafana or a
|
||||
# store that never answers ends and retries. The timer re-runs it, so
|
||||
# the unit must not stay active.
|
||||
name = "the controller's Grafana token is a Viewer account's, stored under Grafana's leaf, and each attempt is bounded";
|
||||
ok =
|
||||
let
|
||||
unit = grafanaRemoteAuthelia.systemd.services.swarm-bao-grafana-controller-token;
|
||||
s = unit.script;
|
||||
in
|
||||
!(lib.elem (toString (unit.serviceConfig.TimeoutStartSec or "infinity")) [
|
||||
"infinity"
|
||||
"0"
|
||||
])
|
||||
&& lib.hasInfix "curl --silent --show-error --connect-timeout 5 --max-time 15" s
|
||||
&& lib.hasInfix "--data '{\"name\":\"swarm-controller\",\"role\":\"Viewer\",\"isDisabled\":false}'" s
|
||||
&& lib.hasInfix "--data '{\"role\":\"Viewer\",\"isDisabled\":false}'" s
|
||||
&& lib.hasInfix "bao kv put secret/swarm/controller/swarm-controller/grafana/token value=-" s
|
||||
&& lib.hasInfix "--unix-socket /run/swarm-grafana/grafana.sock" s
|
||||
&& unit.environment.BAO_CLIENT_CERT == "/etc/pki/bao-grafana-oidc.pem"
|
||||
&& unit.serviceConfig.Restart == "on-failure"
|
||||
&& !(unit.serviceConfig.RemainAfterExit or false)
|
||||
&& lib.elem "swarm-bao-grafana-admin.service" unit.after
|
||||
&& grafanaRemoteAuthelia.systemd.timers ? swarm-bao-grafana-controller-token;
|
||||
}
|
||||
{
|
||||
# Same 403-not-a-miss reason as the matrix and queue arms below: the
|
||||
# reader's grant covers the `services` prefix, so a path outside it is
|
||||
|
|
|
|||
Loading…
Reference in a new issue