vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm metrics store: the read API needs Grafana's bearer or an operator session

Browse source 
The metrics vhost forwarded the store's read API (query, series, labels,
export, vmui, ...) with no credential, so anything that could reach
metrics.<swarm> could read every hive's and agent's metrics. mara ruled on
checked by an auth_request responder, and operators use their authelia
session.

- swarm-metrics-auth (new crate): auth_request responder on the metrics
  store's host. Compares the presented bearer, in constant time, with the
  one stored at swarm/services/<grafana client id>/metrics/bearer, read
  under its own bao leaf and cached for 60s. Static user, the post-#5264
  swarm-logs-auth unit shape.
- swarm-bao-grafana-metrics-bearer: on Grafana's host, creates that bearer
  in bao if absent (create without update, Grafana's existing leaf) and
  writes "Bearer <value>" into the container; the datasource sends it as
  its Authorization header via $__file{}.
- The vhost: read locations take auth_request through a map (a bearer goes
  to swarm-metrics-auth, anything else to authelia, no login redirect);
  vmui takes an authelia session with the login redirect; forceSSL like
  the logs vhost. The authelia subrequest location is renamed from
  /__metrics_push_authz to /__metrics_authelia since it now serves reads.
- Authelia: the collector keeps every path; everyone else is refused the
  collector's push and /metrics paths; group admins is admitted to the
  rest; then deny.
- bao: a swarm-metrics-auth cert role and policy (read on the one leaf),
  its leaf minted by glue-bao-tls, the CN reserved as a hive name, and
  Grafana's grant gains create on the bearer leaf.

Fixes #3596
This commit is contained in:
atlas 2026-10-14 19:53:36 +02:00 • committed by mara
commit 6aab2ebb65
24 changed files with 1126 additions and 200 deletions
Download patch file Download diff file Expand all files Collapse all files

103
nix/host-modules/swarm-grafana.nix
View file

@ -263,6 +263,16 @@ let
adminPasswordPath = "/var/lib/grafana-admin/password";
hostAdminPasswordPath = "/var/lib/nixos-containers/${cfg.machine}${adminPasswordPath}";
# The bearer the metrics datasource presents at the metrics store's query
# API, which `swarm-metrics-auth` on the store's host checks against the
# same store path. Minted once by `swarm-bao-grafana-metrics-bearer` below,
# under the same grant as the admin password. The file holds the whole
# header value, `Bearer <value>`: nixpkgs accepts a `secureJsonData` entry
# only when it is exactly one `$__file{}`.
metricsBearerStorePath = "secret/swarm/services/${cfg.oidc.clientId}/metrics/bearer";
metricsBearerPath = "/var/lib/grafana-metrics/bearer";
hostMetricsBearerPath = "/var/lib/nixos-containers/${cfg.machine}${metricsBearerPath}";
socketPath = "${deployCfg.grafana.socketDir}/grafana.sock";
# Both static NixOS ids, and both checked rather than assumed: nginx has
@ -328,9 +338,12 @@ in
description = ''
Where the provisioned datasource points. Defaults to the metrics
store's swarm name, served by the gateway of whichever host runs the
store, so Grafana and the store need not share a host. The store's
query routes on that vhost take no credential, so the datasource
sends none.
store, so Grafana and the store need not share a host.
The datasource sends the bearer `swarm-bao-grafana-metrics-bearer`
keeps in the swarm secret store, which is what the store's query
routes on that vhost check. Whatever URL this names receives that
bearer.
'';
};
@ -756,6 +769,86 @@ in
'';
};
# The metrics datasource's bearer: created in the store if it holds none,
# then copied into the container, under Grafana's own leaf. Same shape and
# grant as the admin password above. The store's `swarm-metrics-auth`
# reads the same path, so nothing is placed by hand on either host.
systemd.services.swarm-bao-grafana-metrics-bearer = {
description = "create Grafana's metrics store bearer in the swarm secret store if absent and fetch it";
# Ordered after the store where it runs here:
# ./glue-bao-readers-store-order.nix.
before = [ "container@${cfg.machine}.service" ];
wantedBy = [
"multi-user.target"
"container@${cfg.machine}.service"
];
path = [
baoDeploy.package
pkgs.coreutils
pkgs.openssl
pkgs.systemd
];
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
serviceConfig = storeRetry.serviceConfig // {
Type = "oneshot";
RemainAfterExit = true;
SyslogIdentifier = "swarm-bao-grafana-metrics-bearer";
TimeoutStartSec = 30;
};
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
BAO_CLIENT_CERT = baoDeploy.grafanaOidcClientCertFile;
BAO_CLIENT_KEY = baoDeploy.grafanaOidcClientKeyFile;
}
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
BAO_CACERT = baoDeploy.serverCaFile;
};
script = ''
set -euo pipefail
${atomicWriteSecret}
${refreshConsumer}
err="$(mktemp)"
trap 'rm -f "$err"' EXIT
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
echo "could not log in to swarm-bao with Grafana's certificate; leaving Grafana's metrics bearer file as it is." >&2
cat "$err" >&2
exit 1
fi
export BAO_TOKEN
# Read, create on a miss, read again: the admin password's sequence
# above, for its reasons.
if ! bearer="$(bao kv get -field=value ${lib.escapeShellArg metricsBearerStorePath} 2>"$err")"; then
minted="$(openssl rand -hex 32)"
if ! printf '%s' "$minted" | bao kv put -cas=0 ${lib.escapeShellArg metricsBearerStorePath} value=- >/dev/null 2>"$err"; then
echo "could not create ${metricsBearerStorePath}; reading it back in case another run created it." >&2
cat "$err" >&2
fi
if ! bearer="$(bao kv get -field=value ${lib.escapeShellArg metricsBearerStorePath} 2>"$err")"; then
echo "swarm-bao did not return ${metricsBearerStorePath}; Grafana does not start without it." >&2
cat "$err" >&2
exit 1
fi
fi
if [ -z "$bearer" ]; then
echo "swarm-bao returned an empty ${metricsBearerStorePath}; leaving the file as it is." >&2
exit 1
fi
# Owner and mode as for the OIDC secret above, for its reasons.
install -d -m 0755 ${lib.escapeShellArg (builtins.dirOf hostMetricsBearerPath)}
if secret_differs ${lib.escapeShellArg hostMetricsBearerPath} "Bearer $bearer"; then
atomic_write_secret 0400 ${lib.escapeShellArg "${toString config.ids.uids.grafana}:0"} ${lib.escapeShellArg hostMetricsBearerPath} "Bearer $bearer"
fi
# `$__file{}` in a provisioned datasource is read as Grafana starts.
refresh_consumer ${lib.escapeShellArg cfg.machine} grafana.service ${lib.escapeShellArg hostMetricsBearerPath}
'';
};
containers.${cfg.machine} = {
autoStart = true;
ephemeral = false;
@ -1017,6 +1110,10 @@ in
url = deployCfg.grafana.datasourceUrl;
access = "proxy";
isDefault = true;
# The store's query routes admit this bearer or an operator's
# session, and nothing else.
jsonData.httpHeaderName1 = "Authorization";
secureJsonData.httpHeaderValue1 = "$__file{${metricsBearerPath}}";
}
{
name = "VictoriaLogs";