swarm metrics store: the read API needs Grafana's bearer or an operator session
The metrics vhost forwarded the store's read API (query, series, labels,
export, vmui, ...) with no credential, so anything that could reach
metrics.<swarm> could read every hive's and agent's metrics. mara ruled on
checked by an auth_request responder, and operators use their authelia
session.
- swarm-metrics-auth (new crate): auth_request responder on the metrics
store's host. Compares the presented bearer, in constant time, with the
one stored at swarm/services/<grafana client id>/metrics/bearer, read
under its own bao leaf and cached for 60s. Static user, the post-#5264
swarm-logs-auth unit shape.
- swarm-bao-grafana-metrics-bearer: on Grafana's host, creates that bearer
in bao if absent (create without update, Grafana's existing leaf) and
writes "Bearer <value>" into the container; the datasource sends it as
its Authorization header via $__file{}.
- The vhost: read locations take auth_request through a map (a bearer goes
to swarm-metrics-auth, anything else to authelia, no login redirect);
vmui takes an authelia session with the login redirect; forceSSL like
the logs vhost. The authelia subrequest location is renamed from
/__metrics_push_authz to /__metrics_authelia since it now serves reads.
- Authelia: the collector keeps every path; everyone else is refused the
collector's push and /metrics paths; group admins is admitted to the
rest; then deny.
- bao: a swarm-metrics-auth cert role and policy (read on the one leaf),
its leaf minted by glue-bao-tls, the CN reserved as a hive name, and
Grafana's grant gains create on the bearer leaf.
Fixes #3596
This commit is contained in:
atlas
mara
parent
9f0fb71fee
commit
6aab2ebb65
24 changed files with 1126 additions and 200 deletions
|
|
@ -478,36 +478,94 @@ let
|
|||
&& lib.hasInfix "error_page" browser;
|
||||
}
|
||||
{
|
||||
# The store authenticates nothing itself, so every location that
|
||||
# forwards without `auth_request` is open to anything that can reach the
|
||||
# domain. The query location is the positive control: it shows the
|
||||
# filter finds forwarding locations at all.
|
||||
name = "the metrics vhost forwards only read paths without a credential";
|
||||
# The store authenticates nothing itself, so a location that forwards to
|
||||
# it without `auth_request` is open to anything that can reach the
|
||||
# domain. The query location is the control: it shows the filter finds
|
||||
# store-forwarding locations at all.
|
||||
name = "no location on the metrics vhost forwards to the store without auth_request";
|
||||
ok =
|
||||
let
|
||||
locs = allLocal.services.nginx.virtualHosts."metrics.t.local".locations;
|
||||
open = lib.filterAttrs (
|
||||
_: l:
|
||||
l.proxyPass != null
|
||||
&& !(lib.hasInfix "auth_request" l.extraConfig)
|
||||
&& !(lib.hasInfix "internal;" l.extraConfig)
|
||||
toStore = lib.filterAttrs (
|
||||
_: l: l.proxyPass != null && lib.hasPrefix "http://unix:/run/swarm-victoriametrics/" l.proxyPass
|
||||
) locs;
|
||||
writeWords = [
|
||||
"import"
|
||||
"write"
|
||||
"admin"
|
||||
"delete"
|
||||
"insert"
|
||||
"internal"
|
||||
"push"
|
||||
"prometheus"
|
||||
"influx"
|
||||
];
|
||||
open = lib.filterAttrs (_: l: !(lib.hasInfix "auth_request " l.extraConfig)) toStore;
|
||||
in
|
||||
locs."/".proxyPass == null
|
||||
&& locs."/".return == "403"
|
||||
&& open ? "= /api/v1/query"
|
||||
&& lib.all (n: n != "/" && !(lib.any (w: lib.hasInfix w n) writeWords)) (lib.attrNames open);
|
||||
toStore ? "= /api/v1/query"
|
||||
&& open == { }
|
||||
&& locs."/".proxyPass == null
|
||||
&& locs."/".return == "403";
|
||||
}
|
||||
{
|
||||
# The read API's check, link by link: every read location names the
|
||||
# switch with no login redirect (a machine reader would get login HTML
|
||||
# as a 200), the switch rewrites to the map's pick, a bearer reaches
|
||||
# the responder's socket, which the unit binds, and anything else goes
|
||||
# to authelia. vmui is a browser's, so it redirects to the login.
|
||||
name = "the metrics read API sends a bearer to swarm-metrics-auth and a session to authelia";
|
||||
ok =
|
||||
let
|
||||
l = allLocal.services.nginx.virtualHosts."metrics.t.local".locations;
|
||||
http = allLocal.services.nginx.appendHttpConfig;
|
||||
unit = allLocal.systemd.services.swarm-metrics-auth;
|
||||
reads = lib.filterAttrs (n: _: lib.hasPrefix "= /api/v1/" n || lib.hasPrefix "^~ /api/v1/" n) l;
|
||||
in
|
||||
reads ? "= /api/v1/query_range"
|
||||
&& lib.all (
|
||||
r:
|
||||
lib.hasInfix "auth_request /__metrics_read_auth;" r.extraConfig
|
||||
&& !(lib.hasInfix "error_page" r.extraConfig)
|
||||
) (lib.attrValues reads)
|
||||
&& lib.hasInfix "map \$http_authorization \$swarm_metrics_read_auth_target" http
|
||||
&& lib.hasInfix "\"~^Bearer \" /__metrics_grafana_auth;" http
|
||||
&& lib.hasInfix "default /__metrics_authelia;" http
|
||||
&&
|
||||
lib.hasInfix "rewrite ^ \$swarm_metrics_read_auth_target last;"
|
||||
l."= /__metrics_read_auth".extraConfig
|
||||
&& lib.hasInfix "internal;" l."= /__metrics_grafana_auth".extraConfig
|
||||
&& l."= /__metrics_grafana_auth".proxyPass == "http://unix:/run/swarm-metrics-auth/auth.sock:/"
|
||||
&& lib.hasInfix "--socket /run/swarm-metrics-auth/auth.sock" unit.serviceConfig.ExecStart
|
||||
&& lib.hasInfix "auth_request /__metrics_authelia;" l."^~ /vmui/".extraConfig
|
||||
&& lib.hasInfix "error_page 401 =302" l."^~ /vmui/".extraConfig
|
||||
&& l."= /__metrics_authelia".proxyPass == "https://auth.t.local/api/authz/auth-request";
|
||||
}
|
||||
{
|
||||
# Grafana's host creates the bearer and its datasource sends it; the
|
||||
# store's responder reads it. Three modules spell the path, and the
|
||||
# store's grants name it twice: if any one drifts, Grafana sends a value
|
||||
# the responder never reads, or one of them is refused by the store.
|
||||
name = "Grafana's datasource sends the bearer swarm-metrics-auth checks, at one store path";
|
||||
ok =
|
||||
let
|
||||
leaf = "swarm/services/${allLocal.services.hyperhive.swarm.grafana.oidc.clientId}/metrics/bearer";
|
||||
ds =
|
||||
lib.findFirst (d: d.uid == "swarm-victoriametrics") null
|
||||
allLocal.containers.swarm-grafana.config.services.grafana.provision.datasources.settings.datasources;
|
||||
bearerUnit = allLocal.systemd.services.swarm-bao-grafana-metrics-bearer;
|
||||
s = allLocal.systemd.services;
|
||||
in
|
||||
ds.jsonData.httpHeaderName1 == "Authorization"
|
||||
&& ds.secureJsonData.httpHeaderValue1 == "$__file{/var/lib/grafana-metrics/bearer}"
|
||||
&& lib.hasInfix "bao kv put -cas=0 secret/${leaf} value=-" bearerUnit.script
|
||||
&& lib.hasInfix "/var/lib/nixos-containers/swarm-grafana/var/lib/grafana-metrics/bearer \"Bearer $bearer\"" bearerUnit.script
|
||||
&& lib.hasInfix "--store-path ${leaf}" s.swarm-metrics-auth.serviceConfig.ExecStart
|
||||
&& lib.hasInfix "path \"secret/data/${leaf}\" {\n capabilities = [\"read\"]\n}" s.swarm-bao-metrics-auth-policy.script
|
||||
&& lib.hasInfix "path \"secret/data/${leaf}\" {\n capabilities = [\"create\", \"read\"]\n}" s.swarm-bao-grafana-oidc-policy.script;
|
||||
}
|
||||
{
|
||||
# The responder reads the store as itself, with the leaf the store's
|
||||
# host mints for it.
|
||||
name = "swarm-metrics-auth presents its own store leaf";
|
||||
ok =
|
||||
let
|
||||
v = allLocal.services.hyperhive.deploy.victoriametrics;
|
||||
lc = allLocal.systemd.services.swarm-metrics-auth.serviceConfig.LoadCredential;
|
||||
in
|
||||
lib.hasSuffix "/metrics-auth.pem" v.authBaoClientCertFile
|
||||
&& lib.hasSuffix "/metrics-auth-key.pem" v.authBaoClientKeyFile
|
||||
&& builtins.elem "bao-client.pem:${v.authBaoClientCertFile}" lc
|
||||
&& builtins.elem "bao-client-key.pem:${v.authBaoClientKeyFile}" lc
|
||||
&& lib.hasInfix "/metrics-auth.pem ]" allLocal.systemd.services.swarm-bao-pki.script;
|
||||
}
|
||||
{
|
||||
# The read counterpart to the ingest location: an agent queries the log
|
||||
|
|
@ -592,7 +650,7 @@ let
|
|||
# `/run/private/` (0700 root:root), which nginx cannot traverse, so a
|
||||
# socket there answers every proxied request with a 502. Each `unix:`
|
||||
# upstream under `/run/<dir>/` is matched to the units creating `<dir>`.
|
||||
# The first clause is the control: the match finds swarm-logs-auth's.
|
||||
# The first two clauses are the control: the match finds both responders.
|
||||
name = "no unix socket nginx proxies to sits in a DynamicUser's preserved runtime directory";
|
||||
ok =
|
||||
let
|
||||
|
|
@ -626,7 +684,9 @@ let
|
|||
"no"
|
||||
]);
|
||||
in
|
||||
owners ? swarm-logs-auth && !(lib.any (u: hidden u.serviceConfig) (lib.attrValues owners));
|
||||
owners ? swarm-logs-auth
|
||||
&& owners ? swarm-metrics-auth
|
||||
&& !(lib.any (u: hidden u.serviceConfig) (lib.attrValues owners));
|
||||
}
|
||||
{
|
||||
# Every agent bearer the responder has not recently granted costs a
|
||||
|
|
|
|||
Loading…
Reference in a new issue