vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm metrics store: the read API needs Grafana's bearer or an operator session

Browse source 
The metrics vhost forwarded the store's read API (query, series, labels,
export, vmui, ...) with no credential, so anything that could reach
metrics.<swarm> could read every hive's and agent's metrics. mara ruled on
checked by an auth_request responder, and operators use their authelia
session.

- swarm-metrics-auth (new crate): auth_request responder on the metrics
  store's host. Compares the presented bearer, in constant time, with the
  one stored at swarm/services/<grafana client id>/metrics/bearer, read
  under its own bao leaf and cached for 60s. Static user, the post-#5264
  swarm-logs-auth unit shape.
- swarm-bao-grafana-metrics-bearer: on Grafana's host, creates that bearer
  in bao if absent (create without update, Grafana's existing leaf) and
  writes "Bearer <value>" into the container; the datasource sends it as
  its Authorization header via $__file{}.
- The vhost: read locations take auth_request through a map (a bearer goes
  to swarm-metrics-auth, anything else to authelia, no login redirect);
  vmui takes an authelia session with the login redirect; forceSSL like
  the logs vhost. The authelia subrequest location is renamed from
  /__metrics_push_authz to /__metrics_authelia since it now serves reads.
- Authelia: the collector keeps every path; everyone else is refused the
  collector's push and /metrics paths; group admins is admitted to the
  rest; then deny.
- bao: a swarm-metrics-auth cert role and policy (read on the one leaf),
  its leaf minted by glue-bao-tls, the CN reserved as a hive name, and
  Grafana's grant gains create on the bearer leaf.

Fixes #3596
This commit is contained in:
atlas 2026-10-14 19:53:36 +02:00 • committed by mara
commit 6aab2ebb65
24 changed files with 1126 additions and 200 deletions
Download patch file Download diff file Expand all files Collapse all files

59
nix/module-eval/bao-grants.nix
View file

@ -169,7 +169,7 @@ let
_: u: (u.environment.BAO_CLIENT_CERT or null) == granterCertFile
) baoGrantWithConsumers.systemd.services;
# The eighteen units that write a `swarm-*` grant, by name, for the discovery
# The nineteen units that write a `swarm-*` grant, by name, for the discovery
# control below.
grantingUnitNames = [
"swarm-bao-controller-policy"
@ -187,6 +187,7 @@ let
"swarm-bao-hive-leaf-policy"
"swarm-bao-ci-runner-policy"
"swarm-bao-logs-auth-policy"
"swarm-bao-metrics-auth-policy"
"swarm-bao-forge-publisher-policy"
"swarm-bao-review-classifier-policy"
"swarm-bao-builder-policy"
@ -593,10 +594,10 @@ let
# both, and the two are separate principals precisely because a
# dashboard is not entitled to a collector's credential.
#
# The admin password is the one leaf it may create, and without `update`:
# a grant that could replace it would let a second run rotate the
# password Grafana's database already holds.
name = "the Grafana reader reads its own client secret and creates its admin password, and nothing else";
# The admin password and the metrics bearer are the leaves it may create,
# and without `update`: a grant that could replace one would let a second
# run rotate a value Grafana or the metrics store already holds.
name = "the Grafana reader reads its own client secret and creates its admin password and metrics bearer, and nothing else";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-grafana-oidc-policy.script;
@ -613,6 +614,13 @@ let
"read"
];
}
{
path = "secret/data/swarm/services/swarm-grafana/metrics/bearer";
caps = [
"create"
"read"
];
}
]
&& !(lib.hasInfix "secret/data/swarm/agents" s)
&& !(lib.hasInfix "secret/data/swarm/hives" s)
@ -731,6 +739,32 @@ let
in
lib.hasInfix "/logs-auth.pem ]" s && lib.hasInfix "swarm-logs-auth \"\" clientAuth" s;
}
{
# Read-only on the one leaf Grafana's host creates, and nothing else:
# this principal checks a presented bearer and does nothing with it.
name = "the metrics store responder's grant is Grafana's metrics bearer, read-only, and nothing else";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-metrics-auth-policy.script;
in
grantsIn s == [
{
path = "secret/data/swarm/services/swarm-grafana/metrics/bearer";
caps = [ "read" ];
}
]
&& lib.hasInfix "auth/cert/certs/swarm-metrics-auth" s
&& lib.hasInfix "allowed_common_names=swarm-metrics-auth" s
&& lib.hasInfix "token_policies=swarm-metrics-auth" s;
}
{
name = "the PKI unit signs the metrics store responder's leaf under its own subject";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-pki.script;
in
lib.hasInfix "/metrics-auth.pem ]" s && lib.hasInfix "swarm-metrics-auth \"\" clientAuth" s;
}
{
# 🩸 The half that makes the policies above bind: a policy grants only
# through a token that carries it, and a token is minted by a cert-auth
@ -942,24 +976,24 @@ let
}
{
# A store host without the granter's pair writes its grants some other
# way, so none of the eighteen units may exist. Without this arm
# way, so none of the nineteen units may exist. Without this arm
# `lib.mkIf haveGranter` could be dropped from any of them and every other
# case here would still pass.
name = "without the granter's pair none of the eighteen granting units render";
name = "without the granter's pair none of the nineteen granting units render";
ok =
let
s = baoGranterOptOut.systemd.services;
in
lib.all (unit: !(s ? ${unit})) (grantingUnitNames ++ [ "swarm-bao-granter-role" ])
# The control: the same store with the pair renders all eighteen.
# The control: the same store with the pair renders all nineteen.
&& lib.all (unit: baoGrantHere.systemd.services ? ${unit}) grantingUnitNames;
}
{
# 🩸 What replaced the silent skip. With no bootstrap token the eighteen still
# 🩸 What replaced the silent skip. With no bootstrap token the nineteen still
# render, and a refused granter fails them with the step that fixes it.
# A store host that never named a token is told to name one, since the
# unit that sets the granter up renders only where it has.
name = "a store host without a bootstrap token renders the eighteen, each failing loudly with the one-time step";
name = "a store host without a bootstrap token renders the nineteen, each failing loudly with the one-time step";
ok =
let
s = baoGranterNoToken.systemd.services;
@ -1049,6 +1083,7 @@ let
"swarm-services-cert"
"swarm-bao-forwarder-oidc"
"swarm-bao-grafana-admin"
"swarm-bao-grafana-metrics-bearer"
]
++ policyReaders
);
@ -1776,8 +1811,8 @@ let
}
{
# What makes the case above mean something: discovery by the granter's
# certificate reaches all eighteen units, and each yields calls.
name = "the granter-policy check sees all eighteen granting units, and parses calls from each";
# certificate reaches all nineteen units, and each yields calls.
name = "the granter-policy check sees all nineteen granting units, and parses calls from each";
ok =
lib.sort lib.lessThan (lib.attrNames granterUnits) == lib.sort lib.lessThan grantingUnitNames
&& lib.all (u: baoCalls u.script != [ ]) (lib.attrValues granterUnits)

114
nix/module-eval/core-toggle.nix
View file

@ -478,36 +478,94 @@ let
&& lib.hasInfix "error_page" browser;
}
{
# The store authenticates nothing itself, so every location that
# forwards without `auth_request` is open to anything that can reach the
# domain. The query location is the positive control: it shows the
# filter finds forwarding locations at all.
name = "the metrics vhost forwards only read paths without a credential";
# The store authenticates nothing itself, so a location that forwards to
# it without `auth_request` is open to anything that can reach the
# domain. The query location is the control: it shows the filter finds
# store-forwarding locations at all.
name = "no location on the metrics vhost forwards to the store without auth_request";
ok =
let
locs = allLocal.services.nginx.virtualHosts."metrics.t.local".locations;
open = lib.filterAttrs (
_: l:
l.proxyPass != null
&& !(lib.hasInfix "auth_request" l.extraConfig)
&& !(lib.hasInfix "internal;" l.extraConfig)
toStore = lib.filterAttrs (
_: l: l.proxyPass != null && lib.hasPrefix "http://unix:/run/swarm-victoriametrics/" l.proxyPass
) locs;
writeWords = [
"import"
"write"
"admin"
"delete"
"insert"
"internal"
"push"
"prometheus"
"influx"
];
open = lib.filterAttrs (_: l: !(lib.hasInfix "auth_request " l.extraConfig)) toStore;
in
locs."/".proxyPass == null
&& locs."/".return == "403"
&& open ? "= /api/v1/query"
&& lib.all (n: n != "/" && !(lib.any (w: lib.hasInfix w n) writeWords)) (lib.attrNames open);
toStore ? "= /api/v1/query"
&& open == { }
&& locs."/".proxyPass == null
&& locs."/".return == "403";
}
{
# The read API's check, link by link: every read location names the
# switch with no login redirect (a machine reader would get login HTML
# as a 200), the switch rewrites to the map's pick, a bearer reaches
# the responder's socket, which the unit binds, and anything else goes
# to authelia. vmui is a browser's, so it redirects to the login.
name = "the metrics read API sends a bearer to swarm-metrics-auth and a session to authelia";
ok =
let
l = allLocal.services.nginx.virtualHosts."metrics.t.local".locations;
http = allLocal.services.nginx.appendHttpConfig;
unit = allLocal.systemd.services.swarm-metrics-auth;
reads = lib.filterAttrs (n: _: lib.hasPrefix "= /api/v1/" n || lib.hasPrefix "^~ /api/v1/" n) l;
in
reads ? "= /api/v1/query_range"
&& lib.all (
r:
lib.hasInfix "auth_request /__metrics_read_auth;" r.extraConfig
&& !(lib.hasInfix "error_page" r.extraConfig)
) (lib.attrValues reads)
&& lib.hasInfix "map \$http_authorization \$swarm_metrics_read_auth_target" http
&& lib.hasInfix "\"~^Bearer \" /__metrics_grafana_auth;" http
&& lib.hasInfix "default /__metrics_authelia;" http
&&
lib.hasInfix "rewrite ^ \$swarm_metrics_read_auth_target last;"
l."= /__metrics_read_auth".extraConfig
&& lib.hasInfix "internal;" l."= /__metrics_grafana_auth".extraConfig
&& l."= /__metrics_grafana_auth".proxyPass == "http://unix:/run/swarm-metrics-auth/auth.sock:/"
&& lib.hasInfix "--socket /run/swarm-metrics-auth/auth.sock" unit.serviceConfig.ExecStart
&& lib.hasInfix "auth_request /__metrics_authelia;" l."^~ /vmui/".extraConfig
&& lib.hasInfix "error_page 401 =302" l."^~ /vmui/".extraConfig
&& l."= /__metrics_authelia".proxyPass == "https://auth.t.local/api/authz/auth-request";
}
{
# Grafana's host creates the bearer and its datasource sends it; the
# store's responder reads it. Three modules spell the path, and the
# store's grants name it twice: if any one drifts, Grafana sends a value
# the responder never reads, or one of them is refused by the store.
name = "Grafana's datasource sends the bearer swarm-metrics-auth checks, at one store path";
ok =
let
leaf = "swarm/services/${allLocal.services.hyperhive.swarm.grafana.oidc.clientId}/metrics/bearer";
ds =
lib.findFirst (d: d.uid == "swarm-victoriametrics") null
allLocal.containers.swarm-grafana.config.services.grafana.provision.datasources.settings.datasources;
bearerUnit = allLocal.systemd.services.swarm-bao-grafana-metrics-bearer;
s = allLocal.systemd.services;
in
ds.jsonData.httpHeaderName1 == "Authorization"
&& ds.secureJsonData.httpHeaderValue1 == "$__file{/var/lib/grafana-metrics/bearer}"
&& lib.hasInfix "bao kv put -cas=0 secret/${leaf} value=-" bearerUnit.script
&& lib.hasInfix "/var/lib/nixos-containers/swarm-grafana/var/lib/grafana-metrics/bearer \"Bearer $bearer\"" bearerUnit.script
&& lib.hasInfix "--store-path ${leaf}" s.swarm-metrics-auth.serviceConfig.ExecStart
&& lib.hasInfix "path \"secret/data/${leaf}\" {\n capabilities = [\"read\"]\n}" s.swarm-bao-metrics-auth-policy.script
&& lib.hasInfix "path \"secret/data/${leaf}\" {\n capabilities = [\"create\", \"read\"]\n}" s.swarm-bao-grafana-oidc-policy.script;
}
{
# The responder reads the store as itself, with the leaf the store's
# host mints for it.
name = "swarm-metrics-auth presents its own store leaf";
ok =
let
v = allLocal.services.hyperhive.deploy.victoriametrics;
lc = allLocal.systemd.services.swarm-metrics-auth.serviceConfig.LoadCredential;
in
lib.hasSuffix "/metrics-auth.pem" v.authBaoClientCertFile
&& lib.hasSuffix "/metrics-auth-key.pem" v.authBaoClientKeyFile
&& builtins.elem "bao-client.pem:${v.authBaoClientCertFile}" lc
&& builtins.elem "bao-client-key.pem:${v.authBaoClientKeyFile}" lc
&& lib.hasInfix "/metrics-auth.pem ]" allLocal.systemd.services.swarm-bao-pki.script;
}
{
# The read counterpart to the ingest location: an agent queries the log
@ -592,7 +650,7 @@ let
# `/run/private/` (0700 root:root), which nginx cannot traverse, so a
# socket there answers every proxied request with a 502. Each `unix:`
# upstream under `/run/<dir>/` is matched to the units creating `<dir>`.
# The first clause is the control: the match finds swarm-logs-auth's.
# The first two clauses are the control: the match finds both responders.
name = "no unix socket nginx proxies to sits in a DynamicUser's preserved runtime directory";
ok =
let
@ -626,7 +684,9 @@ let
"no"
]);
in
owners ? swarm-logs-auth && !(lib.any (u: hidden u.serviceConfig) (lib.attrValues owners));
owners ? swarm-logs-auth
&& owners ? swarm-metrics-auth
&& !(lib.any (u: hidden u.serviceConfig) (lib.attrValues owners));
}
{
# Every agent bearer the responder has not recently granted costs a

58
nix/module-eval/swarm-otel-identity.nix
View file

@ -287,20 +287,21 @@ let
in
m != null
&& m.proxyPass == "http://unix:/run/swarm-victoriametrics/victoriametrics.sock:/metrics"
&& lib.hasInfix "auth_request /__metrics_push_authz;" m.extraConfig
&& lib.hasInfix "auth_request /__metrics_authelia;" m.extraConfig
&& !(lib.hasInfix "error_page" m.extraConfig)
&& lib.hasInfix "internal;" locs."= /__metrics_push_authz".extraConfig
&& lib.hasInfix "internal;" locs."= /__metrics_authelia".extraConfig
&& locs."/".return == "403"
&& locs."/".proxyPass == null;
}
{
# The metrics store's name admits the collector's client on every path
# and refuses every other subject. The rules carry no `resources`, so no
# path or query spelling misses them and reaches `default_policy`, and
# they come before the apex rule. That is only right while the push
# route and `/metrics` are the vhost's sole authelia-gated locations,
# which the last clause pins.
name = "authelia admits only the collector anywhere on the metrics store's name";
# The metrics store's name admits the collector's client on every path,
# an operator on every path but the collector's two, and nobody else.
# Only the collector's two paths carry `resources`, as prefixes, so no
# path or query spelling of them reaches the operator rule, and no other
# spelling reaches `default_policy`. The rules come before the apex
# rule. The collector's two locations call authelia directly, so a
# bearer there never reaches Grafana's check.
name = "authelia admits the collector anywhere on the metrics store's name and operators only off its two paths";
ok =
let
h = allLocal.services.hyperhive;
@ -311,38 +312,33 @@ let
storeRules = lib.filter (r: r.domain == "metrics.t.local") indexed;
apex = lib.findFirst (r: r.domain == "t.local") null indexed;
policy = subject: autheliaPolicy allLocal subject "metrics.t.local";
paths = [
collectorPaths = [
"/opentelemetry/api/v1/push"
"/opentelemetry/api/v1/push?x=1"
"/metrics"
"/metrics?x=1"
];
readPaths = [
"/api/v1/query"
"/api/v1/query_range?query=up"
"/vmui/"
];
locs = allLocal.services.nginx.virtualHosts.${h.swarm.victoriametrics.domain}.locations;
in
map (r: removeAttrs r [ "i" ]) storeRules == [
{
domain = "metrics.t.local";
policy = "one_factor";
subject = [ collector ];
}
{
lib.length storeRules == 4
&&
removeAttrs (lib.last storeRules) [ "i" ] == {
domain = "metrics.t.local";
policy = "deny";
}
]
&& apex != null
&& lib.all (r: r.i < apex.i) storeRules
&& lib.all (p: policy collector p == "one_factor") paths
&& lib.all (p: policy "user:someone" p == "deny") paths
&& lib.all (p: policy "group:admins" p == "deny") paths
&&
lib.hasInfix "auth_request /__metrics_push_authz;"
locs."= /opentelemetry/api/v1/push".extraConfig
&&
lib.attrNames (lib.filterAttrs (_: l: lib.hasInfix "auth_request" (l.extraConfig or "")) locs) == [
"= /metrics"
"= /opentelemetry/api/v1/push"
];
&& lib.all (p: policy collector p == "one_factor") (collectorPaths ++ readPaths)
&& lib.all (p: policy "user:someone" p == "deny") (collectorPaths ++ readPaths)
&& lib.all (p: policy "group:admins" p == "deny") collectorPaths
&& lib.all (p: policy "group:admins" p == "one_factor") readPaths
&& lib.hasInfix "auth_request /__metrics_authelia;" locs."= /opentelemetry/api/v1/push".extraConfig
&& lib.hasInfix "auth_request /__metrics_authelia;" locs."= /metrics".extraConfig;
}
{
# Every published target admits the collector's client and refuses any
@ -398,7 +394,9 @@ let
let
rules =
autheliaOnly.containers.swarm-authelia.config.services.authelia.instances.swarm.settings.access_control.rules;
metricsRules = lib.filter (r: r ? resources) rules;
# The metrics store's own name is the case above's; its rules carry
# `resources` for its push route and `/metrics` together.
metricsRules = lib.filter (r: r ? resources && r.domain != "metrics.t.local") rules;
re = lib.head (lib.head metricsRules).resources;
table = [
{

11
nix/module-eval/swarm-services-switch.nix
View file

@ -540,7 +540,7 @@ let
{
# No interface but loopback, so the socket is the store's only listener
# the host can reach. Every location that forwards to the store goes
# through it; the authelia subrequest is the one `https://` upstream.
# through it; the internal auth subrequests are the only other upstreams.
# The read, push and scrape locations are named so an empty filter
# can't pass.
name = "the metrics store runs on a netns of its own and the gateway reaches it over its socket";
@ -553,8 +553,10 @@ let
viaSock = path: "http://unix:${sock}:${path}";
locs =
host.services.nginx.virtualHosts.${host.services.hyperhive.swarm.victoriametrics.domain}.locations;
toStore = lib.filter (p: p != null && !(lib.hasPrefix "https://" p)) (
map (l: l.proxyPass) (lib.attrValues locs)
toStore = map (l: l.proxyPass) (
lib.filter (l: l.proxyPass != null && !(lib.hasInfix "internal;" l.extraConfig)) (
lib.attrValues locs
)
);
unit = c.config.systemd.services.victoriametrics;
nginxGid = host.ids.gids.nginx;
@ -749,6 +751,7 @@ let
"swarm-bao-hive-otel-oidc"
"swarm-bao-grafana-oidc"
"swarm-bao-grafana-admin"
"swarm-bao-grafana-metrics-bearer"
"swarm-bao-nats-callout-xkey"
"swarm-bao-ci-runner"
"swarm-bao-otel-oidc"
@ -771,7 +774,7 @@ let
!(lib.any (n: lib.elem n (s.after ++ s.wants ++ s.requires)) storeUnits);
in
lib.all waits readers
&& lib.length readersElsewhere == 6
&& lib.length readersElsewhere == 7
&& lib.all namesNoStoreUnit readersElsewhere
&& !(lib.any (u: storeOnlyNoHive.systemd.services ? ${u}) readers);
}