vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

add swarm-metrics, an agent CLI for the metrics store

Browse source 
swarm-metrics reads the swarm's VictoriaMetrics from inside an agent
container: query, query-range, series, labels and label-values, each one
GET against the store's Prometheus read API through the gateway, printing
the store's JSON. It authenticates the way swarm-logs does, with the
agent's own queue secret read from bao under the agent's leaf, sent as
a swarm-agent bearer.

On the metrics store's host the read API's auth map gains an agent arm
ahead of the Grafana one. It routes to swarm-metrics-agent-auth, a
swarm-logs-auth instance logged in under the existing swarm-metrics-auth
leaf, whose bao grant now includes swarm/agents/+/queue read. The check is
rate-limited per address like the log store's
(deploy.victoriametrics.agentAuth{Rate,Burst,VerdictTtl}).

nix/agent-modules/metrics.nix installs the wrapped binary when the hive
forwards HIVE_AGENT_METRICS_URL (from swarm.victoriametrics.domain) and a
store address, the same gate as swarm-logs. Adds the base:swarm-metrics
skill and the CLI reference with its freshness check.

Fixes #5395
This commit is contained in:
atlas 2026-10-17 00:55:46 +02:00 • committed by mara
commit e92f118bda
30 changed files with 1417 additions and 88 deletions
Download patch file Download diff file Expand all files Collapse all files

41
nix/module-eval/agent-queue-bao.nix
View file

@ -41,19 +41,23 @@ let
agentNoBao = shared.agentBare;
# `swarm-logs` authenticates with the agent's queue secret from the store, so
# the store address gates it beside the query URL.
# `swarm-logs` and `swarm-metrics` authenticate with the agent's queue secret
# from the store, so the store address gates each beside its store URL.
logsUrl = "https://logs.t.local/select/logsql/query";
metricsUrl = "https://metrics.t.local";
agentLogs = agentWith {
services.hyperhive.agent.bao.addr = "https://bao.t.local:8200";
services.hyperhive.agent.logs.queryUrl = logsUrl;
services.hyperhive.agent.metrics.url = metricsUrl;
};
agentLogsNoBao = agentWith { services.hyperhive.agent.logs.queryUrl = logsUrl; };
swarmLogsIn =
machine:
lib.findFirst (
p: (p.name or "") == "swarm-logs-configured"
) null machine.environment.systemPackages;
agentLogsNoBao = agentWith {
services.hyperhive.agent.logs.queryUrl = logsUrl;
services.hyperhive.agent.metrics.url = metricsUrl;
};
configuredIn =
name: machine: lib.findFirst (p: (p.name or "") == name) null machine.environment.systemPackages;
swarmLogsIn = configuredIn "swarm-logs-configured";
swarmMetricsIn = configuredIn "swarm-metrics-configured";
agentBaoIdentity = machine: machine.systemd.services.hive-agent-bao-identity;
cases = [
@ -80,6 +84,27 @@ let
name = "an agent with a log store but no secret store gets no swarm-logs";
ok = swarmLogsIn agentLogsNoBao == null && swarmLogsIn agentLogs != null;
}
{
# The same store identity as `swarm-logs`, and the store's origin.
name = "swarm-metrics is wrapped with the agent's own store identity";
ok =
let
w = (swarmMetricsIn agentLogs).buildCommand;
a = agentLogs.services.hyperhive.agent;
set = k: v: lib.hasInfix "--set ${k} ${lib.escapeShellArg v}" w;
in
set "HIVE_AGENT_METRICS_URL" metricsUrl
&& set "HIVE_AGENT_NAME" a.user.name
&& set "BAO_ADDR" "https://bao.t.local:8200"
&& set "BAO_CLIENT_CERT" a.bao.clientCertFile
&& set "BAO_CLIENT_KEY" a.bao.clientKeyFile
&& set "BAO_CACERT" "/run/credentials/hive-agent.service/hive-agent-bao-server-ca";
}
{
# A metrics URL with nothing to authenticate as installs nothing.
name = "an agent with a metrics store but no secret store gets no swarm-metrics";
ok = swarmMetricsIn agentLogsNoBao == null && swarmMetricsIn agentLogs != null;
}
{
# Both ids or neither: the secret authenticates nobody without the id it
# belongs to, and `swarm-logs` reads both.

14
nix/module-eval/bao-grants.nix
View file

@ -788,16 +788,20 @@ let
lib.hasInfix "/logs-auth.pem ]" s && lib.hasInfix "swarm-logs-auth \"\" clientAuth" s;
}
{
# Read-only on the one leaf Grafana's host creates and on the store
# container forwarder's client secret, and nothing else: this principal
# checks a presented bearer, and its host fetches one secret for the
# container.
name = "the metrics store responder's grant is Grafana's metrics bearer and its container forwarder's client secret, read-only, and nothing else";
# Read-only on every agent's queue secret, the one leaf Grafana's host
# creates and the store container forwarder's client secret, and nothing
# else: this principal checks presented bearers, and its host fetches
# one secret for the container.
name = "the metrics store responder's grant is agents' queue secrets, Grafana's metrics bearer and its container forwarder's client secret, read-only, and nothing else";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-metrics-auth-policy.script;
in
grantsIn s == [
{
path = "secret/data/swarm/agents/+/queue";
caps = [ "read" ];
}
{
path = "secret/data/swarm/services/swarm-grafana/metrics/bearer";
caps = [ "read" ];

31
nix/module-eval/core-toggle.nix
View file

@ -672,15 +672,21 @@ let
{
# The read API's check, link by link: every read location names the
# switch with no login redirect (a machine reader would get login HTML
# as a 200), the switch rewrites to the map's pick, a bearer reaches
# the responder's socket, which the unit binds, and anything else goes
# to authelia. vmui is a browser's, so it redirects to the login.
name = "the metrics read API sends a bearer to swarm-metrics-auth and a session to authelia";
# as a 200), the switch rewrites to the map's pick, an agent bearer
# reaches the agents' responder (listed before the catch-all bearer arm,
# since a map takes the first matching regex), any other bearer reaches
# Grafana's, each socket bound by its unit, and anything else goes to
# authelia. vmui is a browser's, so it redirects to the login.
name = "the metrics read API sends an agent bearer to swarm-metrics-agent-auth, another bearer to swarm-metrics-auth and a session to authelia";
ok =
let
l = allLocal.services.nginx.virtualHosts."metrics.t.local".locations;
http = allLocal.services.nginx.appendHttpConfig;
unit = allLocal.systemd.services.swarm-metrics-auth;
agentUnit = allLocal.systemd.services.swarm-metrics-agent-auth;
agentArm = "\"~^Bearer swarm-agent\\.\" /__metrics_agent_auth;";
bearerArm = "\"~^Bearer \" /__metrics_grafana_auth;";
v = allLocal.services.hyperhive.deploy.victoriametrics;
reads = lib.filterAttrs (n: _: lib.hasPrefix "= /api/v1/" n || lib.hasPrefix "^~ /api/v1/" n) l;
in
reads ? "= /api/v1/query_range"
@ -690,7 +696,9 @@ let
&& !(lib.hasInfix "error_page" r.extraConfig)
) (lib.attrValues reads)
&& lib.hasInfix "map \$http_authorization \$swarm_metrics_read_auth_target" http
&& lib.hasInfix "\"~^Bearer \" /__metrics_grafana_auth;" http
&& lib.hasInfix agentArm http
&& lib.hasInfix bearerArm http
&& !(lib.hasInfix bearerArm (lib.head (lib.splitString agentArm http)))
&& lib.hasInfix "default /__metrics_authelia;" http
&&
lib.hasInfix "rewrite ^ \$swarm_metrics_read_auth_target last;"
@ -698,6 +706,18 @@ let
&& lib.hasInfix "internal;" l."= /__metrics_grafana_auth".extraConfig
&& l."= /__metrics_grafana_auth".proxyPass == "http://unix:/run/swarm-metrics-auth/auth.sock:/"
&& lib.hasInfix "--socket /run/swarm-metrics-auth/auth.sock" unit.serviceConfig.ExecStart
&& lib.hasInfix "internal;" l."= /__metrics_agent_auth".extraConfig
&& l."= /__metrics_agent_auth".proxyPass == "http://unix:/run/swarm-metrics-agent-auth/auth.sock:/"
&& lib.hasInfix "--socket /run/swarm-metrics-agent-auth/auth.sock" agentUnit.serviceConfig.ExecStart
&& lib.hasInfix "/bin/swarm-logs-auth " agentUnit.serviceConfig.ExecStart
&& lib.hasInfix "--store-cert-role swarm-metrics-auth" agentUnit.serviceConfig.ExecStart
&& lib.hasInfix "--verdict-ttl ${toString v.agentAuthVerdictTtl}" agentUnit.serviceConfig.ExecStart
&& agentUnit.serviceConfig.LoadCredential == unit.serviceConfig.LoadCredential
&& lib.hasInfix "limit_req_zone $binary_remote_addr zone=swarm_metrics_agent_auth:1m rate=${v.agentAuthRate};" http
&&
lib.hasInfix "limit_req zone=swarm_metrics_agent_auth burst=${toString v.agentAuthBurst} nodelay;"
l."= /__metrics_agent_auth".extraConfig
&& !(lib.hasInfix "limit_req" l."= /__metrics_grafana_auth".extraConfig)
&& lib.hasInfix "auth_request /__metrics_authelia;" l."^~ /vmui/".extraConfig
&& lib.hasInfix "error_page 401 =302" l."^~ /vmui/".extraConfig
&& l."= /__metrics_authelia".proxyPass == "https://auth.t.local/api/authz/auth-request";
@ -884,6 +904,7 @@ let
owners ? swarm-logs-auth
&& owners ? swarm-logs-grafana-auth
&& owners ? swarm-metrics-auth
&& owners ? swarm-metrics-agent-auth
&& !(lib.any (u: hidden u.serviceConfig) (lib.attrValues owners));
}
{