vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

add swarm-metrics, an agent CLI for the metrics store

Browse source 
swarm-metrics reads the swarm's VictoriaMetrics from inside an agent
container: query, query-range, series, labels and label-values, each one
GET against the store's Prometheus read API through the gateway, printing
the store's JSON. It authenticates the way swarm-logs does, with the
agent's own queue secret read from bao under the agent's leaf, sent as
a swarm-agent bearer.

On the metrics store's host the read API's auth map gains an agent arm
ahead of the Grafana one. It routes to swarm-metrics-agent-auth, a
swarm-logs-auth instance logged in under the existing swarm-metrics-auth
leaf, whose bao grant now includes swarm/agents/+/queue read. The check is
rate-limited per address like the log store's
(deploy.victoriametrics.agentAuth{Rate,Burst,VerdictTtl}).

nix/agent-modules/metrics.nix installs the wrapped binary when the hive
forwards HIVE_AGENT_METRICS_URL (from swarm.victoriametrics.domain) and a
store address, the same gate as swarm-logs. Adds the base:swarm-metrics
skill and the CLI reference with its freshness check.

Fixes #5395
This commit is contained in:
atlas 2026-10-17 00:55:46 +02:00 • committed by mara
commit e92f118bda
30 changed files with 1417 additions and 88 deletions
Download patch file Download diff file Expand all files Collapse all files

135
nix/host-modules/swarm-victoriametrics.nix
View file

@ -38,9 +38,9 @@ let
nginxGid = config.ids.gids.nginx;
# The store's read API, which the vhost forwards to a caller holding
# Grafana's bearer or an operator's session. Each location's `proxy_pass`
# carries the path, so the store receives the path nginx matched, not the
# caller's unnormalised spelling.
# Grafana's bearer, an agent's own bearer, or an operator's session. Each
# location's `proxy_pass` carries the path, so the store receives the path
# nginx matched, not the caller's unnormalised spelling.
#
# Grafana's Prometheus datasource reads the query, series, label, metadata,
# buildinfo, exemplar, rules and alerts routes; the store answers the last
@ -128,6 +128,14 @@ let
# same leaf as `grafanaMetricsBearerLeaf` in ./swarm-bao.nix.
grafanaBearerStorePath = "swarm/services/${hyperhiveCfg.swarm.grafana.oidc.clientId}/metrics/bearer";
# The agents' check on the read API: a `swarm-logs-auth` binary, logging in
# under this host's `swarm-metrics-auth` leaf, whose grant covers every
# agent's queue secret. The log store's host runs `swarm-metrics-auth` under
# its `swarm-logs-auth` leaf the same way.
agentAuthUnit = "swarm-metrics-agent-auth";
agentAuthSocket = "/run/${agentAuthUnit}/auth.sock";
agentAuthCredential = id: "/run/credentials/${agentAuthUnit}.service/${id}";
# ── the container's journal forwarder ───────────────────────────────
#
# With no interface but loopback, the forwarder reaches nothing by name. A
@ -242,7 +250,8 @@ in
example = "/var/lib/swarm-bao-pki/metrics-auth.pem";
description = ''
Client certificate `swarm-metrics-auth` presents to the swarm's secret
store to read Grafana's metrics bearer. Its subject must be
store to read Grafana's metrics bearer, and `swarm-metrics-agent-auth`
to read agents' queue secrets. Its subject must be
{option}`services.hyperhive.deploy.bao.metricsAuthCommonName`.
Required when `deploy.victoriametrics.enable` is set; unset fails at
@ -264,6 +273,47 @@ in
'';
};
options.services.hyperhive.deploy.victoriametrics.agentAuthVerdictTtl = lib.mkOption {
type = lib.types.ints.unsigned;
default = 60;
description = ''
Seconds `swarm-metrics-agent-auth` grants an agent bearer it has
verified again without reading the secret store. A denial is never
remembered.
This is also how long a re-minted or deleted queue secret keeps working
at the metrics store's read API after its last successful check. `0`
checks the store on every query.
'';
};
options.services.hyperhive.deploy.victoriametrics.agentAuthRate = lib.mkOption {
type = lib.types.str;
default = "50r/s";
example = "30r/m";
description = ''
nginx `limit_req` rate, per client address, of read API requests
carrying a `swarm-agent.` bearer, which nginx checks with
`swarm-metrics-agent-auth`. Each such request that the responder has
not granted within
{option}`services.hyperhive.deploy.victoriametrics.agentAuthVerdictTtl`
costs a secret store login and read, whether the bearer is right or not.
Keyed by client address, so every agent on one host shares one budget.
Requests over the rate and its burst are answered 403.
'';
};
options.services.hyperhive.deploy.victoriametrics.agentAuthBurst = lib.mkOption {
type = lib.types.ints.positive;
default = 500;
description = ''
Requests a client address may make above
{option}`services.hyperhive.deploy.victoriametrics.agentAuthRate` before
nginx answers 403, served without delay.
'';
};
config = lib.mkIf deployCfg.victoriametrics.enable {
assertions = [
{
@ -287,7 +337,8 @@ in
services.hyperhive.deploy.victoriametrics.authBaoClientCertFile and
authBaoClientKeyFile must both be set.
swarm-metrics-auth reads Grafana's metrics bearer from the swarm's
swarm-metrics-auth reads Grafana's metrics bearer, and
swarm-metrics-agent-auth agents' queue secrets, from the swarm's
secret store with this certificate. They default to the leaf the
store mints where services.hyperhive.deploy.bao.enable is set on
this host; elsewhere, issue a leaf whose CN is
@ -320,14 +371,18 @@ in
# Which check a read location's `auth_request` gets. `auth_request` takes a
# fixed URI, so it names `/__metrics_read_auth`, which rewrites to the
# target this map picks: a bearer goes to Grafana's check, anything else
# (an operator's session cookie) to authelia. After the forwarder's egress
# target this map picks: an agent's `swarm-agent.` bearer goes to the
# agents' check, any other bearer to Grafana's, anything else (an
# operator's session cookie) to authelia. A map tries its regexes in
# order, so the agent arm comes first. After the forwarder's egress
# listener, `egressServer`.
services.nginx.appendHttpConfig = egressServer + ''
map $http_authorization $swarm_metrics_read_auth_target {
"~^Bearer swarm-agent\." /__metrics_agent_auth;
"~^Bearer " /__metrics_grafana_auth;
default /__metrics_authelia;
}
limit_req_zone $binary_remote_addr zone=swarm_metrics_agent_auth:1m rate=${deployCfg.victoriametrics.agentAuthRate};
'';
# `forceSSL`: authelia answers a plain-http auth subrequest
@ -400,6 +455,22 @@ in
'';
};
# An agent's bearer, checked by `swarm-metrics-agent-auth`. Any
# well-formed agent bearer the responder has not recently granted
# costs a store login and read, so the rate per client address is
# capped here. 403 when over: `auth_request` passes only 401 and 403
# through, and `swarm-metrics` retries a 401.
"= /__metrics_agent_auth" = {
proxyPass = "http://unix:${agentAuthSocket}:/";
extraConfig = ''
internal;
limit_req zone=swarm_metrics_agent_auth burst=${toString deployCfg.victoriametrics.agentAuthBurst} nodelay;
limit_req_status 403;
proxy_pass_request_body off;
proxy_set_header Content-Length "";
'';
};
# The authelia subrequest. For the collector's two locations above it
# admits only the collector's client; for the read and vmui locations
# it admits an operator's session (`metricsStoreRules` in
@ -506,6 +577,56 @@ in
};
};
# The agents' check on the read API, see `agentAuthUnit`.
users.users.${agentAuthUnit} = {
isSystemUser = true;
group = agentAuthUnit;
description = "swarm metrics store auth_request responder for agent credentials";
};
users.groups.${agentAuthUnit} = { };
systemd.services.${agentAuthUnit} = {
description = "swarm metrics store auth_request responder for agent credentials";
wantedBy = [ "multi-user.target" ];
after = [
"network.target"
"swarm-bao-pki.service"
];
serviceConfig = {
ExecStart = lib.concatStringsSep " " [
"${deployCfg.victorialogs.authPackage}/bin/swarm-logs-auth"
"--socket ${agentAuthSocket}"
"--store-cert-role swarm-metrics-auth"
"--verdict-ttl ${toString deployCfg.victoriametrics.agentAuthVerdictTtl}"
];
LoadCredential = lib.mapAttrsToList (id: path: "${id}:${path}") metricsAuthStoreFiles;
# ⚠️ A static user, never `DynamicUser`: see the `swarm-metrics-auth` unit above.
User = agentAuthUnit;
Group = agentAuthUnit;
RuntimeDirectory = agentAuthUnit;
RuntimeDirectoryMode = "0751";
RuntimeDirectoryPreserve = "yes";
Restart = "on-failure";
RestartSec = "5s";
SyslogIdentifier = agentAuthUnit;
ProtectSystem = "strict";
ProtectHome = true;
PrivateTmp = "disconnected";
PrivateDevices = true;
NoNewPrivileges = true;
RestrictSUIDSGID = true;
RemoveIPC = true;
};
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
BAO_CLIENT_CERT = agentAuthCredential "bao-client.pem";
BAO_CLIENT_KEY = agentAuthCredential "bao-client-key.pem";
}
// lib.optionalAttrs (metricsAuthStoreFiles ? "bao-ca.pem") {
BAO_CACERT = agentAuthCredential "bao-ca.pem";
};
};
# nginx runs as its own user under `ProtectSystem=strict`, so it can only
# create the socket where it may write. It removes the socket on a clean
# exit only; a stale one after a crash fails every later `bind`, and with