add swarm-metrics, an agent CLI for the metrics store
swarm-metrics reads the swarm's VictoriaMetrics from inside an agent
container: query, query-range, series, labels and label-values, each one
GET against the store's Prometheus read API through the gateway, printing
the store's JSON. It authenticates the way swarm-logs does, with the
agent's own queue secret read from bao under the agent's leaf, sent as
a swarm-agent bearer.
On the metrics store's host the read API's auth map gains an agent arm
ahead of the Grafana one. It routes to swarm-metrics-agent-auth, a
swarm-logs-auth instance logged in under the existing swarm-metrics-auth
leaf, whose bao grant now includes swarm/agents/+/queue read. The check is
rate-limited per address like the log store's
(deploy.victoriametrics.agentAuth{Rate,Burst,VerdictTtl}).
nix/agent-modules/metrics.nix installs the wrapped binary when the hive
forwards HIVE_AGENT_METRICS_URL (from swarm.victoriametrics.domain) and a
store address, the same gate as swarm-logs. Adds the base:swarm-metrics
skill and the CLI reference with its freshness check.
Fixes #5395
This commit is contained in:
atlas
mara
parent
1e8c398132
commit
e92f118bda
30 changed files with 1417 additions and 88 deletions
|
|
@ -48,6 +48,7 @@ in
|
|||
./matrix.nix
|
||||
./mcp.nix
|
||||
./memory-cap.nix
|
||||
./metrics.nix
|
||||
./network.nix
|
||||
./otel.nix
|
||||
./otel-removed.nix
|
||||
|
|
|
|||
66
nix/agent-modules/metrics.nix
Normal file
66
nix/agent-modules/metrics.nix
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
# `swarm-metrics` on this agent's PATH, configured for this swarm's metrics
|
||||
# store. The sibling of ./logs.nix, in the same shape for the same reasons:
|
||||
# the binary defaults nothing (see `swarm-metrics/README.md`), so it is
|
||||
# installed wrapped, and it is gated on the secret store's address as well as
|
||||
# on the store URL, because the identity it presents is the agent's own queue
|
||||
# secret read from that store.
|
||||
{
|
||||
lib,
|
||||
pkgs,
|
||||
config,
|
||||
...
|
||||
}:
|
||||
let
|
||||
cfg = config.services.hyperhive.agent.metrics;
|
||||
baoCfg = config.services.hyperhive.agent.bao;
|
||||
configured = cfg.url != null && baoCfg.addr != null;
|
||||
|
||||
# The coordinates ./logs.nix gives `swarm-logs`. Paths and a name only: a
|
||||
# value in `--set` lands in the world-readable nix store.
|
||||
wrapperEnv = {
|
||||
HIVE_AGENT_METRICS_URL = cfg.url;
|
||||
HIVE_AGENT_NAME = config.services.hyperhive.agent.user.name;
|
||||
BAO_ADDR = baoCfg.addr;
|
||||
BAO_CLIENT_CERT = baoCfg.clientCertFile;
|
||||
BAO_CLIENT_KEY = baoCfg.clientKeyFile;
|
||||
BAO_CACERT = "/run/credentials/hive-agent.service/hive-agent-bao-server-ca";
|
||||
};
|
||||
|
||||
swarmMetricsConfigured = pkgs.symlinkJoin {
|
||||
name = "swarm-metrics-configured";
|
||||
paths = [ config.services.hyperhive.agent.packages.swarm-metrics ];
|
||||
nativeBuildInputs = [ pkgs.makeWrapper ];
|
||||
postBuild = ''
|
||||
wrapProgram $out/bin/swarm-metrics ${
|
||||
lib.concatStringsSep " " (
|
||||
lib.mapAttrsToList (name: value: "--set ${name} ${lib.escapeShellArg value}") wrapperEnv
|
||||
)
|
||||
}
|
||||
'';
|
||||
};
|
||||
in
|
||||
{
|
||||
options.services.hyperhive.agent.metrics = {
|
||||
url = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
example = "https://metrics.example.com";
|
||||
description = ''
|
||||
The swarm metrics store's gateway origin, as this container reaches
|
||||
it. `swarm-metrics` appends the read API's paths (`/api/v1/query`,
|
||||
...) to it.
|
||||
|
||||
Set by the generated meta flake from the host's own
|
||||
{option}`services.hyperhive.swarm.victoriametrics.domain`.
|
||||
|
||||
`null` means this hive has not been given the metrics store's address
|
||||
for its agents, and `swarm-metrics` is then not installed at all.
|
||||
Not defaulted: a guessed address resolves cleanly to the wrong place.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkIf configured {
|
||||
environment.systemPackages = [ swarmMetricsConfigured ];
|
||||
};
|
||||
}
|
||||
|
|
@ -478,6 +478,21 @@ in
|
|||
touch "$out"
|
||||
'';
|
||||
|
||||
# `swarm-metrics` CLI reference freshness check, the same shape as
|
||||
# `swarm-logs-docs` above.
|
||||
swarm-metrics-docs =
|
||||
pkgs.runCommand "swarm-metrics-docs-fresh" { nativeBuildInputs = [ pkgs.diffutils ]; }
|
||||
''
|
||||
${self.packages.${system}.swarm-metrics}/bin/swarm-metrics markdown-docs > generated.md
|
||||
if ! diff -u ${../docs/tools/swarm-metrics-cli.md} generated.md; then
|
||||
echo "" >&2
|
||||
echo "ERROR: docs/tools/swarm-metrics-cli.md is out of date — regenerate it:" >&2
|
||||
echo " nix build .#swarm-metrics && ./result/bin/swarm-metrics markdown-docs > docs/tools/swarm-metrics-cli.md" >&2
|
||||
exit 1
|
||||
fi
|
||||
touch "$out"
|
||||
'';
|
||||
|
||||
# `hive-forge` CLI reference freshness check — same shape as
|
||||
# `hivectl-docs` above, `hive-forge markdown-docs` (clap-markdown over
|
||||
# its own command tree) instead. Reuses `packages.<system>.hive-forge`
|
||||
|
|
|
|||
|
|
@ -285,8 +285,8 @@ in
|
|||
[ -s ${pkiDir}/logs-auth.pem ] || ${signLeaf} ${pkiDir} logs-auth \
|
||||
${lib.escapeShellArg deployCfg.bao.logsAuthCommonName} "" clientAuth
|
||||
|
||||
# The metrics store's Grafana auth responder, which reads Grafana's
|
||||
# metrics bearer. Minted here for the matrix-ctl leaf's reason: the
|
||||
# The metrics store's auth responders, which read Grafana's metrics
|
||||
# bearer and agents' queue secrets. Minted here for the matrix-ctl leaf's reason: the
|
||||
# metrics store is a swarm singleton, so elsewhere this is the file an
|
||||
# operator copies.
|
||||
[ -s ${pkiDir}/metrics-auth.pem ] || ${signLeaf} ${pkiDir} metrics-auth \
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
# Glue: point the metrics store's Grafana auth responder at the bao leaf minted
|
||||
# for it.
|
||||
# Glue: point the metrics store's auth responders (Grafana's and the agents')
|
||||
# at the bao leaf minted for them.
|
||||
#
|
||||
# ONE PAIRING PER FILE — the swarm-metrics-auth principal ← bao, and nothing
|
||||
# else. Deleting this leaves a metrics store with no responder identity unless
|
||||
|
|
|
|||
|
|
@ -287,6 +287,9 @@ in
|
|||
# `swarm-victorialogs.nix`'s location for the whole reasoning.
|
||||
lib.optionalAttrs haveBaoClientIdentity {
|
||||
HIVE_AGENT_LOGS_QUERY_URL = "https://${config.services.hyperhive.swarm.victorialogs.domain}/select/logsql/query";
|
||||
# The metrics store's origin, for `swarm-metrics`; same credential and
|
||||
# gate. Its read locations answer a bare 401, never a login redirect.
|
||||
HIVE_AGENT_METRICS_URL = "https://${config.services.hyperhive.swarm.victoriametrics.domain}";
|
||||
}
|
||||
//
|
||||
# Where the swarm's secret store is, and the identity this hive presents to
|
||||
|
|
|
|||
|
|
@ -1060,15 +1060,18 @@ let
|
|||
}
|
||||
];
|
||||
|
||||
# The metrics store's responder, which checks the bearer Grafana presents
|
||||
# against the one stored for it; and the host unit that fetches the store
|
||||
# container's journal forwarder secret under the same leaf. Nothing else.
|
||||
# The metrics store's responders under one leaf: the agent one checks the
|
||||
# secret an agent presents against the one stored for it (the same `+` leaf
|
||||
# as the log store's), the other checks the bearer Grafana presents; and the
|
||||
# host unit that fetches the store container's journal forwarder secret.
|
||||
# Nothing else.
|
||||
metricsAuthReaders = [
|
||||
{
|
||||
name = "swarm-metrics-auth";
|
||||
cn = baoDeploy.metricsAuthCommonName;
|
||||
policyText =
|
||||
readStanza "${credentialMountPath}/data/${grafanaMetricsBearerLeaf}"
|
||||
readStanza "${credentialMountPath}/data/swarm/agents/+/queue"
|
||||
+ readStanza "${credentialMountPath}/data/${grafanaMetricsBearerLeaf}"
|
||||
# The metrics store container's journal forwarder, fetched on its host.
|
||||
+ readStanza "${credentialMountPath}/data/swarm/services/${hyperhiveCfg.swarm.victoriametrics.otel.clientId}/oidc/client";
|
||||
}
|
||||
|
|
@ -1977,8 +1980,9 @@ in
|
|||
default = "swarm-metrics-auth";
|
||||
description = ''
|
||||
Subject the store's `swarm-metrics-auth` cert-auth role accepts: the
|
||||
identity the metrics store's Grafana auth responder presents to read
|
||||
the bearer Grafana's datasource sends,
|
||||
identity the metrics store's auth responders present to read agent
|
||||
queue credentials, `swarm/agents/<agent>/queue`, and the bearer
|
||||
Grafana's datasource sends,
|
||||
`swarm/services/<grafana client id>/metrics/bearer`, and the one
|
||||
`swarm-victoriametrics-forwarder-oidc` presents to read
|
||||
`swarm/services/<metrics store forwarder client id>/oidc/client`.
|
||||
|
|
|
|||
|
|
@ -38,9 +38,9 @@ let
|
|||
nginxGid = config.ids.gids.nginx;
|
||||
|
||||
# The store's read API, which the vhost forwards to a caller holding
|
||||
# Grafana's bearer or an operator's session. Each location's `proxy_pass`
|
||||
# carries the path, so the store receives the path nginx matched, not the
|
||||
# caller's unnormalised spelling.
|
||||
# Grafana's bearer, an agent's own bearer, or an operator's session. Each
|
||||
# location's `proxy_pass` carries the path, so the store receives the path
|
||||
# nginx matched, not the caller's unnormalised spelling.
|
||||
#
|
||||
# Grafana's Prometheus datasource reads the query, series, label, metadata,
|
||||
# buildinfo, exemplar, rules and alerts routes; the store answers the last
|
||||
|
|
@ -128,6 +128,14 @@ let
|
|||
# same leaf as `grafanaMetricsBearerLeaf` in ./swarm-bao.nix.
|
||||
grafanaBearerStorePath = "swarm/services/${hyperhiveCfg.swarm.grafana.oidc.clientId}/metrics/bearer";
|
||||
|
||||
# The agents' check on the read API: a `swarm-logs-auth` binary, logging in
|
||||
# under this host's `swarm-metrics-auth` leaf, whose grant covers every
|
||||
# agent's queue secret. The log store's host runs `swarm-metrics-auth` under
|
||||
# its `swarm-logs-auth` leaf the same way.
|
||||
agentAuthUnit = "swarm-metrics-agent-auth";
|
||||
agentAuthSocket = "/run/${agentAuthUnit}/auth.sock";
|
||||
agentAuthCredential = id: "/run/credentials/${agentAuthUnit}.service/${id}";
|
||||
|
||||
# ── the container's journal forwarder ───────────────────────────────
|
||||
#
|
||||
# With no interface but loopback, the forwarder reaches nothing by name. A
|
||||
|
|
@ -242,7 +250,8 @@ in
|
|||
example = "/var/lib/swarm-bao-pki/metrics-auth.pem";
|
||||
description = ''
|
||||
Client certificate `swarm-metrics-auth` presents to the swarm's secret
|
||||
store to read Grafana's metrics bearer. Its subject must be
|
||||
store to read Grafana's metrics bearer, and `swarm-metrics-agent-auth`
|
||||
to read agents' queue secrets. Its subject must be
|
||||
{option}`services.hyperhive.deploy.bao.metricsAuthCommonName`.
|
||||
|
||||
Required when `deploy.victoriametrics.enable` is set; unset fails at
|
||||
|
|
@ -264,6 +273,47 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
options.services.hyperhive.deploy.victoriametrics.agentAuthVerdictTtl = lib.mkOption {
|
||||
type = lib.types.ints.unsigned;
|
||||
default = 60;
|
||||
description = ''
|
||||
Seconds `swarm-metrics-agent-auth` grants an agent bearer it has
|
||||
verified again without reading the secret store. A denial is never
|
||||
remembered.
|
||||
|
||||
This is also how long a re-minted or deleted queue secret keeps working
|
||||
at the metrics store's read API after its last successful check. `0`
|
||||
checks the store on every query.
|
||||
'';
|
||||
};
|
||||
|
||||
options.services.hyperhive.deploy.victoriametrics.agentAuthRate = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "50r/s";
|
||||
example = "30r/m";
|
||||
description = ''
|
||||
nginx `limit_req` rate, per client address, of read API requests
|
||||
carrying a `swarm-agent.` bearer, which nginx checks with
|
||||
`swarm-metrics-agent-auth`. Each such request that the responder has
|
||||
not granted within
|
||||
{option}`services.hyperhive.deploy.victoriametrics.agentAuthVerdictTtl`
|
||||
costs a secret store login and read, whether the bearer is right or not.
|
||||
|
||||
Keyed by client address, so every agent on one host shares one budget.
|
||||
Requests over the rate and its burst are answered 403.
|
||||
'';
|
||||
};
|
||||
|
||||
options.services.hyperhive.deploy.victoriametrics.agentAuthBurst = lib.mkOption {
|
||||
type = lib.types.ints.positive;
|
||||
default = 500;
|
||||
description = ''
|
||||
Requests a client address may make above
|
||||
{option}`services.hyperhive.deploy.victoriametrics.agentAuthRate` before
|
||||
nginx answers 403, served without delay.
|
||||
'';
|
||||
};
|
||||
|
||||
config = lib.mkIf deployCfg.victoriametrics.enable {
|
||||
assertions = [
|
||||
{
|
||||
|
|
@ -287,7 +337,8 @@ in
|
|||
services.hyperhive.deploy.victoriametrics.authBaoClientCertFile and
|
||||
authBaoClientKeyFile must both be set.
|
||||
|
||||
swarm-metrics-auth reads Grafana's metrics bearer from the swarm's
|
||||
swarm-metrics-auth reads Grafana's metrics bearer, and
|
||||
swarm-metrics-agent-auth agents' queue secrets, from the swarm's
|
||||
secret store with this certificate. They default to the leaf the
|
||||
store mints where services.hyperhive.deploy.bao.enable is set on
|
||||
this host; elsewhere, issue a leaf whose CN is
|
||||
|
|
@ -320,14 +371,18 @@ in
|
|||
|
||||
# Which check a read location's `auth_request` gets. `auth_request` takes a
|
||||
# fixed URI, so it names `/__metrics_read_auth`, which rewrites to the
|
||||
# target this map picks: a bearer goes to Grafana's check, anything else
|
||||
# (an operator's session cookie) to authelia. After the forwarder's egress
|
||||
# target this map picks: an agent's `swarm-agent.` bearer goes to the
|
||||
# agents' check, any other bearer to Grafana's, anything else (an
|
||||
# operator's session cookie) to authelia. A map tries its regexes in
|
||||
# order, so the agent arm comes first. After the forwarder's egress
|
||||
# listener, `egressServer`.
|
||||
services.nginx.appendHttpConfig = egressServer + ''
|
||||
map $http_authorization $swarm_metrics_read_auth_target {
|
||||
"~^Bearer swarm-agent\." /__metrics_agent_auth;
|
||||
"~^Bearer " /__metrics_grafana_auth;
|
||||
default /__metrics_authelia;
|
||||
}
|
||||
limit_req_zone $binary_remote_addr zone=swarm_metrics_agent_auth:1m rate=${deployCfg.victoriametrics.agentAuthRate};
|
||||
'';
|
||||
|
||||
# `forceSSL`: authelia answers a plain-http auth subrequest
|
||||
|
|
@ -400,6 +455,22 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
# An agent's bearer, checked by `swarm-metrics-agent-auth`. Any
|
||||
# well-formed agent bearer the responder has not recently granted
|
||||
# costs a store login and read, so the rate per client address is
|
||||
# capped here. 403 when over: `auth_request` passes only 401 and 403
|
||||
# through, and `swarm-metrics` retries a 401.
|
||||
"= /__metrics_agent_auth" = {
|
||||
proxyPass = "http://unix:${agentAuthSocket}:/";
|
||||
extraConfig = ''
|
||||
internal;
|
||||
limit_req zone=swarm_metrics_agent_auth burst=${toString deployCfg.victoriametrics.agentAuthBurst} nodelay;
|
||||
limit_req_status 403;
|
||||
proxy_pass_request_body off;
|
||||
proxy_set_header Content-Length "";
|
||||
'';
|
||||
};
|
||||
|
||||
# The authelia subrequest. For the collector's two locations above it
|
||||
# admits only the collector's client; for the read and vmui locations
|
||||
# it admits an operator's session (`metricsStoreRules` in
|
||||
|
|
@ -506,6 +577,56 @@ in
|
|||
};
|
||||
};
|
||||
|
||||
# The agents' check on the read API, see `agentAuthUnit`.
|
||||
users.users.${agentAuthUnit} = {
|
||||
isSystemUser = true;
|
||||
group = agentAuthUnit;
|
||||
description = "swarm metrics store auth_request responder for agent credentials";
|
||||
};
|
||||
users.groups.${agentAuthUnit} = { };
|
||||
|
||||
systemd.services.${agentAuthUnit} = {
|
||||
description = "swarm metrics store auth_request responder for agent credentials";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [
|
||||
"network.target"
|
||||
"swarm-bao-pki.service"
|
||||
];
|
||||
serviceConfig = {
|
||||
ExecStart = lib.concatStringsSep " " [
|
||||
"${deployCfg.victorialogs.authPackage}/bin/swarm-logs-auth"
|
||||
"--socket ${agentAuthSocket}"
|
||||
"--store-cert-role swarm-metrics-auth"
|
||||
"--verdict-ttl ${toString deployCfg.victoriametrics.agentAuthVerdictTtl}"
|
||||
];
|
||||
LoadCredential = lib.mapAttrsToList (id: path: "${id}:${path}") metricsAuthStoreFiles;
|
||||
# ⚠️ A static user, never `DynamicUser`: see the `swarm-metrics-auth` unit above.
|
||||
User = agentAuthUnit;
|
||||
Group = agentAuthUnit;
|
||||
RuntimeDirectory = agentAuthUnit;
|
||||
RuntimeDirectoryMode = "0751";
|
||||
RuntimeDirectoryPreserve = "yes";
|
||||
Restart = "on-failure";
|
||||
RestartSec = "5s";
|
||||
SyslogIdentifier = agentAuthUnit;
|
||||
ProtectSystem = "strict";
|
||||
ProtectHome = true;
|
||||
PrivateTmp = "disconnected";
|
||||
PrivateDevices = true;
|
||||
NoNewPrivileges = true;
|
||||
RestrictSUIDSGID = true;
|
||||
RemoveIPC = true;
|
||||
};
|
||||
environment = {
|
||||
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||||
BAO_CLIENT_CERT = agentAuthCredential "bao-client.pem";
|
||||
BAO_CLIENT_KEY = agentAuthCredential "bao-client-key.pem";
|
||||
}
|
||||
// lib.optionalAttrs (metricsAuthStoreFiles ? "bao-ca.pem") {
|
||||
BAO_CACERT = agentAuthCredential "bao-ca.pem";
|
||||
};
|
||||
};
|
||||
|
||||
# nginx runs as its own user under `ProtectSystem=strict`, so it can only
|
||||
# create the socket where it may write. It removes the socket on a clean
|
||||
# exit only; a stale one after a crash fails every later `bind`, and with
|
||||
|
|
|
|||
|
|
@ -41,19 +41,23 @@ let
|
|||
|
||||
agentNoBao = shared.agentBare;
|
||||
|
||||
# `swarm-logs` authenticates with the agent's queue secret from the store, so
|
||||
# the store address gates it beside the query URL.
|
||||
# `swarm-logs` and `swarm-metrics` authenticate with the agent's queue secret
|
||||
# from the store, so the store address gates each beside its store URL.
|
||||
logsUrl = "https://logs.t.local/select/logsql/query";
|
||||
metricsUrl = "https://metrics.t.local";
|
||||
agentLogs = agentWith {
|
||||
services.hyperhive.agent.bao.addr = "https://bao.t.local:8200";
|
||||
services.hyperhive.agent.logs.queryUrl = logsUrl;
|
||||
services.hyperhive.agent.metrics.url = metricsUrl;
|
||||
};
|
||||
agentLogsNoBao = agentWith { services.hyperhive.agent.logs.queryUrl = logsUrl; };
|
||||
swarmLogsIn =
|
||||
machine:
|
||||
lib.findFirst (
|
||||
p: (p.name or "") == "swarm-logs-configured"
|
||||
) null machine.environment.systemPackages;
|
||||
agentLogsNoBao = agentWith {
|
||||
services.hyperhive.agent.logs.queryUrl = logsUrl;
|
||||
services.hyperhive.agent.metrics.url = metricsUrl;
|
||||
};
|
||||
configuredIn =
|
||||
name: machine: lib.findFirst (p: (p.name or "") == name) null machine.environment.systemPackages;
|
||||
swarmLogsIn = configuredIn "swarm-logs-configured";
|
||||
swarmMetricsIn = configuredIn "swarm-metrics-configured";
|
||||
|
||||
agentBaoIdentity = machine: machine.systemd.services.hive-agent-bao-identity;
|
||||
cases = [
|
||||
|
|
@ -80,6 +84,27 @@ let
|
|||
name = "an agent with a log store but no secret store gets no swarm-logs";
|
||||
ok = swarmLogsIn agentLogsNoBao == null && swarmLogsIn agentLogs != null;
|
||||
}
|
||||
{
|
||||
# The same store identity as `swarm-logs`, and the store's origin.
|
||||
name = "swarm-metrics is wrapped with the agent's own store identity";
|
||||
ok =
|
||||
let
|
||||
w = (swarmMetricsIn agentLogs).buildCommand;
|
||||
a = agentLogs.services.hyperhive.agent;
|
||||
set = k: v: lib.hasInfix "--set ${k} ${lib.escapeShellArg v}" w;
|
||||
in
|
||||
set "HIVE_AGENT_METRICS_URL" metricsUrl
|
||||
&& set "HIVE_AGENT_NAME" a.user.name
|
||||
&& set "BAO_ADDR" "https://bao.t.local:8200"
|
||||
&& set "BAO_CLIENT_CERT" a.bao.clientCertFile
|
||||
&& set "BAO_CLIENT_KEY" a.bao.clientKeyFile
|
||||
&& set "BAO_CACERT" "/run/credentials/hive-agent.service/hive-agent-bao-server-ca";
|
||||
}
|
||||
{
|
||||
# A metrics URL with nothing to authenticate as installs nothing.
|
||||
name = "an agent with a metrics store but no secret store gets no swarm-metrics";
|
||||
ok = swarmMetricsIn agentLogsNoBao == null && swarmMetricsIn agentLogs != null;
|
||||
}
|
||||
{
|
||||
# Both ids or neither: the secret authenticates nobody without the id it
|
||||
# belongs to, and `swarm-logs` reads both.
|
||||
|
|
|
|||
|
|
@ -788,16 +788,20 @@ let
|
|||
lib.hasInfix "/logs-auth.pem ]" s && lib.hasInfix "swarm-logs-auth \"\" clientAuth" s;
|
||||
}
|
||||
{
|
||||
# Read-only on the one leaf Grafana's host creates and on the store
|
||||
# container forwarder's client secret, and nothing else: this principal
|
||||
# checks a presented bearer, and its host fetches one secret for the
|
||||
# container.
|
||||
name = "the metrics store responder's grant is Grafana's metrics bearer and its container forwarder's client secret, read-only, and nothing else";
|
||||
# Read-only on every agent's queue secret, the one leaf Grafana's host
|
||||
# creates and the store container forwarder's client secret, and nothing
|
||||
# else: this principal checks presented bearers, and its host fetches
|
||||
# one secret for the container.
|
||||
name = "the metrics store responder's grant is agents' queue secrets, Grafana's metrics bearer and its container forwarder's client secret, read-only, and nothing else";
|
||||
ok =
|
||||
let
|
||||
s = baoGrantHere.systemd.services.swarm-bao-metrics-auth-policy.script;
|
||||
in
|
||||
grantsIn s == [
|
||||
{
|
||||
path = "secret/data/swarm/agents/+/queue";
|
||||
caps = [ "read" ];
|
||||
}
|
||||
{
|
||||
path = "secret/data/swarm/services/swarm-grafana/metrics/bearer";
|
||||
caps = [ "read" ];
|
||||
|
|
|
|||
|
|
@ -672,15 +672,21 @@ let
|
|||
{
|
||||
# The read API's check, link by link: every read location names the
|
||||
# switch with no login redirect (a machine reader would get login HTML
|
||||
# as a 200), the switch rewrites to the map's pick, a bearer reaches
|
||||
# the responder's socket, which the unit binds, and anything else goes
|
||||
# to authelia. vmui is a browser's, so it redirects to the login.
|
||||
name = "the metrics read API sends a bearer to swarm-metrics-auth and a session to authelia";
|
||||
# as a 200), the switch rewrites to the map's pick, an agent bearer
|
||||
# reaches the agents' responder (listed before the catch-all bearer arm,
|
||||
# since a map takes the first matching regex), any other bearer reaches
|
||||
# Grafana's, each socket bound by its unit, and anything else goes to
|
||||
# authelia. vmui is a browser's, so it redirects to the login.
|
||||
name = "the metrics read API sends an agent bearer to swarm-metrics-agent-auth, another bearer to swarm-metrics-auth and a session to authelia";
|
||||
ok =
|
||||
let
|
||||
l = allLocal.services.nginx.virtualHosts."metrics.t.local".locations;
|
||||
http = allLocal.services.nginx.appendHttpConfig;
|
||||
unit = allLocal.systemd.services.swarm-metrics-auth;
|
||||
agentUnit = allLocal.systemd.services.swarm-metrics-agent-auth;
|
||||
agentArm = "\"~^Bearer swarm-agent\\.\" /__metrics_agent_auth;";
|
||||
bearerArm = "\"~^Bearer \" /__metrics_grafana_auth;";
|
||||
v = allLocal.services.hyperhive.deploy.victoriametrics;
|
||||
reads = lib.filterAttrs (n: _: lib.hasPrefix "= /api/v1/" n || lib.hasPrefix "^~ /api/v1/" n) l;
|
||||
in
|
||||
reads ? "= /api/v1/query_range"
|
||||
|
|
@ -690,7 +696,9 @@ let
|
|||
&& !(lib.hasInfix "error_page" r.extraConfig)
|
||||
) (lib.attrValues reads)
|
||||
&& lib.hasInfix "map \$http_authorization \$swarm_metrics_read_auth_target" http
|
||||
&& lib.hasInfix "\"~^Bearer \" /__metrics_grafana_auth;" http
|
||||
&& lib.hasInfix agentArm http
|
||||
&& lib.hasInfix bearerArm http
|
||||
&& !(lib.hasInfix bearerArm (lib.head (lib.splitString agentArm http)))
|
||||
&& lib.hasInfix "default /__metrics_authelia;" http
|
||||
&&
|
||||
lib.hasInfix "rewrite ^ \$swarm_metrics_read_auth_target last;"
|
||||
|
|
@ -698,6 +706,18 @@ let
|
|||
&& lib.hasInfix "internal;" l."= /__metrics_grafana_auth".extraConfig
|
||||
&& l."= /__metrics_grafana_auth".proxyPass == "http://unix:/run/swarm-metrics-auth/auth.sock:/"
|
||||
&& lib.hasInfix "--socket /run/swarm-metrics-auth/auth.sock" unit.serviceConfig.ExecStart
|
||||
&& lib.hasInfix "internal;" l."= /__metrics_agent_auth".extraConfig
|
||||
&& l."= /__metrics_agent_auth".proxyPass == "http://unix:/run/swarm-metrics-agent-auth/auth.sock:/"
|
||||
&& lib.hasInfix "--socket /run/swarm-metrics-agent-auth/auth.sock" agentUnit.serviceConfig.ExecStart
|
||||
&& lib.hasInfix "/bin/swarm-logs-auth " agentUnit.serviceConfig.ExecStart
|
||||
&& lib.hasInfix "--store-cert-role swarm-metrics-auth" agentUnit.serviceConfig.ExecStart
|
||||
&& lib.hasInfix "--verdict-ttl ${toString v.agentAuthVerdictTtl}" agentUnit.serviceConfig.ExecStart
|
||||
&& agentUnit.serviceConfig.LoadCredential == unit.serviceConfig.LoadCredential
|
||||
&& lib.hasInfix "limit_req_zone $binary_remote_addr zone=swarm_metrics_agent_auth:1m rate=${v.agentAuthRate};" http
|
||||
&&
|
||||
lib.hasInfix "limit_req zone=swarm_metrics_agent_auth burst=${toString v.agentAuthBurst} nodelay;"
|
||||
l."= /__metrics_agent_auth".extraConfig
|
||||
&& !(lib.hasInfix "limit_req" l."= /__metrics_grafana_auth".extraConfig)
|
||||
&& lib.hasInfix "auth_request /__metrics_authelia;" l."^~ /vmui/".extraConfig
|
||||
&& lib.hasInfix "error_page 401 =302" l."^~ /vmui/".extraConfig
|
||||
&& l."= /__metrics_authelia".proxyPass == "https://auth.t.local/api/authz/auth-request";
|
||||
|
|
@ -884,6 +904,7 @@ let
|
|||
owners ? swarm-logs-auth
|
||||
&& owners ? swarm-logs-grafana-auth
|
||||
&& owners ? swarm-metrics-auth
|
||||
&& owners ? swarm-metrics-agent-auth
|
||||
&& !(lib.any (u: hidden u.serviceConfig) (lib.attrValues owners));
|
||||
}
|
||||
{
|
||||
|
|
|
|||
|
|
@ -43,6 +43,8 @@ let
|
|||
# which is exactly what this list's per-bin packages are for — see
|
||||
# `nix/agent-modules/logs.nix` for what puts it on PATH there.
|
||||
swarm-logs = "hyperhive agent-facing swarm log store query CLI";
|
||||
# The same for the metrics store; `nix/agent-modules/metrics.nix` puts it on PATH.
|
||||
swarm-metrics = "hyperhive agent-facing swarm metrics store query CLI";
|
||||
hive-screen-mcp = "hyperhive screen MCP bridge (screenshot + input for GUI agents)";
|
||||
hive-forge = "hyperhive Forgejo CLI";
|
||||
hive-forge-notify = "hyperhive per-agent Forgejo notification poller daemon";
|
||||
|
|
|
|||
Loading…
Reference in a new issue