From e92f118bdae9cce5087cf2afba1402bfea0d60a5 Mon Sep 17 00:00:00 2001 From: atlas Date: Sat, 17 Oct 2026 00:55:46 +0200 Subject: [PATCH] add swarm-metrics, an agent CLI for the metrics store swarm-metrics reads the swarm's VictoriaMetrics from inside an agent container: query, query-range, series, labels and label-values, each one GET against the store's Prometheus read API through the gateway, printing the store's JSON. It authenticates the way swarm-logs does, with the agent's own queue secret read from bao under the agent's leaf, sent as a swarm-agent bearer. On the metrics store's host the read API's auth map gains an agent arm ahead of the Grafana one. It routes to swarm-metrics-agent-auth, a swarm-logs-auth instance logged in under the existing swarm-metrics-auth leaf, whose bao grant now includes swarm/agents/+/queue read. The check is rate-limited per address like the log store's (deploy.victoriametrics.agentAuth{Rate,Burst,VerdictTtl}). nix/agent-modules/metrics.nix installs the wrapped binary when the hive forwards HIVE_AGENT_METRICS_URL (from swarm.victoriametrics.domain) and a store address, the same gate as swarm-logs. Adds the base:swarm-metrics skill and the CLI reference with its freshness check. Fixes #5395 --- .prettierignore | 1 + CLAUDE.md | 3 +- Cargo.lock | 13 + Cargo.toml | 1 + .../base/skills/swarm-metrics/SKILL.md | 50 +++ docs/process/gotchas.md | 2 +- docs/swarm/credentials.md | 56 +-- docs/swarm/services.md | 28 +- docs/tools/swarm-metrics-cli.md | 123 ++++++ docs/trust-boundary/security.md | 11 +- flake.nix | 1 + hive-c0re/src/meta.rs | 26 ++ nix/agent-modules/default.nix | 1 + nix/agent-modules/metrics.nix | 66 ++++ nix/checks.nix | 15 + nix/host-modules/glue-bao-tls.nix | 4 +- .../glue-metrics-auth-bao-identity.nix | 4 +- nix/host-modules/hive-c0re/environment.nix | 3 + nix/host-modules/swarm-bao.nix | 16 +- nix/host-modules/swarm-victoriametrics.nix | 135 ++++++- nix/module-eval/agent-queue-bao.nix | 41 +- nix/module-eval/bao-grants.nix | 14 +- nix/module-eval/core-toggle.nix | 31 +- nix/packages/default.nix | 2 + swarm-logs-auth/src/main.rs | 20 +- swarm-metrics/Cargo.toml | 27 ++ swarm-metrics/README.md | 60 +++ swarm-metrics/src/auth.rs | 149 ++++++++ swarm-metrics/src/main.rs | 247 ++++++++++++ swarm-metrics/src/query.rs | 355 ++++++++++++++++++ 30 files changed, 1417 insertions(+), 88 deletions(-) create mode 100644 claude-plugins/plugins/base/skills/swarm-metrics/SKILL.md create mode 100644 docs/tools/swarm-metrics-cli.md create mode 100644 nix/agent-modules/metrics.nix create mode 100644 swarm-metrics/Cargo.toml create mode 100644 swarm-metrics/README.md create mode 100644 swarm-metrics/src/auth.rs create mode 100644 swarm-metrics/src/main.rs create mode 100644 swarm-metrics/src/query.rs diff --git a/.prettierignore b/.prettierignore index cda4a07a..a49094a1 100644 --- a/.prettierignore +++ b/.prettierignore @@ -8,6 +8,7 @@ docs/tools/hivectl-cli.md docs/tools/swarmctl-cli.md docs/tools/forge-cli.md docs/tools/swarm-logs-cli.md +docs/tools/swarm-metrics-cli.md # Files with multi-line list-item continuations that prettier strips to col 0. # prettier's `proseWrap: "preserve"` prevents prose reflow but not list-item diff --git a/CLAUDE.md b/CLAUDE.md index 5f8f5ab2..e7b57d43 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -138,10 +138,11 @@ Swarm-level: token against the store; shared by `swarm-nats-auth` and `swarm-logs-auth`. - **`swarm-logs-auth/`** — nginx `auth_request` responder for agent access - to the log store. + to the log store, and to the metrics store as `swarm-metrics-agent-auth`. - **`swarm-metrics-auth/`** — nginx `auth_request` responder for Grafana's bearer on the metrics store's query API. - **`swarm-logs/`** — agent CLI: a LogsQL query in, log lines out. +- **`swarm-metrics/`** — agent CLI: a MetricsQL query in, the store's JSON out. - **`swarm-matrix-client/`** — appservice calls against the swarm homeserver, shared by `swarm-controller` and `swarm-matrix-ctl`. - **`swarm-matrix-ctl/`** — the one binary run inside diff --git a/Cargo.lock b/Cargo.lock index 661101c3..c830a4f9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4944,6 +4944,19 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "swarm-metrics" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "clap-markdown", + "reqwest", + "swarm-queue-client", + "swarm-secret-client", + "tokio", +] + [[package]] name = "swarm-metrics-auth" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index f643c836..b9a22798 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,7 @@ members = [ "swarm-queue-client", "swarm-review-classifier", "swarm-logs", + "swarm-metrics", "swarm-logs-auth", "swarm-metrics-auth", "swarm-secret-client", diff --git a/claude-plugins/plugins/base/skills/swarm-metrics/SKILL.md b/claude-plugins/plugins/base/skills/swarm-metrics/SKILL.md new file mode 100644 index 00000000..3a6e15c7 --- /dev/null +++ b/claude-plugins/plugins/base/skills/swarm-metrics/SKILL.md @@ -0,0 +1,50 @@ +--- +name: swarm-metrics +description: Query the swarm metrics store with `swarm-metrics` when you need to know whether a service's metrics reach VictoriaMetrics, which metric names or labels exist, or what a counter or gauge did over a window. Covers how to match dotted OTLP metric names, how to bound a query, and how to tell an empty answer from a refused request. Use this before running `swarm-metrics` for the first time in a session, and before you report a metric as missing. +--- + +# swarm-metrics + +`swarm-metrics` reads the swarm's VictoriaMetrics through the gateway as you, +the agent. Every verb prints the store's JSON answer, so pipe it into `jq`. + +## Find out what exists first + +Metric names from OTLP keep their dots (`jobq.nodes`, +`hyperhive.vcs.commit.count`), and so do OTLP attribute labels +(`service.name`). Match a dotted metric by `__name__`, and quote a dotted +label name: + +```console +$ swarm-metrics label-values __name__ --match '{__name__=~"jobq\\..*"}' | jq -r '.data[]' +$ swarm-metrics label-values service.name | jq -r '.data[]' +$ swarm-metrics series '{"service.name"="nats"}' --start -1h --limit 20 +``` + +`series`, `labels` and `label-values` cover the last day from 00:00 UTC +unless you pass `--start`/`--end`. Use `--limit` on a broad selector. + +## Query + +```console +$ swarm-metrics query 'count by (__name__) ({__name__=~"hyperhive\\.vcs\\..*"})' +$ swarm-metrics query-range 'sum(rate({__name__="jobq.nodes"}[5m]))' --start -6h --step 5m +``` + +Times take RFC 3339, a unix timestamp or a relative duration (`-1h`). +`query-range` requires `--start`, so a range is always bounded. Inside the +expression, a regex needs its dots escaped twice (`\\.`), once for the +PromQL string and once for the regex. + +## Telling "no series" from "refused" + +- **No matching series**: exit 0, and `.data.result` (or `.data`) is `[]`. +- **Refused** (auth, bad expression, rate limit): non-zero exit, with the + HTTP status and the body on stderr. A 403 is the gateway's rate limit. + +## Before you report a metric missing, run a control + +An empty answer fits a wrong name, a missing label value, or a window too +narrow just as well as a metric that never arrived. Query something that must +exist, such as `label-values service.name`, and widen `--start`. If the +control is also empty, the query is wrong, not the store. diff --git a/docs/process/gotchas.md b/docs/process/gotchas.md index e2ee26f2..3021ef10 100644 --- a/docs/process/gotchas.md +++ b/docs/process/gotchas.md @@ -83,7 +83,7 @@ With both set, systemd creates the runtime directory under `/run/private/` nginx's `connect()` to a socket in it fails with `EACCES` whatever the directory and socket modes are. A unit serving a socket to another user runs as a static user instead (`swarm-controller`, `swarm-logs-auth`, -`swarm-metrics-auth`). +`swarm-metrics-auth`, `swarm-metrics-agent-auth`). `module-eval-core-toggle` fails on any `unix:` upstream of nginx owned by such a unit. diff --git a/docs/swarm/credentials.md b/docs/swarm/credentials.md index f902cab6..5351787d 100644 --- a/docs/swarm/credentials.md +++ b/docs/swarm/credentials.md @@ -55,34 +55,34 @@ of the cell says how. -| store path | minter | reader — pulls at runtime, holds in memory | automatic re-mint | automatic re-pull | -| ----------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `swarm/agents//matrix/main` | `swarm-controller`, with the swarm's appservice token, at agent creation and in a five-minute pass | the agent container itself, under the certificate its hive passed in | ✅ the pass re-mints when the stored token is missing, unknown to the homeserver, or someone else's | ✅ `hive-matrix-daemon` exits when the homeserver rejects its token, and a five-minute timer restarts it, which reads the store again | -| `swarm/agents//matrix/` | `swarm-controller` | the agent container itself, under the certificate its hive passed in | must be stated | ✅ a secondary that didn't come up at start, or whose token the homeserver rejected, is read from the store again two minutes later, then at doubling waits up to sixteen minutes, without restarting the daemon or its other accounts; one that's up and then rejected ends its sync loop, the daemon exits and its timer restarts it | -| `swarm/agents//matrix-recovery/` | `swarm-controller`, a random passphrase for every account listed under the agent's `matrix/`, in a five-minute pass | the agent container itself, under the certificate its hive passed in, when `hive-matrix-daemon` brings the account up | ❌ never replaced: the account's server-side key backup opens only with the passphrase it was enabled with, so the pass writes only to an empty path | ❌ read when the account comes up; an account whose passphrase lands later turns backup on at the next daemon start | -| `swarm/controller/swarm-controller/matrix/appservice-token` | `swarm-matrix-ctl`, inside the `hive-matrix` container, once | `swarm-controller`, under its own certificate | ❌ `swarm-matrix-ctl` mints it once; the container keeps its copy and republishes it when the store's differs | ✅ the controller reads it on every five-minute matrix pass | -| `swarm/controller/swarm-controller/oidc/client` | authelia, at its first boot; `swarm-secret-publish` copies it in | `swarm-controller`, under its own certificate, at start and on refusal | ❌ authelia mints it once. A re-mint is republished by `swarm-secret-publish`'s path unit | ✅ read at start and held in memory; read again when authelia refuses it, at most once per backoff of 5 s doubling to 5 min while the store holds the refused value | -| `swarm/controller/swarm-controller/forge/token` | the forge's `forgejo-swarm-controller-account`, in the `hive-forge` container, once, under the forge publisher's certificate with a write-only grant; it then revokes its earlier tokens | `swarm-controller`, under its own certificate, at start and on refusal | ❌ the forge mints it once and keeps only the token's name in the container, at `/var/lib/forgejo/swarm-controller-token-name`; deleting that file mints a new token on the unit's next start | ✅ the controller reads it at start, and in the background with backoff until it first finds it. When the forge answers 401 it reads it again and sends the request once more with the new token, at most once per backoff of 5 s doubling to 5 min while the store holds the refused value | -| `swarm/controller/swarm-controller/grafana/token` | `swarm-bao-grafana-controller-token` on Grafana's host, under Grafana's certificate: a token for the `swarm-controller` Viewer service account, minted with Grafana's admin password | `swarm-controller`, under its own certificate, on the first `/api/alerts` request and again when Grafana refuses it | ✅ a timer starts the unit again 10 minutes after each run; it keeps a stored token Grafana accepts, and mints a new one when Grafana answers 401 or the store holds none | ✅ the controller holds the token in memory; when Grafana answers 401 it reads the store once more and retries the request | -| `swarm/agents//bao-mtls` | the store's agent PKI mount (`deploy.bao.agentPkiMountPath`), which generates the key, at `swarm-controller`'s request at agent creation | `hive-c0re`, under the hive's own certificate, when it writes the agent's container config; then `hive-agent-bao-identity` in the agent container, under the leaf it holds | ✅ `swarm-controller`'s five-minute pass re-issues a live agent's leaf once it's past half its validity (45 of 90 days, read from the certificate itself) | ✅ `hive-agent-bao-identity` re-reads it every 10 minutes and swaps a new leaf into `/run/hive-agent-bao-identity/`, which every store login in the container presents. It reads under the leaf it holds, so it renews only while that leaf is valid | -| `swarm/agents//queue` | `swarm-controller`, at agent creation | `hive-agent` in the agent container, under the agent's own certificate, held in memory — the identity it presents to the swarm queue, naming that one agent rather than its hive; `swarm-logs` in the agent container, under the same certificate, on every query, as the bearer it presents at the log store; `swarm-nats-auth` and `swarm-logs-auth`, each under its own certificate, to check a presented secret | ✅ `swarm-controller`'s five-minute pass re-mints a live agent's secret once it's 45 days old by `minted_at` on the stored object; a secret with no `minted_at` gets one stamped, value unchanged. The pass skips agents declared `Destroyed` — declaring an agent destroyed deletes every version of the path instead, the undo of the mint rather than another one | ✅ `hive-agent` reads the path before its first connect and again on every reconnect attempt, so a reconnect after a re-mint presents the new secret. An open connection keeps the secret it connected with; after a revocation the agent keeps retrying under the queue client's backoff. `swarm-logs` reads it on every query and once more after a 401. `swarm-logs-auth` keeps granting a secret it verified for `deploy.victorialogs.authVerdictTtl` (60 s by default) without reading the path, so a replaced or deleted secret still works at the log store for up to that long | -| `swarm/agents//forge-token` | `swarm-controller`, at agent creation and in a pass every 5 minutes over every agent with a store identity | the agent container itself, under its own certificate, fetched to `/run/hive-agent-forge-token/token` | ✅ the controller re-mints when the stored token is missing or no longer matches the forge (last eight characters and scopes) | ✅ the agent re-fetches on a 10-minute timer | -| `swarm/agents//forge/