vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

Compare commits

base: vinzenz:42dcf10064bf96cadab378e6884f808f0c20fe6a
Branches Tags
vinzenz:main
..
compare: vinzenz:2186b82485cefa84bb068944f43d37723ed54d08
Branches Tags
vinzenz:main
19 changed files with 5 additions and 944 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.prettierignore
View file

@ -7,7 +7,6 @@
docs/tools/hivectl-cli.md
docs/tools/swarmctl-cli.md
docs/tools/forge-cli.md
docs/tools/swarm-logs-cli.md
# Files with multi-line list-item continuations that prettier strips to col 0.
# prettier's `proseWrap: "preserve"` prevents prose reflow but not list-item

12
Cargo.lock generated
View file

@ -4765,18 +4765,6 @@ dependencies = [
"utoipa-axum",
]
[[package]]
name = "swarm-logs"
version = "0.1.0"
dependencies = [
"anyhow",
"clap",
"clap-markdown",
"reqwest",
"serde_json",
"swarm-queue-client",
]
[[package]]
name = "swarm-nats-auth"
version = "0.1.0"

1
Cargo.toml
View file

@ -28,7 +28,6 @@ members = [
"swarm-controller",
"swarm-nats-auth",
"swarm-queue-client",
"swarm-logs",
"swarm-secret-client",
"swarmctl",
]

7
clippy.toml
View file

@ -1,9 +1,4 @@
# Proper nouns / product names that clippy's `doc_markdown` lint would
# otherwise flag as un-backticked identifiers in doc-comments. `".."`
# keeps clippy's built-in default list (GitHub, OAuth, …) and extends it.
#
# `LogsQL` and `VictoriaLogs` are here rather than backticked at each use
# because several of those uses are clap doc-comments, which ARE the
# `--help` text — backticks would be printed literally to an operator's
# terminal to satisfy a lint about rustdoc rendering.
doc-valid-idents = ["WireGuard", "LogsQL", "VictoriaLogs", ".."]
doc-valid-idents = ["WireGuard", ".."]

57
docs/tools/swarm-logs-cli.md
View file

@ -1,57 +0,0 @@
# Command-Line Help for `swarm-logs`
This document contains the help content for the `swarm-logs` command-line program.
**Command Overview:**
* [`swarm-logs`↴](#swarm-logs)
* [`swarm-logs query`↴](#swarm-logs-query)
## `swarm-logs`
query the swarm's log store from an agent
**Usage:** `swarm-logs <COMMAND>`
###### **Subcommands:**
* `query` — Run a LogsQL query and print the matched log lines
## `swarm-logs query`
Run a LogsQL query and print the matched log lines
**Usage:** `swarm-logs query [OPTIONS] <LOGSQL>`
###### **Arguments:**
* `<LOGSQL>` — The LogsQL query. A bare word is a full-text search for it.
Quote it: LogsQL uses characters the shell also claims, and an unquoted `|` pipes this command into the rest of your query instead of sending it.
###### **Options:**
* `--limit <LIMIT>` — Stop after this many records.
Unset means the store's own default. Worth setting on a broad query — nothing between here and the store narrows one, so a bare word matches across every hive in the swarm.
* `--format <FORMAT>` — What to print for each matched record
Default value: `message`
Possible values:
- `message`:
One log message per line — the `_msg` field and nothing else
- `json`:
The store's response body, unmodified, one JSON object per line
<hr/>
<small><i>
This document was generated automatically by
<a href="https://crates.io/crates/clap-markdown"><code>clap-markdown</code></a>.
</i></small>

1
flake.nix
View file

@ -94,7 +94,6 @@
hive-metric
hive-screen-mcp
hive-subagent-daemon
swarm-logs
assets
frontend
reference-docs

10
hive-c0re/src/meta.rs
View file

@ -706,10 +706,6 @@ const FORWARDED_VARS: &[&str] = &[
// address, which inside a container is the agent itself.
"HIVE_AGENT_NATS_URL",
"HIVE_AGENT_OIDC_TOKEN_ENDPOINT",
// Where an agent's `swarm-logs` queries, computed by the host for the
// same reason as the two above: it is a gateway address, and a container
// cannot derive it from anything it holds.
"HIVE_AGENT_LOGS_QUERY_URL",
"HIVE_FORGE_URL",
"HIVE_FORGE_PUBLIC_URL",
"HIVE_MATRIX_URL",
@ -744,12 +740,6 @@ const FORWARDED_VAR_OPTIONS: &[(&str, &str)] = &[
"HIVE_AGENT_OIDC_TOKEN_ENDPOINT",
"hyperhive.queue.tokenEndpoint",
),
// Read at build time to decide whether `swarm-logs` is installed at all —
// see `nix/agent-modules/logs.nix`. Both halves wired for this map's own
// reason: an agent whose option says no log store and whose env says
// otherwise has a binary that is not on PATH and coordinates that claim
// it is.
("HIVE_AGENT_LOGS_QUERY_URL", "hyperhive.logs.queryUrl"),
("HIVE_FORGE_URL", "hyperhive.forge.url"),
("HIVE_MATRIX_URL", "hyperhive.matrix.url"),
("HYPERHIVE_HIVE_NAME", "hyperhive.hiveName"),

1
nix/agent-modules/default.nix
View file

@ -28,7 +28,6 @@
./forge.nix
./frontend.nix
./github.nix
./logs.nix
./matrix.nix
./mcp.nix
./network.nix

90
nix/agent-modules/logs.nix
View file

@ -1,90 +0,0 @@
# `swarm-logs` on this agent's PATH, configured for this swarm's log store.
#
# The binary reads its coordinates from the environment and defaults nothing
# (see `swarm-logs/README.md`), so it is installed **wrapped** rather than
# bare — the same shape `swarm-controller.nix` installs `swarmctl` in, and for
# the same reason: every value here is derived from an option a module owns,
# and a default inside the binary would be an address we hope points at
# something.
#
# ⚠️ Gated on the queue being configured as well as on the query URL, because
# the identity this CLI presents IS the queue's: one per-hive machine client,
# one credential pair, delivered once. A swarm with a log store and no agent
# credential has nothing for this to authenticate as, and installing a binary
# that can only fail would be worse than leaving it off PATH — an agent would
# read the 401 as "no logs", which is the exact confusion the store's machine
# route was added to end.
{
lib,
pkgs,
config,
...
}:
let
cfg = config.hyperhive.logs;
queueCfg = config.hyperhive.queue;
configured = cfg.queryUrl != null && queueCfg.tokenEndpoint != null;
# The four coordinates `swarm-logs` reads, all-or-none on its side. Bound
# here as one attrset so the wrapper below cannot set three of them: a
# half-set environment is the failure the binary reports as a deployment
# bug, and it should not be reachable from the module that sets it.
#
# 🩸 Two of these are credential **paths**. Neither the id nor the secret
# is read here — a value in `--set` lands in the wrapper script, which is
# in the world-readable nix store.
wrapperEnv = {
HIVE_AGENT_LOGS_QUERY_URL = cfg.queryUrl;
HIVE_AGENT_OIDC_TOKEN_ENDPOINT = queueCfg.tokenEndpoint;
HIVE_AGENT_OIDC_CLIENT_ID_FILE = queueCfg.clientIdFile;
HIVE_AGENT_OIDC_CLIENT_SECRET_FILE = queueCfg.clientSecretFile;
};
# Same shape `swarm-controller.nix` wraps `swarmctl` in — `symlinkJoin` +
# `wrapProgram`, not a `writeShellScriptBin` shim, because the wrapper has
# to leave the binary's `--help`, its exit status and its streams intact.
# The whole point of the CLI is that its stdout pipes into `grep`.
swarmLogsConfigured = pkgs.symlinkJoin {
name = "swarm-logs-configured";
paths = [ config.hyperhive.packages.swarm-logs ];
nativeBuildInputs = [ pkgs.makeWrapper ];
postBuild = ''
wrapProgram $out/bin/swarm-logs ${
lib.concatStringsSep " " (
lib.mapAttrsToList (name: value: "--set ${name} ${lib.escapeShellArg value}") wrapperEnv
)
}
'';
};
in
{
options.hyperhive.logs = {
queryUrl = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "https://logs.example.com/select/logsql/query";
description = ''
The swarm log store's LogsQL query endpoint, as this container
reaches it — the machine route `swarm-victorialogs.nix` puts on the
gateway, not the browser one at `/`.
Set by the generated meta flake from the host's own
{option}`services.hyperhive.swarm.victorialogs.domain`.
⚠️ The **whole URL**, not a host to build one from, because this same
string is also the audience `swarm-logs` mints its token for. Two
spellings of one address present as a valid token refused at the
store — the rule `swarm-otel.nix` already states over its own push
targets.
`null` means this hive has not been given the log store's address for
its agents, and `swarm-logs` is then not installed at all. Deliberately
not defaulted: a guessed address resolves cleanly to the wrong place.
'';
};
};
config = lib.mkIf configured {
environment.systemPackages = [ swarmLogsConfigured ];
};
}

39
nix/agent-modules/queue.nix
View file

@ -23,13 +23,6 @@ let
# side can discover the other's spelling, so a rename is a rename there too.
secretCredential = "hive-queue-agent-secret";
clientIdCredential = "hive-queue-agent-client-id";
# Where systemd materialises this unit's credentials. `%d` above is the
# same directory, but `%d` only expands *inside* a unit — a consumer that
# is not one (the `swarm-logs` wrapper, see ./logs.nix) needs the literal,
# and deriving it from the unit name here is what keeps the two from
# disagreeing about which unit's credentials they mean.
credentialsDir = "/run/credentials/hive-agent.service";
in
{
options.hyperhive.queue = {
@ -66,38 +59,6 @@ in
"no queue coordinates".
'';
};
# One declaration, two readers. The credential ids are this module's —
# `hive_c0re::lifecycle::host_config` and the `LoadCredential=` below are
# the pair that has to agree on them — and a second consumer that spelled
# them again would be a second source of truth for a string whose
# mismatch is a file that is simply not there.
clientIdFile = lib.mkOption {
type = lib.types.str;
readOnly = true;
default = "${credentialsDir}/${clientIdCredential}";
description = ''
Path the agent's OIDC client id is delivered at, for a consumer
outside the harness unit. Read-only: it is a fact about where the
credential lands, not a knob — see {option}`hyperhive.logs.queryUrl`
for the consumer this exists for.
'';
};
clientSecretFile = lib.mkOption {
type = lib.types.str;
readOnly = true;
default = "${credentialsDir}/${secretCredential}";
description = ''
Path the agent's OIDC client secret is delivered at. Read-only for
the same reason as {option}`hyperhive.queue.clientIdFile`.
🩸 A PATH and never a value. The file is `0400` to the agent user and
is read at the moment of a token request; nothing in this tree puts
its contents in an environment variable, where `/proc/<pid>/environ`
would publish them to every process in the container.
'';
};
};
config = lib.mkIf configured {

17
nix/checks.nix
View file

@ -242,23 +242,6 @@ in
touch "$out"
'';
# `swarm-logs` CLI reference freshness check — same shape as
# `hivectl-docs` above, `swarm-logs markdown-docs` (clap-markdown over
# its own command tree) instead. Reuses `packages.<system>.swarm-logs`
# (a per-bin package out of `daemonBins` — see nix/packages/default.nix).
swarm-logs-docs =
pkgs.runCommand "swarm-logs-docs-fresh" { nativeBuildInputs = [ pkgs.diffutils ]; }
''
${self.packages.${system}.swarm-logs}/bin/swarm-logs markdown-docs > generated.md
if ! diff -u ${../docs/tools/swarm-logs-cli.md} generated.md; then
echo "" >&2
echo "ERROR: docs/tools/swarm-logs-cli.md is out of date — regenerate it:" >&2
echo " nix build .#swarm-logs && ./result/bin/swarm-logs markdown-docs > docs/tools/swarm-logs-cli.md" >&2
exit 1
fi
touch "$out"
'';
# `hive-forge` CLI reference freshness check — same shape as
# `hivectl-docs` above, `hive-forge markdown-docs` (clap-markdown over
# its own command tree) instead. Reuses `packages.<system>.hive-forge`

24
nix/host-modules/hive-c0re/environment.nix
View file

@ -302,30 +302,6 @@ in
HIVE_AGENT_NATS_URL = config.services.hyperhive.deploy.hive-controller.queue.agentNatsUrl;
HIVE_AGENT_OIDC_TOKEN_ENDPOINT = config.services.hyperhive.swarm.statusPublish.tokenEndpoint;
}
//
# Where an agent reads the swarm's logs, forwarded the same way and gated on
# the same credential: `swarm-logs` authenticates as the queue's own per-hive
# machine client, so an address with no credential behind it would put a
# binary on PATH that can only ever answer 401 — which an agent reads as "no
# logs", the exact confusion the store's machine route was added to end.
#
# The **machine** route (`^~ /select/logsql/`), not the browser one at `/`:
# that one ends in `error_page 401 =302` and hands a machine caller
# authelia's login page as a 200 with an HTML body. See
# `swarm-victorialogs.nix`'s location for the whole reasoning.
#
# ⚠️ The full URL rather than the domain, because the same string is also
# the audience the token is minted for — `swarm-otel.nix` states that rule
# over its own push targets, and two spellings present as a valid token
# refused at the store.
lib.optionalAttrs
(
config.services.hyperhive.deploy.hive-controller.queue.agentNatsUrl != null
&& config.services.hyperhive.swarm.statusPublish.tokenEndpoint != null
)
{
HIVE_AGENT_LOGS_QUERY_URL = "https://${config.services.hyperhive.swarm.victorialogs.domain}/select/logsql/query";
}
//
# Where the swarm's secret store is, and the identity this hive presents to
# it (hive-c0re::workers::credential). `swarm_secret_client` reads these

39
nix/host-modules/swarm-authelia.nix
View file

@ -144,15 +144,6 @@ let
accessTokenSignedResponseAlg = "RS256";
}) hyperhiveCfg.swarm.hives;
# The one URL an agent's token has to be authorised at beyond the queue:
# the log store's machine query route. Read off `swarm.victorialogs.domain`
# — an unconditional option, declared whether or not this host runs the
# store — rather than restated, because it is also the string `swarm-logs`
# requests as its audience and the string `hive-c0re/environment.nix` hands
# an agent as its endpoint. Three readers, one formula; two spellings
# present as a valid token refused at the store.
logsQueryUrl = "https://${hyperhiveCfg.swarm.victorialogs.domain}/select/logsql/query";
# The identity an agent container presents to the swarm's queue. One per
# HIVE, not per agent, and that is the load-bearing choice rather than a
# shortcut: agents are created at **runtime**, so anything minted per agent
@ -174,38 +165,16 @@ let
# match site, and the assertion below covers the case that ordering cannot.
#
# Mirrors `hiveClients` field for field so the two stay comparable. The
# signing algorithm is not load-bearing here — this client's consumers are
# signing algorithm is not load-bearing here — this client's only consumer is
# the queue's auth-callout responder, which *introspects* rather than
# verifying offline, and authelia's own authz endpoint — but it matches its
# sibling rather than inventing a second answer to a question nobody asked.
# verifying offline — but it matches its sibling rather than inventing a
# second answer to a question nobody asked.
agentClients = lib.mapAttrsToList (name: _: {
id = "${cfg.hiveClientPrefix}${name}${cfg.agentClientSuffix}";
description = "HyperHive agents on hive ${name}";
kind = "machine";
redirectUris = [ ];
# ⚠️ **Two** audiences, where `hiveClients` has one, because an agent
# presents this token at two different kinds of consumer. The queue
# introspects it and reads the client id; the log store's gateway
# location runs `auth_request` against authelia, which authorises a
# bearer token **by the URL being requested** and refuses one whose
# audience does not name it. Measured, not assumed: without the URL
# registered here the token endpoint answers `invalid_target` for it,
# and a token minted with no audience at all is answered by the gateway
# with a bare 401 that names no cause.
audience = [
"${cfg.hiveClientPrefix}${name}${cfg.agentClientSuffix}"
logsQueryUrl
];
# Without it the authz endpoint refuses an otherwise valid token and
# blames the token rather than the missing grant — the same note
# `glue-swarm-otel-oidc-client.nix` carries over the collector's client.
bearerAuthz = true;
# Stated rather than left on authelia's default, for
# `glue-swarm-otel-oidc-client.nix`'s reason: authelia permits only
# basic / JWT methods for a confidential client holding this scope, and
# enforces it in the startup validator. `swarm-queue-client` sends the
# credential as basic auth, so this is also what the minter already does.
tokenEndpointAuthMethod = "client_secret_basic";
audience = [ "${cfg.hiveClientPrefix}${name}${cfg.agentClientSuffix}" ];
accessTokenSignedResponseAlg = "RS256";
}) hyperhiveCfg.swarm.hives;

6
nix/packages/default.nix
View file

@ -37,12 +37,6 @@ let
hive-subagent-daemon = "hyperhive per-agent claude-subagent task runner daemon (serves its MCP tools directly over streamable-http)";
hive-matrix-daemon = "hyperhive per-agent matrix-sdk daemon (serves its MCP tools directly over streamable-http)";
hive-metric = "hyperhive agent-emitted custom metrics CLI";
# The read half of what `hive-metric` writes, from the same seat: an
# agent's CLI for the swarm log store. In this list rather than beside
# `swarmctl` below because it is run *by an agent, inside a container*,
# which is exactly what this list's per-bin packages are for — see
# `nix/agent-modules/logs.nix` for what puts it on PATH there.
swarm-logs = "hyperhive agent-facing swarm log store query CLI";
hive-screen-mcp = "hyperhive screen MCP bridge (screenshot + input for GUI agents)";
hive-forge = "hyperhive Forgejo CLI";
hive-forge-notify = "hyperhive per-agent Forgejo notification poller daemon";

26
swarm-logs/Cargo.toml
View file

@ -1,26 +0,0 @@
[package]
name = "swarm-logs"
version.workspace = true
readme = "README.md"
edition.workspace = true
[[bin]]
name = "swarm-logs"
path = "src/main.rs"
[dependencies]
anyhow.workspace = true
clap.workspace = true
clap-markdown = "0.1"
# The whole point of the crate's dependency list: minting the access token
# is `swarm_queue_client::mint_token_for_blocking`, not a second copy of
# the `client_credentials` request. See `src/auth.rs`.
swarm-queue-client.workspace = true
# Blocking, not async: this binary makes exactly two requests and exits, so
# a reactor would exist only to be torn down. It also matches the token
# minter reached for above, which is `swarm-queue-client`'s blocking one.
reqwest = { workspace = true, features = ["blocking"] }
serde_json.workspace = true
[lints]
workspace = true

92
swarm-logs/README.md
View file

@ -1,92 +0,0 @@
# swarm-logs
An agent's CLI for the swarm log store. One verb — `swarm-logs query
'<LogsQL>'` — so reading logs pipes and greps like any other command
instead of being a hand-rolled token request plus a curl, per query.
Read-only and query-only. There is no ingest path here (that is the
collector's), and no `tail`: streaming is a different LogsQL endpoint with a
different response shape, left for a follow-up rather than folded in.
Distinct from `hive-metric`, the other half of the same surface from an
agent's seat: that one _writes_ a custom metric, this one _reads_ logs.
## ⚠️ There is no scoping
The gateway forwards the query unmodified, so **any authenticated caller
reads the whole swarm's logs** — every hive's, not only its own. That is the
rule in force rather than an omission here; when read permissions exist they
attach at the gateway location, not in this binary.
## How it authenticates
The agent container already holds one identity, the per-hive machine client
`hive-<name>-agent`, delivered as a systemd credential pair. This binary
presents a `client_credentials` access token minted from it as
`Authorization: Bearer`.
Minting is **not** implemented here — `swarm_queue_client::mint_token_for_blocking`
already owns the request, its error type and its CA handling, and a
token-endpoint fix has to be findable in one place. What this crate adds is
the agent-shaped half: the client id arrives as a _file_ beside the secret, so
nothing outside `nix/agent-modules/queue.nix` spells `hive-<name>-agent`
twice. That is the same problem `hive-agent/src/swarm_queue.rs` solves, and
`src/auth.rs` is its `decide` restated over this binary's inputs.
🩸 Every credential here is a **path**. The secret's contents are read at the
moment of the request, inside `swarm-queue-client`, and are never bound to a
name in this crate, logged, or rendered into an error.
### The audience is the URL
`swarm-logs` mints its token with the query URL as the requested audience,
because that is what authelia checks at the authz endpoint the gateway's
`auth_request` calls. The same rule `swarm-otel.nix` states over its own push
targets: one binding for the address and the audience, since two spellings of
one address present as a valid token refused at the store.
⚠️ That requires the agent client to be _registered_ for that audience and to
hold `authelia.bearer.authz`. Both are set in `swarm-authelia.nix`'s
`agentClients`; without them authelia answers `invalid_target` at the token
endpoint, or the gateway answers 401 with no explanation.
## Configuration
Supplied by `nix/agent-modules/logs.nix`, which wraps the binary — the same
shape `swarmctl` is configured in, and for the same reason: every value is
derived from an option that module owns, so a default here would be an address
we _hope_ points at something.
| variable | what |
| ------------------------------------ | ---------------------------------------------------- |
| `HIVE_AGENT_LOGS_QUERY_URL` | the LogsQL query endpoint; also the token's audience |
| `HIVE_AGENT_OIDC_TOKEN_ENDPOINT` | authelia's token endpoint |
| `HIVE_AGENT_OIDC_CLIENT_ID_FILE` | path of the delivered client-id credential |
| `HIVE_AGENT_OIDC_CLIENT_SECRET_FILE` | path of the delivered secret credential |
| `HIVE_AGENT_OIDC_CA_FILE` | optional extra trust anchor; unset in this tree |
`HIVE_AGENT`, the same prefix the harness reads, because it is the same
identity — an agent authenticates as its hive's agent client whether the
caller is the harness or a CLI the agent typed. The first four are all-or-none:
a half-set environment is a deployment bug, and this binary says so by name
rather than behaving like an unconfigured one.
## Output
```console
$ swarm-logs query 'atlas-otel-probe' --limit 5
```
One log message per line by default (the record's `_msg`), which is what a
grep pattern is written against. `--format json` passes the store's NDJSON
through unmodified, for `_time`, `_stream` and `jq`.
A record the projection cannot understand — no string `_msg`, or not JSON at
all — is printed whole rather than dropped. Silently losing a row would make
an incomplete answer indistinguishable from a complete one.
A non-200 is reported with its body and a non-zero exit, never as an empty
result. That is the whole reason the gateway grew a separate machine route:
the operator's browser route answers an unauthenticated caller with authelia's
login page as a **200 with an HTML body**, and a client that reads only the
status code records a query that succeeded and matched nothing.

188
swarm-logs/src/auth.rs
View file

@ -1,188 +0,0 @@
//! The access token this CLI presents to the log store's gateway route.
//!
//! Minting it is **not** implemented here. `swarm-queue-client` already owns
//! the `client_credentials` request against authelia — the basic-auth form,
//! the per-call secret read, the CA-trust builder and the error type that
//! surfaces authelia's own `error_description` — and this crate calls
//! [`swarm_queue_client::mint_token_for_blocking`] rather than writing a
//! second copy of it. A token-endpoint fix has to be findable in one place.
//!
//! What *is* here is the shape of an agent's identity, which differs from the
//! daemons that crate was written for in exactly one way: **the client id
//! arrives as a file, not as a value.** It rides in the same systemd
//! credential pair as the secret so that nothing outside `queue.nix` ever
//! spells `hive-<name>-agent` again. That is the same problem
//! `hive-agent/src/swarm_queue.rs` solves, and [`Config::from_env`] below is
//! that module's `decide` restated over this binary's inputs — including its
//! rule that a *half*-set environment is a deployment bug rather than an
//! absent integration.
//!
//! 🩸 Every credential in here is a **path**. The secret's contents are read
//! by `swarm-queue-client` at the moment of the request and are never bound to
//! a name in this crate, never logged and never rendered into an error.
use std::path::{Path, PathBuf};
use anyhow::{Context as _, Result, bail};
use swarm_queue_client::QueueConfig;
/// Variable prefix for this binary's coordinates.
///
/// `HIVE_AGENT`, deliberately the **same** prefix the harness reads, because
/// it is the same identity: an agent authenticates as its hive's agent client
/// whether the caller is the harness or a CLI the agent typed. A prefix of its
/// own would be a second set of variables carrying one fact, which is the
/// disagreement `queue.nix`'s credential pair exists to prevent.
const ENV_PREFIX: &str = "HIVE_AGENT";
/// Where this CLI queries, and who it queries as.
pub struct Config {
/// Full URL of the LogsQL query endpoint, gateway side.
///
/// The whole URL rather than a host to build one from, because this same
/// string is **also the audience** the token is minted for — the rule
/// `swarm-otel.nix` states over its own push targets ("one binding
/// because the same string is both the address requested and the audience
/// the token is minted for — two spellings present as a valid token
/// refused at the store"). Splitting it would reintroduce exactly that.
pub query_url: String,
/// Everything needed to mint a token, in the form the minter wants it.
pub queue: QueueConfig,
}
impl Config {
/// Read the configuration out of the environment the wrapper sets.
///
/// All four coordinates or none, for the reason
/// [`QueueConfig::from_env`] gives and `hive-agent`'s `swarm_queue`
/// repeats: a half-set environment produces a process that looks
/// configured and never authenticates. Unlike the harness, this binary
/// *errors* on the absent case rather than degrading — a CLI whose only
/// job is the query has nothing left to do without it, and an operator
/// running it wants to be told why rather than handed an empty result
/// they will read as "no logs matched".
pub fn from_env() -> Result<Self> {
let var = |suffix: &str| std::env::var(format!("{ENV_PREFIX}_{suffix}")).ok();
let query_url = var("LOGS_QUERY_URL");
let token_endpoint = var("OIDC_TOKEN_ENDPOINT");
let client_id_file = var("OIDC_CLIENT_ID_FILE");
let client_secret_file = var("OIDC_CLIENT_SECRET_FILE");
// Outside the all-or-none group on purpose, exactly as in
// `QueueConfig::from_env`: a swarm behind a publicly-trusted
// certificate needs no extra anchor, so a CA path with no endpoint is
// meaningless rather than half-configured. Nothing in this tree sets
// it for an agent — a container already trusts the swarm root, which
// `hive_c0re::meta` embeds at build time.
let ca_file = var("OIDC_CA_FILE").map(PathBuf::from);
let (query_url, token_endpoint, client_id_file, client_secret_file) = match (
query_url,
token_endpoint,
client_id_file,
client_secret_file,
) {
(Some(u), Some(t), Some(i), Some(s)) => (u, t, i, s),
(None, None, None, None) => bail!(
"this agent has no swarm log store configured: \
{ENV_PREFIX}_LOGS_QUERY_URL, {ENV_PREFIX}_OIDC_TOKEN_ENDPOINT, \
{ENV_PREFIX}_OIDC_CLIENT_ID_FILE and \
{ENV_PREFIX}_OIDC_CLIENT_SECRET_FILE are all unset"
),
_ => bail!(
"swarm log store half-configured: {ENV_PREFIX}_LOGS_QUERY_URL, \
{ENV_PREFIX}_OIDC_TOKEN_ENDPOINT, {ENV_PREFIX}_OIDC_CLIENT_ID_FILE \
and {ENV_PREFIX}_OIDC_CLIENT_SECRET_FILE are set together or not \
at all — this is a deployment bug, not a missing feature"
),
};
let client_id = read_client_id(Path::new(&client_id_file))?;
Ok(Self {
query_url,
queue: QueueConfig {
// 🩸 Unused by the token mint and deliberately not read from
// the environment. `QueueConfig` is the queue's connect
// config as well as its token config, and this binary needs
// only the second half — taking `HIVE_AGENT_NATS_URL` for it
// would make a broker address a prerequisite for reading
// logs, which is a coupling with no cause. Named so a reader
// of a stack trace is not left wondering.
url: String::from("unused: swarm-logs mints a token and never connects"),
token_endpoint,
client_id,
client_secret_file: PathBuf::from(client_secret_file),
ca_file,
},
})
}
}
/// Read the OIDC client id out of the file the systemd credential landed at.
///
/// Trailing newline stripped and an empty file refused, both copied from
/// `hive-agent`'s `read_client_id`: the writing unit ends the file with a
/// newline, and an id carrying one authenticates as nobody. Where the harness
/// treats absence as the ordinary state of a hive whose credential has not
/// been published yet and carries on, this errors — see [`Config::from_env`]
/// for why a CLI has nothing to carry on with.
fn read_client_id(path: &Path) -> Result<String> {
let raw = std::fs::read_to_string(path)
.with_context(|| format!("reading the OIDC client id from {}", path.display()))?;
let id = raw.trim();
if id.is_empty() {
bail!(
"the OIDC client id at {} is empty — the queue credential has not \
been published to this hive yet",
path.display()
);
}
Ok(id.to_owned())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn client_id_loses_its_trailing_newline() {
let dir = std::env::temp_dir().join(format!("swarm-logs-id-{}", std::process::id()));
std::fs::create_dir_all(&dir).expect("temp dir");
let path = dir.join("client-id");
std::fs::write(&path, "hive-alpha-agent\n").expect("write");
assert_eq!(read_client_id(&path).expect("read"), "hive-alpha-agent");
std::fs::remove_dir_all(&dir).expect("cleanup");
}
/// An empty credential file is the shape a hive has before its secret
/// store has published anything, and it must not become a token request
/// that authenticates as the empty string.
#[test]
fn an_empty_client_id_is_refused_by_name() {
let dir = std::env::temp_dir().join(format!("swarm-logs-empty-{}", std::process::id()));
std::fs::create_dir_all(&dir).expect("temp dir");
let path = dir.join("client-id");
std::fs::write(&path, "\n").expect("write");
let err = read_client_id(&path).expect_err("empty id must not be accepted");
assert!(
err.to_string().contains("is empty"),
"error should say the id is empty, got: {err}"
);
std::fs::remove_dir_all(&dir).expect("cleanup");
}
/// A missing file names the path, because the operator's next move is to
/// look at whether the credential was delivered at all.
#[test]
fn a_missing_client_id_file_names_its_path() {
let path = Path::new("/nonexistent/swarm-logs/client-id");
let err = read_client_id(path).expect_err("missing file must not be accepted");
assert!(
err.to_string()
.contains("/nonexistent/swarm-logs/client-id"),
"error should name the path, got: {err}"
);
}
}

123
swarm-logs/src/main.rs
View file

@ -1,123 +0,0 @@
//! `swarm-logs` — an agent's CLI for the swarm log store.
//!
//! An agent can reach VictoriaLogs only through the gateway, and until this
//! existed the way to read it was to hand-roll a `client_credentials` token
//! request and a curl by hand, per query. That is the gap this closes: one
//! verb, a LogsQL string in, matched log lines out, so the answer pipes into
//! `grep` like any other command's.
//!
//! **Read-only and query-only.** It posts to the store's LogsQL query
//! endpoint and nothing else — there is no ingest path here, and no `tail`.
//! Streaming is a different endpoint with a different response shape and is
//! deliberately left for a follow-up rather than folded in.
//!
//! ⚠️ **The route applies no scoping.** `swarm-victorialogs.nix` forwards the
//! query unmodified, so any authenticated caller reads the whole swarm's
//! logs — every hive's, not just its own. That is the rule in force rather
//! than an omission on this binary's part; when read permissions exist they
//! attach at that location, not here.
//!
//! Distinct from `hive-metric`, which is the other half of the same
//! observability surface from an agent's seat: that one *writes* a custom
//! metric, this one *reads* logs.
mod auth;
mod query;
use anyhow::Result;
use clap::{Parser, Subcommand};
#[derive(Parser)]
#[command(
name = "swarm-logs",
version,
about = "query the swarm's log store from an agent"
)]
struct Cli {
#[command(subcommand)]
command: Verb,
}
#[derive(Subcommand)]
enum Verb {
/// Run a LogsQL query and print the matched log lines.
Query {
/// The LogsQL query. A bare word is a full-text search for it.
///
/// Quote it: LogsQL uses characters the shell also claims, and an
/// unquoted `|` pipes this command into the rest of your query
/// instead of sending it.
logsql: String,
/// Stop after this many records.
///
/// Unset means the store's own default. Worth setting on a broad
/// query — nothing between here and the store narrows one, so a bare
/// word matches across every hive in the swarm.
#[arg(long)]
limit: Option<u64>,
/// What to print for each matched record.
#[arg(long, value_enum, default_value = "message")]
format: query::Format,
},
/// Print this CLI's reference as markdown (source of `docs/tools/swarm-logs-cli.md`).
///
/// No `completions` sibling, unlike `hivectl` and `swarmctl`: those are
/// installed on an operator's interactive host, where a completion script
/// has a shell to be sourced into. This one is run by an agent through a
/// tool call, which has no line editor to complete against — the verb
/// would exist only to be listed.
#[command(hide = true)]
MarkdownDocs,
}
fn main() -> Result<()> {
match Cli::parse().command {
Verb::Query {
logsql,
limit,
format,
} => {
// Config resolved inside this arm and not up front: the two
// verbs below are pure functions of the command tree and must
// keep working on a machine that has no log store wired up at
// all — `markdown-docs` in particular runs in a nix check, where
// no credential exists and none should.
let cfg = auth::Config::from_env()?;
let stdout = std::io::stdout();
query::run(&cfg, &logsql, limit, format, &mut stdout.lock())
}
Verb::MarkdownDocs => {
print!("{}", clap_markdown::help_markdown::<Cli>());
Ok(())
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use clap::CommandFactory as _;
/// clap's own consistency check over the command tree — catches a
/// duplicate long flag or a malformed `about` at test time rather than on
/// the first run of a verb nobody exercised.
#[test]
fn the_command_tree_is_well_formed() {
Cli::command().debug_assert();
}
/// The default matters: it is what an agent gets when it pipes into
/// `grep` without reading the flags first.
#[test]
fn query_defaults_to_projecting_the_message() {
let cli = Cli::try_parse_from(["swarm-logs", "query", "a-marker"]).expect("parses");
let Verb::Query { format, limit, .. } = cli.command else {
panic!("expected the query verb");
};
assert_eq!(format, query::Format::Message);
assert_eq!(limit, None, "no client-side default limit is imposed");
}
}

215
swarm-logs/src/query.rs
View file

@ -1,215 +0,0 @@
//! The one request this binary makes: a LogsQL query against the store's
//! machine route on the gateway.
//!
//! The route is `^~ /select/logsql/`, added by `swarm-victorialogs.nix`
//! specifically so a machine caller is answered with a **401 it cannot
//! mistake for an empty result**. The operator's `/` route ends in
//! `error_page 401 =302` and hands an unauthenticated caller authelia's login
//! page as a 200 with an HTML body — which is why [`run`] below reports a
//! non-200 with its body rather than printing nothing and exiting 0. That
//! failure mode is the reason the route exists; reproducing it in the client
//! would be the whole bug again one layer up.
use std::io::{BufRead as _, BufReader, Write};
use anyhow::{Context as _, Result, bail};
use crate::auth::Config;
/// How the matched records are written to stdout.
#[derive(Clone, Copy, Debug, PartialEq, Eq, clap::ValueEnum)]
pub enum Format {
/// One log message per line — the `_msg` field and nothing else.
///
/// The default because the ask this CLI answers was to *pipe and grep*
/// rather than hand-roll curl, and a grep pattern is written against the
/// message. A record with no `_msg` is passed through as its own JSON
/// object rather than dropped: silently losing a row would make an
/// incomplete answer indistinguishable from a complete one.
Message,
/// The store's response body, unmodified, one JSON object per line.
///
/// For everything [`Format::Message`] leaves out — `_time`, `_stream`
/// and whatever fields the record carries — and for feeding `jq`.
Json,
}
/// Run `query`, writing matched records to `out`.
///
/// Streams rather than collecting: the route applies no scoping, so a loose
/// query matches the whole swarm's logs and buffering the answer to print it
/// is an out-of-memory waiting for a bad `LogsQL`.
pub fn run(
cfg: &Config,
logsql: &str,
limit: Option<u64>,
format: Format,
out: &mut impl Write,
) -> Result<()> {
// Minted per invocation, with the query URL as the audience. Both halves
// are load-bearing: authelia refuses a token carrying no audience at the
// authz endpoint the gateway's `auth_request` calls, and the audience it
// checks is the URL being requested — so the string sent here and the
// string requested below must be one binding, which is why `Config` holds
// exactly one.
let token = swarm_queue_client::mint_token_for_blocking(&cfg.queue, Some(&cfg.query_url))
.map_err(|e| anyhow::anyhow!("{}", swarm_queue_client::chain(&e)))
.context("minting an access token for the swarm log store")?;
let http = build_http_client(cfg)?;
let mut form = vec![("query", logsql.to_owned())];
if let Some(limit) = limit {
form.push(("limit", limit.to_string()));
}
let response = http
.post(&cfg.query_url)
.bearer_auth(&token)
.form(&form)
.send()
.with_context(|| format!("querying the swarm log store at {}", cfg.query_url))?;
let status = response.status();
if !status.is_success() {
// Surfaced, not swallowed. A 401 here is the ordinary shape of "this
// client's identity is not authorised for this route", and the body
// is the half that says which — nginx answers its own HTML for an
// `auth_request` denial, the store answers JSON for a bad LogsQL.
// Truncated because the login-page case is a full HTML document and
// an operator needs the first screen of it, not all of it.
let body = response.text().unwrap_or_default();
let body = body.trim();
let shown: String = body.chars().take(2000).collect();
bail!(
"the swarm log store refused the query ({status}): {shown}{}",
if shown.len() < body.chars().count() {
" […]"
} else {
""
}
);
}
let mut reader = BufReader::new(response);
let mut line = String::new();
loop {
line.clear();
let read = reader
.read_line(&mut line)
.context("reading the query response from the swarm log store")?;
if read == 0 {
break;
}
let line = line.trim_end_matches(['\r', '\n']);
if line.is_empty() {
continue;
}
writeln!(out, "{}", render(line, format)).context("writing to stdout")?;
}
Ok(())
}
/// Render one NDJSON record from the store as the caller asked for it.
///
/// Pure over the line so the projection is testable without a store: what
/// goes wrong here is a record shape, not a request.
fn render(line: &str, format: Format) -> String {
match format {
Format::Json => line.to_owned(),
Format::Message => serde_json::from_str::<serde_json::Value>(line)
.ok()
.and_then(|v| v.get("_msg")?.as_str().map(str::to_owned))
// Anything without a string `_msg` — including a line that is not
// JSON at all — falls back to the line itself. See
// [`Format::Message`]: dropping it would make a partial answer
// look like a whole one.
.unwrap_or_else(|| line.to_owned()),
}
}
/// The HTTP client for the query itself.
///
/// Separate from the one `swarm-queue-client` builds for the token endpoint
/// because they are two hosts — `auth.<swarm>` and `logs.<swarm>` — and the
/// CA-trust knob is declared per endpoint. Same optional anchor, same reason:
/// without it a swarm using its own CA fails at TLS with
/// `invalid peer certificate: UnknownIssuer`, having never looked at the
/// anchor sitting on the host.
///
/// No timeout, deliberately, where the token client has one: a query over a
/// long retention window legitimately takes longer than any number that would
/// be right for a token request, and a client-side cut would truncate a
/// correct answer into a shorter one with no marker saying so.
fn build_http_client(cfg: &Config) -> Result<reqwest::blocking::Client> {
let mut builder = reqwest::blocking::Client::builder();
if let Some(path) = &cfg.queue.ca_file {
let pem = std::fs::read(path).with_context(|| {
format!(
"reading the log-store CA certificate from {}",
path.display()
)
})?;
let cert = reqwest::Certificate::from_pem(&pem).with_context(|| {
format!(
"parsing the log-store CA certificate from {} as PEM",
path.display()
)
})?;
builder = builder.add_root_certificate(cert);
}
builder
.build()
.context("building the log-store HTTP client")
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn json_passes_the_record_through_unmodified() {
let line = r#"{"_time":"2026-09-16T14:14:43Z","_msg":"hello","_stream":"{}"}"#;
assert_eq!(render(line, Format::Json), line);
}
#[test]
fn message_projects_just_the_log_line() {
let line = r#"{"_time":"2026-09-16T14:14:43Z","_msg":"hello there"}"#;
assert_eq!(render(line, Format::Message), "hello there");
}
/// The grep case this CLI exists for: a marker inside `_msg` has to
/// survive the projection intact, punctuation and all.
#[test]
fn message_keeps_a_marker_intact() {
let line = r#"{"_msg":"probe marker-1789568083-6970 emitted"}"#;
assert!(render(line, Format::Message).contains("marker-1789568083-6970"));
}
/// A record the projection cannot understand is still a match, and a
/// caller counting lines must not get a different answer than one reading
/// them. Dropping it would make an incomplete answer look complete.
#[test]
fn a_record_without_a_message_falls_back_to_the_whole_line() {
let line = r#"{"_time":"2026-09-16T14:14:43Z","_stream":"{}"}"#;
assert_eq!(render(line, Format::Message), line);
}
#[test]
fn a_non_json_line_falls_back_to_itself() {
assert_eq!(
render("not json at all", Format::Message),
"not json at all"
);
}
/// `_msg` holding a non-string is the same "shape I cannot project" case
/// as a missing one, and must take the same fallback rather than
/// rendering Rust's debug form of a number.
#[test]
fn a_non_string_message_falls_back_to_the_whole_line() {
let line = r#"{"_msg":42}"#;
assert_eq!(render(line, Format::Message), line);
}
}