diff --git a/Cargo.lock b/Cargo.lock index 1e0f2b33..c694d29e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4765,6 +4765,18 @@ dependencies = [ "utoipa-axum", ] +[[package]] +name = "swarm-logs" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "clap-markdown", + "reqwest", + "serde_json", + "swarm-queue-client", +] + [[package]] name = "swarm-nats-auth" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index a4d1206f..040b3501 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,7 @@ members = [ "swarm-controller", "swarm-nats-auth", "swarm-queue-client", + "swarm-logs", "swarm-secret-client", "swarmctl", ] diff --git a/clippy.toml b/clippy.toml index 17565808..57e33673 100644 --- a/clippy.toml +++ b/clippy.toml @@ -1,4 +1,9 @@ # Proper nouns / product names that clippy's `doc_markdown` lint would # otherwise flag as un-backticked identifiers in doc-comments. `".."` # keeps clippy's built-in default list (GitHub, OAuth, …) and extends it. -doc-valid-idents = ["WireGuard", ".."] +# +# `LogsQL` and `VictoriaLogs` are here rather than backticked at each use +# because several of those uses are clap doc-comments, which ARE the +# `--help` text — backticks would be printed literally to an operator's +# terminal to satisfy a lint about rustdoc rendering. +doc-valid-idents = ["WireGuard", "LogsQL", "VictoriaLogs", ".."] diff --git a/docs/tools/swarm-logs-cli.md b/docs/tools/swarm-logs-cli.md new file mode 100644 index 00000000..29d72c6e --- /dev/null +++ b/docs/tools/swarm-logs-cli.md @@ -0,0 +1,53 @@ +# Command-Line Help for `swarm-logs` + +This document contains the help content for the `swarm-logs` command-line program. + +**Command Overview:** + +- [`swarm-logs`↴](#swarm-logs) +- [`swarm-logs query`↴](#swarm-logs-query) + +## `swarm-logs` + +query the swarm's log store from an agent + +**Usage:** `swarm-logs ` + +###### **Subcommands:** + +- `query` — Run a LogsQL query and print the matched log lines + +## `swarm-logs query` + +Run a LogsQL query and print the matched log lines + +**Usage:** `swarm-logs query [OPTIONS] ` + +###### **Arguments:** + +- `` — The LogsQL query. A bare word is a full-text search for it. + + Quote it: LogsQL uses characters the shell also claims, and an unquoted `|` pipes this command into the rest of your query instead of sending it. + +###### **Options:** + +- `--limit ` — Stop after this many records. + + Unset means the store's own default. Worth setting on a broad query — nothing between here and the store narrows one, so a bare word matches across every hive in the swarm. + +- `--format ` — What to print for each matched record + + Default value: `message` + + Possible values: + - `message`: + One log message per line — the `_msg` field and nothing else + - `json`: + The store's response body, unmodified, one JSON object per line + +
+ + +This document was generated automatically by +clap-markdown. + diff --git a/flake.nix b/flake.nix index a1f0b566..8f831359 100644 --- a/flake.nix +++ b/flake.nix @@ -94,6 +94,7 @@ hive-metric hive-screen-mcp hive-subagent-daemon + swarm-logs assets frontend reference-docs diff --git a/hive-c0re/src/meta.rs b/hive-c0re/src/meta.rs index 8199a958..8dcffa94 100644 --- a/hive-c0re/src/meta.rs +++ b/hive-c0re/src/meta.rs @@ -706,6 +706,10 @@ const FORWARDED_VARS: &[&str] = &[ // address, which inside a container is the agent itself. "HIVE_AGENT_NATS_URL", "HIVE_AGENT_OIDC_TOKEN_ENDPOINT", + // Where an agent's `swarm-logs` queries, computed by the host for the + // same reason as the two above: it is a gateway address, and a container + // cannot derive it from anything it holds. + "HIVE_AGENT_LOGS_QUERY_URL", "HIVE_FORGE_URL", "HIVE_FORGE_PUBLIC_URL", "HIVE_MATRIX_URL", @@ -740,6 +744,12 @@ const FORWARDED_VAR_OPTIONS: &[(&str, &str)] = &[ "HIVE_AGENT_OIDC_TOKEN_ENDPOINT", "hyperhive.queue.tokenEndpoint", ), + // Read at build time to decide whether `swarm-logs` is installed at all — + // see `nix/agent-modules/logs.nix`. Both halves wired for this map's own + // reason: an agent whose option says no log store and whose env says + // otherwise has a binary that is not on PATH and coordinates that claim + // it is. + ("HIVE_AGENT_LOGS_QUERY_URL", "hyperhive.logs.queryUrl"), ("HIVE_FORGE_URL", "hyperhive.forge.url"), ("HIVE_MATRIX_URL", "hyperhive.matrix.url"), ("HYPERHIVE_HIVE_NAME", "hyperhive.hiveName"), diff --git a/nix/agent-modules/default.nix b/nix/agent-modules/default.nix index 8fb48656..281ebb49 100644 --- a/nix/agent-modules/default.nix +++ b/nix/agent-modules/default.nix @@ -28,6 +28,7 @@ ./forge.nix ./frontend.nix ./github.nix + ./logs.nix ./matrix.nix ./mcp.nix ./network.nix diff --git a/nix/agent-modules/logs.nix b/nix/agent-modules/logs.nix new file mode 100644 index 00000000..22d9e1d3 --- /dev/null +++ b/nix/agent-modules/logs.nix @@ -0,0 +1,90 @@ +# `swarm-logs` on this agent's PATH, configured for this swarm's log store. +# +# The binary reads its coordinates from the environment and defaults nothing +# (see `swarm-logs/README.md`), so it is installed **wrapped** rather than +# bare — the same shape `swarm-controller.nix` installs `swarmctl` in, and for +# the same reason: every value here is derived from an option a module owns, +# and a default inside the binary would be an address we hope points at +# something. +# +# ⚠️ Gated on the queue being configured as well as on the query URL, because +# the identity this CLI presents IS the queue's: one per-hive machine client, +# one credential pair, delivered once. A swarm with a log store and no agent +# credential has nothing for this to authenticate as, and installing a binary +# that can only fail would be worse than leaving it off PATH — an agent would +# read the 401 as "no logs", which is the exact confusion the store's machine +# route was added to end. +{ + lib, + pkgs, + config, + ... +}: +let + cfg = config.hyperhive.logs; + queueCfg = config.hyperhive.queue; + configured = cfg.queryUrl != null && queueCfg.tokenEndpoint != null; + + # The four coordinates `swarm-logs` reads, all-or-none on its side. Bound + # here as one attrset so the wrapper below cannot set three of them: a + # half-set environment is the failure the binary reports as a deployment + # bug, and it should not be reachable from the module that sets it. + # + # 🩸 Two of these are credential **paths**. Neither the id nor the secret + # is read here — a value in `--set` lands in the wrapper script, which is + # in the world-readable nix store. + wrapperEnv = { + HIVE_AGENT_LOGS_QUERY_URL = cfg.queryUrl; + HIVE_AGENT_OIDC_TOKEN_ENDPOINT = queueCfg.tokenEndpoint; + HIVE_AGENT_OIDC_CLIENT_ID_FILE = queueCfg.clientIdFile; + HIVE_AGENT_OIDC_CLIENT_SECRET_FILE = queueCfg.clientSecretFile; + }; + + # Same shape `swarm-controller.nix` wraps `swarmctl` in — `symlinkJoin` + + # `wrapProgram`, not a `writeShellScriptBin` shim, because the wrapper has + # to leave the binary's `--help`, its exit status and its streams intact. + # The whole point of the CLI is that its stdout pipes into `grep`. + swarmLogsConfigured = pkgs.symlinkJoin { + name = "swarm-logs-configured"; + paths = [ config.hyperhive.packages.swarm-logs ]; + nativeBuildInputs = [ pkgs.makeWrapper ]; + postBuild = '' + wrapProgram $out/bin/swarm-logs ${ + lib.concatStringsSep " " ( + lib.mapAttrsToList (name: value: "--set ${name} ${lib.escapeShellArg value}") wrapperEnv + ) + } + ''; + }; +in +{ + options.hyperhive.logs = { + queryUrl = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "https://logs.example.com/select/logsql/query"; + description = '' + The swarm log store's LogsQL query endpoint, as this container + reaches it — the machine route `swarm-victorialogs.nix` puts on the + gateway, not the browser one at `/`. + + Set by the generated meta flake from the host's own + {option}`services.hyperhive.swarm.victorialogs.domain`. + + ⚠️ The **whole URL**, not a host to build one from, because this same + string is also the audience `swarm-logs` mints its token for. Two + spellings of one address present as a valid token refused at the + store — the rule `swarm-otel.nix` already states over its own push + targets. + + `null` means this hive has not been given the log store's address for + its agents, and `swarm-logs` is then not installed at all. Deliberately + not defaulted: a guessed address resolves cleanly to the wrong place. + ''; + }; + }; + + config = lib.mkIf configured { + environment.systemPackages = [ swarmLogsConfigured ]; + }; +} diff --git a/nix/agent-modules/queue.nix b/nix/agent-modules/queue.nix index 430c1b7a..4d7c5837 100644 --- a/nix/agent-modules/queue.nix +++ b/nix/agent-modules/queue.nix @@ -23,6 +23,13 @@ let # side can discover the other's spelling, so a rename is a rename there too. secretCredential = "hive-queue-agent-secret"; clientIdCredential = "hive-queue-agent-client-id"; + + # Where systemd materialises this unit's credentials. `%d` above is the + # same directory, but `%d` only expands *inside* a unit — a consumer that + # is not one (the `swarm-logs` wrapper, see ./logs.nix) needs the literal, + # and deriving it from the unit name here is what keeps the two from + # disagreeing about which unit's credentials they mean. + credentialsDir = "/run/credentials/hive-agent.service"; in { options.hyperhive.queue = { @@ -59,6 +66,38 @@ in "no queue coordinates". ''; }; + + # One declaration, two readers. The credential ids are this module's — + # `hive_c0re::lifecycle::host_config` and the `LoadCredential=` below are + # the pair that has to agree on them — and a second consumer that spelled + # them again would be a second source of truth for a string whose + # mismatch is a file that is simply not there. + clientIdFile = lib.mkOption { + type = lib.types.str; + readOnly = true; + default = "${credentialsDir}/${clientIdCredential}"; + description = '' + Path the agent's OIDC client id is delivered at, for a consumer + outside the harness unit. Read-only: it is a fact about where the + credential lands, not a knob — see {option}`hyperhive.logs.queryUrl` + for the consumer this exists for. + ''; + }; + + clientSecretFile = lib.mkOption { + type = lib.types.str; + readOnly = true; + default = "${credentialsDir}/${secretCredential}"; + description = '' + Path the agent's OIDC client secret is delivered at. Read-only for + the same reason as {option}`hyperhive.queue.clientIdFile`. + + 🩸 A PATH and never a value. The file is `0400` to the agent user and + is read at the moment of a token request; nothing in this tree puts + its contents in an environment variable, where `/proc//environ` + would publish them to every process in the container. + ''; + }; }; config = lib.mkIf configured { diff --git a/nix/checks.nix b/nix/checks.nix index 56ee05a0..b99155c6 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -242,6 +242,23 @@ in touch "$out" ''; + # `swarm-logs` CLI reference freshness check — same shape as + # `hivectl-docs` above, `swarm-logs markdown-docs` (clap-markdown over + # its own command tree) instead. Reuses `packages..swarm-logs` + # (a per-bin package out of `daemonBins` — see nix/packages/default.nix). + swarm-logs-docs = + pkgs.runCommand "swarm-logs-docs-fresh" { nativeBuildInputs = [ pkgs.diffutils ]; } + '' + ${self.packages.${system}.swarm-logs}/bin/swarm-logs markdown-docs > generated.md + if ! diff -u ${../docs/tools/swarm-logs-cli.md} generated.md; then + echo "" >&2 + echo "ERROR: docs/tools/swarm-logs-cli.md is out of date — regenerate it:" >&2 + echo " nix build .#swarm-logs && ./result/bin/swarm-logs markdown-docs > docs/tools/swarm-logs-cli.md" >&2 + exit 1 + fi + touch "$out" + ''; + # `hive-forge` CLI reference freshness check — same shape as # `hivectl-docs` above, `hive-forge markdown-docs` (clap-markdown over # its own command tree) instead. Reuses `packages..hive-forge` diff --git a/nix/host-modules/hive-c0re/environment.nix b/nix/host-modules/hive-c0re/environment.nix index 254fa978..268d014b 100644 --- a/nix/host-modules/hive-c0re/environment.nix +++ b/nix/host-modules/hive-c0re/environment.nix @@ -302,6 +302,30 @@ in HIVE_AGENT_NATS_URL = config.services.hyperhive.deploy.hive-controller.queue.agentNatsUrl; HIVE_AGENT_OIDC_TOKEN_ENDPOINT = config.services.hyperhive.swarm.statusPublish.tokenEndpoint; } +// + # Where an agent reads the swarm's logs, forwarded the same way and gated on + # the same credential: `swarm-logs` authenticates as the queue's own per-hive + # machine client, so an address with no credential behind it would put a + # binary on PATH that can only ever answer 401 — which an agent reads as "no + # logs", the exact confusion the store's machine route was added to end. + # + # The **machine** route (`^~ /select/logsql/`), not the browser one at `/`: + # that one ends in `error_page 401 =302` and hands a machine caller + # authelia's login page as a 200 with an HTML body. See + # `swarm-victorialogs.nix`'s location for the whole reasoning. + # + # ⚠️ The full URL rather than the domain, because the same string is also + # the audience the token is minted for — `swarm-otel.nix` states that rule + # over its own push targets, and two spellings present as a valid token + # refused at the store. + lib.optionalAttrs + ( + config.services.hyperhive.deploy.hive-controller.queue.agentNatsUrl != null + && config.services.hyperhive.swarm.statusPublish.tokenEndpoint != null + ) + { + HIVE_AGENT_LOGS_QUERY_URL = "https://${config.services.hyperhive.swarm.victorialogs.domain}/select/logsql/query"; + } // # Where the swarm's secret store is, and the identity this hive presents to # it (hive-c0re::workers::credential). `swarm_secret_client` reads these diff --git a/nix/host-modules/swarm-authelia.nix b/nix/host-modules/swarm-authelia.nix index cb68143d..3b03796d 100644 --- a/nix/host-modules/swarm-authelia.nix +++ b/nix/host-modules/swarm-authelia.nix @@ -144,6 +144,15 @@ let accessTokenSignedResponseAlg = "RS256"; }) hyperhiveCfg.swarm.hives; + # The one URL an agent's token has to be authorised at beyond the queue: + # the log store's machine query route. Read off `swarm.victorialogs.domain` + # — an unconditional option, declared whether or not this host runs the + # store — rather than restated, because it is also the string `swarm-logs` + # requests as its audience and the string `hive-c0re/environment.nix` hands + # an agent as its endpoint. Three readers, one formula; two spellings + # present as a valid token refused at the store. + logsQueryUrl = "https://${hyperhiveCfg.swarm.victorialogs.domain}/select/logsql/query"; + # The identity an agent container presents to the swarm's queue. One per # HIVE, not per agent, and that is the load-bearing choice rather than a # shortcut: agents are created at **runtime**, so anything minted per agent @@ -165,16 +174,38 @@ let # match site, and the assertion below covers the case that ordering cannot. # # Mirrors `hiveClients` field for field so the two stay comparable. The - # signing algorithm is not load-bearing here — this client's only consumer is + # signing algorithm is not load-bearing here — this client's consumers are # the queue's auth-callout responder, which *introspects* rather than - # verifying offline — but it matches its sibling rather than inventing a - # second answer to a question nobody asked. + # verifying offline, and authelia's own authz endpoint — but it matches its + # sibling rather than inventing a second answer to a question nobody asked. agentClients = lib.mapAttrsToList (name: _: { id = "${cfg.hiveClientPrefix}${name}${cfg.agentClientSuffix}"; description = "HyperHive agents on hive ${name}"; kind = "machine"; redirectUris = [ ]; - audience = [ "${cfg.hiveClientPrefix}${name}${cfg.agentClientSuffix}" ]; + # ⚠️ **Two** audiences, where `hiveClients` has one, because an agent + # presents this token at two different kinds of consumer. The queue + # introspects it and reads the client id; the log store's gateway + # location runs `auth_request` against authelia, which authorises a + # bearer token **by the URL being requested** and refuses one whose + # audience does not name it. Measured, not assumed: without the URL + # registered here the token endpoint answers `invalid_target` for it, + # and a token minted with no audience at all is answered by the gateway + # with a bare 401 that names no cause. + audience = [ + "${cfg.hiveClientPrefix}${name}${cfg.agentClientSuffix}" + logsQueryUrl + ]; + # Without it the authz endpoint refuses an otherwise valid token and + # blames the token rather than the missing grant — the same note + # `glue-swarm-otel-oidc-client.nix` carries over the collector's client. + bearerAuthz = true; + # Stated rather than left on authelia's default, for + # `glue-swarm-otel-oidc-client.nix`'s reason: authelia permits only + # basic / JWT methods for a confidential client holding this scope, and + # enforces it in the startup validator. `swarm-queue-client` sends the + # credential as basic auth, so this is also what the minter already does. + tokenEndpointAuthMethod = "client_secret_basic"; accessTokenSignedResponseAlg = "RS256"; }) hyperhiveCfg.swarm.hives; diff --git a/nix/packages/default.nix b/nix/packages/default.nix index c0a534a8..ac18933d 100644 --- a/nix/packages/default.nix +++ b/nix/packages/default.nix @@ -37,6 +37,12 @@ let hive-subagent-daemon = "hyperhive per-agent claude-subagent task runner daemon (serves its MCP tools directly over streamable-http)"; hive-matrix-daemon = "hyperhive per-agent matrix-sdk daemon (serves its MCP tools directly over streamable-http)"; hive-metric = "hyperhive agent-emitted custom metrics CLI"; + # The read half of what `hive-metric` writes, from the same seat: an + # agent's CLI for the swarm log store. In this list rather than beside + # `swarmctl` below because it is run *by an agent, inside a container*, + # which is exactly what this list's per-bin packages are for — see + # `nix/agent-modules/logs.nix` for what puts it on PATH there. + swarm-logs = "hyperhive agent-facing swarm log store query CLI"; hive-screen-mcp = "hyperhive screen MCP bridge (screenshot + input for GUI agents)"; hive-forge = "hyperhive Forgejo CLI"; hive-forge-notify = "hyperhive per-agent Forgejo notification poller daemon"; diff --git a/swarm-logs/Cargo.toml b/swarm-logs/Cargo.toml new file mode 100644 index 00000000..ca59942b --- /dev/null +++ b/swarm-logs/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "swarm-logs" +version.workspace = true +readme = "README.md" +edition.workspace = true + +[[bin]] +name = "swarm-logs" +path = "src/main.rs" + +[dependencies] +anyhow.workspace = true +clap.workspace = true +clap-markdown = "0.1" +# The whole point of the crate's dependency list: minting the access token +# is `swarm_queue_client::mint_token_for_blocking`, not a second copy of +# the `client_credentials` request. See `src/auth.rs`. +swarm-queue-client.workspace = true +# Blocking, not async: this binary makes exactly two requests and exits, so +# a reactor would exist only to be torn down. It also matches the token +# minter reached for above, which is `swarm-queue-client`'s blocking one. +reqwest = { workspace = true, features = ["blocking"] } +serde_json.workspace = true + +[lints] +workspace = true diff --git a/swarm-logs/README.md b/swarm-logs/README.md new file mode 100644 index 00000000..82e92809 --- /dev/null +++ b/swarm-logs/README.md @@ -0,0 +1,92 @@ +# swarm-logs + +An agent's CLI for the swarm log store. One verb — `swarm-logs query +''` — so reading logs pipes and greps like any other command +instead of being a hand-rolled token request plus a curl, per query. + +Read-only and query-only. There is no ingest path here (that is the +collector's), and no `tail`: streaming is a different LogsQL endpoint with a +different response shape, left for a follow-up rather than folded in. + +Distinct from `hive-metric`, the other half of the same surface from an +agent's seat: that one _writes_ a custom metric, this one _reads_ logs. + +## ⚠️ There is no scoping + +The gateway forwards the query unmodified, so **any authenticated caller +reads the whole swarm's logs** — every hive's, not only its own. That is the +rule in force rather than an omission here; when read permissions exist they +attach at the gateway location, not in this binary. + +## How it authenticates + +The agent container already holds one identity, the per-hive machine client +`hive--agent`, delivered as a systemd credential pair. This binary +presents a `client_credentials` access token minted from it as +`Authorization: Bearer`. + +Minting is **not** implemented here — `swarm_queue_client::mint_token_for_blocking` +already owns the request, its error type and its CA handling, and a +token-endpoint fix has to be findable in one place. What this crate adds is +the agent-shaped half: the client id arrives as a _file_ beside the secret, so +nothing outside `nix/agent-modules/queue.nix` spells `hive--agent` +twice. That is the same problem `hive-agent/src/swarm_queue.rs` solves, and +`src/auth.rs` is its `decide` restated over this binary's inputs. + +🩸 Every credential here is a **path**. The secret's contents are read at the +moment of the request, inside `swarm-queue-client`, and are never bound to a +name in this crate, logged, or rendered into an error. + +### The audience is the URL + +`swarm-logs` mints its token with the query URL as the requested audience, +because that is what authelia checks at the authz endpoint the gateway's +`auth_request` calls. The same rule `swarm-otel.nix` states over its own push +targets: one binding for the address and the audience, since two spellings of +one address present as a valid token refused at the store. + +⚠️ That requires the agent client to be _registered_ for that audience and to +hold `authelia.bearer.authz`. Both are set in `swarm-authelia.nix`'s +`agentClients`; without them authelia answers `invalid_target` at the token +endpoint, or the gateway answers 401 with no explanation. + +## Configuration + +Supplied by `nix/agent-modules/logs.nix`, which wraps the binary — the same +shape `swarmctl` is configured in, and for the same reason: every value is +derived from an option that module owns, so a default here would be an address +we _hope_ points at something. + +| variable | what | +| ------------------------------------ | ---------------------------------------------------- | +| `HIVE_AGENT_LOGS_QUERY_URL` | the LogsQL query endpoint; also the token's audience | +| `HIVE_AGENT_OIDC_TOKEN_ENDPOINT` | authelia's token endpoint | +| `HIVE_AGENT_OIDC_CLIENT_ID_FILE` | path of the delivered client-id credential | +| `HIVE_AGENT_OIDC_CLIENT_SECRET_FILE` | path of the delivered secret credential | +| `HIVE_AGENT_OIDC_CA_FILE` | optional extra trust anchor; unset in this tree | + +`HIVE_AGENT`, the same prefix the harness reads, because it is the same +identity — an agent authenticates as its hive's agent client whether the +caller is the harness or a CLI the agent typed. The first four are all-or-none: +a half-set environment is a deployment bug, and this binary says so by name +rather than behaving like an unconfigured one. + +## Output + +```console +$ swarm-logs query 'atlas-otel-probe' --limit 5 +``` + +One log message per line by default (the record's `_msg`), which is what a +grep pattern is written against. `--format json` passes the store's NDJSON +through unmodified, for `_time`, `_stream` and `jq`. + +A record the projection cannot understand — no string `_msg`, or not JSON at +all — is printed whole rather than dropped. Silently losing a row would make +an incomplete answer indistinguishable from a complete one. + +A non-200 is reported with its body and a non-zero exit, never as an empty +result. That is the whole reason the gateway grew a separate machine route: +the operator's browser route answers an unauthenticated caller with authelia's +login page as a **200 with an HTML body**, and a client that reads only the +status code records a query that succeeded and matched nothing. diff --git a/swarm-logs/src/auth.rs b/swarm-logs/src/auth.rs new file mode 100644 index 00000000..40368b93 --- /dev/null +++ b/swarm-logs/src/auth.rs @@ -0,0 +1,188 @@ +//! The access token this CLI presents to the log store's gateway route. +//! +//! Minting it is **not** implemented here. `swarm-queue-client` already owns +//! the `client_credentials` request against authelia — the basic-auth form, +//! the per-call secret read, the CA-trust builder and the error type that +//! surfaces authelia's own `error_description` — and this crate calls +//! [`swarm_queue_client::mint_token_for_blocking`] rather than writing a +//! second copy of it. A token-endpoint fix has to be findable in one place. +//! +//! What *is* here is the shape of an agent's identity, which differs from the +//! daemons that crate was written for in exactly one way: **the client id +//! arrives as a file, not as a value.** It rides in the same systemd +//! credential pair as the secret so that nothing outside `queue.nix` ever +//! spells `hive--agent` again. That is the same problem +//! `hive-agent/src/swarm_queue.rs` solves, and [`Config::from_env`] below is +//! that module's `decide` restated over this binary's inputs — including its +//! rule that a *half*-set environment is a deployment bug rather than an +//! absent integration. +//! +//! 🩸 Every credential in here is a **path**. The secret's contents are read +//! by `swarm-queue-client` at the moment of the request and are never bound to +//! a name in this crate, never logged and never rendered into an error. + +use std::path::{Path, PathBuf}; + +use anyhow::{Context as _, Result, bail}; +use swarm_queue_client::QueueConfig; + +/// Variable prefix for this binary's coordinates. +/// +/// `HIVE_AGENT`, deliberately the **same** prefix the harness reads, because +/// it is the same identity: an agent authenticates as its hive's agent client +/// whether the caller is the harness or a CLI the agent typed. A prefix of its +/// own would be a second set of variables carrying one fact, which is the +/// disagreement `queue.nix`'s credential pair exists to prevent. +const ENV_PREFIX: &str = "HIVE_AGENT"; + +/// Where this CLI queries, and who it queries as. +pub struct Config { + /// Full URL of the LogsQL query endpoint, gateway side. + /// + /// The whole URL rather than a host to build one from, because this same + /// string is **also the audience** the token is minted for — the rule + /// `swarm-otel.nix` states over its own push targets ("one binding + /// because the same string is both the address requested and the audience + /// the token is minted for — two spellings present as a valid token + /// refused at the store"). Splitting it would reintroduce exactly that. + pub query_url: String, + /// Everything needed to mint a token, in the form the minter wants it. + pub queue: QueueConfig, +} + +impl Config { + /// Read the configuration out of the environment the wrapper sets. + /// + /// All four coordinates or none, for the reason + /// [`QueueConfig::from_env`] gives and `hive-agent`'s `swarm_queue` + /// repeats: a half-set environment produces a process that looks + /// configured and never authenticates. Unlike the harness, this binary + /// *errors* on the absent case rather than degrading — a CLI whose only + /// job is the query has nothing left to do without it, and an operator + /// running it wants to be told why rather than handed an empty result + /// they will read as "no logs matched". + pub fn from_env() -> Result { + let var = |suffix: &str| std::env::var(format!("{ENV_PREFIX}_{suffix}")).ok(); + + let query_url = var("LOGS_QUERY_URL"); + let token_endpoint = var("OIDC_TOKEN_ENDPOINT"); + let client_id_file = var("OIDC_CLIENT_ID_FILE"); + let client_secret_file = var("OIDC_CLIENT_SECRET_FILE"); + + // Outside the all-or-none group on purpose, exactly as in + // `QueueConfig::from_env`: a swarm behind a publicly-trusted + // certificate needs no extra anchor, so a CA path with no endpoint is + // meaningless rather than half-configured. Nothing in this tree sets + // it for an agent — a container already trusts the swarm root, which + // `hive_c0re::meta` embeds at build time. + let ca_file = var("OIDC_CA_FILE").map(PathBuf::from); + + let (query_url, token_endpoint, client_id_file, client_secret_file) = match ( + query_url, + token_endpoint, + client_id_file, + client_secret_file, + ) { + (Some(u), Some(t), Some(i), Some(s)) => (u, t, i, s), + (None, None, None, None) => bail!( + "this agent has no swarm log store configured: \ + {ENV_PREFIX}_LOGS_QUERY_URL, {ENV_PREFIX}_OIDC_TOKEN_ENDPOINT, \ + {ENV_PREFIX}_OIDC_CLIENT_ID_FILE and \ + {ENV_PREFIX}_OIDC_CLIENT_SECRET_FILE are all unset" + ), + _ => bail!( + "swarm log store half-configured: {ENV_PREFIX}_LOGS_QUERY_URL, \ + {ENV_PREFIX}_OIDC_TOKEN_ENDPOINT, {ENV_PREFIX}_OIDC_CLIENT_ID_FILE \ + and {ENV_PREFIX}_OIDC_CLIENT_SECRET_FILE are set together or not \ + at all — this is a deployment bug, not a missing feature" + ), + }; + + let client_id = read_client_id(Path::new(&client_id_file))?; + + Ok(Self { + query_url, + queue: QueueConfig { + // 🩸 Unused by the token mint and deliberately not read from + // the environment. `QueueConfig` is the queue's connect + // config as well as its token config, and this binary needs + // only the second half — taking `HIVE_AGENT_NATS_URL` for it + // would make a broker address a prerequisite for reading + // logs, which is a coupling with no cause. Named so a reader + // of a stack trace is not left wondering. + url: String::from("unused: swarm-logs mints a token and never connects"), + token_endpoint, + client_id, + client_secret_file: PathBuf::from(client_secret_file), + ca_file, + }, + }) + } +} + +/// Read the OIDC client id out of the file the systemd credential landed at. +/// +/// Trailing newline stripped and an empty file refused, both copied from +/// `hive-agent`'s `read_client_id`: the writing unit ends the file with a +/// newline, and an id carrying one authenticates as nobody. Where the harness +/// treats absence as the ordinary state of a hive whose credential has not +/// been published yet and carries on, this errors — see [`Config::from_env`] +/// for why a CLI has nothing to carry on with. +fn read_client_id(path: &Path) -> Result { + let raw = std::fs::read_to_string(path) + .with_context(|| format!("reading the OIDC client id from {}", path.display()))?; + let id = raw.trim(); + if id.is_empty() { + bail!( + "the OIDC client id at {} is empty — the queue credential has not \ + been published to this hive yet", + path.display() + ); + } + Ok(id.to_owned()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn client_id_loses_its_trailing_newline() { + let dir = std::env::temp_dir().join(format!("swarm-logs-id-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("temp dir"); + let path = dir.join("client-id"); + std::fs::write(&path, "hive-alpha-agent\n").expect("write"); + assert_eq!(read_client_id(&path).expect("read"), "hive-alpha-agent"); + std::fs::remove_dir_all(&dir).expect("cleanup"); + } + + /// An empty credential file is the shape a hive has before its secret + /// store has published anything, and it must not become a token request + /// that authenticates as the empty string. + #[test] + fn an_empty_client_id_is_refused_by_name() { + let dir = std::env::temp_dir().join(format!("swarm-logs-empty-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("temp dir"); + let path = dir.join("client-id"); + std::fs::write(&path, "\n").expect("write"); + let err = read_client_id(&path).expect_err("empty id must not be accepted"); + assert!( + err.to_string().contains("is empty"), + "error should say the id is empty, got: {err}" + ); + std::fs::remove_dir_all(&dir).expect("cleanup"); + } + + /// A missing file names the path, because the operator's next move is to + /// look at whether the credential was delivered at all. + #[test] + fn a_missing_client_id_file_names_its_path() { + let path = Path::new("/nonexistent/swarm-logs/client-id"); + let err = read_client_id(path).expect_err("missing file must not be accepted"); + assert!( + err.to_string() + .contains("/nonexistent/swarm-logs/client-id"), + "error should name the path, got: {err}" + ); + } +} diff --git a/swarm-logs/src/main.rs b/swarm-logs/src/main.rs new file mode 100644 index 00000000..6096559d --- /dev/null +++ b/swarm-logs/src/main.rs @@ -0,0 +1,123 @@ +//! `swarm-logs` — an agent's CLI for the swarm log store. +//! +//! An agent can reach VictoriaLogs only through the gateway, and until this +//! existed the way to read it was to hand-roll a `client_credentials` token +//! request and a curl by hand, per query. That is the gap this closes: one +//! verb, a LogsQL string in, matched log lines out, so the answer pipes into +//! `grep` like any other command's. +//! +//! **Read-only and query-only.** It posts to the store's LogsQL query +//! endpoint and nothing else — there is no ingest path here, and no `tail`. +//! Streaming is a different endpoint with a different response shape and is +//! deliberately left for a follow-up rather than folded in. +//! +//! ⚠️ **The route applies no scoping.** `swarm-victorialogs.nix` forwards the +//! query unmodified, so any authenticated caller reads the whole swarm's +//! logs — every hive's, not just its own. That is the rule in force rather +//! than an omission on this binary's part; when read permissions exist they +//! attach at that location, not here. +//! +//! Distinct from `hive-metric`, which is the other half of the same +//! observability surface from an agent's seat: that one *writes* a custom +//! metric, this one *reads* logs. + +mod auth; +mod query; + +use anyhow::Result; +use clap::{Parser, Subcommand}; + +#[derive(Parser)] +#[command( + name = "swarm-logs", + version, + about = "query the swarm's log store from an agent" +)] +struct Cli { + #[command(subcommand)] + command: Verb, +} + +#[derive(Subcommand)] +enum Verb { + /// Run a LogsQL query and print the matched log lines. + Query { + /// The LogsQL query. A bare word is a full-text search for it. + /// + /// Quote it: LogsQL uses characters the shell also claims, and an + /// unquoted `|` pipes this command into the rest of your query + /// instead of sending it. + logsql: String, + + /// Stop after this many records. + /// + /// Unset means the store's own default. Worth setting on a broad + /// query — nothing between here and the store narrows one, so a bare + /// word matches across every hive in the swarm. + #[arg(long)] + limit: Option, + + /// What to print for each matched record. + #[arg(long, value_enum, default_value = "message")] + format: query::Format, + }, + + /// Print this CLI's reference as markdown (source of `docs/tools/swarm-logs-cli.md`). + /// + /// No `completions` sibling, unlike `hivectl` and `swarmctl`: those are + /// installed on an operator's interactive host, where a completion script + /// has a shell to be sourced into. This one is run by an agent through a + /// tool call, which has no line editor to complete against — the verb + /// would exist only to be listed. + #[command(hide = true)] + MarkdownDocs, +} + +fn main() -> Result<()> { + match Cli::parse().command { + Verb::Query { + logsql, + limit, + format, + } => { + // Config resolved inside this arm and not up front: the two + // verbs below are pure functions of the command tree and must + // keep working on a machine that has no log store wired up at + // all — `markdown-docs` in particular runs in a nix check, where + // no credential exists and none should. + let cfg = auth::Config::from_env()?; + let stdout = std::io::stdout(); + query::run(&cfg, &logsql, limit, format, &mut stdout.lock()) + } + Verb::MarkdownDocs => { + print!("{}", clap_markdown::help_markdown::()); + Ok(()) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use clap::CommandFactory as _; + + /// clap's own consistency check over the command tree — catches a + /// duplicate long flag or a malformed `about` at test time rather than on + /// the first run of a verb nobody exercised. + #[test] + fn the_command_tree_is_well_formed() { + Cli::command().debug_assert(); + } + + /// The default matters: it is what an agent gets when it pipes into + /// `grep` without reading the flags first. + #[test] + fn query_defaults_to_projecting_the_message() { + let cli = Cli::try_parse_from(["swarm-logs", "query", "a-marker"]).expect("parses"); + let Verb::Query { format, limit, .. } = cli.command else { + panic!("expected the query verb"); + }; + assert_eq!(format, query::Format::Message); + assert_eq!(limit, None, "no client-side default limit is imposed"); + } +} diff --git a/swarm-logs/src/query.rs b/swarm-logs/src/query.rs new file mode 100644 index 00000000..66f6cd93 --- /dev/null +++ b/swarm-logs/src/query.rs @@ -0,0 +1,215 @@ +//! The one request this binary makes: a LogsQL query against the store's +//! machine route on the gateway. +//! +//! The route is `^~ /select/logsql/`, added by `swarm-victorialogs.nix` +//! specifically so a machine caller is answered with a **401 it cannot +//! mistake for an empty result**. The operator's `/` route ends in +//! `error_page 401 =302` and hands an unauthenticated caller authelia's login +//! page as a 200 with an HTML body — which is why [`run`] below reports a +//! non-200 with its body rather than printing nothing and exiting 0. That +//! failure mode is the reason the route exists; reproducing it in the client +//! would be the whole bug again one layer up. + +use std::io::{BufRead as _, BufReader, Write}; + +use anyhow::{Context as _, Result, bail}; + +use crate::auth::Config; + +/// How the matched records are written to stdout. +#[derive(Clone, Copy, Debug, PartialEq, Eq, clap::ValueEnum)] +pub enum Format { + /// One log message per line — the `_msg` field and nothing else. + /// + /// The default because the ask this CLI answers was to *pipe and grep* + /// rather than hand-roll curl, and a grep pattern is written against the + /// message. A record with no `_msg` is passed through as its own JSON + /// object rather than dropped: silently losing a row would make an + /// incomplete answer indistinguishable from a complete one. + Message, + /// The store's response body, unmodified, one JSON object per line. + /// + /// For everything [`Format::Message`] leaves out — `_time`, `_stream` + /// and whatever fields the record carries — and for feeding `jq`. + Json, +} + +/// Run `query`, writing matched records to `out`. +/// +/// Streams rather than collecting: the route applies no scoping, so a loose +/// query matches the whole swarm's logs and buffering the answer to print it +/// is an out-of-memory waiting for a bad `LogsQL`. +pub fn run( + cfg: &Config, + logsql: &str, + limit: Option, + format: Format, + out: &mut impl Write, +) -> Result<()> { + // Minted per invocation, with the query URL as the audience. Both halves + // are load-bearing: authelia refuses a token carrying no audience at the + // authz endpoint the gateway's `auth_request` calls, and the audience it + // checks is the URL being requested — so the string sent here and the + // string requested below must be one binding, which is why `Config` holds + // exactly one. + let token = swarm_queue_client::mint_token_for_blocking(&cfg.queue, Some(&cfg.query_url)) + .map_err(|e| anyhow::anyhow!("{}", swarm_queue_client::chain(&e))) + .context("minting an access token for the swarm log store")?; + + let http = build_http_client(cfg)?; + + let mut form = vec![("query", logsql.to_owned())]; + if let Some(limit) = limit { + form.push(("limit", limit.to_string())); + } + + let response = http + .post(&cfg.query_url) + .bearer_auth(&token) + .form(&form) + .send() + .with_context(|| format!("querying the swarm log store at {}", cfg.query_url))?; + + let status = response.status(); + if !status.is_success() { + // Surfaced, not swallowed. A 401 here is the ordinary shape of "this + // client's identity is not authorised for this route", and the body + // is the half that says which — nginx answers its own HTML for an + // `auth_request` denial, the store answers JSON for a bad LogsQL. + // Truncated because the login-page case is a full HTML document and + // an operator needs the first screen of it, not all of it. + let body = response.text().unwrap_or_default(); + let body = body.trim(); + let shown: String = body.chars().take(2000).collect(); + bail!( + "the swarm log store refused the query ({status}): {shown}{}", + if shown.len() < body.chars().count() { + " […]" + } else { + "" + } + ); + } + + let mut reader = BufReader::new(response); + let mut line = String::new(); + loop { + line.clear(); + let read = reader + .read_line(&mut line) + .context("reading the query response from the swarm log store")?; + if read == 0 { + break; + } + let line = line.trim_end_matches(['\r', '\n']); + if line.is_empty() { + continue; + } + writeln!(out, "{}", render(line, format)).context("writing to stdout")?; + } + Ok(()) +} + +/// Render one NDJSON record from the store as the caller asked for it. +/// +/// Pure over the line so the projection is testable without a store: what +/// goes wrong here is a record shape, not a request. +fn render(line: &str, format: Format) -> String { + match format { + Format::Json => line.to_owned(), + Format::Message => serde_json::from_str::(line) + .ok() + .and_then(|v| v.get("_msg")?.as_str().map(str::to_owned)) + // Anything without a string `_msg` — including a line that is not + // JSON at all — falls back to the line itself. See + // [`Format::Message`]: dropping it would make a partial answer + // look like a whole one. + .unwrap_or_else(|| line.to_owned()), + } +} + +/// The HTTP client for the query itself. +/// +/// Separate from the one `swarm-queue-client` builds for the token endpoint +/// because they are two hosts — `auth.` and `logs.` — and the +/// CA-trust knob is declared per endpoint. Same optional anchor, same reason: +/// without it a swarm using its own CA fails at TLS with +/// `invalid peer certificate: UnknownIssuer`, having never looked at the +/// anchor sitting on the host. +/// +/// No timeout, deliberately, where the token client has one: a query over a +/// long retention window legitimately takes longer than any number that would +/// be right for a token request, and a client-side cut would truncate a +/// correct answer into a shorter one with no marker saying so. +fn build_http_client(cfg: &Config) -> Result { + let mut builder = reqwest::blocking::Client::builder(); + if let Some(path) = &cfg.queue.ca_file { + let pem = std::fs::read(path).with_context(|| { + format!( + "reading the log-store CA certificate from {}", + path.display() + ) + })?; + let cert = reqwest::Certificate::from_pem(&pem).with_context(|| { + format!( + "parsing the log-store CA certificate from {} as PEM", + path.display() + ) + })?; + builder = builder.add_root_certificate(cert); + } + builder + .build() + .context("building the log-store HTTP client") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn json_passes_the_record_through_unmodified() { + let line = r#"{"_time":"2026-09-16T14:14:43Z","_msg":"hello","_stream":"{}"}"#; + assert_eq!(render(line, Format::Json), line); + } + + #[test] + fn message_projects_just_the_log_line() { + let line = r#"{"_time":"2026-09-16T14:14:43Z","_msg":"hello there"}"#; + assert_eq!(render(line, Format::Message), "hello there"); + } + + /// The grep case this CLI exists for: a marker inside `_msg` has to + /// survive the projection intact, punctuation and all. + #[test] + fn message_keeps_a_marker_intact() { + let line = r#"{"_msg":"probe marker-1789568083-6970 emitted"}"#; + assert!(render(line, Format::Message).contains("marker-1789568083-6970")); + } + + /// A record the projection cannot understand is still a match, and a + /// caller counting lines must not get a different answer than one reading + /// them. Dropping it would make an incomplete answer look complete. + #[test] + fn a_record_without_a_message_falls_back_to_the_whole_line() { + let line = r#"{"_time":"2026-09-16T14:14:43Z","_stream":"{}"}"#; + assert_eq!(render(line, Format::Message), line); + } + + #[test] + fn a_non_json_line_falls_back_to_itself() { + assert_eq!( + render("not json at all", Format::Message), + "not json at all" + ); + } + + /// `_msg` holding a non-string is the same "shape I cannot project" case + /// as a missing one, and must take the same fallback rather than + /// rendering Rust's debug form of a number. + #[test] + fn a_non_string_message_falls_back_to_the_whole_line() { + let line = r#"{"_msg":42}"#; + assert_eq!(render(line, Format::Message), line); + } +}