diff --git a/.prettierignore b/.prettierignore index 6a9b2fb4..8cfbf2f9 100644 --- a/.prettierignore +++ b/.prettierignore @@ -7,7 +7,6 @@ docs/tools/hivectl-cli.md docs/tools/swarmctl-cli.md docs/tools/forge-cli.md -docs/tools/swarm-logs-cli.md # Files with multi-line list-item continuations that prettier strips to col 0. # prettier's `proseWrap: "preserve"` prevents prose reflow but not list-item diff --git a/Cargo.lock b/Cargo.lock index c694d29e..1e0f2b33 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4765,18 +4765,6 @@ dependencies = [ "utoipa-axum", ] -[[package]] -name = "swarm-logs" -version = "0.1.0" -dependencies = [ - "anyhow", - "clap", - "clap-markdown", - "reqwest", - "serde_json", - "swarm-queue-client", -] - [[package]] name = "swarm-nats-auth" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 040b3501..a4d1206f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,7 +28,6 @@ members = [ "swarm-controller", "swarm-nats-auth", "swarm-queue-client", - "swarm-logs", "swarm-secret-client", "swarmctl", ] diff --git a/clippy.toml b/clippy.toml index 57e33673..17565808 100644 --- a/clippy.toml +++ b/clippy.toml @@ -1,9 +1,4 @@ # Proper nouns / product names that clippy's `doc_markdown` lint would # otherwise flag as un-backticked identifiers in doc-comments. `".."` # keeps clippy's built-in default list (GitHub, OAuth, …) and extends it. -# -# `LogsQL` and `VictoriaLogs` are here rather than backticked at each use -# because several of those uses are clap doc-comments, which ARE the -# `--help` text — backticks would be printed literally to an operator's -# terminal to satisfy a lint about rustdoc rendering. -doc-valid-idents = ["WireGuard", "LogsQL", "VictoriaLogs", ".."] +doc-valid-idents = ["WireGuard", ".."] diff --git a/docs/tools/swarm-logs-cli.md b/docs/tools/swarm-logs-cli.md deleted file mode 100644 index f6f1d2b1..00000000 --- a/docs/tools/swarm-logs-cli.md +++ /dev/null @@ -1,57 +0,0 @@ -# Command-Line Help for `swarm-logs` - -This document contains the help content for the `swarm-logs` command-line program. - -**Command Overview:** - -* [`swarm-logs`↴](#swarm-logs) -* [`swarm-logs query`↴](#swarm-logs-query) - -## `swarm-logs` - -query the swarm's log store from an agent - -**Usage:** `swarm-logs ` - -###### **Subcommands:** - -* `query` — Run a LogsQL query and print the matched log lines - - - -## `swarm-logs query` - -Run a LogsQL query and print the matched log lines - -**Usage:** `swarm-logs query [OPTIONS] ` - -###### **Arguments:** - -* `` — The LogsQL query. A bare word is a full-text search for it. - - Quote it: LogsQL uses characters the shell also claims, and an unquoted `|` pipes this command into the rest of your query instead of sending it. - -###### **Options:** - -* `--limit ` — Stop after this many records. - - Unset means the store's own default. Worth setting on a broad query — nothing between here and the store narrows one, so a bare word matches across every hive in the swarm. -* `--format ` — What to print for each matched record - - Default value: `message` - - Possible values: - - `message`: - One log message per line — the `_msg` field and nothing else - - `json`: - The store's response body, unmodified, one JSON object per line - - - - -
- - - This document was generated automatically by - clap-markdown. - diff --git a/flake.nix b/flake.nix index 8f831359..a1f0b566 100644 --- a/flake.nix +++ b/flake.nix @@ -94,7 +94,6 @@ hive-metric hive-screen-mcp hive-subagent-daemon - swarm-logs assets frontend reference-docs diff --git a/hive-c0re/src/meta.rs b/hive-c0re/src/meta.rs index 8dcffa94..8199a958 100644 --- a/hive-c0re/src/meta.rs +++ b/hive-c0re/src/meta.rs @@ -706,10 +706,6 @@ const FORWARDED_VARS: &[&str] = &[ // address, which inside a container is the agent itself. "HIVE_AGENT_NATS_URL", "HIVE_AGENT_OIDC_TOKEN_ENDPOINT", - // Where an agent's `swarm-logs` queries, computed by the host for the - // same reason as the two above: it is a gateway address, and a container - // cannot derive it from anything it holds. - "HIVE_AGENT_LOGS_QUERY_URL", "HIVE_FORGE_URL", "HIVE_FORGE_PUBLIC_URL", "HIVE_MATRIX_URL", @@ -744,12 +740,6 @@ const FORWARDED_VAR_OPTIONS: &[(&str, &str)] = &[ "HIVE_AGENT_OIDC_TOKEN_ENDPOINT", "hyperhive.queue.tokenEndpoint", ), - // Read at build time to decide whether `swarm-logs` is installed at all — - // see `nix/agent-modules/logs.nix`. Both halves wired for this map's own - // reason: an agent whose option says no log store and whose env says - // otherwise has a binary that is not on PATH and coordinates that claim - // it is. - ("HIVE_AGENT_LOGS_QUERY_URL", "hyperhive.logs.queryUrl"), ("HIVE_FORGE_URL", "hyperhive.forge.url"), ("HIVE_MATRIX_URL", "hyperhive.matrix.url"), ("HYPERHIVE_HIVE_NAME", "hyperhive.hiveName"), diff --git a/nix/agent-modules/default.nix b/nix/agent-modules/default.nix index 281ebb49..8fb48656 100644 --- a/nix/agent-modules/default.nix +++ b/nix/agent-modules/default.nix @@ -28,7 +28,6 @@ ./forge.nix ./frontend.nix ./github.nix - ./logs.nix ./matrix.nix ./mcp.nix ./network.nix diff --git a/nix/agent-modules/logs.nix b/nix/agent-modules/logs.nix deleted file mode 100644 index 22d9e1d3..00000000 --- a/nix/agent-modules/logs.nix +++ /dev/null @@ -1,90 +0,0 @@ -# `swarm-logs` on this agent's PATH, configured for this swarm's log store. -# -# The binary reads its coordinates from the environment and defaults nothing -# (see `swarm-logs/README.md`), so it is installed **wrapped** rather than -# bare — the same shape `swarm-controller.nix` installs `swarmctl` in, and for -# the same reason: every value here is derived from an option a module owns, -# and a default inside the binary would be an address we hope points at -# something. -# -# ⚠️ Gated on the queue being configured as well as on the query URL, because -# the identity this CLI presents IS the queue's: one per-hive machine client, -# one credential pair, delivered once. A swarm with a log store and no agent -# credential has nothing for this to authenticate as, and installing a binary -# that can only fail would be worse than leaving it off PATH — an agent would -# read the 401 as "no logs", which is the exact confusion the store's machine -# route was added to end. -{ - lib, - pkgs, - config, - ... -}: -let - cfg = config.hyperhive.logs; - queueCfg = config.hyperhive.queue; - configured = cfg.queryUrl != null && queueCfg.tokenEndpoint != null; - - # The four coordinates `swarm-logs` reads, all-or-none on its side. Bound - # here as one attrset so the wrapper below cannot set three of them: a - # half-set environment is the failure the binary reports as a deployment - # bug, and it should not be reachable from the module that sets it. - # - # 🩸 Two of these are credential **paths**. Neither the id nor the secret - # is read here — a value in `--set` lands in the wrapper script, which is - # in the world-readable nix store. - wrapperEnv = { - HIVE_AGENT_LOGS_QUERY_URL = cfg.queryUrl; - HIVE_AGENT_OIDC_TOKEN_ENDPOINT = queueCfg.tokenEndpoint; - HIVE_AGENT_OIDC_CLIENT_ID_FILE = queueCfg.clientIdFile; - HIVE_AGENT_OIDC_CLIENT_SECRET_FILE = queueCfg.clientSecretFile; - }; - - # Same shape `swarm-controller.nix` wraps `swarmctl` in — `symlinkJoin` + - # `wrapProgram`, not a `writeShellScriptBin` shim, because the wrapper has - # to leave the binary's `--help`, its exit status and its streams intact. - # The whole point of the CLI is that its stdout pipes into `grep`. - swarmLogsConfigured = pkgs.symlinkJoin { - name = "swarm-logs-configured"; - paths = [ config.hyperhive.packages.swarm-logs ]; - nativeBuildInputs = [ pkgs.makeWrapper ]; - postBuild = '' - wrapProgram $out/bin/swarm-logs ${ - lib.concatStringsSep " " ( - lib.mapAttrsToList (name: value: "--set ${name} ${lib.escapeShellArg value}") wrapperEnv - ) - } - ''; - }; -in -{ - options.hyperhive.logs = { - queryUrl = lib.mkOption { - type = lib.types.nullOr lib.types.str; - default = null; - example = "https://logs.example.com/select/logsql/query"; - description = '' - The swarm log store's LogsQL query endpoint, as this container - reaches it — the machine route `swarm-victorialogs.nix` puts on the - gateway, not the browser one at `/`. - - Set by the generated meta flake from the host's own - {option}`services.hyperhive.swarm.victorialogs.domain`. - - ⚠️ The **whole URL**, not a host to build one from, because this same - string is also the audience `swarm-logs` mints its token for. Two - spellings of one address present as a valid token refused at the - store — the rule `swarm-otel.nix` already states over its own push - targets. - - `null` means this hive has not been given the log store's address for - its agents, and `swarm-logs` is then not installed at all. Deliberately - not defaulted: a guessed address resolves cleanly to the wrong place. - ''; - }; - }; - - config = lib.mkIf configured { - environment.systemPackages = [ swarmLogsConfigured ]; - }; -} diff --git a/nix/agent-modules/queue.nix b/nix/agent-modules/queue.nix index 4d7c5837..430c1b7a 100644 --- a/nix/agent-modules/queue.nix +++ b/nix/agent-modules/queue.nix @@ -23,13 +23,6 @@ let # side can discover the other's spelling, so a rename is a rename there too. secretCredential = "hive-queue-agent-secret"; clientIdCredential = "hive-queue-agent-client-id"; - - # Where systemd materialises this unit's credentials. `%d` above is the - # same directory, but `%d` only expands *inside* a unit — a consumer that - # is not one (the `swarm-logs` wrapper, see ./logs.nix) needs the literal, - # and deriving it from the unit name here is what keeps the two from - # disagreeing about which unit's credentials they mean. - credentialsDir = "/run/credentials/hive-agent.service"; in { options.hyperhive.queue = { @@ -66,38 +59,6 @@ in "no queue coordinates". ''; }; - - # One declaration, two readers. The credential ids are this module's — - # `hive_c0re::lifecycle::host_config` and the `LoadCredential=` below are - # the pair that has to agree on them — and a second consumer that spelled - # them again would be a second source of truth for a string whose - # mismatch is a file that is simply not there. - clientIdFile = lib.mkOption { - type = lib.types.str; - readOnly = true; - default = "${credentialsDir}/${clientIdCredential}"; - description = '' - Path the agent's OIDC client id is delivered at, for a consumer - outside the harness unit. Read-only: it is a fact about where the - credential lands, not a knob — see {option}`hyperhive.logs.queryUrl` - for the consumer this exists for. - ''; - }; - - clientSecretFile = lib.mkOption { - type = lib.types.str; - readOnly = true; - default = "${credentialsDir}/${secretCredential}"; - description = '' - Path the agent's OIDC client secret is delivered at. Read-only for - the same reason as {option}`hyperhive.queue.clientIdFile`. - - 🩸 A PATH and never a value. The file is `0400` to the agent user and - is read at the moment of a token request; nothing in this tree puts - its contents in an environment variable, where `/proc//environ` - would publish them to every process in the container. - ''; - }; }; config = lib.mkIf configured { diff --git a/nix/checks.nix b/nix/checks.nix index b99155c6..56ee05a0 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -242,23 +242,6 @@ in touch "$out" ''; - # `swarm-logs` CLI reference freshness check — same shape as - # `hivectl-docs` above, `swarm-logs markdown-docs` (clap-markdown over - # its own command tree) instead. Reuses `packages..swarm-logs` - # (a per-bin package out of `daemonBins` — see nix/packages/default.nix). - swarm-logs-docs = - pkgs.runCommand "swarm-logs-docs-fresh" { nativeBuildInputs = [ pkgs.diffutils ]; } - '' - ${self.packages.${system}.swarm-logs}/bin/swarm-logs markdown-docs > generated.md - if ! diff -u ${../docs/tools/swarm-logs-cli.md} generated.md; then - echo "" >&2 - echo "ERROR: docs/tools/swarm-logs-cli.md is out of date — regenerate it:" >&2 - echo " nix build .#swarm-logs && ./result/bin/swarm-logs markdown-docs > docs/tools/swarm-logs-cli.md" >&2 - exit 1 - fi - touch "$out" - ''; - # `hive-forge` CLI reference freshness check — same shape as # `hivectl-docs` above, `hive-forge markdown-docs` (clap-markdown over # its own command tree) instead. Reuses `packages..hive-forge` diff --git a/nix/host-modules/hive-c0re/environment.nix b/nix/host-modules/hive-c0re/environment.nix index 268d014b..254fa978 100644 --- a/nix/host-modules/hive-c0re/environment.nix +++ b/nix/host-modules/hive-c0re/environment.nix @@ -302,30 +302,6 @@ in HIVE_AGENT_NATS_URL = config.services.hyperhive.deploy.hive-controller.queue.agentNatsUrl; HIVE_AGENT_OIDC_TOKEN_ENDPOINT = config.services.hyperhive.swarm.statusPublish.tokenEndpoint; } -// - # Where an agent reads the swarm's logs, forwarded the same way and gated on - # the same credential: `swarm-logs` authenticates as the queue's own per-hive - # machine client, so an address with no credential behind it would put a - # binary on PATH that can only ever answer 401 — which an agent reads as "no - # logs", the exact confusion the store's machine route was added to end. - # - # The **machine** route (`^~ /select/logsql/`), not the browser one at `/`: - # that one ends in `error_page 401 =302` and hands a machine caller - # authelia's login page as a 200 with an HTML body. See - # `swarm-victorialogs.nix`'s location for the whole reasoning. - # - # ⚠️ The full URL rather than the domain, because the same string is also - # the audience the token is minted for — `swarm-otel.nix` states that rule - # over its own push targets, and two spellings present as a valid token - # refused at the store. - lib.optionalAttrs - ( - config.services.hyperhive.deploy.hive-controller.queue.agentNatsUrl != null - && config.services.hyperhive.swarm.statusPublish.tokenEndpoint != null - ) - { - HIVE_AGENT_LOGS_QUERY_URL = "https://${config.services.hyperhive.swarm.victorialogs.domain}/select/logsql/query"; - } // # Where the swarm's secret store is, and the identity this hive presents to # it (hive-c0re::workers::credential). `swarm_secret_client` reads these diff --git a/nix/host-modules/swarm-authelia.nix b/nix/host-modules/swarm-authelia.nix index 3b03796d..cb68143d 100644 --- a/nix/host-modules/swarm-authelia.nix +++ b/nix/host-modules/swarm-authelia.nix @@ -144,15 +144,6 @@ let accessTokenSignedResponseAlg = "RS256"; }) hyperhiveCfg.swarm.hives; - # The one URL an agent's token has to be authorised at beyond the queue: - # the log store's machine query route. Read off `swarm.victorialogs.domain` - # — an unconditional option, declared whether or not this host runs the - # store — rather than restated, because it is also the string `swarm-logs` - # requests as its audience and the string `hive-c0re/environment.nix` hands - # an agent as its endpoint. Three readers, one formula; two spellings - # present as a valid token refused at the store. - logsQueryUrl = "https://${hyperhiveCfg.swarm.victorialogs.domain}/select/logsql/query"; - # The identity an agent container presents to the swarm's queue. One per # HIVE, not per agent, and that is the load-bearing choice rather than a # shortcut: agents are created at **runtime**, so anything minted per agent @@ -174,38 +165,16 @@ let # match site, and the assertion below covers the case that ordering cannot. # # Mirrors `hiveClients` field for field so the two stay comparable. The - # signing algorithm is not load-bearing here — this client's consumers are + # signing algorithm is not load-bearing here — this client's only consumer is # the queue's auth-callout responder, which *introspects* rather than - # verifying offline, and authelia's own authz endpoint — but it matches its - # sibling rather than inventing a second answer to a question nobody asked. + # verifying offline — but it matches its sibling rather than inventing a + # second answer to a question nobody asked. agentClients = lib.mapAttrsToList (name: _: { id = "${cfg.hiveClientPrefix}${name}${cfg.agentClientSuffix}"; description = "HyperHive agents on hive ${name}"; kind = "machine"; redirectUris = [ ]; - # ⚠️ **Two** audiences, where `hiveClients` has one, because an agent - # presents this token at two different kinds of consumer. The queue - # introspects it and reads the client id; the log store's gateway - # location runs `auth_request` against authelia, which authorises a - # bearer token **by the URL being requested** and refuses one whose - # audience does not name it. Measured, not assumed: without the URL - # registered here the token endpoint answers `invalid_target` for it, - # and a token minted with no audience at all is answered by the gateway - # with a bare 401 that names no cause. - audience = [ - "${cfg.hiveClientPrefix}${name}${cfg.agentClientSuffix}" - logsQueryUrl - ]; - # Without it the authz endpoint refuses an otherwise valid token and - # blames the token rather than the missing grant — the same note - # `glue-swarm-otel-oidc-client.nix` carries over the collector's client. - bearerAuthz = true; - # Stated rather than left on authelia's default, for - # `glue-swarm-otel-oidc-client.nix`'s reason: authelia permits only - # basic / JWT methods for a confidential client holding this scope, and - # enforces it in the startup validator. `swarm-queue-client` sends the - # credential as basic auth, so this is also what the minter already does. - tokenEndpointAuthMethod = "client_secret_basic"; + audience = [ "${cfg.hiveClientPrefix}${name}${cfg.agentClientSuffix}" ]; accessTokenSignedResponseAlg = "RS256"; }) hyperhiveCfg.swarm.hives; diff --git a/nix/packages/default.nix b/nix/packages/default.nix index ac18933d..c0a534a8 100644 --- a/nix/packages/default.nix +++ b/nix/packages/default.nix @@ -37,12 +37,6 @@ let hive-subagent-daemon = "hyperhive per-agent claude-subagent task runner daemon (serves its MCP tools directly over streamable-http)"; hive-matrix-daemon = "hyperhive per-agent matrix-sdk daemon (serves its MCP tools directly over streamable-http)"; hive-metric = "hyperhive agent-emitted custom metrics CLI"; - # The read half of what `hive-metric` writes, from the same seat: an - # agent's CLI for the swarm log store. In this list rather than beside - # `swarmctl` below because it is run *by an agent, inside a container*, - # which is exactly what this list's per-bin packages are for — see - # `nix/agent-modules/logs.nix` for what puts it on PATH there. - swarm-logs = "hyperhive agent-facing swarm log store query CLI"; hive-screen-mcp = "hyperhive screen MCP bridge (screenshot + input for GUI agents)"; hive-forge = "hyperhive Forgejo CLI"; hive-forge-notify = "hyperhive per-agent Forgejo notification poller daemon"; diff --git a/swarm-logs/Cargo.toml b/swarm-logs/Cargo.toml deleted file mode 100644 index ca59942b..00000000 --- a/swarm-logs/Cargo.toml +++ /dev/null @@ -1,26 +0,0 @@ -[package] -name = "swarm-logs" -version.workspace = true -readme = "README.md" -edition.workspace = true - -[[bin]] -name = "swarm-logs" -path = "src/main.rs" - -[dependencies] -anyhow.workspace = true -clap.workspace = true -clap-markdown = "0.1" -# The whole point of the crate's dependency list: minting the access token -# is `swarm_queue_client::mint_token_for_blocking`, not a second copy of -# the `client_credentials` request. See `src/auth.rs`. -swarm-queue-client.workspace = true -# Blocking, not async: this binary makes exactly two requests and exits, so -# a reactor would exist only to be torn down. It also matches the token -# minter reached for above, which is `swarm-queue-client`'s blocking one. -reqwest = { workspace = true, features = ["blocking"] } -serde_json.workspace = true - -[lints] -workspace = true diff --git a/swarm-logs/README.md b/swarm-logs/README.md deleted file mode 100644 index 82e92809..00000000 --- a/swarm-logs/README.md +++ /dev/null @@ -1,92 +0,0 @@ -# swarm-logs - -An agent's CLI for the swarm log store. One verb — `swarm-logs query -''` — so reading logs pipes and greps like any other command -instead of being a hand-rolled token request plus a curl, per query. - -Read-only and query-only. There is no ingest path here (that is the -collector's), and no `tail`: streaming is a different LogsQL endpoint with a -different response shape, left for a follow-up rather than folded in. - -Distinct from `hive-metric`, the other half of the same surface from an -agent's seat: that one _writes_ a custom metric, this one _reads_ logs. - -## ⚠️ There is no scoping - -The gateway forwards the query unmodified, so **any authenticated caller -reads the whole swarm's logs** — every hive's, not only its own. That is the -rule in force rather than an omission here; when read permissions exist they -attach at the gateway location, not in this binary. - -## How it authenticates - -The agent container already holds one identity, the per-hive machine client -`hive--agent`, delivered as a systemd credential pair. This binary -presents a `client_credentials` access token minted from it as -`Authorization: Bearer`. - -Minting is **not** implemented here — `swarm_queue_client::mint_token_for_blocking` -already owns the request, its error type and its CA handling, and a -token-endpoint fix has to be findable in one place. What this crate adds is -the agent-shaped half: the client id arrives as a _file_ beside the secret, so -nothing outside `nix/agent-modules/queue.nix` spells `hive--agent` -twice. That is the same problem `hive-agent/src/swarm_queue.rs` solves, and -`src/auth.rs` is its `decide` restated over this binary's inputs. - -🩸 Every credential here is a **path**. The secret's contents are read at the -moment of the request, inside `swarm-queue-client`, and are never bound to a -name in this crate, logged, or rendered into an error. - -### The audience is the URL - -`swarm-logs` mints its token with the query URL as the requested audience, -because that is what authelia checks at the authz endpoint the gateway's -`auth_request` calls. The same rule `swarm-otel.nix` states over its own push -targets: one binding for the address and the audience, since two spellings of -one address present as a valid token refused at the store. - -⚠️ That requires the agent client to be _registered_ for that audience and to -hold `authelia.bearer.authz`. Both are set in `swarm-authelia.nix`'s -`agentClients`; without them authelia answers `invalid_target` at the token -endpoint, or the gateway answers 401 with no explanation. - -## Configuration - -Supplied by `nix/agent-modules/logs.nix`, which wraps the binary — the same -shape `swarmctl` is configured in, and for the same reason: every value is -derived from an option that module owns, so a default here would be an address -we _hope_ points at something. - -| variable | what | -| ------------------------------------ | ---------------------------------------------------- | -| `HIVE_AGENT_LOGS_QUERY_URL` | the LogsQL query endpoint; also the token's audience | -| `HIVE_AGENT_OIDC_TOKEN_ENDPOINT` | authelia's token endpoint | -| `HIVE_AGENT_OIDC_CLIENT_ID_FILE` | path of the delivered client-id credential | -| `HIVE_AGENT_OIDC_CLIENT_SECRET_FILE` | path of the delivered secret credential | -| `HIVE_AGENT_OIDC_CA_FILE` | optional extra trust anchor; unset in this tree | - -`HIVE_AGENT`, the same prefix the harness reads, because it is the same -identity — an agent authenticates as its hive's agent client whether the -caller is the harness or a CLI the agent typed. The first four are all-or-none: -a half-set environment is a deployment bug, and this binary says so by name -rather than behaving like an unconfigured one. - -## Output - -```console -$ swarm-logs query 'atlas-otel-probe' --limit 5 -``` - -One log message per line by default (the record's `_msg`), which is what a -grep pattern is written against. `--format json` passes the store's NDJSON -through unmodified, for `_time`, `_stream` and `jq`. - -A record the projection cannot understand — no string `_msg`, or not JSON at -all — is printed whole rather than dropped. Silently losing a row would make -an incomplete answer indistinguishable from a complete one. - -A non-200 is reported with its body and a non-zero exit, never as an empty -result. That is the whole reason the gateway grew a separate machine route: -the operator's browser route answers an unauthenticated caller with authelia's -login page as a **200 with an HTML body**, and a client that reads only the -status code records a query that succeeded and matched nothing. diff --git a/swarm-logs/src/auth.rs b/swarm-logs/src/auth.rs deleted file mode 100644 index 40368b93..00000000 --- a/swarm-logs/src/auth.rs +++ /dev/null @@ -1,188 +0,0 @@ -//! The access token this CLI presents to the log store's gateway route. -//! -//! Minting it is **not** implemented here. `swarm-queue-client` already owns -//! the `client_credentials` request against authelia — the basic-auth form, -//! the per-call secret read, the CA-trust builder and the error type that -//! surfaces authelia's own `error_description` — and this crate calls -//! [`swarm_queue_client::mint_token_for_blocking`] rather than writing a -//! second copy of it. A token-endpoint fix has to be findable in one place. -//! -//! What *is* here is the shape of an agent's identity, which differs from the -//! daemons that crate was written for in exactly one way: **the client id -//! arrives as a file, not as a value.** It rides in the same systemd -//! credential pair as the secret so that nothing outside `queue.nix` ever -//! spells `hive--agent` again. That is the same problem -//! `hive-agent/src/swarm_queue.rs` solves, and [`Config::from_env`] below is -//! that module's `decide` restated over this binary's inputs — including its -//! rule that a *half*-set environment is a deployment bug rather than an -//! absent integration. -//! -//! 🩸 Every credential in here is a **path**. The secret's contents are read -//! by `swarm-queue-client` at the moment of the request and are never bound to -//! a name in this crate, never logged and never rendered into an error. - -use std::path::{Path, PathBuf}; - -use anyhow::{Context as _, Result, bail}; -use swarm_queue_client::QueueConfig; - -/// Variable prefix for this binary's coordinates. -/// -/// `HIVE_AGENT`, deliberately the **same** prefix the harness reads, because -/// it is the same identity: an agent authenticates as its hive's agent client -/// whether the caller is the harness or a CLI the agent typed. A prefix of its -/// own would be a second set of variables carrying one fact, which is the -/// disagreement `queue.nix`'s credential pair exists to prevent. -const ENV_PREFIX: &str = "HIVE_AGENT"; - -/// Where this CLI queries, and who it queries as. -pub struct Config { - /// Full URL of the LogsQL query endpoint, gateway side. - /// - /// The whole URL rather than a host to build one from, because this same - /// string is **also the audience** the token is minted for — the rule - /// `swarm-otel.nix` states over its own push targets ("one binding - /// because the same string is both the address requested and the audience - /// the token is minted for — two spellings present as a valid token - /// refused at the store"). Splitting it would reintroduce exactly that. - pub query_url: String, - /// Everything needed to mint a token, in the form the minter wants it. - pub queue: QueueConfig, -} - -impl Config { - /// Read the configuration out of the environment the wrapper sets. - /// - /// All four coordinates or none, for the reason - /// [`QueueConfig::from_env`] gives and `hive-agent`'s `swarm_queue` - /// repeats: a half-set environment produces a process that looks - /// configured and never authenticates. Unlike the harness, this binary - /// *errors* on the absent case rather than degrading — a CLI whose only - /// job is the query has nothing left to do without it, and an operator - /// running it wants to be told why rather than handed an empty result - /// they will read as "no logs matched". - pub fn from_env() -> Result { - let var = |suffix: &str| std::env::var(format!("{ENV_PREFIX}_{suffix}")).ok(); - - let query_url = var("LOGS_QUERY_URL"); - let token_endpoint = var("OIDC_TOKEN_ENDPOINT"); - let client_id_file = var("OIDC_CLIENT_ID_FILE"); - let client_secret_file = var("OIDC_CLIENT_SECRET_FILE"); - - // Outside the all-or-none group on purpose, exactly as in - // `QueueConfig::from_env`: a swarm behind a publicly-trusted - // certificate needs no extra anchor, so a CA path with no endpoint is - // meaningless rather than half-configured. Nothing in this tree sets - // it for an agent — a container already trusts the swarm root, which - // `hive_c0re::meta` embeds at build time. - let ca_file = var("OIDC_CA_FILE").map(PathBuf::from); - - let (query_url, token_endpoint, client_id_file, client_secret_file) = match ( - query_url, - token_endpoint, - client_id_file, - client_secret_file, - ) { - (Some(u), Some(t), Some(i), Some(s)) => (u, t, i, s), - (None, None, None, None) => bail!( - "this agent has no swarm log store configured: \ - {ENV_PREFIX}_LOGS_QUERY_URL, {ENV_PREFIX}_OIDC_TOKEN_ENDPOINT, \ - {ENV_PREFIX}_OIDC_CLIENT_ID_FILE and \ - {ENV_PREFIX}_OIDC_CLIENT_SECRET_FILE are all unset" - ), - _ => bail!( - "swarm log store half-configured: {ENV_PREFIX}_LOGS_QUERY_URL, \ - {ENV_PREFIX}_OIDC_TOKEN_ENDPOINT, {ENV_PREFIX}_OIDC_CLIENT_ID_FILE \ - and {ENV_PREFIX}_OIDC_CLIENT_SECRET_FILE are set together or not \ - at all — this is a deployment bug, not a missing feature" - ), - }; - - let client_id = read_client_id(Path::new(&client_id_file))?; - - Ok(Self { - query_url, - queue: QueueConfig { - // 🩸 Unused by the token mint and deliberately not read from - // the environment. `QueueConfig` is the queue's connect - // config as well as its token config, and this binary needs - // only the second half — taking `HIVE_AGENT_NATS_URL` for it - // would make a broker address a prerequisite for reading - // logs, which is a coupling with no cause. Named so a reader - // of a stack trace is not left wondering. - url: String::from("unused: swarm-logs mints a token and never connects"), - token_endpoint, - client_id, - client_secret_file: PathBuf::from(client_secret_file), - ca_file, - }, - }) - } -} - -/// Read the OIDC client id out of the file the systemd credential landed at. -/// -/// Trailing newline stripped and an empty file refused, both copied from -/// `hive-agent`'s `read_client_id`: the writing unit ends the file with a -/// newline, and an id carrying one authenticates as nobody. Where the harness -/// treats absence as the ordinary state of a hive whose credential has not -/// been published yet and carries on, this errors — see [`Config::from_env`] -/// for why a CLI has nothing to carry on with. -fn read_client_id(path: &Path) -> Result { - let raw = std::fs::read_to_string(path) - .with_context(|| format!("reading the OIDC client id from {}", path.display()))?; - let id = raw.trim(); - if id.is_empty() { - bail!( - "the OIDC client id at {} is empty — the queue credential has not \ - been published to this hive yet", - path.display() - ); - } - Ok(id.to_owned()) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn client_id_loses_its_trailing_newline() { - let dir = std::env::temp_dir().join(format!("swarm-logs-id-{}", std::process::id())); - std::fs::create_dir_all(&dir).expect("temp dir"); - let path = dir.join("client-id"); - std::fs::write(&path, "hive-alpha-agent\n").expect("write"); - assert_eq!(read_client_id(&path).expect("read"), "hive-alpha-agent"); - std::fs::remove_dir_all(&dir).expect("cleanup"); - } - - /// An empty credential file is the shape a hive has before its secret - /// store has published anything, and it must not become a token request - /// that authenticates as the empty string. - #[test] - fn an_empty_client_id_is_refused_by_name() { - let dir = std::env::temp_dir().join(format!("swarm-logs-empty-{}", std::process::id())); - std::fs::create_dir_all(&dir).expect("temp dir"); - let path = dir.join("client-id"); - std::fs::write(&path, "\n").expect("write"); - let err = read_client_id(&path).expect_err("empty id must not be accepted"); - assert!( - err.to_string().contains("is empty"), - "error should say the id is empty, got: {err}" - ); - std::fs::remove_dir_all(&dir).expect("cleanup"); - } - - /// A missing file names the path, because the operator's next move is to - /// look at whether the credential was delivered at all. - #[test] - fn a_missing_client_id_file_names_its_path() { - let path = Path::new("/nonexistent/swarm-logs/client-id"); - let err = read_client_id(path).expect_err("missing file must not be accepted"); - assert!( - err.to_string() - .contains("/nonexistent/swarm-logs/client-id"), - "error should name the path, got: {err}" - ); - } -} diff --git a/swarm-logs/src/main.rs b/swarm-logs/src/main.rs deleted file mode 100644 index 6096559d..00000000 --- a/swarm-logs/src/main.rs +++ /dev/null @@ -1,123 +0,0 @@ -//! `swarm-logs` — an agent's CLI for the swarm log store. -//! -//! An agent can reach VictoriaLogs only through the gateway, and until this -//! existed the way to read it was to hand-roll a `client_credentials` token -//! request and a curl by hand, per query. That is the gap this closes: one -//! verb, a LogsQL string in, matched log lines out, so the answer pipes into -//! `grep` like any other command's. -//! -//! **Read-only and query-only.** It posts to the store's LogsQL query -//! endpoint and nothing else — there is no ingest path here, and no `tail`. -//! Streaming is a different endpoint with a different response shape and is -//! deliberately left for a follow-up rather than folded in. -//! -//! ⚠️ **The route applies no scoping.** `swarm-victorialogs.nix` forwards the -//! query unmodified, so any authenticated caller reads the whole swarm's -//! logs — every hive's, not just its own. That is the rule in force rather -//! than an omission on this binary's part; when read permissions exist they -//! attach at that location, not here. -//! -//! Distinct from `hive-metric`, which is the other half of the same -//! observability surface from an agent's seat: that one *writes* a custom -//! metric, this one *reads* logs. - -mod auth; -mod query; - -use anyhow::Result; -use clap::{Parser, Subcommand}; - -#[derive(Parser)] -#[command( - name = "swarm-logs", - version, - about = "query the swarm's log store from an agent" -)] -struct Cli { - #[command(subcommand)] - command: Verb, -} - -#[derive(Subcommand)] -enum Verb { - /// Run a LogsQL query and print the matched log lines. - Query { - /// The LogsQL query. A bare word is a full-text search for it. - /// - /// Quote it: LogsQL uses characters the shell also claims, and an - /// unquoted `|` pipes this command into the rest of your query - /// instead of sending it. - logsql: String, - - /// Stop after this many records. - /// - /// Unset means the store's own default. Worth setting on a broad - /// query — nothing between here and the store narrows one, so a bare - /// word matches across every hive in the swarm. - #[arg(long)] - limit: Option, - - /// What to print for each matched record. - #[arg(long, value_enum, default_value = "message")] - format: query::Format, - }, - - /// Print this CLI's reference as markdown (source of `docs/tools/swarm-logs-cli.md`). - /// - /// No `completions` sibling, unlike `hivectl` and `swarmctl`: those are - /// installed on an operator's interactive host, where a completion script - /// has a shell to be sourced into. This one is run by an agent through a - /// tool call, which has no line editor to complete against — the verb - /// would exist only to be listed. - #[command(hide = true)] - MarkdownDocs, -} - -fn main() -> Result<()> { - match Cli::parse().command { - Verb::Query { - logsql, - limit, - format, - } => { - // Config resolved inside this arm and not up front: the two - // verbs below are pure functions of the command tree and must - // keep working on a machine that has no log store wired up at - // all — `markdown-docs` in particular runs in a nix check, where - // no credential exists and none should. - let cfg = auth::Config::from_env()?; - let stdout = std::io::stdout(); - query::run(&cfg, &logsql, limit, format, &mut stdout.lock()) - } - Verb::MarkdownDocs => { - print!("{}", clap_markdown::help_markdown::()); - Ok(()) - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use clap::CommandFactory as _; - - /// clap's own consistency check over the command tree — catches a - /// duplicate long flag or a malformed `about` at test time rather than on - /// the first run of a verb nobody exercised. - #[test] - fn the_command_tree_is_well_formed() { - Cli::command().debug_assert(); - } - - /// The default matters: it is what an agent gets when it pipes into - /// `grep` without reading the flags first. - #[test] - fn query_defaults_to_projecting_the_message() { - let cli = Cli::try_parse_from(["swarm-logs", "query", "a-marker"]).expect("parses"); - let Verb::Query { format, limit, .. } = cli.command else { - panic!("expected the query verb"); - }; - assert_eq!(format, query::Format::Message); - assert_eq!(limit, None, "no client-side default limit is imposed"); - } -} diff --git a/swarm-logs/src/query.rs b/swarm-logs/src/query.rs deleted file mode 100644 index 66f6cd93..00000000 --- a/swarm-logs/src/query.rs +++ /dev/null @@ -1,215 +0,0 @@ -//! The one request this binary makes: a LogsQL query against the store's -//! machine route on the gateway. -//! -//! The route is `^~ /select/logsql/`, added by `swarm-victorialogs.nix` -//! specifically so a machine caller is answered with a **401 it cannot -//! mistake for an empty result**. The operator's `/` route ends in -//! `error_page 401 =302` and hands an unauthenticated caller authelia's login -//! page as a 200 with an HTML body — which is why [`run`] below reports a -//! non-200 with its body rather than printing nothing and exiting 0. That -//! failure mode is the reason the route exists; reproducing it in the client -//! would be the whole bug again one layer up. - -use std::io::{BufRead as _, BufReader, Write}; - -use anyhow::{Context as _, Result, bail}; - -use crate::auth::Config; - -/// How the matched records are written to stdout. -#[derive(Clone, Copy, Debug, PartialEq, Eq, clap::ValueEnum)] -pub enum Format { - /// One log message per line — the `_msg` field and nothing else. - /// - /// The default because the ask this CLI answers was to *pipe and grep* - /// rather than hand-roll curl, and a grep pattern is written against the - /// message. A record with no `_msg` is passed through as its own JSON - /// object rather than dropped: silently losing a row would make an - /// incomplete answer indistinguishable from a complete one. - Message, - /// The store's response body, unmodified, one JSON object per line. - /// - /// For everything [`Format::Message`] leaves out — `_time`, `_stream` - /// and whatever fields the record carries — and for feeding `jq`. - Json, -} - -/// Run `query`, writing matched records to `out`. -/// -/// Streams rather than collecting: the route applies no scoping, so a loose -/// query matches the whole swarm's logs and buffering the answer to print it -/// is an out-of-memory waiting for a bad `LogsQL`. -pub fn run( - cfg: &Config, - logsql: &str, - limit: Option, - format: Format, - out: &mut impl Write, -) -> Result<()> { - // Minted per invocation, with the query URL as the audience. Both halves - // are load-bearing: authelia refuses a token carrying no audience at the - // authz endpoint the gateway's `auth_request` calls, and the audience it - // checks is the URL being requested — so the string sent here and the - // string requested below must be one binding, which is why `Config` holds - // exactly one. - let token = swarm_queue_client::mint_token_for_blocking(&cfg.queue, Some(&cfg.query_url)) - .map_err(|e| anyhow::anyhow!("{}", swarm_queue_client::chain(&e))) - .context("minting an access token for the swarm log store")?; - - let http = build_http_client(cfg)?; - - let mut form = vec![("query", logsql.to_owned())]; - if let Some(limit) = limit { - form.push(("limit", limit.to_string())); - } - - let response = http - .post(&cfg.query_url) - .bearer_auth(&token) - .form(&form) - .send() - .with_context(|| format!("querying the swarm log store at {}", cfg.query_url))?; - - let status = response.status(); - if !status.is_success() { - // Surfaced, not swallowed. A 401 here is the ordinary shape of "this - // client's identity is not authorised for this route", and the body - // is the half that says which — nginx answers its own HTML for an - // `auth_request` denial, the store answers JSON for a bad LogsQL. - // Truncated because the login-page case is a full HTML document and - // an operator needs the first screen of it, not all of it. - let body = response.text().unwrap_or_default(); - let body = body.trim(); - let shown: String = body.chars().take(2000).collect(); - bail!( - "the swarm log store refused the query ({status}): {shown}{}", - if shown.len() < body.chars().count() { - " […]" - } else { - "" - } - ); - } - - let mut reader = BufReader::new(response); - let mut line = String::new(); - loop { - line.clear(); - let read = reader - .read_line(&mut line) - .context("reading the query response from the swarm log store")?; - if read == 0 { - break; - } - let line = line.trim_end_matches(['\r', '\n']); - if line.is_empty() { - continue; - } - writeln!(out, "{}", render(line, format)).context("writing to stdout")?; - } - Ok(()) -} - -/// Render one NDJSON record from the store as the caller asked for it. -/// -/// Pure over the line so the projection is testable without a store: what -/// goes wrong here is a record shape, not a request. -fn render(line: &str, format: Format) -> String { - match format { - Format::Json => line.to_owned(), - Format::Message => serde_json::from_str::(line) - .ok() - .and_then(|v| v.get("_msg")?.as_str().map(str::to_owned)) - // Anything without a string `_msg` — including a line that is not - // JSON at all — falls back to the line itself. See - // [`Format::Message`]: dropping it would make a partial answer - // look like a whole one. - .unwrap_or_else(|| line.to_owned()), - } -} - -/// The HTTP client for the query itself. -/// -/// Separate from the one `swarm-queue-client` builds for the token endpoint -/// because they are two hosts — `auth.` and `logs.` — and the -/// CA-trust knob is declared per endpoint. Same optional anchor, same reason: -/// without it a swarm using its own CA fails at TLS with -/// `invalid peer certificate: UnknownIssuer`, having never looked at the -/// anchor sitting on the host. -/// -/// No timeout, deliberately, where the token client has one: a query over a -/// long retention window legitimately takes longer than any number that would -/// be right for a token request, and a client-side cut would truncate a -/// correct answer into a shorter one with no marker saying so. -fn build_http_client(cfg: &Config) -> Result { - let mut builder = reqwest::blocking::Client::builder(); - if let Some(path) = &cfg.queue.ca_file { - let pem = std::fs::read(path).with_context(|| { - format!( - "reading the log-store CA certificate from {}", - path.display() - ) - })?; - let cert = reqwest::Certificate::from_pem(&pem).with_context(|| { - format!( - "parsing the log-store CA certificate from {} as PEM", - path.display() - ) - })?; - builder = builder.add_root_certificate(cert); - } - builder - .build() - .context("building the log-store HTTP client") -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn json_passes_the_record_through_unmodified() { - let line = r#"{"_time":"2026-09-16T14:14:43Z","_msg":"hello","_stream":"{}"}"#; - assert_eq!(render(line, Format::Json), line); - } - - #[test] - fn message_projects_just_the_log_line() { - let line = r#"{"_time":"2026-09-16T14:14:43Z","_msg":"hello there"}"#; - assert_eq!(render(line, Format::Message), "hello there"); - } - - /// The grep case this CLI exists for: a marker inside `_msg` has to - /// survive the projection intact, punctuation and all. - #[test] - fn message_keeps_a_marker_intact() { - let line = r#"{"_msg":"probe marker-1789568083-6970 emitted"}"#; - assert!(render(line, Format::Message).contains("marker-1789568083-6970")); - } - - /// A record the projection cannot understand is still a match, and a - /// caller counting lines must not get a different answer than one reading - /// them. Dropping it would make an incomplete answer look complete. - #[test] - fn a_record_without_a_message_falls_back_to_the_whole_line() { - let line = r#"{"_time":"2026-09-16T14:14:43Z","_stream":"{}"}"#; - assert_eq!(render(line, Format::Message), line); - } - - #[test] - fn a_non_json_line_falls_back_to_itself() { - assert_eq!( - render("not json at all", Format::Message), - "not json at all" - ); - } - - /// `_msg` holding a non-string is the same "shape I cannot project" case - /// as a missing one, and must take the same fallback rather than - /// rendering Rust's debug form of a number. - #[test] - fn a_non_string_message_falls_back_to_the_whole_line() { - let line = r#"{"_msg":42}"#; - assert_eq!(render(line, Format::Message), line); - } -}