diff --git a/docs/swarm/ui.md b/docs/swarm/ui.md index b7ef3651..2e60a130 100644 --- a/docs/swarm/ui.md +++ b/docs/swarm/ui.md @@ -16,6 +16,7 @@ domain, covers host-level detail for one hive. | `/jobs` | the controller's job graph — where agent creation and credential mints show progress | | `/schedules` | the scheduled prompts: add, edit, pause, resume and remove | | `/issues` | a cross-repo issue report | +| `/review-tryout` | try a review classification against a real PR | Everything it shows comes from [`swarm-controller`](../../swarm-controller/README.md). An agent created here or with `swarmctl agent create` starts `paused`; set it @@ -46,6 +47,26 @@ page. Selecting outside it, or Escape, closes it. You can stop or pause the agent, or link an account, without leaving the terminal. A subagent's tab opens its agent's details. +### Review try-out + +`/review-tryout` shows how the [review classifier](review-classifier.md) +answers a classification on a real PR, before you put it in a repo's config. +Give it a repo as `owner/name` and a PR number, then write a classification: a +name, a description, a polarity (positive or negative) and a threshold from 0 +to 100. **add a classification** adds more, and **run** starts the run. + +A run takes a minute or more. The page says it's running and shows how long it +has been, then shows the review's table: for each classification the verdict, +the model's certainty, the threshold, whether the verdict counted and the +reason. A classification that couldn't run shows its error where the reason +would be. Under the table are the overall verdict, approve or request changes, +and the head commit the run read. A failed run shows its error, for example when the swarm +has no API key yet. + +What you typed stays after a run, so you can edit the classification and run it +again. The page saves nothing and posts nothing to the forge. It shows no token +counts, cache counts or cost. + ### Scheduled prompts `/schedules` has one card per scheduled prompt. The **name** is the card's diff --git a/frontend/packages/swarm-ui/src/App.tsx b/frontend/packages/swarm-ui/src/App.tsx index bd20222d..44493a1e 100644 --- a/frontend/packages/swarm-ui/src/App.tsx +++ b/frontend/packages/swarm-ui/src/App.tsx @@ -8,6 +8,7 @@ import { ComponentsPage } from "./pages/ComponentsPage.js"; import { JobsPage } from "./pages/JobsPage.js"; import { HivesPage } from "./pages/HivesPage.js"; import { IssueReportPage } from "./pages/IssueReportPage.js"; +import { ReviewTryOutPage } from "./pages/review-tryout/ReviewTryOutPage.js"; import { SchedulesPage } from "./pages/schedules/SchedulesPage.js"; import { Panel } from "./ui/panel/Panel.js"; @@ -33,6 +34,7 @@ export function App() { + diff --git a/frontend/packages/swarm-ui/src/pages/review-tryout/ClassificationEditor.tsx b/frontend/packages/swarm-ui/src/pages/review-tryout/ClassificationEditor.tsx new file mode 100644 index 00000000..bdcc2eec --- /dev/null +++ b/frontend/packages/swarm-ui/src/pages/review-tryout/ClassificationEditor.tsx @@ -0,0 +1,65 @@ +// — the fields of one review classification: a name, a +// description (the prompt the model answers), which answer passes, and how +// certain the model has to be for its answer to count. +import { Button } from "../../ui/button/Button.js"; +import { SelectField } from "../../ui/select-field/SelectField.js"; +import { TextAreaField } from "../../ui/text-area-field/TextAreaField.js"; +import { TextField } from "../../ui/text-field/TextField.js"; +import type { ClassificationDraft, Polarity } from "./TryOutTypes.js"; +import "./ReviewTryOutPage.css"; + +const POLARITY_OPTIONS = [ + { value: "positive", label: "positive — the PR should match" }, + { value: "negative", label: "negative — the PR should not match" }, +]; + +export function ClassificationEditor({ + draft, + onChange, + onRemove, +}: { + draft: ClassificationDraft; + onChange: (next: ClassificationDraft) => void; + /** Absent when this is the only classification: one is the minimum. */ + onRemove?: () => void; +}) { + const id = `tryout-${draft.key}`; + return ( +
+ onChange({ ...draft, name })} + /> + onChange({ ...draft, description })} + /> +
+ + onChange({ ...draft, polarity: polarity as Polarity }) + } + /> + onChange({ ...draft, threshold })} + /> + {onRemove ? : null} +
+
+ ); +} diff --git a/frontend/packages/swarm-ui/src/pages/review-tryout/ReviewTryOutPage.css b/frontend/packages/swarm-ui/src/pages/review-tryout/ReviewTryOutPage.css new file mode 100644 index 00000000..1d51e603 --- /dev/null +++ b/frontend/packages/swarm-ui/src/pages/review-tryout/ReviewTryOutPage.css @@ -0,0 +1,66 @@ +/* The form panel and the result panel under it, apart from each other. */ +.ui-tryout-page { + display: flex; + flex-direction: column; + gap: 1em; +} +/* The try-out form: the repo and PR on one line, then one block per + classification, then the actions. */ +.ui-tryout-form { + display: flex; + flex-direction: column; + gap: 1em; + margin-top: 1em; +} +.ui-tryout-target { + display: flex; + flex-wrap: wrap; + gap: 0.75em; +} +.ui-tryout-target > * { + flex: 1 1 14em; +} +.ui-tryout-classification { + display: flex; + flex-direction: column; + gap: 0.6em; + padding: 0.75em; + border: 1px solid var(--border); + border-radius: 0.5em; +} +.ui-tryout-classification-row { + display: flex; + flex-wrap: wrap; + align-items: flex-end; + gap: 0.75em; +} +.ui-tryout-classification-row > * { + flex: 1 1 12em; +} +.ui-tryout-classification-row > button { + flex: none; +} +.ui-tryout-actions { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 0.75em; +} +.ui-tryout-hint { + margin: 0; + color: var(--muted); +} +.ui-tryout-running { + margin: 0; + color: var(--muted); +} +.ui-tryout-row-error { + color: var(--red); +} +.ui-tryout-overall, +.ui-tryout-sha { + margin: 0.75em 0 0; +} +.ui-tryout-sha { + color: var(--muted); +} diff --git a/frontend/packages/swarm-ui/src/pages/review-tryout/ReviewTryOutPage.tsx b/frontend/packages/swarm-ui/src/pages/review-tryout/ReviewTryOutPage.tsx new file mode 100644 index 00000000..fa52c568 --- /dev/null +++ b/frontend/packages/swarm-ui/src/pages/review-tryout/ReviewTryOutPage.tsx @@ -0,0 +1,233 @@ +// — write a review classification and see how the model +// answers it on a real PR, before it goes into a repo's config +// (`docs/swarm/review-classifier.md`, "Try-out"). Pick a repo and a PR, write +// one or more classifications, run: the page posts them to +// swarm-controller's `POST /api/review-classifier/try-out`, which answers +// `202 {id}`, and polls `GET .../try-out/{id}` until the run is `done` or +// `failed`. A run takes a minute or more, so the page says it is running. +// +// The inputs stay as typed after a run, so the operator can edit and run +// again. The page stores nothing (not even in the browser) and posts nothing +// to the forge. +import { useEffect, useState } from "preact/hooks"; +import { ApiErrorPanel } from "@hive/shared/api-error-panel.js"; +import { readApiError, type ProblemDetails } from "@hive/shared/api-error.js"; +import { Button } from "../../ui/button/Button.js"; +import { Panel } from "../../ui/panel/Panel.js"; +import { TextField } from "../../ui/text-field/TextField.js"; +import { ClassificationEditor } from "./ClassificationEditor.js"; +import { TryOutResult } from "./TryOutResult.js"; +import { + draftProblem, + prNumber, + repoProblem, + toBody, + type ClassificationDraft, + type TryOutResult as Result, +} from "./TryOutTypes.js"; +import "./ReviewTryOutPage.css"; + +const TRY_OUT = "/api/review-classifier/try-out"; +const POLL_MS = 2000; +// More classifications than this are not offered a row; the backend has its +// own cap and says so with a 400, which the page shows as it is. +const MAX_CLASSIFICATIONS = 10; + +let nextKey = 0; +function blankDraft(): ClassificationDraft { + return { + key: nextKey++, + name: "", + description: "", + polarity: "positive", + threshold: "80", + }; +} + +export function ReviewTryOutPage() { + const [repo, setRepo] = useState(""); + const [pr, setPr] = useState(""); + const [drafts, setDrafts] = useState(() => [ + blankDraft(), + ]); + // The id of the run being polled; null when none is in flight. + const [runId, setRunId] = useState(null); + const [starting, setStarting] = useState(false); + const [result, setResult] = useState(null); + const [error, setError] = useState(null); + // Whole seconds since the run started, so a long wait shows it is alive. + const [elapsed, setElapsed] = useState(0); + + const running = starting || runId !== null; + + useEffect(() => { + if (!running) return; + setElapsed(0); + const t = setInterval(() => setElapsed((s) => s + 1), 1000); + return () => clearInterval(t); + }, [running]); + + // Polls the run until it leaves `running`. Cancelled by the next run, or by + // leaving the page. + useEffect(() => { + if (runId === null) return; + let cancelled = false; + let timer: ReturnType | undefined; + const stop = (problem: ProblemDetails) => { + setError(problem); + setRunId(null); + }; + async function poll() { + try { + const res = await fetch( + `${TRY_OUT}/${encodeURIComponent(runId ?? "")}`, + ); + if (cancelled) return; + if (res.status === 404) { + stop({ + detail: "the result expired before it was read: run it again", + }); + return; + } + if (!res.ok) { + const problem = await readApiError(res); + if (!cancelled) stop(problem); + return; + } + const body = (await res.json()) as Result; + if (cancelled) return; + setResult(body); + if (body.state === "running") + timer = setTimeout(() => void poll(), POLL_MS); + else setRunId(null); + } catch (e: unknown) { + if (!cancelled) stop({ detail: String(e) }); + } + } + void poll(); + return () => { + cancelled = true; + if (timer !== undefined) clearTimeout(timer); + }; + }, [runId]); + + const problem = + repoProblem(repo) ?? + (prNumber(pr) === null ? "a PR number is a whole number above 0" : null) ?? + drafts.map(draftProblem).find((p) => p !== null) ?? + null; + + async function run(e: Event) { + e.preventDefault(); + const number = prNumber(pr); + if (problem !== null || number === null) return; + setStarting(true); + setError(null); + setResult(null); + try { + const res = await fetch(TRY_OUT, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + repo: repo.trim(), + pr: number, + classifications: drafts.map(toBody), + }), + }); + if (!res.ok) { + setError(await readApiError(res)); + return; + } + setRunId(((await res.json()) as { id: string }).id); + } catch (err: unknown) { + setError({ detail: String(err) }); + } finally { + setStarting(false); + } + } + + function changeDraft(next: ClassificationDraft) { + setDrafts((ds) => ds.map((d) => (d.key === next.key ? next : d))); + } + + return ( +
+ +

+ Run a review classification against a real PR and see how the model + answers it. Nothing is posted to the forge, and nothing is saved: + change the classification and run it again until it answers the way + you want. +

+
void run(e)}> +
+ + +
+ {drafts.map((d) => ( + 1 + ? () => setDrafts((ds) => ds.filter((x) => x.key !== d.key)) + : undefined + } + /> + ))} +
+ + +
+ {problem !== null ?

{problem}

: null} + +
+ {running || result || error ? ( + + {running ? ( +

+ Running — a run takes a minute or more ({elapsed}s so far). +

+ ) : null} + {error ? ( + + ) : null} + {result?.state === "failed" ? ( + + ) : null} + {result?.state === "done" ? : null} + + ) : null} +
+ ); +} diff --git a/frontend/packages/swarm-ui/src/pages/review-tryout/TryOutResult.tsx b/frontend/packages/swarm-ui/src/pages/review-tryout/TryOutResult.tsx new file mode 100644 index 00000000..668e1f30 --- /dev/null +++ b/frontend/packages/swarm-ui/src/pages/review-tryout/TryOutResult.tsx @@ -0,0 +1,72 @@ +// — a finished try-out: one row per classification, the +// overall verdict under them, and the head commit the run read. The columns +// are the review's own table (`docs/swarm/review-classifier.md`); no token +// counts, cache counts or cost, here or anywhere in the UI. +import { Badge } from "@hive/shared/badge.js"; +import { Table, type TableColumn } from "../../ui/table/Table.js"; +import type { TryOutResult as Result, TryOutRow } from "./TryOutTypes.js"; + +const VERDICT_LABEL: Record = { + match: "match", + "no-match": "no match", + "not-applicable": "not applicable", +}; + +const COLUMNS: TableColumn[] = [ + { key: "name", header: "classification", render: (r) => r.name }, + { key: "polarity", header: "polarity", render: (r) => r.polarity }, + { + key: "verdict", + header: "verdict", + render: (r) => (r.error ? "—" : VERDICT_LABEL[r.verdict]), + }, + { + key: "certainty", + header: "certainty", + render: (r) => (r.error ? "—" : r.certainty), + }, + { key: "threshold", header: "threshold", render: (r) => r.threshold }, + { + key: "counted", + header: "counted", + // A counted row shows whether it passed its polarity; one below its + // threshold does not count and has no pass or fail. + render: (r) => + r.error ? "—" : r.counted ? (r.passed ? "yes ✅" : "yes ❌") : "no", + }, + { + key: "reason", + header: "reason", + cellClass: "ui-table-prose", + // A row that could not run says why where the reason would be. + render: (r) => + r.error ? {r.error} : r.reason, + }, +]; + +export function TryOutResult({ result }: { result: Result }) { + const rows = result.rows ?? []; + return ( +
+ r.name} + storageKey="swarm-ui:review-tryout:table-filters" + emptyMessage="the run returned no classifications" + /> + {result.overall ? ( +

+ Overall: + +

+ ) : null} +

+ Ran on head {result.head_sha.slice(0, 12)} +

+ + ); +} diff --git a/frontend/packages/swarm-ui/src/pages/review-tryout/TryOutTypes.ts b/frontend/packages/swarm-ui/src/pages/review-tryout/TryOutTypes.ts new file mode 100644 index 00000000..94d0de63 --- /dev/null +++ b/frontend/packages/swarm-ui/src/pages/review-tryout/TryOutTypes.ts @@ -0,0 +1,91 @@ +// Wire shapes of the review classifier's try-out endpoints +// (`POST /api/review-classifier/try-out`, `GET .../try-out/{id}`), and the +// checks the form makes before a request is worth sending. The backend stays +// the real gate: these only keep an obviously bad form from a round trip. + +export type Polarity = "positive" | "negative"; + +/** One classification as the form holds it: every field as the user typed it. */ +export interface ClassificationDraft { + /** Local only, so a row keeps its identity when another is removed. */ + key: number; + name: string; + description: string; + polarity: Polarity; + threshold: string; +} + +/** The request's classification: what the backend is sent. */ +export interface ClassificationBody { + name: string; + description: string; + polarity: Polarity; + threshold: number; +} + +export type Verdict = "match" | "no-match" | "not-applicable"; + +export interface TryOutRow { + name: string; + polarity: Polarity; + verdict: Verdict; + certainty: number; + threshold: number; + /** Whether the verdict counted: its certainty reached the threshold. */ + counted: boolean; + /** For a counted row, whether it passed its polarity. */ + passed: boolean; + reason: string; + /** Set when this classification could not be run. */ + error?: string; +} + +export type TryOutOverall = "approve" | "request-changes"; + +export interface TryOutResult { + state: "running" | "done" | "failed"; + head_sha: string; + rows?: TryOutRow[]; + overall?: TryOutOverall; + error?: string; +} + +const REPO = /^[^/\s]+\/[^/\s]+$/; + +export function repoProblem(repo: string): string | null { + return REPO.test(repo.trim()) ? null : "a repo is owner/name"; +} + +export function prNumber(pr: string): number | null { + const n = Number(pr.trim()); + return /^\d+$/.test(pr.trim()) && n > 0 ? n : null; +} + +/** The threshold as a number, or null when it is not a whole 0–100. */ +export function thresholdValue(text: string): number | null { + const t = text.trim(); + if (!/^\d+$/.test(t)) return null; + const n = Number(t); + return n <= 100 ? n : null; +} + +/** Why a classification is not ready to send, or null when it is. */ +export function draftProblem(d: ClassificationDraft): string | null { + if (d.name.trim() === "") return "give each classification a name"; + if (d.description.trim() === "") { + return `${d.name.trim()} needs a description`; + } + if (thresholdValue(d.threshold) === null) { + return `${d.name.trim()}: the threshold is a whole number from 0 to 100`; + } + return null; +} + +export function toBody(d: ClassificationDraft): ClassificationBody { + return { + name: d.name.trim(), + description: d.description, + polarity: d.polarity, + threshold: thresholdValue(d.threshold) ?? 0, + }; +} diff --git a/frontend/packages/swarm-ui/src/shell/Shell.tsx b/frontend/packages/swarm-ui/src/shell/Shell.tsx index 7989fc6f..324d7ad9 100644 --- a/frontend/packages/swarm-ui/src/shell/Shell.tsx +++ b/frontend/packages/swarm-ui/src/shell/Shell.tsx @@ -51,6 +51,7 @@ const NAV_ITEMS: { href: string; label: string; accent: string }[] = [ { href: "/jobs", label: "jobs", accent: "var(--pink)" }, { href: "/schedules", label: "schedules", accent: "var(--cyan)" }, { href: "/issues", label: "issues", accent: "var(--yellow)" }, + { href: "/review-tryout", label: "review try-out", accent: "var(--red)" }, { href: "/components", label: "components", accent: "var(--blue)" }, ]; @@ -61,7 +62,7 @@ const NAV_ITEMS: { href: string; label: string; accent: string }[] = [ // modifier alongside the base class, not a replacement, so every other // route's layout is untouched. `/agents` joined for the same reason — // its own table view (plus the list+detail split) hits the same cap. -const WIDE_BODY_ROUTES = new Set(["/issues", "/agents"]); +const WIDE_BODY_ROUTES = new Set(["/issues", "/agents", "/review-tryout"]); // `/agents/:name/terminal` (and a subagent's, under // `/agents/:name/subagents/:subagent/`) is full-bleed: the terminal fills