diff --git a/ui/storybook/prototypes/jev-task-creation/JevTaskComposer.tsx b/ui/storybook/prototypes/jev-task-creation/JevTaskComposer.tsx index 2a9e098dc64..961cb8b511b 100644 --- a/ui/storybook/prototypes/jev-task-creation/JevTaskComposer.tsx +++ b/ui/storybook/prototypes/jev-task-creation/JevTaskComposer.tsx @@ -40,6 +40,8 @@ export type JevTaskComposerProps = { jevUnavailable?: boolean; /** Pre-set the assignee as if the user already chose it. Jev never overwrites it. */ presetAssigneeId?: string; + /** Agents to treat as paused. Jev never routes to them and the fallback skips them. */ + pausedAgentIds?: string[]; /** Types `initialPrompt` in character by character to show suggestions arriving. */ typeOnMount?: boolean; mobile?: boolean; @@ -81,9 +83,15 @@ export function JevTaskComposer({ titleLatencyMs = 1200, jevUnavailable = false, presetAssigneeId, + pausedAgentIds, typeOnMount = false, mobile = false, }: JevTaskComposerProps) { + const pausedKey = (pausedAgentIds ?? []).join(","); + const agents = useMemo( + () => JEV_AGENTS.map((agent) => (pausedKey.split(",").includes(agent.id) ? { ...agent, paused: true } : agent)), + [pausedKey], + ); const [prompt, setPrompt] = useState(typeOnMount ? "" : initialPrompt); const [routingStatus, setRoutingStatus] = useState("idle"); const [routing, setRouting] = useState(null); @@ -117,7 +125,7 @@ export function JevTaskComposer({ const ignoreAbort = (error: unknown) => error instanceof DOMException && error.name === "AbortError"; setRoutingStatus("thinking"); - routeTaskWithJev(text, { latencyMs, fail: jevUnavailable, signal: controller.signal }) + routeTaskWithJev(text, { latencyMs, fail: jevUnavailable, signal: controller.signal, agents }) .then((result) => { setRouting(result); setRoutingStatus("ready"); }) .catch((error: unknown) => { if (!ignoreAbort(error)) setRoutingStatus("error"); }); @@ -125,7 +133,7 @@ export function JevTaskComposer({ draftTaskTitle(text, { latencyMs: titleLatencyMs, signal: controller.signal }) .then((title) => { setSuggestedTitle(title); setTitleStatus("ready"); }) .catch((error: unknown) => { if (!ignoreAbort(error)) setTitleStatus("error"); }); - }, [latencyMs, titleLatencyMs, jevUnavailable]); + }, [latencyMs, titleLatencyMs, jevUnavailable, agents]); // Suggest-first: re-run after the user pauses typing. useEffect(() => { @@ -245,6 +253,7 @@ export function JevTaskComposer({ status={routingStatus} promptLength={prompt.trim().length} prompt={createdPrompt || prompt.trim()} + agents={agents} routing={routing} effective={effective} fromJev={fromJev} @@ -382,6 +391,7 @@ function RoutingPanel({ status, promptLength, prompt, + agents, routing, effective, fromJev, @@ -396,6 +406,7 @@ function RoutingPanel({ status: Status; promptLength: number; prompt: string; + agents: typeof JEV_AGENTS; routing: JevRouting | null; effective: Effective; fromJev: (field: Exclude) => boolean; @@ -405,11 +416,17 @@ function RoutingPanel({ onToggleDetails: () => void; onRetry: () => void; }) { - const unsureOwner = routing !== null && fromJev("assignee") && routing.confidence.assignee < JEV_CONFIDENCE_THRESHOLD; + // Below the confidence threshold the task goes to the org's fallback owner (see `fallbackAssigneeId`). + const fallbackOwner = routing?.assigneeSource === "fallback" && fromJev("assignee") + ? JEV_AGENTS.find((agent) => agent.id === routing.assigneeId) ?? null + : null; const statusLine = (() => { if (status === "thinking" && !routing) return created ? `${JEV_NAME} is routing this task…` : `${JEV_NAME} is reading your request…`; - if (unsureOwner) return `${JEV_NAME} is ${percent(routing!.confidence.assignee)} sure about the owner. Pick one:`; + if (fallbackOwner) { + const why = fallbackOwner.reportsTo === null ? "who reports to the board" : "the first agent in your org"; + return `${JEV_NAME} wasn't sure who should own this (${percent(routing!.confidence.assignee)}), so it goes to ${fallbackOwner.name}, ${why}.`; + } if (status === "idle" && flow === "instant" && !created) return `${JEV_NAME} picks the type and owner after you start. You can change anything later.`; if (status === "idle" && promptLength > 0 && promptLength < JEV_MIN_PROMPT_LENGTH) return "Keep going. Suggestions start once there's a bit more to go on."; return null; @@ -466,11 +483,11 @@ function RoutingPanel({ {assignee.name} : Assignee} > - {(close) => JEV_AGENTS.map((agent) => ( + {(close) => agents.map((agent) => ( {agent.name} - {agent.title} + + {agent.title}{agent.paused ? " · Paused" : ""}{agent.id === fallbackOwner?.id ? " · Fallback owner" : ""} + ))} @@ -525,9 +544,9 @@ function RoutingPanel({ ) : null} - {unsureOwner && alternates.length > 0 ? ( + {fallbackOwner && alternates.length > 0 ? (
- Or: + {JEV_NAME}'s best guesses: {alternates.map((agent) => ( - {detailsOpen ? : null} + {detailsOpen ? : null}
) : null} @@ -554,12 +573,17 @@ const FIELD_LABELS: Record = { type: "Type", assignee: "Assign function choiceLabel(field: JevField, routing: JevRouting): string { if (field === "type") return JEV_TASK_TYPES.find((option) => option.value === routing.type)?.label ?? routing.type; - if (field === "assignee") return JEV_AGENTS.find((agent) => agent.id === routing.assigneeId)?.name ?? routing.assigneeId; + if (field === "assignee") { + const name = (id: string) => JEV_AGENTS.find((agent) => agent.id === id)?.name ?? id; + return routing.assigneeSource === "fallback" + ? `${name(routing.assigneeId)} (fallback; ${JEV_NAME} leaned ${name(routing.jevAssigneeId)})` + : name(routing.assigneeId); + } if (field === "project") return JEV_PROJECTS.find((project) => project.id === routing.projectId)?.name ?? "No project"; return workModeMetaFor(routing.workMode).label; } -function ConfidenceDetails({ routing, prompt }: { routing: JevRouting; prompt: string }) { +function ConfidenceDetails({ routing, prompt, agents }: { routing: JevRouting; prompt: string; agents: typeof JEV_AGENTS }) { const fields: JevField[] = ["type", "assignee", "project", "workMode"]; return (
@@ -593,7 +617,7 @@ function ConfidenceDetails({ routing, prompt }: { routing: JevRouting; prompt: s
Request sent to {JEV_NAME} 
-          {`POST https://openrouter.ai/api/alpha/decisions\n${JSON.stringify(buildJevDecisionRequest(prompt), null, 2)}`}
+          {`POST https://openrouter.ai/api/alpha/decisions\n${JSON.stringify(buildJevDecisionRequest(prompt, agents), null, 2)}`}
diff --git a/ui/storybook/prototypes/jev-task-creation/README.md b/ui/storybook/prototypes/jev-task-creation/README.md index 98b1585a2b6..057674ec02e 100644 --- a/ui/storybook/prototypes/jev-task-creation/README.md +++ b/ui/storybook/prototypes/jev-task-creation/README.md @@ -33,8 +33,13 @@ Two models do the work, because they are good at different things: - Suggested values carry a sparkle. When you change a value, the sparkle goes away and the value is never overwritten. The undo icon restores the suggestion. - Every chip menu shows Jev's probability for each option. -- Below the confidence threshold (`JEV_CONFIDENCE_THRESHOLD`, 0.6), the composer - says how sure Jev is and offers the next likely owners as one-click buttons. +- **Low-confidence fallback.** When Jev's assignee confidence is below + `JEV_CONFIDENCE_THRESHOLD` (0.6), the task goes to the org's first active agent + that reports to the board (`reportsTo: null`, earliest `createdAt`). If no + active agent reports to the board, it goes to the org's first active agent. + Paused agents are skipped, and Jev never sees them as options. The composer says + so in one line and offers Jev's best guesses as one-click buttons + (`fallbackAssigneeId` in `jev-classifier.ts`; stories 09 and 09b). - **How sure Jev is** lists the confidence for each decision, the token cost, and the exact request body sent to Jev. - If Jev fails, the title still arrives, the chips work as manual pickers, and @@ -70,15 +75,14 @@ too (`draftTitle`). No API is called and no task is persisted. instance's OpenRouter key, and a small chat model for the title, in parallel. Keep the key on the server. 2. Return a `JevRouting`-shaped payload, so this UI doesn't change. -3. Record the chosen values and Jev's confidences on the created task's activity - entry, so the board can audit routing decisions. +3. Record the chosen values, Jev's confidences, and whether the fallback owner + was used on the created task's activity entry, so the board can audit routing + decisions. 4. Label 100–200 real tasks and tune the confidence threshold against them, as the OpenRouter classification cookbook recommends. ## Open questions -- Should a task auto-assign in the instant flow even when Jev's confidence is - low, or wait for the user to pick? - Does task **type** need to be a real field? Today `IssueWorkMode` and labels cover part of it. - Should Jev's criteria for each agent come from agent instructions or role diff --git a/ui/storybook/prototypes/jev-task-creation/jev-classifier.ts b/ui/storybook/prototypes/jev-task-creation/jev-classifier.ts index 7e8d56617c8..a38e253cfcd 100644 --- a/ui/storybook/prototypes/jev-task-creation/jev-classifier.ts +++ b/ui/storybook/prototypes/jev-task-creation/jev-classifier.ts @@ -19,7 +19,7 @@ import type { IssueWorkMode } from "@paperclipai/shared"; export const JEV_NAME = "Jev"; export const JEV_MODEL = "typesafe/jev-1.13"; -/** Below this confidence, the UI asks the user to pick between likely owners. */ +/** Below this assignee confidence, the task goes to the fallback owner (see `fallbackAssigneeId`). */ export const JEV_CONFIDENCE_THRESHOLD = 0.6; /** Prompts need a little substance before routing starts. */ export const JEV_MIN_PROMPT_LENGTH = 18; @@ -59,14 +59,24 @@ export const JEV_TASK_TYPES: { value: JevTaskType; label: string; hint: string } { value: "ops", label: "Ops", hint: "Admin, release, or coordination work" }, ]; -export type JevAgent = { id: string; name: string; role: string; title: string; owns: string }; +export type JevAgent = { + id: string; + name: string; + role: string; + title: string; + owns: string; + /** `null` means the agent reports directly to the board. */ + reportsTo: string | null; + createdAt: string; + paused?: boolean; +}; export const JEV_AGENTS: JevAgent[] = [ - { id: "agent-codex", name: "CodexCoder", role: "engineer", title: "Senior Product Engineer", owns: "Product engineering: bugs and features in the app and API." }, - { id: "agent-design-system", name: "DesignSystemCoder", role: "designer", title: "Design System Engineer", owns: "UI, UX, visual polish, and the design system." }, - { id: "agent-qa", name: "QAChecker", role: "qa", title: "QA Engineer", owns: "Verification, reproduction, and release testing." }, - { id: "agent-cto", name: "CTO", role: "cto", title: "CTO", owns: "Architecture decisions and open-ended technical questions." }, - { id: "agent-darnold", name: "Darnold", role: "general", title: "Chief of Staff", owns: "Coordination, admin, and operational follow-ups." }, + { id: "agent-codex", name: "CodexCoder", role: "engineer", title: "Senior Product Engineer", owns: "Product engineering: bugs and features in the app and API.", reportsTo: "agent-cto", createdAt: "2026-04-02T09:00:00Z" }, + { id: "agent-design-system", name: "DesignSystemCoder", role: "designer", title: "Design System Engineer", owns: "UI, UX, visual polish, and the design system.", reportsTo: "agent-cto", createdAt: "2026-04-06T09:00:00Z" }, + { id: "agent-qa", name: "QAChecker", role: "qa", title: "QA Engineer", owns: "Verification, reproduction, and release testing.", reportsTo: "agent-cto", createdAt: "2026-04-03T09:00:00Z" }, + { id: "agent-cto", name: "CTO", role: "cto", title: "CTO", owns: "Architecture decisions and open-ended technical questions.", reportsTo: null, createdAt: "2026-04-01T09:00:00Z" }, + { id: "agent-darnold", name: "Darnold", role: "general", title: "Chief of Staff", owns: "Coordination, admin, and operational follow-ups.", reportsTo: null, createdAt: "2026-04-10T09:00:00Z" }, ]; export type JevProject = { id: string; name: string; description: string }; @@ -82,12 +92,14 @@ export const JEV_NO_PROJECT = "none"; // --------------------------------------------------------------------------- // Request builder: this is what a server would send to OpenRouter. -export function buildJevDecisionRequest(prompt: string): JevDecisionRequest { +export function buildJevDecisionRequest(prompt: string, agents: JevAgent[] = JEV_AGENTS): JevDecisionRequest { + // Paused agents can't take work, so Jev never sees them as options. + const available = agents.filter((agent) => !agent.paused); return { model: JEV_MODEL, state: { task_prompt: prompt, - agents: JEV_AGENTS.map(({ id, name, title, owns }) => ({ id, name, title, owns })), + agents: available.map(({ id, name, title, owns }) => ({ id, name, title, owns })), projects: JEV_PROJECTS.map(({ id, name, description }) => ({ id, name, description })), }, questions: { @@ -99,7 +111,7 @@ export function buildJevDecisionRequest(prompt: string): JevDecisionRequest { assignee: { type: "choice", instructions: "Which agent should own this task?", - criteria: Object.fromEntries(JEV_AGENTS.map((agent) => [agent.id, `${agent.name}, ${agent.title}. ${agent.owns}`])), + criteria: Object.fromEntries(available.map((agent) => [agent.id, `${agent.name}, ${agent.title}. ${agent.owns}`])), }, project: { type: "choice", @@ -186,10 +198,14 @@ export function simulateJevDecisions(request: JevDecisionRequest): JevDecisionRe if (Object.values(typeScores).every((value) => value === 0)) typeScores.research = 0.6; const typeProbabilities = softmax(typeScores); - const assigneeProbabilities: Record = Object.fromEntries(JEV_AGENTS.map((agent) => [agent.id, 0])); + const assigneeQuestion = request.questions.assignee as JevChoiceQuestion; + const assigneeProbabilities: Record = Object.fromEntries(Object.keys(assigneeQuestion.criteria).map((id) => [id, 0.02])); for (const [type, probability] of Object.entries(typeProbabilities)) { - assigneeProbabilities[TYPE_OWNER[type as JevTaskType]]! += probability; + const owner = TYPE_OWNER[type as JevTaskType]; + if (owner in assigneeProbabilities) assigneeProbabilities[owner]! += probability; } + const assigneeTotal = Object.values(assigneeProbabilities).reduce((sum, value) => sum + value, 0); + for (const id of Object.keys(assigneeProbabilities)) assigneeProbabilities[id]! /= assigneeTotal; const projectScores: Record = { ...Object.fromEntries(Object.entries(PROJECT_KEYWORDS).map(([id, keywords]) => [id, score(text, keywords)])), @@ -219,27 +235,44 @@ export function simulateJevDecisions(request: JevDecisionRequest): JevDecisionRe export type JevField = "type" | "assignee" | "project" | "workMode"; +/** + * Who gets a task when Jev isn't confident about the owner: the org's first + * active agent that reports to the board, or else its first active agent. + */ +export function fallbackAssigneeId(agents: JevAgent[]): string | null { + const active = agents.filter((agent) => !agent.paused).sort((a, b) => a.createdAt.localeCompare(b.createdAt)); + return (active.find((agent) => agent.reportsTo === null) ?? active[0])?.id ?? null; +} + export type JevRouting = { type: JevTaskType; + /** The owner to use: Jev's pick, or the fallback when Jev is unsure. */ assigneeId: string; + assigneeSource: "jev" | "fallback"; + /** Jev's own top pick, kept even when the fallback is used. */ + jevAssigneeId: string; projectId: string | null; workMode: IssueWorkMode; confidence: Record; probabilities: Record>; - /** Likely owners other than the top pick, shown when assignee confidence is low. */ + /** Jev's likely owners other than the one assigned, offered when Jev is unsure. */ alternateAssigneeIds: string[]; usage: JevDecisionResponse["usage"]; }; -export function routingFromJev(response: JevDecisionResponse): JevRouting { +export function routingFromJev(response: JevDecisionResponse, agents: JevAgent[] = JEV_AGENTS): JevRouting { const { task_type: type, assignee, project, work_mode: mode } = response.answers as Record; + const fallbackId = assignee!.confidence < JEV_CONFIDENCE_THRESHOLD ? fallbackAssigneeId(agents) : null; + const assigneeId = fallbackId ?? assignee!.choice; const alternateAssigneeIds = Object.entries(assignee!.probabilities) - .filter(([id, probability]) => id !== assignee!.choice && probability >= 0.1) + .filter(([id, probability]) => id !== assigneeId && probability >= 0.1) .slice(0, 2) .map(([id]) => id); return { type: type!.choice as JevTaskType, - assigneeId: assignee!.choice, + assigneeId, + assigneeSource: fallbackId ? "fallback" : "jev", + jevAssigneeId: assignee!.choice, projectId: project!.choice === JEV_NO_PROJECT ? null : project!.choice, workMode: mode!.choice as IssueWorkMode, confidence: { type: type!.confidence, assignee: assignee!.confidence, project: project!.confidence, workMode: mode!.confidence }, @@ -250,6 +283,7 @@ export function routingFromJev(response: JevDecisionResponse): JevRouting { } export type LatencyOptions = { latencyMs?: number; fail?: boolean; signal?: AbortSignal }; +export type RouteOptions = LatencyOptions & { agents?: JevAgent[] }; function delay(produce: () => T, { latencyMs = 400, fail = false, signal }: LatencyOptions, failure: string): Promise { return new Promise((resolve, reject) => { @@ -262,8 +296,8 @@ function delay(produce: () => T, { latencyMs = 400, fail = false, signal }: L } /** Jev is a fast decision model, so routing usually lands before the title. */ -export function routeTaskWithJev(prompt: string, options: LatencyOptions = {}): Promise { - return delay(() => routingFromJev(simulateJevDecisions(buildJevDecisionRequest(prompt))), options, `${JEV_NAME} is unavailable right now.`); +export function routeTaskWithJev(prompt: string, { agents = JEV_AGENTS, ...options }: RouteOptions = {}): Promise { + return delay(() => routingFromJev(simulateJevDecisions(buildJevDecisionRequest(prompt, agents)), agents), options, `${JEV_NAME} is unavailable right now.`); } // --------------------------------------------------------------------------- @@ -275,7 +309,7 @@ const CURATED_TITLES: Record = { "should we move": "Evaluate moving heartbeats to a queue", "the new task dialog feels": "Tighten spacing in the new task dialog", "before friday's release": "Run release smoke tests before Friday", - "something is off": "Investigate last month's numbers", + "the invite emails": "Sort out the invite emails", "rotate the": "Rotate the GitHub App private key", "plan the migration": "Plan phased migration of runtime sessions to the new adapter", }; diff --git a/ui/storybook/stories/jev-task-creation.stories.tsx b/ui/storybook/stories/jev-task-creation.stories.tsx index 1d3991edf8a..d23cf5dc7e3 100644 --- a/ui/storybook/stories/jev-task-creation.stories.tsx +++ b/ui/storybook/stories/jev-task-creation.stories.tsx @@ -8,7 +8,7 @@ const PROMPTS = { research: "Should we move heartbeat scheduling onto a queue? How would that change wake latency for agents?", design: "The new task dialog feels cramped on mobile. The spacing and typography around the property chips need a design pass.", qa: "Before Friday's release, run the smoke tests and verify that invite links still work on authenticated private mode.", - ambiguous: "Something is off with the numbers for last month, can you take a look", + ambiguous: "The invite emails look wrong, can you sort it out", planning: "Plan the migration of agent runtime sessions to the new adapter, in phases, so nothing breaks mid-run.", }; @@ -23,7 +23,7 @@ const meta = { "Prototype: prompt-first task creation. You write what you need. Jev (`typesafe/jev-1.13`, TypeSafe's decision model on OpenRouter) answers " + "`choice` questions for task type, assignee, project, and work mode, with a confidence and per-option probabilities. Jev doesn't generate text, so a separate small text model drafts the title, like Claude Code naming a session. " + "Suggested values carry a sparkle. Anything you change is yours and is never overwritten; the undo icon restores the suggestion. " + - "Below the confidence threshold, likely alternate owners appear as one-click buttons. **Instant** stories create first; the title and routing arrive a moment later. " + + "Below the confidence threshold the task goes to the org's first active agent that reports to the board (else its first active agent), and Jev's best guesses appear as one-click buttons. **Instant** stories create first; the title and routing arrive a moment later. " + "Jev's answers are simulated locally in the documented response shape (`prototypes/jev-task-creation/jev-classifier.ts`); **How sure Jev is → Request sent to Jev** shows the real request body. No API is called and no task is persisted.", }, }, @@ -51,9 +51,13 @@ export const Design: Story = { name: "06 · Design → DesignSystemCoder", args: export const QA: Story = { name: "07 · QA → QAChecker", args: { initialPrompt: PROMPTS.qa } }; export const LargeScope: Story = { name: "08 · Large scope → Plan mode", args: { initialPrompt: PROMPTS.planning } }; export const LowConfidence: Story = { - name: "09 · Unsure · Jev offers alternate owners", + name: "09 · Unsure · Goes to the agent reporting to the board", args: { initialPrompt: PROMPTS.ambiguous }, }; +export const LowConfidenceBoardAgentPaused: Story = { + name: "09b · Unsure, CTO paused · Next board-reporting agent", + args: { initialPrompt: PROMPTS.ambiguous, pausedAgentIds: ["agent-cto"] }, +}; export const UserOverride: Story = { name: "10 · You chose the assignee · Jev keeps it", args: { initialPrompt: PROMPTS.bug, presetAssigneeId: "agent-qa" },