From faff66d41d43c8339dd33f835043a09fa9837e7b Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 21:48:50 +0000 Subject: [PATCH] storybook: describe agents to Jev from stored Paperclip context Build each agent's assignee criteria from the context Paperclip already stores, highest fidelity first: title and role, capabilities, project leads, assigned skills, recent finished tasks, and customized instruction excerpts. Empty sources are skipped and the default instructions template is never sent. The details panel reports which sources were available and token estimates now come from the request size. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01SWn6etZjfGz6p8NWYkCQve --- .../jev-task-creation/JevTaskComposer.tsx | 22 ++- .../prototypes/jev-task-creation/README.md | 31 +++- .../jev-task-creation/jev-classifier.ts | 142 ++++++++++++++++-- 3 files changed, 175 insertions(+), 20 deletions(-) diff --git a/ui/storybook/prototypes/jev-task-creation/JevTaskComposer.tsx b/ui/storybook/prototypes/jev-task-creation/JevTaskComposer.tsx index e16d76b6347..bb369366be1 100644 --- a/ui/storybook/prototypes/jev-task-creation/JevTaskComposer.tsx +++ b/ui/storybook/prototypes/jev-task-creation/JevTaskComposer.tsx @@ -15,6 +15,7 @@ import { JEV_NAME, JEV_NO_PROJECT, JEV_PROJECTS, + agentContextCoverage, buildJevDecisionRequest, draftTaskTitle, routeTaskWithJev, @@ -585,12 +586,13 @@ function ConfidenceDetails({ routing, prompt, agents }: { routing: JevRouting; p {JEV_MODEL} returns a choice and probabilities, not reasons. Open a chip to see every option's probability. Anything you change stays as you set it.

+ !agent.paused)} />

{routing.usage.input_tokens} input tokens · ${routing.usage.cost.toFixed(6)} · output tokens are free

Request sent to {JEV_NAME} - 
+        
           {`POST https://openrouter.ai/api/alpha/decisions\n${JSON.stringify(buildJevDecisionRequest(prompt, agents), null, 2)}`}
@@ -598,6 +600,24 @@ function ConfidenceDetails({ routing, prompt, agents }: { routing: JevRouting; p ); } +/** Which agent context sources fed the assignee decision, highest fidelity first. */ +function AgentContextLine({ agents }: { agents: typeof JEV_AGENTS }) { + const coverage = agentContextCoverage(agents); + const sources = [ + ["finished tasks", coverage.history], + ["project leads", coverage.leads], + ["custom instructions", coverage.instructions], + ["skills", coverage.skills], + ["capabilities", coverage.capabilities], + ] as const; + return ( +

+ Agent context sent: title and role for all {coverage.total} + {sources.map(([label, count]) => `, ${label} for ${count}`).join("")}. +

+ ); +} + function PropertyChip({ label, display, diff --git a/ui/storybook/prototypes/jev-task-creation/README.md b/ui/storybook/prototypes/jev-task-creation/README.md index 2f2e7865c28..497e6c6c2af 100644 --- a/ui/storybook/prototypes/jev-task-creation/README.md +++ b/ui/storybook/prototypes/jev-task-creation/README.md @@ -63,9 +63,29 @@ Two models do the work, because they are good at different things: ## What's real and what's simulated `jev-classifier.ts` builds the real Decisions API request body -(`buildJevDecisionRequest`): the prompt, the company's agents, and its projects go -in `state`, with four `choice` questions. The request and response types follow -the OpenRouter docs. +(`buildJevDecisionRequest`): the prompt, the company's agents and projects in +`state`, and three `choice` questions. The request and response types follow the +OpenRouter docs. + +Each agent's option in the assignee question is described by +`describeAgentForJev`. It uses the context Paperclip already stores, highest +fidelity first, and skips any source that is empty: + +1. Name, title, and role. These are always present, but rarely decide anything. +2. `agents.capabilities`, when set. Built-in agents and imports fill it; the + new-agent flow can't, so hand-made agents usually lack it. +3. Projects it leads (`projects.leadAgentId`). +4. Assigned skills (`adapterConfig.paperclipSkillSync`). +5. Its last 10 finished tasks (`issues.assigneeAgentId`, status done). This is + the strongest signal, because it shows what the agent really does. +6. An excerpt of its `AGENTS.md` instructions, only when customized. The default + template says nothing about the agent, so it is left out. + +The fixtures have realistic gaps: CodexCoder is hand-made with no capabilities +and default instructions, the CTO comes from the teams catalog, and Darnold is a +built-in agent. **How sure Jev is** says which sources were available, and +**Request sent to Jev** shows the full body. Five agents come to roughly 900 +input tokens, far under Jev's 32K limit. The response is simulated (`simulateJevDecisions`): keyword signals go through a softmax to produce probabilities in Jev's response shape. The title is simulated @@ -88,5 +108,6 @@ too (`draftTitle`). No API is called and no task is persisted. ## Open questions -- Should Jev's criteria for each agent come from agent instructions or role - metadata, rather than a hand-written "owns" line? +- Should the new-agent flow and the configuration page let people set + `capabilities`? It is the one field built for describing an agent, and today + hand-made agents can't set it. diff --git a/ui/storybook/prototypes/jev-task-creation/jev-classifier.ts b/ui/storybook/prototypes/jev-task-creation/jev-classifier.ts index 6cb6745e916..7e1a40d68f7 100644 --- a/ui/storybook/prototypes/jev-task-creation/jev-classifier.ts +++ b/ui/storybook/prototypes/jev-task-creation/jev-classifier.ts @@ -49,12 +49,29 @@ export type JevDecisionResponse = { // --------------------------------------------------------------------------- // Company fixtures (match the Storybook company) +/** + * An agent as Paperclip stores it, reduced to the fields that describe what it + * does. Ranked by how well they separate agents for routing: + * + * 1. `recentDoneTaskTitles`: issues it finished (`issues.assigneeAgentId`, status done). + * 2. Projects it leads (`projects.leadAgentId`, on `JevProject`). + * 3. `customInstructionsExcerpt`: its `AGENTS.md` bundle, only when customized. + * 4. `skills`: assigned company skills (`adapterConfig.paperclipSkillSync`). + * 5. `capabilities`: free text (`agents.capabilities`). Set by built-in agents and + * imports; the new-agent flow can't set it, so hand-made agents usually lack it. + * 6. `name`, `title`, `role`: always present, rarely decisive. + */ export type JevAgent = { id: string; name: string; role: string; title: string; - owns: string; + capabilities: string | null; + skills: string[]; + /** Null while the agent still uses the default instructions template, which says nothing about it. */ + customInstructionsExcerpt: string | null; + /** Most recent first. Empty for new agents. */ + recentDoneTaskTitles: string[]; /** `null` means the agent reports directly to the board. */ reportsTo: string | null; createdAt: string; @@ -62,21 +79,116 @@ export type JevAgent = { }; export const JEV_AGENTS: JevAgent[] = [ - { id: "agent-codex", name: "CodexCoder", role: "engineer", title: "Senior Product Engineer", owns: "Product engineering: bugs and features in the app and API.", reportsTo: "agent-cto", createdAt: "2026-04-02T09:00:00Z" }, - { id: "agent-design-system", name: "DesignSystemCoder", role: "designer", title: "Design System Engineer", owns: "UI, UX, visual polish, and the design system.", reportsTo: "agent-cto", createdAt: "2026-04-06T09:00:00Z" }, - { id: "agent-qa", name: "QAChecker", role: "qa", title: "QA Engineer", owns: "Verification, reproduction, and release testing.", reportsTo: "agent-cto", createdAt: "2026-04-03T09:00:00Z" }, - { id: "agent-cto", name: "CTO", role: "cto", title: "CTO", owns: "Architecture decisions and open-ended technical questions.", reportsTo: null, createdAt: "2026-04-01T09:00:00Z" }, - { id: "agent-darnold", name: "Darnold", role: "general", title: "Chief of Staff", owns: "Coordination, admin, and operational follow-ups.", reportsTo: null, createdAt: "2026-04-10T09:00:00Z" }, + { + id: "agent-codex", + name: "CodexCoder", + role: "engineer", + title: "Senior Product Engineer", + // Hand-created: no capabilities, default instructions. History carries it. + capabilities: null, + skills: ["paperclip", "github-pr-workflow"], + customInstructionsExcerpt: null, + recentDoneTaskTitles: [ + "Fix inbox unread badge after archiving", + "Retry issue checkout on 409 conflicts", + "Paginate the activity feed API", + "Fix duplicate toast on task create", + ], + reportsTo: "agent-cto", + createdAt: "2026-04-02T09:00:00Z", + }, + { + id: "agent-design-system", + name: "DesignSystemCoder", + role: "designer", + title: "Design System Engineer", + capabilities: "Owns the design system: tokens, shared components, and Storybook coverage.", + skills: ["paperclip", "design-guide"], + customInstructionsExcerpt: "You maintain the token layer in ui/src/index.css and the shared components. Every UI change must pass the token gates. Review spacing, typography, and dark mode on every surface you touch.", + recentDoneTaskTitles: ["Tokenize spacing in dialogs", "Add dark-mode stories for badges", "Tighten mobile layout of the inbox"], + reportsTo: "agent-cto", + createdAt: "2026-04-06T09:00:00Z", + }, + { + id: "agent-qa", + name: "QAChecker", + role: "qa", + title: "QA Engineer", + capabilities: null, + skills: ["paperclip", "release-smoke"], + customInstructionsExcerpt: null, + recentDoneTaskTitles: ["Run release smoke for v1.14", "Reproduce invite link 404 in private mode", "Verify budget hard-stop pauses agents"], + reportsTo: "agent-cto", + createdAt: "2026-04-03T09:00:00Z", + }, + { + id: "agent-cto", + name: "CTO", + role: "cto", + title: "Chief Technology Officer", + // Hired from the teams catalog: catalog instructions and skills, no capabilities text. + capabilities: null, + skills: ["github-pr-workflow", "task-planning", "doc-maintenance"], + customInstructionsExcerpt: "You own technical direction. Break large work into planned child issues for the engineers, review architecture proposals, and answer technical questions from the board.", + recentDoneTaskTitles: ["Decide between a queue and cron for routines", "Review the adapter plugin API proposal"], + reportsTo: null, + createdAt: "2026-04-01T09:00:00Z", + }, + { + id: "agent-darnold", + name: "Darnold", + role: "general", + title: "Chief of Staff", + // Built-in agent: capabilities come from its short purpose. + capabilities: "Prepares concise operational briefs for the board and agent company.", + skills: ["paperclip"], + customInstructionsExcerpt: null, + recentDoneTaskTitles: ["Weekly board brief", "Follow up on overdue approvals", "Send the invoice reminder to finance"], + reportsTo: null, + createdAt: "2026-04-10T09:00:00Z", + }, ]; -export type JevProject = { id: string; name: string; description: string }; +export type JevProject = { id: string; name: string; description: string; leadAgentId: string | null }; export const JEV_PROJECTS: JevProject[] = [ - { id: "project-board-ui", name: "Board UI", description: "The operator-facing web app: pages, dialogs, inbox, mobile." }, - { id: "project-agent-runtime", name: "Agent Runtime", description: "Adapters, heartbeats, runners, and agent sessions." }, - { id: "project-budget-guardrails", name: "Budget Guardrails", description: "Spend limits, cost tracking, and billing." }, + { id: "project-board-ui", name: "Board UI", description: "The operator-facing web app: pages, dialogs, inbox, mobile.", leadAgentId: "agent-codex" }, + { id: "project-agent-runtime", name: "Agent Runtime", description: "Adapters, heartbeats, runners, and agent sessions.", leadAgentId: "agent-cto" }, + { id: "project-budget-guardrails", name: "Budget Guardrails", description: "Spend limits, cost tracking, and billing.", leadAgentId: null }, ]; +/** How many finished task titles to send per agent. */ +export const JEV_RECENT_TASKS_PER_AGENT = 10; +const INSTRUCTIONS_EXCERPT_MAX = 280; + +/** One line per agent for Jev's choice criteria, highest-fidelity context first; empty sources are skipped. */ +export function describeAgentForJev(agent: JevAgent, projects: JevProject[] = JEV_PROJECTS): string { + const leads = projects.filter((project) => project.leadAgentId === agent.id).map((project) => project.name); + const recent = agent.recentDoneTaskTitles.slice(0, JEV_RECENT_TASKS_PER_AGENT); + const instructions = agent.customInstructionsExcerpt?.slice(0, INSTRUCTIONS_EXCERPT_MAX); + return [ + `${agent.name}, ${agent.title} (${agent.role}).`, + agent.capabilities ? agent.capabilities : null, + leads.length ? `Leads ${leads.join(", ")}.` : null, + agent.skills.length ? `Skills: ${agent.skills.join(", ")}.` : null, + recent.length ? `Recently finished: ${recent.map((title) => `"${title}"`).join("; ")}.` : null, + instructions ? `Instructions: ${instructions}` : null, + ].filter(Boolean).join(" "); +} + +/** Which context sources were available for the agents sent to Jev. */ +export function agentContextCoverage(agents: JevAgent[], projects: JevProject[] = JEV_PROJECTS) { + const count = (predicate: (agent: JevAgent) => boolean) => agents.filter(predicate).length; + return { + total: agents.length, + history: count((agent) => agent.recentDoneTaskTitles.length > 0), + leads: count((agent) => projects.some((project) => project.leadAgentId === agent.id)), + instructions: count((agent) => agent.customInstructionsExcerpt !== null), + skills: count((agent) => agent.skills.length > 0), + capabilities: count((agent) => agent.capabilities !== null), + }; +} + export const JEV_NO_PROJECT = "none"; // --------------------------------------------------------------------------- @@ -89,8 +201,9 @@ export function buildJevDecisionRequest(prompt: string, agents: JevAgent[] = JEV model: JEV_MODEL, state: { task_prompt: prompt, - agents: available.map(({ id, name, title, owns }) => ({ id, name, title, owns })), - projects: JEV_PROJECTS.map(({ id, name, description }) => ({ id, name, description })), + // Each agent's full description lives in the assignee criteria, so it is sent once. + agents: available.map(({ id, name, title, reportsTo }) => ({ id, name, title, reports_to: reportsTo ?? "board" })), + projects: JEV_PROJECTS.map(({ id, name, description, leadAgentId }) => ({ id, name, description, lead_agent_id: leadAgentId })), }, questions: { work_mode: { @@ -105,7 +218,7 @@ export function buildJevDecisionRequest(prompt: string, agents: JevAgent[] = JEV assignee: { type: "choice", instructions: "Which agent should own this task?", - criteria: Object.fromEntries(available.map((agent) => [agent.id, `${agent.name}, ${agent.title}. ${agent.owns}`])), + criteria: Object.fromEntries(available.map((agent) => [agent.id, describeAgentForJev(agent)])), }, project: { type: "choice", @@ -205,7 +318,8 @@ export function simulateJevDecisions(request: JevDecisionRequest): JevDecisionRe const isInvestigation = /^(investigate|research|find out|figure out|look into|compare|evaluate)\b/.test(text); const modeScores = { standard: 1, planning: isLarge ? 3 : 0, ask: isQuestion ? 3 : isInvestigation ? 1.6 : 0 }; - const inputTokens = 380 + Math.round(prompt.length / 4); + // Rough estimate: about four characters per token. + const inputTokens = Math.round(JSON.stringify(request).length / 4); return { model: request.model, answers: {