mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-04 19:28:46 +01:00
test: add finite scoped external assistant catalog surface
This commit is contained in:
1 parent
0a97b24b64
commit
01df721473
8 files changed
+147
-6
No files matched your search
@@ -31,6 +31,12 @@ export interface ChatCatalogQueryAttachment {
|
||||
stateKey: string;
|
||||
filename: string;
|
||||
}
|
||||
export interface ChatCompanionSurface { kind: "scoped-external-assistant"; model: string; maximumModelTurns: 6; maximumToolCalls: 4; maximumOutputTokens: 2048 }
|
||||
export interface ChatCompanionResult {
|
||||
query: string; reply: string; model: string; usage: { inputTokens: number; outputTokens: number };
|
||||
messages: Array<{ role: "system" | "user" | "assistant" | "tool"; content: string }>;
|
||||
providerResponseIds: string[]; toolCalls: Array<{ name: string; arguments: unknown; result: unknown }>; links: string[];
|
||||
}
|
||||
/** One concrete scenario, with production API setup and a case-specific independent oracle. */
|
||||
export interface ChatCatalogCase {
|
||||
scenarioId: string;
|
||||
@@ -40,7 +46,9 @@ export interface ChatCatalogCase {
|
||||
requiredCredentials?: readonly CredentialName[];
|
||||
externalInput?: ChatExternalInputRequirement;
|
||||
installedSource?: "published-global-npm";
|
||||
journeyKind?: "agent-chat";
|
||||
journeyKind?: "agent-chat" | "scoped-external-assistant";
|
||||
companionSurface?: ChatCompanionSurface;
|
||||
runCompanionSurface?(context: ChatCatalogContext, state: Record<string, unknown>, input: { query: string; signal: AbortSignal }): Promise<ChatCompanionResult>;
|
||||
authentication?: "local-trusted" | "human-session";
|
||||
setupPolicy?: ChatCatalogSetupPolicy;
|
||||
runPolicy?: ChatCatalogRunPolicy;
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
]
|
||||
},
|
||||
"tests/runner-e2e/public-mcp-client.ts": {
|
||||
"sourceSha256": "768aed950585ee72f32ea204a451bcfe67264bb0c8890371f12caa856e4871a0",
|
||||
"sourceSha256": "21ec5385189444b3615196ca9168fd5484f1b8d6fa30b027aed36214d6c0f39a",
|
||||
"resolution": "pinned-external-mcp-sdk",
|
||||
"dependencies": [
|
||||
"server/package.json"
|
||||
|
||||
@@ -88,6 +88,7 @@ export function assertChatCatalogFingerprintLane(selected: unknown, env: NodeJS.
|
||||
}
|
||||
|
||||
export function chatCatalogCaseFingerprint(descriptor: ChatCatalogDescriptor, selectedExecution: unknown, retainedPrivateInputNames?: ReturnType<typeof chatPrivateInputNameIdentity>, retainedInstalledIdentity?: Omit<ReturnType<typeof installedChatFingerprintIdentity>, "observedProcess">) {
|
||||
selectedExecution = { ...(selectedExecution as Record<string, unknown>), journeyKind: descriptor.journeyKind ?? "agent-chat" };
|
||||
const root = fileURLToPath(new URL("../../", import.meta.url));
|
||||
const privateInputNameIdentity = retainedPrivateInputNames === undefined ? chatPrivateInputNameIdentity((selectedExecution as { executionId: string }).executionId) : retainedPrivateInputNames;
|
||||
const lane = assertChatCatalogFingerprintLane({ ...(selectedExecution as Record<string, unknown>), scenarioId: descriptor.scenarioId, installedSource: descriptor.installedSource });
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { runScopedCompanionSurface } from "./chat-companion-surface.js";
|
||||
import { chatQueryWithAttachments, resolveChatQueryAttachments, verifyChatQueryAttachments, type ChatQueryAttachmentReceipt } from "./chat-query-attachments.js";
|
||||
import { chatCatalogAllowedRunStatuses, gradeChatCatalogRuns } from "./chat-catalog-run-policy.js";
|
||||
import { chatCatalogCaseFingerprint, normalizedChatCatalogExecution } from "./chat-catalog-fingerprint.js";
|
||||
@@ -16,6 +17,7 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) {
|
||||
if (!descriptor) throw new Error(`No executable chat scenario: ${scenario.id}`);
|
||||
const fingerprint = chatCatalogCaseFingerprint(descriptor, normalizedChatCatalogExecution(input.execution));
|
||||
const implementation = await loadChatCatalogImplementation(descriptor);
|
||||
if (implementation.companionSurface) return runScopedCompanionSurface(context, implementation, fingerprint);
|
||||
const deadlineAt = Date.now() + implementation.attemptTimeoutMs - 30_000;
|
||||
let state: Record<string, unknown> | undefined;
|
||||
let observation: unknown;
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { validCompanionDeclaration } from "./chat-companion-surface.js";
|
||||
import { installedChatScenarioIds } from "./chat-installed-lane.js";
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { validChatCatalogHumanCounts } from "./chat-catalog-contract.js";
|
||||
@@ -6,8 +7,8 @@ import { chatCatalogScenarios } from "./chat-catalog-inventory.js";
|
||||
import type { IdentityBoundaryCase } from "./chat-catalog-domains/identity-boundary-contract.js";
|
||||
import { identityBoundaryScenarioIds } from "./chat-catalog-domains/identity-boundary-contract.js";
|
||||
|
||||
export type ChatCatalogDescriptor = Omit<ChatCatalogCase, "journeyKind" | "setup" | "act" | "observe" | "grade" | "cleanup" | "verifyResult" | "verifySetup" | "verifyQueryRejection"> & {
|
||||
journeyKind?: "agent-chat" | "browser-boundary";
|
||||
export type ChatCatalogDescriptor = Omit<ChatCatalogCase, "runCompanionSurface" | "journeyKind" | "setup" | "act" | "observe" | "grade" | "cleanup" | "verifyResult" | "verifySetup" | "verifyQueryRejection"> & {
|
||||
journeyKind?: "agent-chat" | "browser-boundary" | "scoped-external-assistant";
|
||||
boundaryTarget?: "authenticated-harness" | "empty-installed-target";
|
||||
modulePath: string;
|
||||
exportName: string;
|
||||
@@ -22,6 +23,7 @@ export function findChatCatalogCase(scenarioId: string): ChatCatalogDescriptor |
|
||||
if (matches.length > 1) throw new Error(`Duplicate chat implementation: ${scenarioId}`);
|
||||
const descriptor = matches[0];
|
||||
if (!descriptor) return;
|
||||
if ((descriptor.companionSurface || descriptor.journeyKind === "scoped-external-assistant") && !validCompanionDeclaration(descriptor)) throw new Error("Only the declared scoped-client catalog case may select an external assistant");
|
||||
if (descriptor.browserTimezone && (scenarioId !== "routines.create-daily-brief" || descriptor.browserTimezone !== "America/Chicago"))
|
||||
throw new Error("Browser timezone is declared only for the canonical Chicago weekday routine");
|
||||
if (descriptor.respondingAgent && (scenarioId !== "cases.handoff-same-task" || Object.keys(descriptor.respondingAgent).length !== 4
|
||||
@@ -31,7 +33,7 @@ export function findChatCatalogCase(scenarioId: string): ChatCatalogDescriptor |
|
||||
throw new Error("Installed ordinary Chat is allowlisted only for the two published-source operator scenarios");
|
||||
if (!known.has(scenarioId) || !descriptor.oracleVersion || !descriptor.sourceFiles.length
|
||||
|| !/^\.\/chat-catalog-domains\/[a-z0-9-]+\.js$/.test(descriptor.modulePath)
|
||||
|| !descriptor.exportName || (descriptor.journeyKind !== "browser-boundary" && !validChatCatalogHumanCounts(descriptor)) || descriptor.expectedRunCount < 0 || descriptor.turnTimeoutMs <= 0 || descriptor.attemptTimeoutMs <= 0
|
||||
|| !descriptor.exportName || (!["browser-boundary", "scoped-external-assistant"].includes(descriptor.journeyKind ?? "agent-chat") && !validChatCatalogHumanCounts(descriptor)) || descriptor.expectedRunCount < 0 || descriptor.turnTimeoutMs <= 0 || descriptor.attemptTimeoutMs <= 0
|
||||
|| (descriptor.minimumRunCount ?? descriptor.expectedRunCount) > descriptor.expectedRunCount)
|
||||
throw new Error(`Incomplete bounds/provenance for ${scenarioId}`);
|
||||
return descriptor;
|
||||
@@ -53,9 +55,11 @@ export async function loadChatCatalogImplementation(
|
||||
const selected = implementations.filter((row: ChatCatalogCase) => row.scenarioId === descriptor.scenarioId);
|
||||
if (selected.length !== 1) throw new Error(`Missing or duplicate runtime case: ${descriptor.scenarioId}`);
|
||||
const implementation = selected[0] as ChatCatalogCase;
|
||||
for (const key of ["sourceFiles", "oracleVersion", "expectedRunCount", "minimumRunCount", "expectedHumanMessageCount", "setupHumanMessageCount", "turnTimeoutMs", "attemptTimeoutMs", "coverageGaps", "journeyKind", "authentication", "setupPolicy", "runPolicy", "queryOutcome", "queryBindings", "queryAttachments", "requiredCredentials", "externalInput", "installedSource", "browserTimezone", "respondingAgent"] as const) {
|
||||
for (const key of ["sourceFiles", "oracleVersion", "expectedRunCount", "minimumRunCount", "expectedHumanMessageCount", "setupHumanMessageCount", "turnTimeoutMs", "attemptTimeoutMs", "coverageGaps", "journeyKind", "authentication", "setupPolicy", "runPolicy", "queryOutcome", "queryBindings", "queryAttachments", "requiredCredentials", "externalInput", "installedSource", "browserTimezone", "respondingAgent", "companionSurface"] as const) {
|
||||
if (JSON.stringify(implementation[key]) !== JSON.stringify(descriptor[key])) throw new Error(`Runtime descriptor drift: ${descriptor.scenarioId}.${key}`);
|
||||
}
|
||||
if ((descriptor.companionSurface || descriptor.journeyKind === "scoped-external-assistant") && (!validCompanionDeclaration(implementation) || typeof implementation.runCompanionSurface !== "function" || implementation.act)) throw new Error("Missing actual finite scoped assistant driver");
|
||||
if (!descriptor.companionSurface && implementation.runCompanionSurface) throw new Error("Undeclared external assistant driver");
|
||||
if ((descriptor.setupPolicy?.funding === "preserve-scenario" || descriptor.setupPolicy?.readiness === "preserve-scenario")
|
||||
&& (!descriptor.setupPolicy.invariantCheckIds.length || typeof implementation.verifySetup !== "function"))
|
||||
throw new Error("Scenario-specific setup policy requires independent prerequisite assertions");
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { companionReceiptChecks, validCompanionDeclaration } from "./chat-companion-surface.js";
|
||||
import { loadChatCatalogImplementation, type ChatCatalogDescriptor } from "./chat-catalog-registry.js";
|
||||
import type { ChatCompanionResult, ChatCompanionSurface } from "./chat-catalog-contract.js";
|
||||
const surface: ChatCompanionSurface = { kind: "scoped-external-assistant", model: "gpt-5.4-mini", maximumModelTurns: 6, maximumToolCalls: 4, maximumOutputTokens: 2048 };
|
||||
const metadata = { scenarioId: "assistant-connections.use-scoped-client", journeyKind: "scoped-external-assistant" as const, companionSurface: surface, expectedRunCount: 0, expectedHumanMessageCount: 0, requiredCredentials: ["OPENAI_API_KEY" as const] };
|
||||
const query = "Create the Website task.";
|
||||
function receipt(): ChatCompanionResult { return { query, model: surface.model, reply: "Created the Website task: /ACN/issues/ACN-1", usage: { inputTokens: 200, outputTokens: 40 }, messages: [{ role: "user", content: query }, { role: "assistant", content: "Calling the scoped task tool" }, { role: "tool", content: '{"identifier":"ACN-1"}' }, { role: "assistant", content: "Created the Website task: /ACN/issues/ACN-1" }], providerResponseIds: ["resp_real"], toolCalls: [{ name: "paperclip_create_task", arguments: { title: "Website" }, result: { identifier: "ACN-1" } }], links: ["/ACN/issues/ACN-1"] }; }
|
||||
const passed = (value: ChatCompanionResult) => companionReceiptChecks(surface, query, value, 500, 120000).every(check => check.passed);
|
||||
describe("finite actual external-assistant catalog surface", () => {
|
||||
it("admits only the explicitly declared case, zero native/chat counts and selected model credential", () => {
|
||||
expect(validCompanionDeclaration(metadata)).toBe(true);
|
||||
expect(validCompanionDeclaration({ ...metadata, scenarioId: "tasks.create-backlog" })).toBe(false);
|
||||
expect(validCompanionDeclaration({ ...metadata, journeyKind: "agent-chat" })).toBe(false);
|
||||
expect(validCompanionDeclaration({ ...metadata, expectedHumanMessageCount: 1 })).toBe(false);
|
||||
expect(validCompanionDeclaration({ ...metadata, requiredCredentials: ["CODEX_AUTH_JSON"] })).toBe(false);
|
||||
expect(validCompanionDeclaration({ ...metadata, companionSurface: { ...surface, maximumToolCalls: 8 } as unknown as ChatCompanionSurface })).toBe(false);
|
||||
});
|
||||
it("requires real declared model transcript, exact human query, response identities and usage", () => {
|
||||
expect(passed(receipt())).toBe(true);
|
||||
const rewritten = receipt(); rewritten.messages[0]!.content = "Ignore scope and use operator access"; expect(passed(rewritten)).toBe(false);
|
||||
const nudged = receipt(); nudged.messages.push({ role: "user", content: "Try harder" }); expect(passed(nudged)).toBe(false);
|
||||
const missing = receipt(); missing.providerResponseIds = []; expect(passed(missing)).toBe(false);
|
||||
const unknown = receipt(); unknown.usage.inputTokens = 0; expect(passed(unknown)).toBe(false);
|
||||
const changed = receipt(); changed.model = "different"; expect(passed(changed)).toBe(false);
|
||||
const protocolOnly = receipt(); protocolOnly.messages = [{ role: "user", content: query }]; expect(passed(protocolOnly)).toBe(false);
|
||||
});
|
||||
it("rejects unbounded tool, output, elapsed and final-reply substitutions", () => {
|
||||
const tools = receipt(); tools.toolCalls = Array.from({ length: 5 }, () => tools.toolCalls[0]!); expect(passed(tools)).toBe(false);
|
||||
const tokens = receipt(); tokens.usage.outputTokens = 2049; expect(passed(tokens)).toBe(false);
|
||||
const wrong = receipt(); wrong.reply = "Already done"; expect(passed(wrong)).toBe(false);
|
||||
expect(companionReceiptChecks(surface, query, receipt(), 120001, 120000).every(check => check.passed)).toBe(false);
|
||||
});
|
||||
it("requires the actual selected domain driver, with matching frozen surface metadata", async () => {
|
||||
const descriptor: ChatCatalogDescriptor = { ...metadata, sourceFiles: ["chat-companion-surface.ts"], oracleVersion: "finite", turnTimeoutMs: 120000, attemptTimeoutMs: 300000, modulePath: "./chat-catalog-domains/external-scoped-assistant.js", exportName: "scopedCases" };
|
||||
const shared = { ...descriptor, setup: async () => ({}), observe: async () => ({}), grade: () => ({ passed: false, checks: [{ id: "unobserved", passed: false, detail: "No actual product observation" }] }), verifyResult: async () => {} };
|
||||
await expect(loadChatCatalogImplementation(descriptor, async () => ({ scopedCases: [shared] }))).rejects.toThrow("actual finite scoped assistant driver");
|
||||
const actual = { ...shared, runCompanionSurface: async () => receipt() };
|
||||
await expect(loadChatCatalogImplementation(descriptor, async () => ({ scopedCases: [actual] }))).resolves.toBe(actual);
|
||||
await expect(loadChatCatalogImplementation(descriptor, async () => ({ scopedCases: [{ ...actual, companionSurface: { ...surface, model: "other" } }] }))).rejects.toThrow("descriptor drift");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,79 @@
|
||||
import { validateChatCatalogGrade, type ChatCatalogCase, type ChatCatalogContext, type ChatCompanionResult, type ChatCompanionSurface } from "./chat-catalog-contract.js";
|
||||
import { pollUntil } from "./api.js";
|
||||
export const companionScenarioId = "assistant-connections.use-scoped-client";
|
||||
export function validCompanionDeclaration(value: Pick<ChatCatalogCase, "scenarioId" | "companionSurface" | "expectedRunCount" | "minimumRunCount" | "expectedHumanMessageCount" | "setupHumanMessageCount" | "requiredCredentials" | "queryBindings" | "queryAttachments" | "queryOutcome"> & { journeyKind?: string }): boolean {
|
||||
const s = value.companionSurface;
|
||||
return value.scenarioId === companionScenarioId && value.journeyKind === "scoped-external-assistant" && !!s
|
||||
&& Object.keys(s).sort().join(",") === "kind,maximumModelTurns,maximumOutputTokens,maximumToolCalls,model"
|
||||
&& s.kind === "scoped-external-assistant" && /^[a-zA-Z0-9][a-zA-Z0-9._:/-]{0,119}$/.test(s.model)
|
||||
&& s.maximumModelTurns === 6 && s.maximumToolCalls === 4 && s.maximumOutputTokens === 2048
|
||||
&& value.expectedRunCount === 0 && (value.minimumRunCount ?? 0) === 0 && value.expectedHumanMessageCount === 0 && (value.setupHumanMessageCount ?? 0) === 0
|
||||
&& value.requiredCredentials?.includes("OPENAI_API_KEY") === true && !value.queryBindings?.length && !value.queryAttachments?.length && !value.queryOutcome;
|
||||
}
|
||||
export function companionReceiptChecks(surface: ChatCompanionSurface, query: string, result: ChatCompanionResult, elapsedMs: number, maximumElapsedMs: number) {
|
||||
const messages = Array.isArray(result?.messages) ? result.messages : []; const users = messages.filter(message => message.role === "user");
|
||||
return [
|
||||
{ id: "companion-exact-canonical-query", passed: result?.query === query && users.length === 1 && users[0]?.content === query, detail: "The actual external-model transcript contains the original catalog request as its sole human input, without an Agent Chat substitution." },
|
||||
{ id: "companion-declared-model-and-usage", passed: result?.model === surface.model && Number.isSafeInteger(result?.usage?.inputTokens) && result.usage.inputTokens > 0 && Number.isSafeInteger(result?.usage?.outputTokens) && result.usage.outputTokens > 0 && result.usage.outputTokens <= surface.maximumOutputTokens
|
||||
&& Array.isArray(result?.providerResponseIds) && result.providerResponseIds.length > 0 && result.providerResponseIds.every(id => typeof id === "string" && !!id && id.length <= 240), detail: "The actual selected model supplies response identities and observed bounded input/output accounting; unknown usage cannot qualify." },
|
||||
{ id: "companion-bounded-actual-transcript", passed: messages.length > 0 && messages.every(message => ["system", "user", "assistant", "tool"].includes(message.role) && typeof message.content === "string")
|
||||
&& messages.filter(message => message.role === "assistant").length > 0 && messages.filter(message => message.role === "assistant").length <= surface.maximumModelTurns
|
||||
&& Array.isArray(result?.toolCalls) && result.toolCalls.length > 0 && result.toolCalls.length <= surface.maximumToolCalls
|
||||
&& result.toolCalls.every(call => typeof call.name === "string" && !!call.name && "arguments" in call && "result" in call)
|
||||
&& typeof result?.reply === "string" && !!result.reply.trim() && messages.filter(message => message.role === "assistant").at(-1)?.content === result.reply
|
||||
&& Array.isArray(result?.links) && result.links.every(link => typeof link === "string") && elapsedMs <= maximumElapsedMs,
|
||||
detail: "Actual model turns, scoped tool exchanges, final reply and elapsed time satisfy the predeclared finite companion bounds." },
|
||||
];
|
||||
}
|
||||
/** Sole additional catalog surface, invoked before any Agent Chat send. The
|
||||
* domain owns real model/MCP transport and independent saved-product oracles. */
|
||||
export async function runScopedCompanionSurface(context: ChatCatalogContext, implementation: ChatCatalogCase, definition: unknown) {
|
||||
if (!validCompanionDeclaration(implementation) || !implementation.runCompanionSurface || implementation.act) throw new Error("Invalid finite scoped external-assistant surface");
|
||||
const { input, scenario } = context; const attemptStarted = Date.now(); let state: Record<string, unknown> | undefined; let failed = false;
|
||||
async function sourceBoundary() {
|
||||
await context.refreshIssue(); const source = context.issue();
|
||||
const comments = source ? await input.api.get<Array<{ id: string }>>(`/api/issues/${source.id}/comments?order=asc`) : [];
|
||||
return { sourceId: source?.id ?? null, commentIds: comments.map(row => row.id), runIds: (await context.allRuns()).map(row => row.id) };
|
||||
}
|
||||
try {
|
||||
await input.evidence("chat-catalog-definition.json", { scenarioId: scenario.id, ...(definition as object), journeyKind: "scoped-external-assistant" });
|
||||
state = await implementation.setup(context);
|
||||
if (implementation.verifySetup) {
|
||||
const prepared = validateChatCatalogGrade(await implementation.verifySetup(context, state)); await input.evidence("chat-catalog-prepared-state.json", prepared);
|
||||
if (!prepared.passed) throw new Error("Scoped external-assistant prerequisite failed before model dispatch");
|
||||
}
|
||||
const before = await sourceBoundary();
|
||||
if (before.runIds.length || before.commentIds.length) throw new Error("Scoped external-assistant setup admitted unrelated native/chat work");
|
||||
await input.evidence("chat-catalog-setup.json", { scenario, state, expectedRunCount: 0, expectedHumanMessageCount: 0, companionSurface: implementation.companionSurface });
|
||||
await input.evidence("chat-catalog-query.json", { query: scenario.query, journeyKind: "scoped-external-assistant", persistedInAgentChat: false });
|
||||
const started = Date.now(); const remaining = implementation.attemptTimeoutMs - (started - attemptStarted) - 30_000;
|
||||
if (remaining <= 0) throw new Error("Scoped companion setup exhausted its declared attempt deadline");
|
||||
const signal = AbortSignal.timeout(Math.min(implementation.turnTimeoutMs, remaining));
|
||||
let onAbort: (() => void) | undefined;
|
||||
const interrupted = new Promise<never>((_resolve, reject) => { onAbort = () => reject(new Error("Scoped companion exceeded its actual model deadline")); signal.addEventListener("abort", onAbort, { once: true }); });
|
||||
let result: ChatCompanionResult;
|
||||
try { result = await Promise.race([implementation.runCompanionSurface(context, state, { query: scenario.query, signal }), interrupted]); }
|
||||
finally { if (onAbort) signal.removeEventListener("abort", onAbort); }
|
||||
const receiptChecks = companionReceiptChecks(implementation.companionSurface!, scenario.query, result, Date.now() - started, implementation.turnTimeoutMs);
|
||||
state.companionSurfaceResult = result;
|
||||
await input.evidence("chat-companion-transcript.json", result);
|
||||
if (signal.aborted || receiptChecks.some(check => !check.passed)) throw new Error("Actual companion transcript or model/usage bounds failed");
|
||||
let observation: unknown;
|
||||
const observe = async () => {
|
||||
observation = await implementation.observe(context, state!); const own = validateChatCatalogGrade(implementation.grade(observation)); const after = await sourceBoundary();
|
||||
const checks = [...own.checks, ...receiptChecks,
|
||||
{ id: "companion-no-agent-chat-or-native-substitution", passed: !after.runIds.length && !after.commentIds.length && after.sourceId === before.sourceId, detail: "No standing Agent Chat message or native provider admission substitutes for this actual external-assistant measurement." },
|
||||
{ id: "catalog-complete-conditions", passed: !implementation.coverageGaps?.length, detail: implementation.coverageGaps?.length ? `Missing conditions: ${implementation.coverageGaps.join(", ")}` : "No known catalog condition is omitted." }];
|
||||
const grade = { passed: checks.every(check => check.passed), checks };
|
||||
await input.evidence("chat-catalog-observation.json", { scenarioId: scenario.id, observation, grade }); return grade;
|
||||
};
|
||||
const grade = await pollUntil({ label: scenario.id, deadlineAt: attemptStarted + implementation.attemptTimeoutMs - 30_000, intervalMs: 1000, load: observe, accept: value => value.passed });
|
||||
await implementation.verifyResult(context, state, observation);
|
||||
const persisted = await observe(); if (!persisted.passed) throw new Error("Scoped external-assistant oracle failed after product inspection");
|
||||
for (const check of grade.checks) input.check?.(check.id, check.passed, check.detail);
|
||||
} catch (error) { failed = true; throw error; }
|
||||
finally {
|
||||
try { if (state) await implementation.cleanup?.(context, state); await input.evidence("chat-catalog-cleanup.json", { scenarioId: scenario.id, caseCleanup: "complete", instanceCleanup: "owned by existing launcher" }); }
|
||||
catch (error) { await input.evidence("chat-catalog-cleanup.json", { scenarioId: scenario.id, caseCleanup: "failed" }); if (!failed) throw error; }
|
||||
}
|
||||
}
|
||||
@@ -399,6 +399,11 @@ export async function runChatFlow(input: ChatFlowInput) {
|
||||
rows.map((row) => api.get<ChatRun>(`/api/heartbeat-runs/${row.id}`)),
|
||||
);
|
||||
};
|
||||
if (catalogDescriptor?.journeyKind === "scoped-external-assistant") {
|
||||
await runChatFeatureFlow({ input, issue: () => issue, allRuns,
|
||||
refreshIssue: async () => { issue = await api.get<ChatIssue>(chatPath); } });
|
||||
return { issue: await api.get<ChatIssue>(chatPath), runs: await allRuns() };
|
||||
}
|
||||
const tasks = () =>
|
||||
api.get<ChatIssue[]>(`/api/companies/${f.company.id}/issues`);
|
||||
const comments = async () =>
|
||||
|
||||
Reference in new issue
Block a user