mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-24 18:19:05 +02:00
55 lines
4.4 KiB
TypeScript
55 lines
4.4 KiB
TypeScript
import { isDeepStrictEqual } from "node:util";
|
|
import { createHash } from "node:crypto";
|
|
import { readFileSync } from "node:fs";
|
|
import { continuationScenario, continuationTasks, necessaryQuestionContract } from "./continuation-cases.js";
|
|
import { questionResumedOnce, validResumeQuestionForm } from "./question-resume-scoring.js";
|
|
import type { ContinuationCheckpoint } from "./continuation-scoring.js";
|
|
import type { RunnerTaskFixture } from "./types.js";
|
|
|
|
type Row = Record<string, any>;
|
|
export const necessaryQuestionTask: RunnerTaskFixture = {
|
|
...continuationTasks.find(task => task.id === "answer-updates-scope")!,
|
|
id: "necessary-free-text", label: "Necessary free-text question and one continuation",
|
|
expectedRunCount: 2, minimumExpectedRunCount: 1, attemptTimeoutMs: { local: 10 * 60_000, daytona: 15 * 60_000 }, turnTimeoutMs: 4 * 60_000,
|
|
buildTitle: nonce => `Booking confirmation ${nonce}`,
|
|
buildPrompt: nonce => continuationScenario("necessary-free-text", nonce).prompt,
|
|
buildVisibleMarker: nonce => continuationScenario("necessary-free-text", nonce).marker,
|
|
};
|
|
|
|
export function gradeNecessaryQuestion(checkpoints: ContinuationCheckpoint[], answer: string) {
|
|
const initial = checkpoints.find(point => point.phase === "initial"), final = checkpoints.find(point => point.phase === "final");
|
|
const cards = (point?: ContinuationCheckpoint) => (point?.interactions ?? []).filter((value: any) => value.kind === "ask_user_questions") as Row[];
|
|
const pending = cards(initial), finished = cards(final), question = pending[0], resolved = finished[0];
|
|
const field = question?.payload?.questionSet?.questions?.[0], contract = necessaryQuestionContract();
|
|
const check = (id: string, passed: boolean, detail: string) => ({ id, passed: Boolean(passed), detail });
|
|
const response = resolved?.result?.answers;
|
|
return [
|
|
check("necessary-question.form", checkpoints.length === 2 && !!initial && pending.length === 1 && question?.status === "pending" && initial.documents.length === 0 && initial.attachments.length === 0 &&
|
|
validResumeQuestionForm(initial, question, false) && field?.required === true && field.prompt === contract.prompt &&
|
|
[question.payload.questionSet.description, field.helpText].includes(contract.context),
|
|
"One bound text-only question must show the required prompt and its explanation before delivery."),
|
|
check("necessary-question.visible", initial?.questionPresentation?.interactionId === question?.id &&
|
|
initial?.questionPresentation?.promptVisible === true && initial?.questionPresentation?.contextVisible === true,
|
|
"The browser must observe the actual question and context, not only its title."),
|
|
check("necessary-question.answer", finished.length === 1 && resolved?.id === question?.id && resolved?.sourceRunId === question?.sourceRunId &&
|
|
isDeepStrictEqual(question?.payload, resolved?.payload) && resolved?.status === "answered" &&
|
|
typeof resolved.resolvedByUserId === "string" && !!resolved.resolvedByUserId && !resolved.resolvedByAgentId &&
|
|
Array.isArray(response) && response.length === 1 && response[0]?.questionId === field?.id && response[0]?.otherText === answer &&
|
|
!response[0]?.optionIds?.length,
|
|
"One real user answer must remain attached to the unchanged question."),
|
|
check("necessary-question.one-continuation", questionResumedOnce(initial, final, question) && !!final && final.runs.length <= 2 &&
|
|
final.runs.every((run: Row) => run.status === "succeeded" && run.runtimeMode === "native" && run.nativePhase === "committed" && run.resultJson?.finalizationPhase === "committed"),
|
|
"The answer must resume its bound provider run or admit exactly one bound semantic successor, without duplicate work."),
|
|
check("necessary-question.result", final?.issue.status === "done" && final.documents.length === 1 &&
|
|
!!final.documents[0]?.latestRevisionId && final.documents[0]?.body.trim() === `Booking confirmed: ${answer}` && final.outputLinkVisible === true,
|
|
"The saved confirmation must contain the exact answer and have an accessible link in the rendered task."),
|
|
];
|
|
}
|
|
|
|
export function necessaryQuestionDigest() {
|
|
const hash = createHash("sha256");
|
|
for (const file of ["necessary-question.ts", "continuation-cases.ts", "continuation-flow.ts", "continuation-scoring.ts", "question-resume-scoring.ts", "stock-harness.ts"])
|
|
hash.update(file).update(readFileSync(new URL(file, import.meta.url)));
|
|
return hash.digest("hex");
|
|
}
|