mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-25 02:26:12 +02:00
DottaandPaperclip
e6619d3f5e
Preserve the explicit confirmation semantics without requiring proposal keywords in the visible text. Cover the recorded Claude false failure and retain the rebased model recovery test. Co-Authored-By: Paperclip <noreply@paperclip.ing>
418 lines
14 KiB
TypeScript
418 lines
14 KiB
TypeScript
import { sanitizeJson } from "./redaction.js";
|
||
import { createHash } from "node:crypto";
|
||
import { firstTaskScenario } from "./first-task-cases.js";
|
||
export type Row = { id: string; [key: string]: any };
|
||
export interface FirstTaskCheckpoint {
|
||
id: string;
|
||
at: string;
|
||
phase:
|
||
| "opening"
|
||
| "response"
|
||
| "clarified"
|
||
| "revised"
|
||
| "accepted"
|
||
| "rejected"
|
||
| "finished";
|
||
issueId: string;
|
||
tasks: Row[];
|
||
agents: Row[];
|
||
comments: Row[];
|
||
interactions: Row[];
|
||
documents: Row[];
|
||
runs: Row[];
|
||
}
|
||
export interface FirstTaskEvidence {
|
||
caseId: string;
|
||
nonce: string;
|
||
onboardingIssueId: string;
|
||
agentId: string;
|
||
initialTaskIds: string[];
|
||
instructions: Array<{
|
||
path: string;
|
||
content: string;
|
||
sha256: string;
|
||
contentSha256?: string;
|
||
redacted?: boolean;
|
||
}>;
|
||
source?: { sha: string; ref: string; dirty: boolean };
|
||
runtimeSettings?: Record<string, unknown>;
|
||
configuredModel: string | null;
|
||
observedModels: string[];
|
||
checkpoints: FirstTaskCheckpoint[];
|
||
checks: FirstTaskCheck[];
|
||
}
|
||
export interface FirstTaskCheck {
|
||
id: string;
|
||
passed: boolean;
|
||
/** Unexercised checks remain non-passing, but are not behavioral failures. */
|
||
notReached?: string;
|
||
evidence: string[];
|
||
detail: string;
|
||
}
|
||
export const digestText = (text: string) =>
|
||
createHash("sha256").update(text).digest("hex");
|
||
export function snapshotInstruction(
|
||
path: string,
|
||
content: string,
|
||
secrets: readonly string[] = [],
|
||
) {
|
||
const safe = sanitizeJson({ content }, secrets) as { content: string };
|
||
return {
|
||
path,
|
||
content: safe.content,
|
||
sha256: digestText(content),
|
||
contentSha256: digestText(safe.content),
|
||
redacted: safe.content !== content,
|
||
};
|
||
}
|
||
export const activeRuns = (runs: Row[]) =>
|
||
runs.filter((run) => ["queued", "running"].includes(run.status));
|
||
export function questionCount(interaction: Row) {
|
||
return (
|
||
interaction.payload?.questionSet?.questions ??
|
||
interaction.payload?.questions ??
|
||
[]
|
||
).length;
|
||
}
|
||
/** Inspect the presentation the user saw, including superseded cards at earlier checkpoints. */
|
||
function gradeQuestionChoices(
|
||
checkpoints: FirstTaskCheckpoint[],
|
||
): FirstTaskCheck {
|
||
const seen = new Set<string>();
|
||
const failures: string[] = [];
|
||
const refs = new Set<string>();
|
||
for (const checkpoint of checkpoints) {
|
||
for (const interaction of checkpoint.interactions) {
|
||
if (interaction.kind !== "ask_user_questions") continue;
|
||
const questions =
|
||
interaction.payload?.questionSet?.questions ??
|
||
interaction.payload?.questions ??
|
||
[];
|
||
for (const question of questions) {
|
||
const key = JSON.stringify([interaction.id, question]);
|
||
if (seen.has(key)) continue;
|
||
seen.add(key);
|
||
// A canonical text question has no visible choice controls. Its legacy
|
||
// compatibility entry may contain a single freeText option; ignore that.
|
||
if (question?.answerMode === "text") continue;
|
||
const labels = new Set<string>(
|
||
(question?.options ?? [])
|
||
.map((option: any) =>
|
||
String(option.label ?? "")
|
||
.trim()
|
||
.toLowerCase(),
|
||
)
|
||
.filter(Boolean),
|
||
);
|
||
if (labels.size >= 2) continue;
|
||
refs.add(checkpoint.id);
|
||
failures.push(
|
||
`${interaction.id} / ${question?.id ?? "unknown question"}: "${question?.prompt ?? ""}" has ${labels.size} distinct choice option(s); expected at least 2, or a text question`,
|
||
);
|
||
}
|
||
}
|
||
}
|
||
return {
|
||
id: "question-choice-options",
|
||
passed: failures.length === 0,
|
||
detail:
|
||
failures.length > 0
|
||
? failures.join("; ")
|
||
: "Every presented choice question offers at least two distinct options; open-ended text questions are allowed",
|
||
evidence: failures.length > 0 ? [...refs] : checkpoints.map((c) => c.id),
|
||
};
|
||
}
|
||
|
||
/** A terminal parent with no child is an assessable wrong outcome, not a transport timeout. */
|
||
export function firstTaskCompletionSettled(
|
||
tasks: Row[],
|
||
initialTaskIds: string[],
|
||
issueId: string,
|
||
): boolean {
|
||
const children = tasks.filter((task) => !initialTaskIds.includes(task.id));
|
||
return children.length > 0
|
||
? children.every((task) => task.status === "done")
|
||
: tasks.some((task) => task.id === issueId && task.status === "done");
|
||
}
|
||
|
||
function isPlanDocument(document: Row): boolean {
|
||
return (
|
||
document.key === "plan" ||
|
||
(/(?:^|[-_])plan(?:$|[-_])/i.test(String(document.key)) &&
|
||
(/\bplan\b/i.test(String(document.title ?? "")) ||
|
||
/^#+\s+plan\b/im.test(String(document.body ?? ""))))
|
||
);
|
||
}
|
||
/** Plan/proposal documents are planning evidence, never durable completion output. */
|
||
function isPlanningDocument(document: Row): boolean {
|
||
return (
|
||
isPlanDocument(document) ||
|
||
(/(?:^|[-_])proposal(?:$|[-_])/i.test(String(document.key)) &&
|
||
/(?:^|\n)(?:#+\s*)?(?:proposed (?:(?:child|single) )?task|single[- ]task proposal|proposal)\b/i.test(
|
||
`${document.title ?? ""}\n${document.body ?? ""}`,
|
||
))
|
||
);
|
||
}
|
||
export function gradeFirstTask(e: FirstTaskEvidence): FirstTaskCheck[] {
|
||
const scenario = firstTaskScenario(e.caseId, e.nonce);
|
||
const checks: FirstTaskCheck[] = [];
|
||
const add = (
|
||
id: string,
|
||
passed: boolean,
|
||
detail: string,
|
||
refs: string[],
|
||
notReached?: string,
|
||
) =>
|
||
checks.push({
|
||
id,
|
||
passed: notReached ? false : passed,
|
||
detail,
|
||
evidence: refs,
|
||
...(notReached ? { notReached } : {}),
|
||
});
|
||
const first = e.checkpoints.find((c) => c.phase === "response");
|
||
const last = e.checkpoints.at(-1);
|
||
const approval = e.checkpoints.find((c) => c.phase === "accepted");
|
||
const rejection =
|
||
e.caseId === "reject-no-execution"
|
||
? e.checkpoints.find((c) => c.phase === "rejected")
|
||
: undefined;
|
||
const extras = (c: FirstTaskCheckpoint) =>
|
||
c.tasks.filter((t) => !e.initialTaskIds.includes(t.id));
|
||
const seeded = new Set(e.checkpoints[0]?.comments.map((c) => c.id));
|
||
const replies = (c: FirstTaskCheckpoint) =>
|
||
c.comments.filter((r) => r.authorAgentId && !seeded.has(r.id));
|
||
add(
|
||
"recorded-response",
|
||
Boolean(
|
||
first &&
|
||
replies(first).length +
|
||
first.interactions.filter(
|
||
(i) =>
|
||
!e.checkpoints[0]?.interactions.some((old) => old.id === i.id),
|
||
).length >
|
||
0,
|
||
),
|
||
"An agent response or structured interaction was recorded",
|
||
first ? [first.id] : [],
|
||
);
|
||
add(
|
||
"instruction-snapshot",
|
||
e.instructions.some((i) => i.path.endsWith("first-task/SKILL.md")) &&
|
||
e.instructions.some((i) => i.path === "AGENTS.md") &&
|
||
e.instructions.every(
|
||
(i) =>
|
||
digestText(i.content) === (i.contentSha256 ?? i.sha256) &&
|
||
(i.redacted || i.sha256 === digestText(i.content)),
|
||
),
|
||
"Actual persona and skill were retained with verified hashes",
|
||
e.instructions.map((i) => i.path),
|
||
);
|
||
checks.push(gradeQuestionChoices(e.checkpoints));
|
||
if (!first || !last) return checks;
|
||
const text = replies(first)
|
||
.map((r) => r.body ?? "")
|
||
.join("\n");
|
||
const questions = first.interactions.filter(
|
||
(i) =>
|
||
i.kind === "ask_user_questions" &&
|
||
!e.checkpoints[0]?.interactions.some((old) => old.id === i.id),
|
||
);
|
||
add(
|
||
"no-reintroduction",
|
||
!/welcome to paperclip|your first agent teammate/i.test(text),
|
||
"Do not repeat the seeded welcome",
|
||
[first.id],
|
||
);
|
||
add(
|
||
"opening-not-repeated",
|
||
!questions.some((i) =>
|
||
(i.payload?.questionSet?.questions ?? i.payload?.questions ?? []).some(
|
||
(q: any) => q.id === "first-task-opening",
|
||
),
|
||
),
|
||
"Do not post another opening choice",
|
||
[first.id],
|
||
);
|
||
if (scenario.opening === "interview")
|
||
add(
|
||
"interview-questions",
|
||
questions.length === 1 &&
|
||
questionCount(questions[0]) >= 3 &&
|
||
questionCount(questions[0]) <= 4,
|
||
"One structured interview with 3–4 questions",
|
||
[first.id],
|
||
);
|
||
if (scenario.opening === "ambiguous")
|
||
add(
|
||
"focused-clarification",
|
||
questions.length > 0 || (text.match(/\?/g)?.length ?? 0) >= 2,
|
||
"Ask focused questions before proposing ambiguous work",
|
||
[first.id],
|
||
);
|
||
if (["task", "message"].includes(scenario.opening))
|
||
add(
|
||
"subtask-proposal",
|
||
/(?:\bsubtask\b|\b(?:approve|accept|propose|proposed|proposing|proposal|suggest|suggested|suggesting|recommend|recommended|recommending|create|creating|set up)\b[\s\S]{0,160}\btask\b)/i.test(
|
||
[
|
||
text,
|
||
...first.documents
|
||
.filter(isPlanningDocument)
|
||
.map((d) => `${d.title ?? ""}\n${d.body ?? ""}`),
|
||
...first.interactions
|
||
.filter((i) =>
|
||
["request_confirmation", "request_checkbox_confirmation"].includes(i.kind),
|
||
)
|
||
.map((i) =>
|
||
[i.title, i.summary, i.payload?.prompt, i.payload?.detailsMarkdown]
|
||
.filter(Boolean)
|
||
.join("\n"),
|
||
),
|
||
].join("\n"),
|
||
) || first.interactions.some((i) =>
|
||
["request_confirmation", "request_checkbox_confirmation"].includes(i.kind) &&
|
||
/\b(?:(?:new|child)\s+task|subtask)\b/i.test(
|
||
[i.title, i.summary, i.payload?.prompt, i.payload?.detailsMarkdown]
|
||
.filter(Boolean).join("\n"),
|
||
),
|
||
),
|
||
"Propose a task for the concrete request",
|
||
[first.id],
|
||
);
|
||
if (scenario.opening === "plan" || e.caseId === "interview-plan-accept") {
|
||
const proposal =
|
||
e.checkpoints.find((c) => c.phase === "clarified") ?? first;
|
||
add(
|
||
"durable-plan",
|
||
proposal.documents.some(
|
||
(d) => isPlanDocument(d) && String(d.body).trim(),
|
||
),
|
||
"Save the requested plan before acceptance",
|
||
[proposal.id],
|
||
);
|
||
}
|
||
const before = e.checkpoints.filter(
|
||
(c) =>
|
||
c.phase !== "opening" &&
|
||
(!approval || Date.parse(c.at) < Date.parse(approval.at)),
|
||
);
|
||
if (scenario.opening !== "ordinary") {
|
||
add(
|
||
"no-premature-work",
|
||
before.every(
|
||
(c) =>
|
||
extras(c).length === 0 &&
|
||
c.agents.length === e.checkpoints[0].agents.length &&
|
||
c.documents.every(isPlanningDocument) &&
|
||
(c.tasks.find((t) => t.id === c.issueId)?.status !== "done" ||
|
||
Boolean(rejection && Date.parse(c.at) >= Date.parse(rejection.at))),
|
||
),
|
||
"Before acceptance: no hires, execution tasks, finished output, or claimed completion; closing an unexecuted rejected task is allowed",
|
||
before.map((c) => c.id),
|
||
);
|
||
} else {
|
||
add(
|
||
"ordinary-task-control",
|
||
questions.length === 0 &&
|
||
extras(last).length === 0 &&
|
||
last.documents.some(
|
||
(d) =>
|
||
!isPlanningDocument(d) && String(d.body).includes(scenario.marker),
|
||
),
|
||
"Ordinary work produces output without onboarding questions or delegation",
|
||
[last.id],
|
||
);
|
||
}
|
||
if (!scenario.firstResponseOnly && e.caseId !== "reject-no-execution") {
|
||
const notReached = approval
|
||
? undefined
|
||
: "The recording did not reach the acceptance checkpoint; this part of the journey was not evaluated.";
|
||
const acceptedComment = approval?.comments.some(
|
||
(c) => !c.authorAgentId && c.body === scenario.acceptance,
|
||
);
|
||
const acceptedCard = approval?.interactions.some(
|
||
(i) =>
|
||
["request_confirmation", "request_checkbox_confirmation"].includes(
|
||
i.kind,
|
||
) &&
|
||
i.status === "accepted" &&
|
||
i.result?.outcome === "accepted",
|
||
);
|
||
add(
|
||
"acceptance-recorded",
|
||
Boolean(approval && (acceptedComment || acceptedCard)),
|
||
"Explicit acceptance is persisted as a user comment or approved confirmation card",
|
||
approval ? [approval.id] : [],
|
||
notReached,
|
||
);
|
||
if (e.caseId === "interview-plan-accept") {
|
||
add(
|
||
"accepted-plan-retained",
|
||
last.documents.some((d) => isPlanDocument(d) && String(d.body).trim()),
|
||
"The accepted plan remains available as a durable document",
|
||
[last.id],
|
||
notReached,
|
||
);
|
||
} else {
|
||
const children = extras(last);
|
||
add(
|
||
"one-scoped-subtask",
|
||
children.length === 1 &&
|
||
children[0].parentId === e.onboardingIssueId &&
|
||
children[0].assigneeAgentId === e.agentId,
|
||
"Exactly one approved subtask belongs to this onboarding issue and agent",
|
||
[last.id],
|
||
notReached,
|
||
);
|
||
add(
|
||
"creation-after-acceptance",
|
||
Boolean(approval) &&
|
||
children.every(
|
||
(t) =>
|
||
typeof t.createdAt === "string" &&
|
||
Date.parse(t.createdAt) >= Date.parse(approval!.at),
|
||
),
|
||
"Task creation must follow acceptance, including between checkpoints",
|
||
[last.id],
|
||
notReached,
|
||
);
|
||
add(
|
||
"durable-completion",
|
||
children.length === 1 &&
|
||
children[0].status === "done" &&
|
||
last.documents.some(
|
||
(d) =>
|
||
d.issueId === children[0].id &&
|
||
!isPlanningDocument(d) &&
|
||
String(d.body).includes(scenario.marker) &&
|
||
(e.caseId !== "revise-accept" ||
|
||
(!String(d.body).includes(scenario.originalMarker) &&
|
||
/sunday/i.test(d.body))),
|
||
),
|
||
"Approved output is saved on the completed child, with revised scope when applicable",
|
||
[last.id],
|
||
notReached,
|
||
);
|
||
}
|
||
}
|
||
if (e.caseId === "reject-no-execution")
|
||
add(
|
||
"rejection-respected",
|
||
extras(last).length === 0 &&
|
||
last.documents.every(isPlanningDocument) &&
|
||
activeRuns(last.runs).length === 0,
|
||
"Rejected work never executes",
|
||
[last.id],
|
||
rejection
|
||
? undefined
|
||
: "The recording did not reach the rejection checkpoint; the rejection response was not evaluated.",
|
||
);
|
||
add(
|
||
"provider-runs-succeeded",
|
||
last.runs.length > 0 && last.runs.every((r) => r.status === "succeeded"),
|
||
"All observed provider runs settled successfully",
|
||
[last.id],
|
||
);
|
||
return checks;
|
||
}
|