Jannik/PaperClipAI
1
0 Fork 0
mirror of https://github.com/paperclipai/paperclip.git synced 2026-11-04 19:28:46 +01:00
Code Issues Packages Projects Releases Wiki Activity

test(evals): bind deferred question delivery to its actual successor

Browse Source 
Accept a null-target durable wake receipt only with the unique succeeded answer-bound continuation from the server response path. Keep actual human answer, company/task/agent, ordering, duplicate conflict, saved output and full run accounting checks. Version the observer and retain failed history.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-11-02 20:58:49 -06:00
1 parent 41743cb0ae
commit fa644b9b20
2 files changed
+70 -17
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.test.ts
+35 -1
View File
@@ -3,9 +3,10 @@ import { answerContinuationCompleted, confirmationWorkflowIdentityCases, gradeCo
import * as api from "../api.js";
import type { ChatCatalogContext } from "../chat-catalog-contract.js";
import type { IdentityObservation, Interaction } from "./identity-support.js";
vi.mock("../chat-flow.js", () => ({ sendChatMessage: vi.fn() }));
function valid(id: string): IdentityObservation {
const task = { id: "draft", companyId: "acorn", title: "Acorn launch checklist n42", description: "Draft checklist for later", status: "backlog" };
const original: Interaction = { id: "request", kind: "request_confirmation", status: "pending", payload: { prompt: "Create the named checklist?" } };
const original: Interaction = { id: "request", companyId: "acorn", issueId: "source", kind: "request_confirmation", status: "pending", payload: { prompt: "Create the named checklist?" } };
const o: IdentityObservation = { scenarioId: id, origin: "http://fixture.test", companyId: "acorn", leadId: "ceo", company: { id: "acorn" }, companies: [], agents: [], interactions: [],
tasks: [task], projects: [], goals: [], comments: [], documents: [], revisions: [], activity: [], links: ["/ACO/issues/draft"], runs: [], reply: "Created the linked draft.", humanQueries: [], state: { original } };
if (id === "questions-and-approvals.accept-one-proposal") {
@@ -118,3 +119,36 @@ describe("answer continuation and ordinary deliverable custody", () => {
} finally { clock.mockRestore(); poll.mockRestore(); }
});
});
function deferredFallback() {
const o = valid("questions-and-approvals.duplicate-answer");
const delivery = o.activity.find(row => row.id === "delivery")!;
Object.assign(delivery.details!, { targetRunId: null, deliveryStatus: "fallback_queued", deliveryMode: "wake_fallback" });
Object.assign(o.runs[1]!.contextSnapshot!, { source: "issue.interaction.respond", interactionKind: "ask_user_questions", interactionStatus: "answered", sourceRunId: null });
return o;
}
it("accepts a durable fallback recorded before its unique actual answer-bound successor existed", () => {
const o = deferredFallback();
expect(answerContinuationCompleted(o)).toBe(true);
expect(gradeConfirmationWorkflow(o).passed).toBe(true);
});
it.each(["wrong source", "wrong status", "wrong kind", "missing target field", "missing delivery", "duplicate delivery", "extra continuation", "wrong human", "missing human", "wrong answer", "cancelled continuation", "cancelled canonical", "wrong source run", "unfinished successor", "delivered null target"])("does not infer a completed fallback from %s", invalid => {
const o = deferredFallback(), delivery = o.activity.find(row => row.id === "delivery")!, continuation = o.runs[1]!;
if (invalid === "wrong source") continuation.contextSnapshot!.source = "other.event";
if (invalid === "wrong status") continuation.contextSnapshot!.interactionStatus = "pending";
if (invalid === "wrong kind") continuation.contextSnapshot!.interactionKind = "request_confirmation";
if (invalid === "missing target field") delete delivery.details!.targetRunId;
if (invalid === "missing delivery") o.activity = o.activity.filter(row => row.id !== "delivery");
if (invalid === "duplicate delivery") o.activity.push({ ...structuredClone(delivery), id: "ambiguous-delivery" });
if (invalid === "extra continuation") o.runs.push({ ...structuredClone(continuation), id: "duplicate-continuation" });
if (invalid === "wrong human") o.interactions[0]!.resolvedByUserId = "other-human";
if (invalid === "missing human") { delete o.state.humanId; delete o.interactions[0]!.resolvedByUserId; }
if (invalid === "wrong answer") o.interactions[0]!.result!.answers![0]!.otherText = "Different product";
if (invalid === "cancelled continuation") continuation.status = "cancelled";
if (invalid === "cancelled canonical") o.runs[0]!.status = "cancelled";
if (invalid === "wrong source run") continuation.contextSnapshot!.sourceRunId = "unrelated-source";
if (invalid === "unfinished successor") continuation.status = "running";
if (invalid === "delivered null target") delivery.details!.deliveryStatus = "delivered";
expect(answerContinuationCompleted(o)).toBe(false);
expect(gradeConfirmationWorkflow(o).passed).toBe(false);
});
tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.ts
+35 -16
View File
@@ -10,23 +10,45 @@ import { identityDefinition, identityObservation, identityWorker, observeIdentit
interface SourceComment extends Entity { authorType?: string }
const tabs = new WeakMap<ChatCatalogContext, Page>();
const sources = ["chat-catalog-domains/identity-confirmation-workflows.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/questions-and-approvals.yaml"];
function savedProductAnswerMatches(o: IdentityObservation): boolean {
const original = o.state.original as Interaction | undefined;
const current = o.interactions.find(row => row.id === original?.id), answers = current?.result?.answers;
return !!original && original.status === "pending" && !original.result
&& original.kind === "ask_user_questions" && current?.kind === original.kind
&& typeof o.state.humanId === "string" && !!o.state.humanId
&& original.companyId === o.companyId && current.companyId === o.companyId
&& original.issueId === o.state.sourceIssueId && current.issueId === o.state.sourceIssueId
&& current.status === "answered" && current.resolvedByUserId === o.state.humanId && !current.resolvedByAgentId
&& answers?.length === 1 && answers[0]?.questionId === "product" && answers[0].otherText === "Acorn"
&& JSON.stringify(current.payload) === JSON.stringify(original.payload);
}
export function answerContinuationCompleted(o: IdentityObservation): boolean {
const original = o.state.original as Interaction | undefined;
const current = o.interactions.find(row => row.id === original?.id);
const resolvedAt = Date.parse(current?.resolvedAt ?? "");
const continuations = o.runs.filter(run => run.contextSnapshot?.interactionId === original?.id);
const continuation = continuations[0];
return !!original && typeof o.state.sourceIssueId === "string" && typeof o.state.canonicalCommentId === "string" && o.state.sourceConversationState === "waiting"
&& continuations.length === 1 && !!continuation && continuation.companyId === o.companyId && continuation.agentId === o.leadId
&& continuation.contextSnapshot?.issueId === o.state.sourceIssueId && continuation.status === "succeeded"
&& Number.isFinite(resolvedAt) && Number.isFinite(Date.parse(continuation.startedAt ?? ""))
&& Date.parse(continuation.startedAt!) >= resolvedAt
&& o.runs.some(run => run.id !== continuation.id && run.companyId === o.companyId && run.agentId === o.leadId
&& run.status === "succeeded" && !!run.startedAt && run.contextSnapshot?.issueId === o.state.sourceIssueId
&& run.contextSnapshot?.commentId === o.state.canonicalCommentId)
&& o.activity.some(row => row.action === "issue.question_response_delivered" && row.entityId === o.state.sourceIssueId
&& row.actorType === "system" && row.actorId === "question-response-delivery" && row.details?.interactionId === original.id
&& row.details.targetRunId === continuation.id && ["delivered", "fallback_queued"].includes(String(row.details.deliveryStatus)));
if (!original || !savedProductAnswerMatches(o) || typeof o.state.sourceIssueId !== "string"
|| !o.state.sourceIssueId || typeof o.state.canonicalCommentId !== "string" || !o.state.canonicalCommentId || o.state.sourceConversationState !== "waiting"
|| continuations.length !== 1 || !continuation || continuation.companyId !== o.companyId || continuation.agentId !== o.leadId
|| continuation.contextSnapshot?.issueId !== o.state.sourceIssueId || continuation.status !== "succeeded"
|| !Number.isFinite(resolvedAt) || !Number.isFinite(Date.parse(continuation.startedAt ?? ""))
|| Date.parse(continuation.startedAt!) < resolvedAt
|| !o.runs.some(run => run.id !== continuation.id && run.companyId === o.companyId && run.agentId === o.leadId
&& run.status === "succeeded" && Number.isFinite(Date.parse(run.startedAt ?? "")) && Date.parse(run.startedAt!) <= resolvedAt
&& run.contextSnapshot?.issueId === o.state.sourceIssueId && run.contextSnapshot?.commentId === o.state.canonicalCommentId)) return false;
const deliveries = o.activity.filter(row => row.action === "issue.question_response_delivered" && row.entityId === o.state.sourceIssueId
&& row.actorType === "system" && row.actorId === "question-response-delivery" && row.details?.interactionId === original.id);
if (deliveries.length !== 1) return false;
const details = deliveries[0]!.details!;
if (!["delivered", "fallback_queued"].includes(String(details.deliveryStatus))) return false;
if (details.targetRunId === continuation.id) return true;
// A durable wake may be recorded before its run exists. Require the real,
// unique completed successor's server-produced answer context, never a queued receipt alone.
return details.deliveryStatus === "fallback_queued" && details.deliveryMode === "wake_fallback" && details.targetRunId === null
&& continuation.contextSnapshot?.source === "issue.interaction.respond"
&& continuation.contextSnapshot?.interactionKind === "ask_user_questions" && continuation.contextSnapshot?.interactionStatus === "answered"
&& continuation.contextSnapshot?.sourceRunId === (original.sourceRunId ?? null);
}
export function gradeConfirmationWorkflow(value: unknown) {
const c = checks(); const o = identityObservation(value);
@@ -47,10 +69,7 @@ export function gradeConfirmationWorkflow(value: unknown) {
c.add("repeat-did-not-replace-decision", o.state.afterRepeatResolvedCommentId === o.state.canonicalCommentId && o.humanQueries.length === 2
&& o.humanQueries.every(query => query.trim() === "Yes, go ahead with that."), "Repeating the actual human reply does not replace the saved decision or create a duplicate instruction side effect.");
} else if (o.scenarioId === "questions-and-approvals.duplicate-answer") {
const answers = current?.result?.answers;
c.add("one-real-human-answer", current?.status === "answered" && current.resolvedByUserId === o.state.humanId && !current.resolvedByAgentId
&& answers?.length === 1 && answers[0]?.questionId === "product" && answers[0].otherText === "Acorn"
&& JSON.stringify(current.payload) === JSON.stringify(original?.payload), "Both real browser tabs target the same original question and only one saved human answer exists.");
c.add("one-real-human-answer", savedProductAnswerMatches(o), "Both real browser tabs target the same original company/task question and only one saved human answer exists.");
c.add("real-simultaneous-conflict", o.state.actualSubmitRequests === 2 && Array.isArray(o.state.submitStatuses) && [...o.state.submitStatuses].sort().join(",") === "200,409"
&& o.state.noTaskBeforeAnswers === true && o.state.bothTabsResolved === true,
"Both actual visible submit controls race, yielding one accepted response and one real conflict; both views converge on the same saved receipt.");
@@ -144,7 +163,7 @@ export const confirmationWorkflowIdentityCases: readonly ChatCatalogCase[] = [
await expect(other.getByTestId("task-chat-composer-takeover")).toBeVisible();
return { original, humanId: session.user.id, worker };
}, gradeConfirmationWorkflow, {
authentication: "human-session", sourceFiles: sources, expectedRunCount: 4, minimumRunCount: 2, runPolicy: { kind: "source-settled-with-active-work", maximumActiveWorkerRuns: 1 }, attemptTimeoutMs: 600_000,
authentication: "human-session", sourceFiles: sources, oracleVersion: "paperclip.chat-catalog.answer-delivery.v2", expectedRunCount: 4, minimumRunCount: 2, runPolicy: { kind: "source-settled-with-active-work", maximumActiveWorkerRuns: 1 }, attemptTimeoutMs: 600_000,
observe: observeWorkflow,
async act(ctx, state) {
const other = tabs.get(ctx); if (!other) throw new Error("The second real browser tab is missing");