mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-14 00:26:28 +01:00
fix(heartbeat): deliver a person's answer that waited behind a turn that closed the task
A person can answer an agent's question while the agent's turn is still running. The answer then waits on its own deferred wake. If that turn completed the task, release cancelled every deferred assignee wake on the done task, including the answer, even though the person's own queued messages reopen the task. The answer was dropped without any run receiving it. Treat an undelivered person's answer that was saved before the task was completed as owed input: reopen the done task straight to in_progress (an answer continues the assignee's own execution) and promote the answer wake. Answers saved after completion, tool-action outcomes, already-delivered answers, agent-requested wakes and cancelled tasks keep the old behavior. Eval steering.approval-delivery (oracle v7): also accept Acorn work that the worker's settling turn saved after the answer was saved, while still requiring exactly one later turn bound to the answer. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
42fd790e80
commit
8a4eaab2c0
8 files changed
+238
-9
No files matched your search
@@ -5451,7 +5451,7 @@
|
||||
"chat-catalog-domains/runtime-support.ts",
|
||||
"chat-scenarios/steering.yaml"
|
||||
],
|
||||
"oracleVersion": "paperclip.chat-catalog.runtime.approval-delivery.v6",
|
||||
"oracleVersion": "paperclip.chat-catalog.runtime.approval-delivery.v7",
|
||||
"coverageGaps": [],
|
||||
"expectedRunCount": 5,
|
||||
"minimumRunCount": 4,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { expect, it } from "vitest";
|
||||
import { approvalResponseListing, classifyApprovalSuccessors, gradeApprovalQueue, leadRelayEntries, ordinaryApprovalEntries, pendingInputYield } from "./runtime-approval-queue.js";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { acornWorkSource, approvalResponseListing, classifyApprovalSuccessors, gradeApprovalQueue, leadRelayEntries, ordinaryApprovalEntries, pendingInputYield } from "./runtime-approval-queue.js";
|
||||
function valid() {
|
||||
const payload = { questionSet: { questions: [{ id: "name", prompt: "What is the product name?" }] } };
|
||||
return { scenarioId: "steering.approval-delivery", companyId: "company", leadId: "ceo", agents: [], links: [], humanQueries: [], reply: "", tasks: [{ id: "website" }], activity: [],
|
||||
@@ -100,3 +100,26 @@ it("keeps only the person's own messages as editable ordinary entries", () => {
|
||||
expect(ordinaryApprovalEntries(queue, "human").map(row => row.comment.id)).toEqual(["mine"]);
|
||||
expect(leadRelayEntries(queue, "ceo").map(row => row.comment.id)).toEqual(["relay"]);
|
||||
});
|
||||
|
||||
// Retained runner-opencode attempt (gha-43129293316): the settling turn read the answered question itself and saved the
|
||||
// Acorn document after the answer was saved; the answer must still reach one bound turn (product fix in this change).
|
||||
describe("where the Acorn document came from", () => {
|
||||
const answeredAt = "2026-11-12T01:03:19.592Z";
|
||||
const doc = (runId: string, createdAt: string, body = "# Acorn website delivery\n\nProduct name: **Acorn**", agent = "morgan") => ({
|
||||
revisions: [{ id: "rev", documentId: "doc", createdByRunId: runId, createdByAgentId: agent, createdAt, body }], documents: [{ id: "doc", latestRevisionId: "rev" }] });
|
||||
const source = (input: ReturnType<typeof doc>, extra: Partial<Parameters<typeof acornWorkSource>[0]> = {}) => acornWorkSource({ ...input, workerId: "morgan",
|
||||
successorRunIds: ["bound", "relay"], settlingRunId: "settling", answeredAt, ...extra });
|
||||
it("accepts the bound turn's document", () => expect(source(doc("bound", "2026-11-12T01:05:30.000Z"))).toBe("delivered_turn"));
|
||||
it("accepts the lead relay turn's document", () => expect(source(doc("relay", "2026-11-12T01:05:00.000Z"))).toBe("delivered_turn"));
|
||||
it("accepts the settling turn's document saved after the answer (the retained shape)", () => expect(source(doc("settling", "2026-11-12T01:04:21.660Z"))).toBe("settling_turn_after_answer"));
|
||||
it("accepts a reworded document that still names Acorn", () => expect(source(doc("settling", "2026-11-12T01:04:21.660Z", "Website outline for Acorn: hero, features, contact."))).toBe("settling_turn_after_answer"));
|
||||
it("rejects a settling-turn document saved before the answer existed", () => expect(source(doc("settling", "2026-11-12T01:02:00.000Z"))).toBeNull());
|
||||
it("rejects a document that does not use the answered name", () => expect(source(doc("bound", "2026-11-12T01:05:30.000Z", "# Website delivery\n\nProduct name: Birch"))).toBeNull());
|
||||
it("rejects a name that only appears inside another word", () => expect(source(doc("bound", "2026-11-12T01:05:30.000Z", "Acorns are a placeholder."))).toBeNull());
|
||||
it("rejects a document saved by another agent", () => expect(source(doc("bound", "2026-11-12T01:05:30.000Z", undefined, "ceo"))).toBeNull());
|
||||
it("rejects a revision that is no longer the document's current one", () => { const d = doc("bound", "2026-11-12T01:05:30.000Z"); d.documents[0]!.latestRevisionId = "newer";
|
||||
expect(source(d)).toBeNull(); });
|
||||
it("rejects a document from an unrelated turn", () => expect(source(doc("other", "2026-11-12T01:05:30.000Z"))).toBeNull());
|
||||
it("rejects the settling turn when the answer time is unknown", () => expect(source(doc("settling", "2026-11-12T01:04:21.660Z"), { answeredAt: null })).toBeNull());
|
||||
it("still fails the case when the settling turn used the answer but no turn was bound to it", () => { const o = valid(); o.state.consumingRunCount = 0; expect(gradeApprovalQueue(o).passed).toBe(false); });
|
||||
});
|
||||
@@ -52,6 +52,23 @@ export function classifyApprovalSuccessors(runs: SuccessorRun[], questionId: str
|
||||
}
|
||||
return { bound, relay, other };
|
||||
}
|
||||
type SavedRevision = { id?: unknown; documentId?: unknown; createdByRunId?: unknown; createdByAgentId?: unknown; createdAt?: unknown; body?: unknown };
|
||||
/**
|
||||
* Where the current Acorn document came from. A later worker turn (the one bound to the answer, or the lead's relay
|
||||
* turn) counts. So does the worker's settling turn, but only for a revision it saved after the answer was saved: an
|
||||
* agent may read the answered question on its own task, and the answer still reaches exactly one bound turn later.
|
||||
* A revision saved before the answer existed, by another agent, or no longer current does not count.
|
||||
*/
|
||||
export function acornWorkSource(input: { revisions: SavedRevision[]; documents: Array<{ id?: unknown; latestRevisionId?: unknown }>; workerId: string;
|
||||
successorRunIds: string[]; settlingRunId: string; answeredAt: unknown }) {
|
||||
const answered = typeof input.answeredAt === "string" ? Date.parse(input.answeredAt) : NaN;
|
||||
const current = input.revisions.filter(row => row.createdByAgentId === input.workerId && typeof row.body === "string" && /\bAcorn\b/.test(row.body)
|
||||
&& input.documents.some(doc => doc.id === row.documentId && doc.latestRevisionId === row.id));
|
||||
if (current.some(row => typeof row.createdByRunId === "string" && input.successorRunIds.includes(row.createdByRunId))) return "delivered_turn" as const;
|
||||
if (Number.isFinite(answered) && current.some(row => row.createdByRunId === input.settlingRunId && typeof row.createdAt === "string" && Date.parse(row.createdAt) > answered))
|
||||
return "settling_turn_after_answer" as const;
|
||||
return null;
|
||||
}
|
||||
export function gradeApprovalQueue(value: unknown) {
|
||||
const c = checks(), o = runtimeObservation(value);
|
||||
c.add("real-question-response-and-ordinary-queue-evidence", !!o, "The actual native question, human response, separate queue mutations and once-only consuming native input are required."); if (!o) return c.result();
|
||||
@@ -118,7 +135,7 @@ export const runtimeApprovalQueueCases = [runtimeDefinition("steering.approval-d
|
||||
humanId: session.user.id, ordinaryQueueOwned: true, ordinaryQueuedBeforeAnswer: true };
|
||||
}, gradeApprovalQueue, {
|
||||
setupPolicy: runtimeQueueCases[0]!.setupPolicy, verifySetup: runtimeQueueCases[0]!.verifySetup,
|
||||
authentication: "human-session", expectedRunCount: 5, minimumRunCount: 4, attemptTimeoutMs: 900_000, oracleVersion: "paperclip.chat-catalog.runtime.approval-delivery.v6",
|
||||
authentication: "human-session", expectedRunCount: 5, minimumRunCount: 4, attemptTimeoutMs: 900_000, oracleVersion: "paperclip.chat-catalog.runtime.approval-delivery.v7",
|
||||
sourceFiles: ["chat-catalog-domains/runtime-approval-queue.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/steering.yaml"],
|
||||
async act(ctx, state) {
|
||||
const canonicalPhase = await pollUntil({ label: "Canonical naming answer guidance while real worker remains active", deadlineAt: Date.now() + 150_000,
|
||||
@@ -236,8 +253,10 @@ export const runtimeApprovalQueueCases = [runtimeDefinition("steering.approval-d
|
||||
},
|
||||
async observe(ctx, state) { const o = await observeRuntime(ctx, state);
|
||||
const question = state.questionBefore as IssueThreadInteraction; state.questionAfter = (await ctx.input.api.get<IssueThreadInteraction[]>(`/api/issues/${(state.task as Entity).id}/interactions`)).find(row => row.id === question.id);
|
||||
const successorIds = [state.consumerRunId, ...((state.relayRunIds as string[] | undefined) ?? [])];
|
||||
state.actualAcornResult = o.revisions.some(row => successorIds.includes(row.createdByRunId) && row.createdByAgentId === (state.worker as Entity).id && /Acorn/i.test(row.body ?? "") && o.documents.some(doc => doc.id === row.documentId && doc.latestRevisionId === row.id));
|
||||
const successorRunIds = [state.consumerRunId, ...((state.relayRunIds as string[] | undefined) ?? [])].filter((id): id is string => typeof id === "string");
|
||||
state.acornWorkSource = acornWorkSource({ revisions: o.revisions, documents: o.documents, workerId: (state.worker as Entity).id, successorRunIds,
|
||||
settlingRunId: String(state.workerRunId), answeredAt: (state.questionAfter as IssueThreadInteraction | undefined)?.resolvedAt });
|
||||
state.actualAcornResult = state.acornWorkSource !== null;
|
||||
state.initialWorkRetained = o.revisions.some(row => row.createdByRunId === state.workerRunId && (row.body ?? "").includes(String(state.initialMarker))); return o; },
|
||||
async verifyResult(ctx, state) { const source = ctx.input.page.url(), task = state.task as Entity;
|
||||
await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" }); await ctx.input.page.reload({ waitUntil: "domcontentloaded" });
|
||||
|
||||
Reference in new issue
Block a user