mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-01 23:43:16 +01:00
Merge current master native chat recovery into deferred receipts
Co-Authored-By: Paperclip <noreply@paperclip.ing> * commit '11b49125db5ea1b274e69faf9d1abe057cdbb551': fix(chat): recover committed native Agent Chat replies (#18465) perf(heartbeat): drive retry promotion from durable deadlines (#18387) fix(ui): make the Coordinator composer footer opaque (#18455) fix(testops): verify unexpanded result job runs (#18462) fix(runner): preserve OpenCode written-answer defaults (#18456) fix(native): fence retries when replaying an audited Stop (#18457) test: await async outcomes and drive saved deadlines deterministically (#18416)
This commit is contained in:
commit
d6d0b67e8a
18 files changed
+522
-46
No files matched your search
@@ -0,0 +1,149 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { agents, authUsers, companies, completionContracts, createDb, heartbeatRunEvents, heartbeatRuns, issueComments, issues,
|
||||
nativeRunFinalizations, nativeRunResults, type Db } from "@paperclipai/db";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
|
||||
import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js";
|
||||
import { NativeRunCoordinatorStore } from "./native-run-coordinator-store.js";
|
||||
import { finalizeNativeRun, recoverCommittedNativeChatResponse } from "./native-run-finalizer.js";
|
||||
import { nativeResponsePresentationWork } from "./native-response-presentation-work.js";
|
||||
|
||||
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_AGENT_CHAT_RESPONSE_TEST_DATABASE_URL");
|
||||
const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
describe.skipIf(!support.supported)("committed Agent Chat reply recovery", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
|
||||
let db: Db;
|
||||
beforeAll(async () => {
|
||||
if (!external) database = await startEmbeddedPostgresTestDatabase("agent-chat-response-");
|
||||
db = createDb(external ?? database!.connectionString);
|
||||
}, 60_000);
|
||||
afterAll(async () => { await db?.$client.end(); await database?.cleanup(); });
|
||||
|
||||
async function fixture(conversation = true) {
|
||||
const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID();
|
||||
const sessionId = randomUUID(), runnerId = randomUUID(), userId = `fixture-${randomUUID()}`, wakeId = randomUUID();
|
||||
await db.insert(authUsers).values({ id: userId, name: "Conversation owner", email: `${userId}@example.test`,
|
||||
createdAt: new Date("2020-01-01T00:00:00Z"), updatedAt: new Date("2020-01-01T00:00:00Z") });
|
||||
await db.insert(companies).values({ id: companyId, name: "Reply recovery fixture", issuePrefix: companyId.slice(0, 8) });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Conversation agent", adapterType: "paperclip_runner" });
|
||||
await db.insert(issues).values({ id: issueId, companyId, title: "Conversation reply", status: "in_progress", assigneeAgentId: agentId,
|
||||
...(conversation ? { conversationAgentId: agentId, conversationUserId: userId, conversationState: "active", conversationSessionGeneration: 0 } : {}) });
|
||||
await db.insert(issueComments).values({ id: wakeId, companyId, issueId, authorUserId: userId, body: "Hire one teammate and tell me when ready.",
|
||||
createdAt: new Date("2020-01-01T00:00:00Z") });
|
||||
const [contract] = await db.insert(completionContracts).values({ companyId, issueId, revision: 1,
|
||||
schemaVersion: "paperclip.completion-contract.v1", policyVersion: "test", risk: "standard", completionAuthority: "server_arbiter",
|
||||
incompleteCriteriaPolicy: "preserve_non_terminal", contractJson: { revision: "test", objective: "Answer the user", criteria: [] },
|
||||
canonicalSha256: "fixture-conversation-contract", createdByActorType: "system", createdByActorId: "test" }).returning();
|
||||
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, nativeIssueId: issueId, nativeSessionId: sessionId,
|
||||
runnerInstanceId: runnerId, completionContractId: contract.id, completionContractSha256: contract.canonicalSha256,
|
||||
status: "running", runtimeMode: "native", responsibleUserId: userId,
|
||||
contextSnapshot: { issueId, source: "issue.comment", wakeCommentId: wakeId, ...(conversation ? { conversationSessionGeneration: 0 } : {}) } });
|
||||
await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId));
|
||||
const port = new NativeRunCoordinatorStore(db, { companyId, issueId, runId, agentId, normalizedSessionId: sessionId,
|
||||
runnerSourceInstanceId: runnerId, completionContractId: contract.id, completionContractSha256: contract.canonicalSha256,
|
||||
completionContractRevision: "test", completionContractCriterionIds: [] });
|
||||
const reply = "The teammate is hired and ready for your next instruction.";
|
||||
const prpEvent = { runId, normalizedSessionId: sessionId, turnId: "hire-turn", sourceEventId: "fixture-final-answer",
|
||||
payload: { kind: "agentMessage", channel: "final", providerPhase: "final", text: reply } };
|
||||
await db.insert(heartbeatRunEvents).values({ companyId, runId, agentId, seq: 170, eventType: "item.completed", payload: { prpEvent } });
|
||||
await port.completeRun({ result: { schema: "paperclip.run_result.v1", reportedWorkDisposition: "done",
|
||||
summary: "A completed hire, with its final message preserved separately.", completionClaim: { contractRevision: "test",
|
||||
objectiveSatisfied: true, criteria: [], remainingWork: [] }, evidence: [], verification: [], attentionRequests: [], artifacts: [] },
|
||||
terminal: { schema: "paperclip.prp.terminal.v1", runTerminalState: "succeeded", turnTerminalState: "completed", reportedWorkDisposition: "done" }, turnId: "hire-turn" });
|
||||
await finalizeNativeRun({ db, runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: false });
|
||||
await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date("2020-01-01T00:02:00Z") }).where(eq(heartbeatRuns.id, runId));
|
||||
expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)))[0].phase).toBe("committed");
|
||||
const input = { companyId, issueId, runId };
|
||||
return { ...input, input, agentId, userId, wakeId, prpEvent, reply };
|
||||
}
|
||||
const comments = (runId: string) => db.select().from(issueComments).where(eq(issueComments.createdByRunId, runId));
|
||||
const issue = async (issueId: string) => (await db.select().from(issues).where(eq(issues.id, issueId)))[0];
|
||||
|
||||
it("finds a committed final Agent Chat turn, persists the exact reply once and only then settles waiting", async () => {
|
||||
const f = await fixture();
|
||||
expect(await comments(f.runId)).toEqual([]);
|
||||
expect(await issue(f.issueId)).toMatchObject({ status: "in_progress", conversationState: "active" });
|
||||
const work = nativeResponsePresentationWork(db), entries = await work.load(f.input);
|
||||
expect(entries).toEqual([expect.objectContaining({ id: f.runId })]);
|
||||
await expect(work.check(entries[0])).resolves.toBeNull();
|
||||
expect(await comments(f.runId)).toEqual([expect.objectContaining({ body: f.reply, authorAgentId: f.agentId, createdByRunId: f.runId })]);
|
||||
expect(await issue(f.issueId)).toMatchObject({ status: "in_review", conversationState: "waiting" });
|
||||
expect(await work.load(f.input)).toEqual([]);
|
||||
await expect(recoverCommittedNativeChatResponse(db, f.input)).resolves.toBe(false);
|
||||
expect(await comments(f.runId)).toHaveLength(1);
|
||||
});
|
||||
it("keeps a later user message actionable after recovering the earlier exact reply", async () => {
|
||||
const f = await fixture();
|
||||
await db.insert(issueComments).values({ companyId: f.companyId, issueId: f.issueId, authorUserId: f.userId, body: "Another instruction",
|
||||
createdAt: new Date("2020-01-01T00:01:00Z") });
|
||||
await expect(recoverCommittedNativeChatResponse(db, f.input)).resolves.toBe(true);
|
||||
expect(await comments(f.runId)).toHaveLength(1);
|
||||
expect(await issue(f.issueId)).toMatchObject({ status: "in_progress", conversationState: "active" });
|
||||
});
|
||||
it("does not duplicate a reply under concurrent recovery", async () => {
|
||||
const f = await fixture();
|
||||
expect((await Promise.all([recoverCommittedNativeChatResponse(db, f.input), recoverCommittedNativeChatResponse(db, f.input)])).sort())
|
||||
.toEqual([false, true]);
|
||||
expect(await comments(f.runId)).toHaveLength(1);
|
||||
});
|
||||
it.each(["skip", "generation", "agent", "user", "execution", "waiting", "turn", "event-run", "session", "channel", "kind", "digest", "partial"])(
|
||||
"keeps %s mismatches unchanged instead of publishing an unauthorized or incomplete reply", async fault => {
|
||||
const f = await fixture();
|
||||
if (["skip", "generation"].includes(fault)) await db.update(heartbeatRuns).set({ contextSnapshot: { issueId: f.issueId,
|
||||
source: "issue.comment", wakeCommentId: f.wakeId, conversationSessionGeneration: fault === "generation" ? 1 : 0,
|
||||
...(fault === "skip" ? { skipIssueComment: true } : {}) } }).where(eq(heartbeatRuns.id, f.runId));
|
||||
if (fault === "user") await db.update(heartbeatRuns).set({ responsibleUserId: "another-user" }).where(eq(heartbeatRuns.id, f.runId));
|
||||
if (fault === "agent") {
|
||||
const other = randomUUID(); await db.insert(agents).values({ id: other, companyId: f.companyId, name: "Other agent" });
|
||||
await db.update(issues).set({ conversationAgentId: other, assigneeAgentId: other }).where(eq(issues.id, f.issueId));
|
||||
}
|
||||
if (fault === "execution") {
|
||||
const other = randomUUID(); await db.insert(heartbeatRuns).values({ id: other, companyId: f.companyId, agentId: f.agentId });
|
||||
await db.update(issues).set({ executionRunId: other }).where(eq(issues.id, f.issueId));
|
||||
}
|
||||
if (fault === "waiting") await db.update(issues).set({ executionRunId: null, status: "in_review", conversationState: "waiting" }).where(eq(issues.id, f.issueId));
|
||||
if (["turn", "event-run", "session", "channel", "kind"].includes(fault)) {
|
||||
const event = structuredClone(f.prpEvent);
|
||||
if (fault === "turn") event.turnId = "foreign-turn";
|
||||
if (fault === "event-run") event.runId = randomUUID();
|
||||
if (fault === "session") event.normalizedSessionId = randomUUID();
|
||||
if (fault === "channel") event.payload.channel = "analysis";
|
||||
if (fault === "kind") event.payload.kind = "reasoning";
|
||||
await db.update(heartbeatRunEvents).set({ payload: { prpEvent: event } }).where(and(eq(heartbeatRunEvents.runId, f.runId), eq(heartbeatRunEvents.seq, 170)));
|
||||
}
|
||||
if (fault === "digest") await db.update(nativeRunResults).set({ canonicalSha256: "invalid-digest" }).where(eq(nativeRunResults.runId, f.runId));
|
||||
if (fault === "partial") await db.update(nativeRunFinalizations).set({ phase: "copying_back" }).where(eq(nativeRunFinalizations.runId, f.runId));
|
||||
const before = await issue(f.issueId);
|
||||
await expect(recoverCommittedNativeChatResponse(db, f.input)).resolves.toBe(false);
|
||||
expect(await comments(f.runId)).toEqual([]); expect(await issue(f.issueId)).toEqual(before);
|
||||
},
|
||||
);
|
||||
it.each([false, true])("preserves a selected reply without replacing or resurrecting it (deleted=%s)", async deleted => {
|
||||
const f = await fixture();
|
||||
const [saved] = await db.insert(issueComments).values({ companyId: f.companyId, issueId: f.issueId, authorAgentId: f.agentId,
|
||||
createdByRunId: f.runId, body: "Already selected", ...(deleted ? { deletedAt: new Date("2020-01-01T00:03:00Z") } : {}) }).returning();
|
||||
await db.update(heartbeatRuns).set({ resultJson: { presentationDecision: { commentId: saved.id, commentAction: "create" } } }).where(eq(heartbeatRuns.id, f.runId));
|
||||
await expect(recoverCommittedNativeChatResponse(db, f.input)).resolves.toBe(false);
|
||||
expect(await comments(f.runId)).toEqual([saved]);
|
||||
});
|
||||
it("never crosses company scope and keeps ordinary internal task response recovery intact", async () => {
|
||||
const f = await fixture(false);
|
||||
await expect(recoverCommittedNativeChatResponse(db, { ...f.input, companyId: randomUUID() })).resolves.toBe(false);
|
||||
const work = nativeResponsePresentationWork(db), entries = await work.load(f.input);
|
||||
expect(entries).toHaveLength(1); await expect(work.check(entries[0])).resolves.toBeNull();
|
||||
expect(await comments(f.runId)).toEqual([expect.objectContaining({ body: f.reply })]);
|
||||
expect((await issue(f.issueId)).conversationAgentId).toBeNull();
|
||||
});
|
||||
it("rolls back a partial reply write without changing the waiting state or consuming recovery", async () => {
|
||||
const f = await fixture(), abort = new Error("fixture-commit-aborted");
|
||||
const failing = new Proxy(db, { get(target, key) { return key === "transaction"
|
||||
? (callback: Parameters<Db["transaction"]>[0]) => db.transaction(async tx => { await callback(tx); throw abort; })
|
||||
: Reflect.get(target, key); } });
|
||||
await expect(recoverCommittedNativeChatResponse(failing, f.input)).rejects.toBe(abort);
|
||||
expect(await comments(f.runId)).toEqual([]);
|
||||
expect(await issue(f.issueId)).toMatchObject({ status: "in_progress", conversationState: "active" });
|
||||
await expect(recoverCommittedNativeChatResponse(db, f.input)).resolves.toBe(true);
|
||||
expect(await comments(f.runId)).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
@@ -1,6 +1,6 @@
|
||||
import { QueryBuilder } from "drizzle-orm/pg-core";
|
||||
import { and, asc, eq, gt, inArray, isNotNull, isNull, notExists, or, sql } from "drizzle-orm";
|
||||
import { heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults, statusDecisions, statusDecisionEffects } from "@paperclipai/db";
|
||||
import { and, asc, eq, exists, gt, inArray, isNotNull, isNull, notExists, or, sql } from "drizzle-orm";
|
||||
import { heartbeatRunEvents, heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults, statusDecisions, statusDecisionEffects } from "@paperclipai/db";
|
||||
import type { NativeResponsePresentationScope } from "../native-response-presentation-notifications.js";
|
||||
|
||||
export function nativeResponsePresentationCandidates(scope?: NativeResponsePresentationScope, after?: string, runId?: string) {
|
||||
@@ -30,7 +30,24 @@ export function nativeResponsePresentationCandidates(scope?: NativeResponsePrese
|
||||
sql`${nativeRunResults.resultJson} #>> '{terminal,turnTerminalState}' = 'completed'`,
|
||||
or(
|
||||
and(eq(heartbeatRuns.status, "succeeded"),
|
||||
sql`(coalesce(${issues.conversationAgentId}::text, '') = '' or coalesce(${issues.conversationUserId}, '') = '')`,
|
||||
or(sql`(coalesce(${issues.conversationAgentId}::text, '') = '' or coalesce(${issues.conversationUserId}, '') = '')`,
|
||||
and(eq(statusDecisions.reasonCode, "conversation_turn_finished"),
|
||||
eq(issues.conversationAgentId, heartbeatRuns.agentId), eq(issues.conversationUserId, heartbeatRuns.responsibleUserId),
|
||||
eq(issues.status, "in_progress"), eq(issues.conversationState, "active"),
|
||||
or(isNull(issues.executionRunId), eq(issues.executionRunId, heartbeatRuns.id)),
|
||||
sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issues.id}::text`,
|
||||
sql`jsonb_typeof(${heartbeatRuns.contextSnapshot}->'conversationSessionGeneration') = 'number'`,
|
||||
sql`${heartbeatRuns.contextSnapshot}->>'conversationSessionGeneration' = ${issues.conversationSessionGeneration}::text`,
|
||||
exists(db.select({ id: heartbeatRunEvents.id }).from(heartbeatRunEvents).where(and(
|
||||
eq(heartbeatRunEvents.companyId, heartbeatRuns.companyId), eq(heartbeatRunEvents.runId, heartbeatRuns.id),
|
||||
eq(heartbeatRunEvents.agentId, heartbeatRuns.agentId), eq(heartbeatRunEvents.eventType, "item.completed"),
|
||||
sql`${heartbeatRunEvents.payload} #>> '{prpEvent,runId}' = ${heartbeatRuns.id}::text`,
|
||||
sql`${heartbeatRunEvents.payload} #>> '{prpEvent,normalizedSessionId}' = ${heartbeatRuns.nativeSessionId}::text`,
|
||||
sql`${heartbeatRunEvents.payload} #>> '{prpEvent,turnId}' = ${nativeRunResults.turnId}`,
|
||||
sql`${heartbeatRunEvents.payload} #>> '{prpEvent,payload,kind}' = 'agentMessage'`,
|
||||
sql`${heartbeatRunEvents.payload} #>> '{prpEvent,payload,channel}' = 'final'`,
|
||||
sql`${heartbeatRunEvents.payload} #>> '{prpEvent,payload,text}' ~ '[^[:space:]]'`,
|
||||
))))),
|
||||
sql`${nativeRunResults.resultJson} #>> '{result,reportedWorkDisposition}' = 'done'`,
|
||||
sql`${nativeRunResults.resultJson} #>> '{terminal,reportedWorkDisposition}' = 'done'`),
|
||||
and(sql`${nativeRunResults.resultJson} #>> '{result,reportedWorkDisposition}' = 'yielded'`,
|
||||
|
||||
@@ -29,7 +29,7 @@ import { readPersistedNativeFailureEvidence } from "./native-provider-failure-ev
|
||||
import { dismissAutomaticCompletionReviews } from "./automatic-completion-reviews.js";
|
||||
import { findSupersededExternalReviewRequests } from "./superseded-external-review-requests.js";
|
||||
import { getNativeReviewAssignment, readNativeReviewAssignmentContext } from "./native-review-participant.js";
|
||||
import { conversationNativeDecision, isConversation } from "../agent-conversations.js";
|
||||
import { conversationNativeDecision, isConversation, settleConversationTurnForRun } from "../agent-conversations.js";
|
||||
import { issueTreeControlService } from "../issue-tree-control.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { preserveNativeWorkspaceExportLease } from "./native-workspace-export-resume.js";
|
||||
@@ -747,6 +747,7 @@ async function recoverCommittedNativeChatResponseInScope(
|
||||
): Promise<boolean> {
|
||||
let agentId: string | null = null;
|
||||
let materialized = false;
|
||||
let conversationReply = false;
|
||||
const entered = beginRunPhase("response.transaction.entry");
|
||||
try {
|
||||
materialized = await measureRunPhase("response.transaction", () => db.transaction(async (transaction) => {
|
||||
@@ -789,6 +790,7 @@ async function recoverCommittedNativeChatResponseInScope(
|
||||
id: heartbeatRuns.id,
|
||||
companyId: heartbeatRuns.companyId,
|
||||
agentId: heartbeatRuns.agentId,
|
||||
responsibleUserId: heartbeatRuns.responsibleUserId,
|
||||
nativeIssueId: heartbeatRuns.nativeIssueId,
|
||||
nativeSessionId: heartbeatRuns.nativeSessionId,
|
||||
completionContractId: heartbeatRuns.completionContractId,
|
||||
@@ -803,6 +805,8 @@ async function recoverCommittedNativeChatResponseInScope(
|
||||
contextSnapshot: sql<Record<string, unknown> | null>`case when ${heartbeatRuns.contextSnapshot} is null then null
|
||||
else jsonb_build_object(
|
||||
'skipIssueComment', ${heartbeatRuns.contextSnapshot}->'skipIssueComment',
|
||||
'issueId', ${heartbeatRuns.contextSnapshot}->'issueId',
|
||||
'conversationSessionGeneration', ${heartbeatRuns.contextSnapshot}->'conversationSessionGeneration',
|
||||
'source', ${heartbeatRuns.contextSnapshot}->'source',
|
||||
'externalChatContinuation', ${heartbeatRuns.contextSnapshot}->'externalChatContinuation',
|
||||
'paperclipWake', jsonb_build_object('externalInteractionContinuation',
|
||||
@@ -909,12 +913,16 @@ async function recoverCommittedNativeChatResponseInScope(
|
||||
agentId = run.agentId;
|
||||
return true;
|
||||
}
|
||||
// Copyback can be owned by reconciliation before the live heartbeat
|
||||
// reaches its presentation step. Recover a completed internal task's
|
||||
// exact final reply from the same accepted turn, without rerunning work
|
||||
// or granting external-chat publication authority.
|
||||
const conversation = isConversation(issue);
|
||||
const currentConversation = conversation && decision.reasonCode === "conversation_turn_finished"
|
||||
&& issue.conversationAgentId === run.agentId && issue.conversationUserId === run.responsibleUserId
|
||||
&& issue.status === "in_progress" && issue.conversationState === "active"
|
||||
&& (!issue.executionRunId || issue.executionRunId === run.id)
|
||||
&& run.contextSnapshot?.issueId === issue.id
|
||||
&& typeof run.contextSnapshot.conversationSessionGeneration === "number"
|
||||
&& run.contextSnapshot.conversationSessionGeneration === issue.conversationSessionGeneration;
|
||||
if (
|
||||
!isConversation(issue) &&
|
||||
(!conversation || currentConversation) &&
|
||||
run.status === "succeeded" &&
|
||||
record(run.contextSnapshot).skipIssueComment !== true &&
|
||||
!isExternalChatPresentationContext(run.contextSnapshot) &&
|
||||
@@ -929,6 +937,9 @@ async function recoverCommittedNativeChatResponseInScope(
|
||||
eq(heartbeatRunEvents.companyId, input.companyId),
|
||||
eq(heartbeatRunEvents.runId, run.id),
|
||||
eq(heartbeatRunEvents.eventType, "item.completed"),
|
||||
conversation ? and(eq(heartbeatRunEvents.agentId, run.agentId),
|
||||
sql`${heartbeatRunEvents.payload}->'prpEvent'->>'runId' = ${run.id}`,
|
||||
sql`${heartbeatRunEvents.payload}->'prpEvent'->>'normalizedSessionId' = ${run.nativeSessionId}`) : undefined,
|
||||
sql`${heartbeatRunEvents.payload}->'prpEvent'->>'turnId' = ${accepted.turnId}`,
|
||||
sql`${heartbeatRunEvents.payload} #>> '{prpEvent,payload,kind}' = 'agentMessage'`,
|
||||
sql`${heartbeatRunEvents.payload} #>> '{prpEvent,payload,channel}' = 'final'`,
|
||||
@@ -963,6 +974,7 @@ async function recoverCommittedNativeChatResponseInScope(
|
||||
})}::jsonb`, updatedAt: new Date(),
|
||||
}).where(eq(heartbeatRuns.id, run.id));
|
||||
agentId = run.agentId;
|
||||
conversationReply = conversation;
|
||||
return true;
|
||||
}
|
||||
if (
|
||||
@@ -1039,12 +1051,14 @@ async function recoverCommittedNativeChatResponseInScope(
|
||||
} finally {
|
||||
entered();
|
||||
}
|
||||
if (materialized && agentId)
|
||||
if (materialized && agentId) {
|
||||
if (conversationReply) await settleConversationTurnForRun(db, input.companyId, input.runId);
|
||||
publishChatPublicationCommitSignal({
|
||||
...input,
|
||||
agentId,
|
||||
eventType: "run.presentation.resolved",
|
||||
});
|
||||
}
|
||||
return materialized;
|
||||
}
|
||||
|
||||
|
||||
@@ -99,6 +99,7 @@ import { buildNativeHeartbeatPreparationSpans, createNativeRunTrace } from "./na
|
||||
import { NativeRunnerOwnershipUnverifiedError } from "./native-runner-ownership.js";
|
||||
import type { AdapterRuntimeEvent } from "../../adapters/index.js";
|
||||
import type { AdapterUsageCheckpoint } from "@paperclipai/adapter-utils";
|
||||
import { acknowledgedNativeStopExecutionHasStopped, isAcknowledgedNativeStop } from "../acknowledged-native-stop.js";
|
||||
|
||||
vi.mock("node:fs", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("node:fs")>();
|
||||
@@ -7852,6 +7853,27 @@ describe("native session cancellation", () => {
|
||||
expect(state.persistActivity).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ companyId: "another-company" }, { runId: "another-run" }, { issueId: "another-issue" },
|
||||
{ scope: "issue" }, { effects: ["cancel_issue"] }, { intentAuditId: null },
|
||||
])("does not fence a failed retry for a foreign or stale Stop receipt %j", async changed => {
|
||||
const intent = { schema: "paperclip.native-cancellation.v1", intentId: "native-cancellation:earlier-stop",
|
||||
...execution.binding, scope: "run", reasonCode: "cancellation_run_only", effects: ["release_run_resources"],
|
||||
dispatchState: "acknowledged", dispatched: true, intentAuditId: "earlier-intent-audit",
|
||||
acknowledgementAuditId: "earlier-ack-audit", ...changed };
|
||||
const coordinator = { runId: execution.binding.runId, assessmentId: null, phase: "retryable_failure" };
|
||||
const persistence = cancellationDb({ status: "failed", coordinator, resultJson: { nativeCancellation: intent } });
|
||||
const before = structuredClone(persistence.getResultJson());
|
||||
state.cancel.mockClear();
|
||||
state.persistActivity.mockClear();
|
||||
await expect(cancelNativeSession(execution.binding.runId, "retry Stop", { db: persistence.db, scope: "run" })).rejects.toThrow();
|
||||
expect(persistence.updates).toEqual([]);
|
||||
expect(coordinator.phase).toBe("retryable_failure");
|
||||
expect(persistence.getResultJson()).toEqual(before);
|
||||
expect(state.cancel).not.toHaveBeenCalled();
|
||||
expect(state.persistActivity).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not mistake a recovered running attempt for the failed retry that reserved Stop", async () => {
|
||||
const id = "11111111-1111-4111-8111-111111111111";
|
||||
const persistence = cancellationDb({ status: "running",
|
||||
@@ -7877,6 +7899,39 @@ describe("native session cancellation", () => {
|
||||
expect(persistence.getResultJsonUpdateCount()).toBe(writes);
|
||||
});
|
||||
|
||||
it("fences a failed retry whose earlier Stop is already acknowledged without changing its receipts", async () => {
|
||||
const intent = { schema: "paperclip.native-cancellation.v1", intentId: "native-cancellation:earlier-stop",
|
||||
...execution.binding, scope: "run", reasonCode: "cancellation_run_only",
|
||||
effects: ["release_run_resources"], dispatchState: "acknowledged", dispatched: true,
|
||||
intentAuditId: "earlier-intent-audit", acknowledgementAuditId: "earlier-ack-audit",
|
||||
acknowledgedAt: "2026-01-01T00:00:00.000Z", decisionId: null, priorCoordinatorDecisionId: null };
|
||||
const coordinator = { runId: execution.binding.runId, assessmentId: null, phase: "retryable_failure",
|
||||
nextAttemptAt: new Date(Date.now() + 30_000) };
|
||||
const persistence = cancellationDb({ status: "failed", coordinator,
|
||||
resultJson: { nativeCancellation: intent, executionCancellation: { state: "requested", requestedAt: "later-stop" } } });
|
||||
const before = structuredClone(persistence.getResultJson());
|
||||
state.cancel.mockClear();
|
||||
state.persistActivity.mockClear();
|
||||
for (let attempt = 0; attempt < 2; attempt++) {
|
||||
await expect(cancelNativeSession(execution.binding.runId, "retry Stop", { db: persistence.db, scope: "run" }))
|
||||
.resolves.toMatchObject({ dispatched: true, auditId: intent.intentAuditId });
|
||||
expect(coordinator).toMatchObject({ phase: "terminal_failure", failureCode: "native_retry_cancelled", nextAttemptAt: null });
|
||||
expect(persistence.getResultJson()).toEqual(before);
|
||||
}
|
||||
expect(persistence.updates.filter(update => update.table === nativeRunFinalizations)).toHaveLength(1);
|
||||
expect(persistence.getResultJsonUpdateCount()).toBe(0);
|
||||
expect(state.cancel).not.toHaveBeenCalled();
|
||||
expect(state.persistActivity).not.toHaveBeenCalled();
|
||||
const stopped = { id: execution.binding.runId, companyId: execution.binding.companyId,
|
||||
nativeIssueId: execution.binding.issueId, status: "cancelled", processPid: null, processGroupId: null,
|
||||
resultJson: { ...persistence.getResultJson(), cancelledByActorType: "user", cancelledByUserId: "operator" } };
|
||||
const lease = { provider: "daytona", providerLeaseId: "unconfirmed-provider-resource", status: "active",
|
||||
releasedAt: null, cleanupStatus: null, metadata: {} };
|
||||
const leaseDb = { select: () => ({ from: () => ({ where: async () => [lease] }) }) } as unknown as Db;
|
||||
expect(isAcknowledgedNativeStop(stopped)).toBe(true);
|
||||
await expect(acknowledgedNativeStopExecutionHasStopped(leaseDb, stopped)).resolves.toBe(false);
|
||||
});
|
||||
|
||||
it("uses the reserved caller UUID and permits same-intent and default retries", async () => {
|
||||
const id = "11111111-1111-4111-8111-111111111111";
|
||||
const persistence = cancellationDb({ resultJson: { startupCancellation: { cancellationRequestId: id } } });
|
||||
|
||||
@@ -7541,6 +7541,27 @@ export async function cancelNativeSession(
|
||||
runId, companyId: cancellationContext.companyId, issueId: cancellationContext.issueId,
|
||||
resultJson, requestId: requestedId, scope: options.scope ?? "run",
|
||||
}, coordinator)) throw conflict("Native retry is no longer cancellable");
|
||||
const fenceRetryCancellation = async () => {
|
||||
if (coordinator.phase !== "retryable_failure") return;
|
||||
await tx
|
||||
.update(nativeRunFinalizations)
|
||||
.set({
|
||||
phase: "terminal_failure",
|
||||
nextAttemptAt: null,
|
||||
leaseOwner: null,
|
||||
leaseExpiresAt: null,
|
||||
recoveryState: "blocked",
|
||||
failureCode: "native_retry_cancelled",
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(
|
||||
and(
|
||||
eq(nativeRunFinalizations.runId, runId),
|
||||
eq(nativeRunFinalizations.companyId, cancellationContext.companyId),
|
||||
eq(nativeRunFinalizations.phase, "retryable_failure"),
|
||||
),
|
||||
);
|
||||
};
|
||||
const existing = record(resultJson.nativeCancellation);
|
||||
const existingIntentId =
|
||||
typeof existing.intentId === "string" && existing.intentId.length > 0
|
||||
@@ -7564,6 +7585,10 @@ export async function cancelNativeSession(
|
||||
: null;
|
||||
if (!existingAuditId)
|
||||
throw new Error("native_cancellation_intent_audit_missing");
|
||||
if (retryCancellation && coordinator.phase === "retryable_failure") {
|
||||
await notifyNativeResultCommit(tx, { companyId: cancellationContext.companyId, runId });
|
||||
await fenceRetryCancellation();
|
||||
}
|
||||
return {
|
||||
intentId: existingIntentId,
|
||||
auditId: existingAuditId,
|
||||
@@ -7643,24 +7668,7 @@ export async function cancelNativeSession(
|
||||
// Cancellation is also an authority fence for a durable retry whose
|
||||
// preceding provider has already failed. No in-memory session is needed.
|
||||
await notifyNativeResultCommit(tx, { companyId: cancellationContext.companyId, runId });
|
||||
await tx
|
||||
.update(nativeRunFinalizations)
|
||||
.set({
|
||||
phase: "terminal_failure",
|
||||
nextAttemptAt: null,
|
||||
leaseOwner: null,
|
||||
leaseExpiresAt: null,
|
||||
recoveryState: "blocked",
|
||||
failureCode: "native_retry_cancelled",
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(
|
||||
and(
|
||||
eq(nativeRunFinalizations.runId, runId),
|
||||
eq(nativeRunFinalizations.companyId, cancellationContext.companyId),
|
||||
eq(nativeRunFinalizations.phase, "retryable_failure"),
|
||||
),
|
||||
);
|
||||
await fenceRetryCancellation();
|
||||
intentPublication = activity.publication;
|
||||
return {
|
||||
intentId,
|
||||
|
||||
Reference in new issue
Block a user