fix: retry reviewed chat setup after temporary contention (#19787)

Retry reviewed chat preparation with the original accepted input.

Bound retries for authorization proof waits that expire before provider
work. Preserve the original answer receipt and queue later wakes
separately. Keep plan confirmation semantics by assigning the interaction
marker only to question answer delivery and legacy question payloads.

Verification: 433 tests across four backend suites, all 17 project-chat
browser tests, and all server source/test typecheck layers passed. The
plan approval regression was reproduced on the previous head before the
fix. All applicable CI passed and exact-head Greptile Apex scored 5/5.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

server/src/services/native-runtime/external-chat-question-response-retry.ts

@@ -0,0 +1,52 @@
+import { and, eq } from "drizzle-orm";
+import { heartbeatRuns, type Db } from "@paperclipai/db";
+import { INTERACTION_CONTINUATION_INFRA_RETRY_REASON } from "../../modules/run-dispatch/index.js";
+import { executionFailureRetryCount } from "../execution-recovery-attempt.js";
+import { REVIEWED_CHAT_BINDING_NOT_READY } from "../reviewed-chat-binding-diagnostics.js";
+import { nativeSha256 } from "./canonical.js";
+
+export const EXTERNAL_CHAT_ANSWER_ADMISSION_KEYS = [
+  "source", "issueId", "interactionId", "interactionKind", "interactionStatus",
+  "sourceRunId", "sourceCommentId", "externalChatContinuation",
+  "wakeCommentId", "wakeCommentIds", "commentId",
+] as const;
+
+type Run = typeof heartbeatRuns.$inferSelect;
+type LockMode = "blocking" | "nonblocking" | "read";
+const record = (value: unknown): Record<string, unknown> =>
+  value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
+
+export async function resolveQuestionResponseRetryAdmission(
+  tx: Db,
+  run: Run,
+  lockMode: LockMode,
+): Promise<Run | null> {
+  const seen = new Set<string>();
+  let current = run;
+  while (current.retryOfRunId) {
+    const context = record(current.contextSnapshot);
+    if (seen.size >= 2 || seen.has(current.id) || current.scopeKind !== "issue" || current.issueId !== context.issueId ||
+        current.issueId !== run.issueId || current.scheduledRetryReason !== INTERACTION_CONTINUATION_INFRA_RETRY_REASON ||
+        context.retryOfRunId !== current.retryOfRunId || context.retryReason !== INTERACTION_CONTINUATION_INFRA_RETRY_REASON ||
+        context.wakeReason !== INTERACTION_CONTINUATION_INFRA_RETRY_REASON ||
+        context.scheduledRetryAttempt !== current.scheduledRetryAttempt ||
+        current.scheduledRetryAttempt === null || current.scheduledRetryAttempt < 1 || current.scheduledRetryAttempt > 2) return null;
+    seen.add(current.id);
+    const query = tx.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, current.retryOfRunId),
+      eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.agentId, run.agentId)));
+    const [parent] = await (lockMode === "read" ? query : lockMode === "nonblocking"
+      ? query.for("update", { noWait: true }) : query.for("update"));
+    if (!parent || seen.has(parent.id)) return null;
+    const parentContext = record(parent.contextSnapshot);
+    const recovery = record(record(parent.resultJson).executionRecovery);
+    if (parent.scopeKind !== "issue" || parent.issueId !== run.issueId || parent.status !== "failed" || !parent.finishedAt ||
+        parent.errorCode !== REVIEWED_CHAT_BINDING_NOT_READY || parent.runtimeMode !== "legacy" ||
+        parent.nativeIssueId !== null || parent.nativeSessionId !== null || parent.runnerInstanceId !== null ||
+        parent.processPid !== null || parent.processGroupId !== null || parent.externalRunId !== null ||
+        recovery.kind !== "bootstrap" || recovery.providerWorkStarted !== false ||
+        executionFailureRetryCount(parent) + 1 !== current.scheduledRetryAttempt ||
+        EXTERNAL_CHAT_ANSWER_ADMISSION_KEYS.some(key => nativeSha256(parentContext[key] ?? null) !== nativeSha256(context[key] ?? null))) return null;
+    current = parent;
+  }
+  return record(current.contextSnapshot).wakeReason === "issue_commented" ? current : null;
+}

server/src/services/native-runtime/external-chat-question-response.ts

@@ -28,6 +28,8 @@ import {
 import { questionResponseDeliveryValues } from "../question-response-delivery.js";
 import { voiceSessionStore, type VoiceTransaction, currentVoiceCredentialFingerprint } from "../voice/voice-session-store.js";
 import { nativeSha256 } from "./canonical.js";
+import { EXTERNAL_CHAT_ANSWER_ADMISSION_KEYS, resolveQuestionResponseRetryAdmission } from "./external-chat-question-response-retry.js";
+import { INTERACTION_CONTINUATION_INFRA_RETRY_REASON } from "../../modules/run-dispatch/index.js";
 
 export const EXTERNAL_CHAT_QUESTION_RESPONSE_KEY =
   "paperclipExternalChatQuestionResponse";
@@ -221,7 +223,7 @@ async function resolveQuestionResponseChain(
   if (
     context.source !== "issue.interaction.respond" ||
     context.issueId !== binding.issueId ||
-    context.wakeReason !== "issue_commented" ||
+    (context.wakeReason !== "issue_commented" && context.wakeReason !== INTERACTION_CONTINUATION_INFRA_RETRY_REASON) ||
     context.interactionKind !== "ask_user_questions" ||
     context.interactionStatus !== "answered" ||
     context.externalChatContinuation !== true ||
@@ -251,23 +253,14 @@ async function resolveQuestionResponseChain(
       : runQuery.for("update"));
   if (!run?.wakeupRequestId) return null;
   const admitted = record(run.contextSnapshot);
-  for (const key of [
-    "source",
-    "issueId",
-    "interactionId",
-    "interactionKind",
-    "interactionStatus",
-    "sourceRunId",
-    "sourceCommentId",
-    "externalChatContinuation",
-    "wakeCommentId",
-    "wakeCommentIds",
-  ]) {
+  for (const key of [...EXTERNAL_CHAT_ANSWER_ADMISSION_KEYS, "wakeReason"]) {
     if (
       nativeSha256(admitted[key] ?? null) !== nativeSha256(context[key] ?? null)
     )
       return null;
   }
+  const admission = await resolveQuestionResponseRetryAdmission(tx, run, lockMode);
+  if (!admission?.wakeupRequestId) return null;
 
   const query = tx
     .select({
@@ -308,10 +301,10 @@ async function resolveQuestionResponseChain(
     .innerJoin(
       agentWakeupRequests,
       and(
-        eq(agentWakeupRequests.id, run.wakeupRequestId),
+        eq(agentWakeupRequests.id, admission.wakeupRequestId),
         eq(agentWakeupRequests.companyId, issueThreadInteractions.companyId),
         eq(agentWakeupRequests.agentId, binding.agentId),
-        eq(agentWakeupRequests.runId, binding.runId),
+        eq(agentWakeupRequests.runId, admission.id),
       ),
     )
     .where(
@@ -420,7 +413,7 @@ async function resolveQuestionResponseChain(
     !(
       (delivery.status === "fallback_queued" &&
         delivery.deliveryMode === "wake_fallback" &&
-        delivery.targetRunId === binding.runId) ||
+        delivery.targetRunId === admission.id) ||
       (delivery.status === "delivering" &&
         delivery.targetRunId === null &&
         delivery.deliveryMode === null)

server/src/services/native-runtime/external-chat-wait.integration.test.ts

@@ -88,7 +88,7 @@ import {
   isExternalChatWaitAuthorizationContention,
   resolveExternalChatResponseWaitAuthorization,
 } from "./chat-attachment-reuse.js";
-import { attestReviewedExternalChatRun, buildPaperclipWakePayload } from "../heartbeat.js";
+import { attestReviewedExternalChatRun, buildPaperclipWakePayload, heartbeatService } from "../heartbeat.js";
 import { voiceCredentialFingerprint } from "../voice/voice-session-store.js";
 import { nativeSha256 } from "./canonical.js";
 import { questionResponseDeliveryValues } from "../question-response-delivery.js";
@@ -1297,6 +1297,100 @@ describe("native external-chat response wait", () => {
     },
   );
 
+  async function answerAttestationRetry(attempts: number) {
+    const native = await seedAnsweredChatTurn();
+    const [admitted] = await db.insert(heartbeatRuns).values({ companyId: native.companyId, agentId: native.agentId,
+      scopeKind: "issue", issueId: native.issueId, status: "running", runtimeMode: "legacy",
+      responsibleUserId: native.userId, wakeupRequestId: native.wakeId, contextSnapshot: native.context }).returning();
+    await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, native.runId));
+    await db.update(agentWakeupRequests).set({ runId: admitted!.id }).where(eq(agentWakeupRequests.id, native.wakeId));
+    await db.update(issueQuestionResponseDeliveries).set({ targetRunId: admitted!.id }).where(eq(issueQuestionResponseDeliveries.id, native.responseDeliveryId));
+    const fixture = { ...native, runId: admitted!.id };
+    await db.update(issues).set({ responsibleUserId: fixture.userId, executionRunId: fixture.runId }).where(eq(issues.id, fixture.issueId));
+    let predecessorId = fixture.runId;
+    let child: typeof heartbeatRuns.$inferSelect | undefined;
+    for (let attempt = 1; attempt <= attempts; attempt += 1) {
+      await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date(), runtimeMode: "legacy",
+        nativeIssueId: null, nativeSessionId: null, runnerInstanceId: null, scopeKind: "issue", issueId: fixture.issueId,
+        responsibleUserId: fixture.userId, errorCode: "reviewed_chat_execution_binding_not_ready",
+        resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } },
+      }).where(eq(heartbeatRuns.id, predecessorId));
+      const scheduled = await heartbeatService(db).scheduleBoundedRetry(predecessorId, {
+        now: new Date(), random: () => 0, retryReason: "interaction_continuation_infra_retry",
+        wakeReason: "interaction_continuation_infra_retry", maxAttempts: 2,
+      });
+      expect(scheduled.outcome).toBe("scheduled");
+      if (scheduled.outcome !== "scheduled") throw new Error("expected the admitted answer retry");
+      child = scheduled.run;
+      predecessorId = child.id;
+    }
+    await db.update(heartbeatRuns).set({ status: "running" }).where(eq(heartbeatRuns.id, child!.id));
+    return { fixture, child: child!, binding: { ...fixture, runId: child!.id }, context: structuredClone(child!.contextSnapshot!) };
+  }
+
+  it.each([1, 2])("attests answered retry %s against its unchanged original user receipt", async (attempts) => {
+    const { fixture, child, binding, context } = await answerAttestationRetry(attempts);
+    const [delivery] = await db.select().from(issueQuestionResponseDeliveries).where(eq(issueQuestionResponseDeliveries.id, fixture.responseDeliveryId));
+    const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, fixture.wakeId));
+    expect(await attestReviewedExternalChatRun({ db, ...binding, contextSnapshot: context })).toBe(true);
+    context.paperclipExternalChatExecutionBound = true;
+    context.paperclipWake = await buildPaperclipWakePayload({ db, companyId: fixture.companyId,
+      agentId: fixture.agentId, runId: child.id, contextSnapshot: context });
+    await db.update(heartbeatRuns).set({ contextSnapshot: context }).where(eq(heartbeatRuns.id, child.id));
+    expect(await resolveExternalChatQuestionResponse(db, binding, context, "read"))
+      .toMatchObject({ interactionIds: [fixture.interactionId], marker: { responseDeliveryId: fixture.responseDeliveryId } });
+    expect(await materializeExternalChatQuestionResponseInput({ db, binding, contextSnapshot: context }))
+      .toEqual([expect.objectContaining({ interactionId: fixture.interactionId })]);
+    expect((await db.select().from(issueQuestionResponseDeliveries).where(eq(issueQuestionResponseDeliveries.id, fixture.responseDeliveryId)))[0]).toEqual(delivery);
+    expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, fixture.wakeId)))[0]).toEqual(wake);
+  });
+
+  it.each(["revoked_identity", "receipt_actor", "delivery_target", "parent_batch", "provider_started", "parent_session", "unrelated_retry", "foreign_parent", "foreign_task"] as const)(
+    "rejects answered retry authority after %s", async (change) => {
+      const { fixture, child, binding, context } = await answerAttestationRetry(1);
+      if (change === "revoked_identity") await db.update(chatIdentityLinks).set({ status: "revoked" }).where(eq(chatIdentityLinks.principalId, fixture.principalId));
+      if (change === "receipt_actor") await db.update(agentWakeupRequests).set({ requestedByActorId: "different-user" }).where(eq(agentWakeupRequests.id, fixture.wakeId));
+      if (change === "delivery_target") await db.update(issueQuestionResponseDeliveries).set({ targetRunId: child.id }).where(eq(issueQuestionResponseDeliveries.id, fixture.responseDeliveryId));
+      if (change === "parent_batch") await db.update(heartbeatRuns).set({ contextSnapshot: { ...fixture.context, wakeCommentIds: [randomUUID()] } }).where(eq(heartbeatRuns.id, fixture.runId));
+      if (change === "provider_started") await db.update(heartbeatRuns).set({ resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: true } } }).where(eq(heartbeatRuns.id, fixture.runId));
+      if (change === "parent_session") await db.update(heartbeatRuns).set({ nativeSessionId: randomUUID() }).where(eq(heartbeatRuns.id, fixture.runId));
+      if (change === "unrelated_retry") await db.update(heartbeatRuns).set({ scheduledRetryReason: "transient_failure" }).where(eq(heartbeatRuns.id, child.id));
+      if (change === "foreign_parent" || change === "foreign_task") {
+        const [company] = change === "foreign_parent" ? await db.insert(companies).values({ name: "Foreign retry", issuePrefix: `F${randomUUID().replaceAll("-", "")}` }).returning() : [];
+        const [actor] = company ? await db.insert(agents).values({ companyId: company.id, name: "Foreign agent", adapterType: "process" }).returning() : [];
+        const companyId = company?.id ?? fixture.companyId;
+        const [other] = await db.insert(issues).values({ companyId, title: "Another task" }).returning();
+        const [foreign] = await db.insert(heartbeatRuns).values({ companyId, agentId: actor?.id ?? fixture.agentId,
+          scopeKind: "issue", issueId: other!.id, finishedAt: new Date(),
+          status: "failed", errorCode: "reviewed_chat_execution_binding_not_ready", contextSnapshot: fixture.context,
+          resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } }).returning();
+        context.retryOfRunId = foreign!.id;
+        await db.update(heartbeatRuns).set({ retryOfRunId: foreign!.id, contextSnapshot: context }).where(eq(heartbeatRuns.id, child.id));
+      }
+      expect(await attestReviewedExternalChatRun({ db, ...binding, contextSnapshot: context })).toBe(false);
+      expect(context.paperclipExternalChatQuestionResponse).toBeUndefined();
+    },
+  );
+
+  it("exhausts answered retries and rejects a forged child beyond their authority bound", async () => {
+    const { fixture, child } = await answerAttestationRetry(2);
+    await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date(),
+      errorCode: "reviewed_chat_execution_binding_not_ready",
+      resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } },
+    }).where(eq(heartbeatRuns.id, child.id));
+    expect(await heartbeatService(db).scheduleBoundedRetry(child.id, { retryReason: "interaction_continuation_infra_retry",
+      wakeReason: "interaction_continuation_infra_retry", maxAttempts: 3 })).toMatchObject({ outcome: "retry_exhausted" });
+    const [wake] = await db.insert(agentWakeupRequests).values({ companyId: fixture.companyId, agentId: fixture.agentId,
+      source: "automation", status: "claimed" }).returning();
+    const context = { ...child.contextSnapshot, retryOfRunId: child.id, scheduledRetryAttempt: 3,
+      executionRetryAccounting: { version: 1, failureRetries: 3, maxTurnContinuations: 0 } };
+    const [forged] = await db.insert(heartbeatRuns).values({ ...child, id: randomUUID(), status: "running",
+      wakeupRequestId: wake!.id, retryOfRunId: child.id, scheduledRetryAttempt: 3, contextSnapshot: context }).returning();
+    await db.update(agentWakeupRequests).set({ runId: forged!.id }).where(eq(agentWakeupRequests.id, wake!.id));
+    await db.update(issues).set({ executionRunId: forged!.id }).where(eq(issues.id, fixture.issueId));
+    expect(await attestReviewedExternalChatRun({ db, ...fixture, runId: forged!.id, contextSnapshot: context })).toBe(false);
+  });
+
   async function attestAnswer(
     fixture:
       | Awaited<ReturnType<typeof seedAnsweredChatTurn>>