mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-10 11:46:36 +01:00
fix: retry reviewed chat setup after temporary contention (#19787)
Retry reviewed chat preparation with the original accepted input. Bound retries for authorization proof waits that expire before provider work. Preserve the original answer receipt and queue later wakes separately. Keep plan confirmation semantics by assigning the interaction marker only to question answer delivery and legacy question payloads. Verification: 433 tests across four backend suites, all 17 project-chat browser tests, and all server source/test typecheck layers passed. The plan approval regression was reproduced on the previous head before the fix. All applicable CI passed and exact-head Greptile Apex scored 5/5. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
27170dca49
commit
bf12b2d8ad
16 files changed
+405
-28
No files matched your search
@@ -189,7 +189,7 @@ import type { budgetService } from "../budgets.js";
|
||||
import { isBoardOwnedBudgetBlock } from "../budget-block-ownership.js";
|
||||
import type { instanceSettingsService } from "../instance-settings.js";
|
||||
import type { createRunDispatch } from "../../modules/run-dispatch/index.js";
|
||||
import type { PostCommitEffect as WakeQueuePostCommitEffect, createWakeQueue } from "../../modules/wake-queue/index.js";
|
||||
import { hasRetainedExternalChatRetry, type PostCommitEffect as WakeQueuePostCommitEffect, type createWakeQueue } from "../../modules/wake-queue/index.js";
|
||||
|
||||
type HeartbeatRun = typeof heartbeatRuns.$inferSelect;
|
||||
type HeartbeatRunState = ReturnType<typeof createHeartbeatRunState>;
|
||||
@@ -4566,7 +4566,8 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
// A reviewed chat run attests one conversation, so never fold another
|
||||
// conversation's reply into it; that reply queues its own run instead.
|
||||
const coalescedTargetRun = rawCoalescedTarget &&
|
||||
wakeChatConversationsConflict(rawCoalescedTarget.contextSnapshot, enrichedContextSnapshot)
|
||||
(hasRetainedExternalChatRetry(rawCoalescedTarget.contextSnapshot) ||
|
||||
wakeChatConversationsConflict(rawCoalescedTarget.contextSnapshot, enrichedContextSnapshot))
|
||||
? null
|
||||
: filterZombieCoalesceTarget(rawCoalescedTarget, liveRunExecutions);
|
||||
|
||||
|
||||
@@ -289,6 +289,42 @@ describePostgres("heartbeat retry module database wiring", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("bounds answered chat attestation retries across restarts and escalates their exhaustion", async () => {
|
||||
const { run, agent, companyId } = await fixture();
|
||||
const [wake] = await db.insert(agentWakeupRequests).values({ companyId, agentId: agent.id,
|
||||
runId: run.id, source: "automation", status: "failed" }).returning();
|
||||
const contextSnapshot = { ...run.contextSnapshot, source: "issue.interaction.respond", externalChatContinuation: true,
|
||||
interactionId: randomUUID(), interactionStatus: "answered", interactionKind: "ask_user_questions",
|
||||
mutation: "interaction", wakeReason: "issue_commented", wakeCommentIds: [randomUUID()] };
|
||||
const [failed] = await db.update(heartbeatRuns).set({ wakeupRequestId: wake!.id,
|
||||
errorCode: "reviewed_chat_execution_binding_not_ready", contextSnapshot,
|
||||
}).where(eq(heartbeatRuns.id, run.id)).returning();
|
||||
let predecessor = failed!;
|
||||
for (const attempt of [1, 2]) {
|
||||
const { retries, deps } = boundRetries();
|
||||
expect(await retries.scheduleInteractionContinuationInfrastructureRetryIfEligible(predecessor, agent))
|
||||
.toMatchObject({ outcome: "scheduled", attempt, maxAttempts: 2 });
|
||||
const successors = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, predecessor.id));
|
||||
expect(successors).toHaveLength(1);
|
||||
const successor = successors[0]!;
|
||||
expect(successor).toMatchObject({ status: "scheduled_retry", scheduledRetryAttempt: attempt,
|
||||
scheduledRetryReason: legacy.INTERACTION_CONTINUATION_INFRA_RETRY_REASON,
|
||||
contextSnapshot: { ...contextSnapshot, wakeReason: legacy.INTERACTION_CONTINUATION_INFRA_WAKE_REASON } });
|
||||
expect(deps.escalatePlanApprovalResumeFailureNeedsAttention).not.toHaveBeenCalled();
|
||||
const [next] = await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date(),
|
||||
errorCode: "reviewed_chat_execution_binding_not_ready", resultJson: run.resultJson,
|
||||
}).where(eq(heartbeatRuns.id, successor.id)).returning();
|
||||
predecessor = next!;
|
||||
}
|
||||
const { retries, deps } = boundRetries();
|
||||
expect(await retries.scheduleInteractionContinuationInfrastructureRetryIfEligible(predecessor, agent))
|
||||
.toMatchObject({ outcome: "retry_exhausted", maxAttempts: 2 });
|
||||
expect(deps.escalatePlanApprovalResumeFailureNeedsAttention).toHaveBeenCalledOnce();
|
||||
expect(deps.escalatePlanApprovalResumeFailureNeedsAttention)
|
||||
.toHaveBeenCalledWith(expect.objectContaining({ attempt: 2, maxAttempts: 2, run: predecessor }));
|
||||
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId))).toHaveLength(3);
|
||||
});
|
||||
|
||||
it("keeps the accepted interaction owner and its catalog provider deadline", async () => {
|
||||
const { run, agent } = await fixture();
|
||||
const retryAt = new Date(now.getTime() + 3600_000);
|
||||
|
||||
@@ -28,6 +28,7 @@ import { REUSABLE_SANDBOX_RESUME_FAILED_CODE } from "../reusable-sandbox-resume-
|
||||
import { PLUGIN_WORKER_UNAVAILABLE_CODE } from "../plugin-worker-unavailable-error.js";
|
||||
import { ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE } from "../environment-provider-unavailable-error.js";
|
||||
import { REPOSITORY_CATALOG_UNAVAILABLE_CODE } from "../repository-catalog-unavailable-error.js";
|
||||
import { REVIEWED_CHAT_BINDING_NOT_READY } from "../reviewed-chat-binding-diagnostics.js";
|
||||
import { RUNNER_REVISION_MISMATCH_CODE, RUNNER_REVISION_MISMATCH_RETRY, RUNNER_REVISION_WAIT_RETRY_REASON } from "../runner-revision-mismatch.js";
|
||||
import { CONTROLLER_LEASE_LOST_FAILURE_CODE } from "../legacy-controller-lease.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
@@ -143,7 +144,7 @@ export function isTransientWorkspacePreparationCode(code: string | null | undefi
|
||||
|| code === REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE || code === REUSABLE_SANDBOX_RESUME_FAILED_CODE
|
||||
|| code === REMOTE_WORKSPACE_RESTORE_FAILED_CODE || code === PLUGIN_WORKER_UNAVAILABLE_CODE
|
||||
|| code === RUNNER_REVISION_MISMATCH_CODE || code === ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE
|
||||
|| code === REPOSITORY_CATALOG_UNAVAILABLE_CODE;
|
||||
|| code === REPOSITORY_CATALOG_UNAVAILABLE_CODE || code === REVIEWED_CHAT_BINDING_NOT_READY;
|
||||
}
|
||||
|
||||
export const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS =
|
||||
@@ -359,6 +360,7 @@ function isRetryableInteractionContinuationInfrastructureFailure(
|
||||
run.errorCode === PLUGIN_WORKER_UNAVAILABLE_CODE ||
|
||||
run.errorCode === ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE ||
|
||||
run.errorCode === REPOSITORY_CATALOG_UNAVAILABLE_CODE ||
|
||||
run.errorCode === REVIEWED_CHAT_BINDING_NOT_READY ||
|
||||
run.errorCode === RUNNER_REVISION_MISMATCH_CODE
|
||||
) {
|
||||
return true;
|
||||
|
||||
@@ -16,7 +16,7 @@ import { createHash, randomUUID } from "node:crypto";
|
||||
import { appendHeartbeatRunEvent } from "../heartbeat-run-events.js";
|
||||
import { createNativeRunLogCoverageVerifier, formatNativeRunLogEvent } from "../native-runtime/native-run-log-export.js";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { activityLog, agents, agentWakeupRequests, budgetReservations, caseHandoffs, cases, companies, completionContracts, costEvents, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, heartbeatRunEvents, issueComments, issueRecoveryActions, issues, nativeRunFinalizations } from "@paperclipai/db";
|
||||
import { activityLog, agents, agentWakeupRequests, budgetReservations, caseHandoffs, cases, companies, completionContracts, costEvents, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, heartbeatRunEvents, issueComments, issueQuestionResponseDeliveries, issueRecoveryActions, issues, nativeRunFinalizations } from "@paperclipai/db";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it, onTestFinished, vi } from "vitest";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
|
||||
@@ -30,6 +30,7 @@ import { RemoteWorkspaceOperationError, RemoteWorkspaceOperationTimeoutError, Re
|
||||
import { EnvironmentRunError, environmentRunOrchestrator } from "../environment-run-orchestrator.js";
|
||||
import { ReusableSandboxResumeError } from "../reusable-sandbox-resume-error.js";
|
||||
import { PluginWorkerUnavailableError } from "../plugin-worker-unavailable-error.js";
|
||||
import { ReviewedChatBindingDeniedError, ReviewedChatBindingNotReadyError } from "../reviewed-chat-binding-diagnostics.js";
|
||||
import { environmentTaskErrorCodeData, JsonRpcCallError, PluginEnvironmentTaskError, PluginWorkspaceOperationError, workspaceOperationErrorData } from "@paperclipai/plugin-sdk";
|
||||
import { RUNNER_REVISION_MISMATCH_RETRY } from "../runner-revision-mismatch.js";
|
||||
import { workspaceProviderCallFailure } from "../workspace-provider-failure.js";
|
||||
@@ -46,6 +47,9 @@ import { buildNativeCompletionContract } from "../native-runtime/completion-cont
|
||||
import { readPendingNativeRunStop } from "../native-runtime/native-cancellation-request.js";
|
||||
import { reserveRunBudget } from "../budget-reservations.js";
|
||||
import { hasAcknowledgedNativeReassignmentStopIntent } from "../acknowledged-native-stop.js";
|
||||
import { questionResponseDeliveryService } from "../question-response-delivery.js";
|
||||
import { issueThreadInteractionService } from "../issue-thread-interactions.js";
|
||||
import { heartbeatService } from "../heartbeat.js";
|
||||
|
||||
vi.mock("../live-events.js", () => ({ publishLiveEvent: vi.fn() }));
|
||||
|
||||
@@ -1044,6 +1048,82 @@ describe.skipIf(!support.supported && !process.env.PAPERCLIP_TEST_DATABASE_URL)(
|
||||
} else expect(f.deps.scheduleBoundedRetryForRun).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(["fresh", "started", "untrusted", "denied", "retained", "session", "runner"] as const)("bounds only an owned pre-dispatch chat binding wait: %s", async scenario => {
|
||||
const f = await fixture({ runtimeMode: scenario === "retained" ? "native" : "legacy",
|
||||
...(scenario === "session" ? { nativeSessionId: randomUUID() } : {}),
|
||||
...(scenario === "runner" ? { runnerInstanceId: randomUUID() } : {}) });
|
||||
const error = scenario === "untrusted" ? new Error("reviewed_chat_execution_binding_not_ready")
|
||||
: scenario === "denied" ? new ReviewedChatBindingDeniedError("principal_denied")
|
||||
: new ReviewedChatBindingNotReadyError();
|
||||
f.deps.scheduleBoundedRetryForRun.mockResolvedValueOnce({ outcome: "scheduled" } as never);
|
||||
await f.completion.failRunSetup(f.setup(error, scenario === "started"));
|
||||
const failed = await read(f.run.id);
|
||||
expect(failed).toMatchObject({ status: "failed", errorCode: scenario === "fresh" ? "reviewed_chat_execution_binding_not_ready" : "setup_failed" });
|
||||
if (scenario === "fresh") {
|
||||
expect(failed.resultJson).toMatchObject({ executionRecovery: { kind: "bootstrap", providerWorkStarted: false } });
|
||||
expect(f.deps.scheduleBoundedRetryForRun).toHaveBeenCalledWith(expect.objectContaining({ id: f.run.id }), f.agent);
|
||||
} else expect(f.deps.scheduleBoundedRetryForRun).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps an answered chat interaction on its own bounded continuation retry", async () => {
|
||||
const f = await fixture({ contextSnapshot: { source: "issue.interaction.respond", externalChatContinuation: true,
|
||||
interactionId: "interaction-1", interactionStatus: "answered", interactionKind: "ask_user_questions",
|
||||
mutation: "interaction", wakeReason: "issue_commented" } });
|
||||
await f.completion.failRunSetup(f.setup(new ReviewedChatBindingNotReadyError()));
|
||||
expect(await read(f.run.id)).toMatchObject({ status: "failed", errorCode: "reviewed_chat_execution_binding_not_ready" });
|
||||
expect(f.deps.scheduleInteractionContinuationInfrastructureRetryIfEligible)
|
||||
.toHaveBeenCalledWith(expect.objectContaining({ id: f.run.id }), f.agent);
|
||||
expect(f.deps.scheduleBoundedRetryForRun).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([false, true])("selects the answer retry from the real delivery producer and heartbeat queue (legacy context=%s)", async legacy => {
|
||||
const f = await fixture();
|
||||
await db.update(agents).set({ adapterType: "paperclip_runner", status: "idle",
|
||||
runtimeConfig: { heartbeat: { maxConcurrentRuns: 1 } } }).where(eq(agents.id, f.agent.id));
|
||||
const [issue] = await db.insert(issues).values({ companyId: f.run.companyId, title: "Answer delivery retry",
|
||||
status: "in_progress", assigneeAgentId: f.agent.id, responsibleUserId: "local-board" }).returning();
|
||||
const [questionRun] = await db.insert(heartbeatRuns).values({ companyId: f.run.companyId, agentId: f.agent.id,
|
||||
issueId: issue.id, nativeIssueId: issue.id, runtimeMode: "native", status: "succeeded", finishedAt: new Date(),
|
||||
contextSnapshot: { issueId: issue.id, source: "chat:telegram" } }).returning();
|
||||
const interactions = issueThreadInteractionService(db);
|
||||
const interaction = await interactions.create(issue, { kind: "ask_user_questions", sourceRunId: questionRun.id,
|
||||
continuationPolicy: "wake_assignee", payload: { version: 1, questions: [{ id: "choice", prompt: "Choose",
|
||||
selectionMode: "single", required: true, options: [{ id: "yes", label: "Yes" }] }] } },
|
||||
{ agentId: f.agent.id, runId: questionRun.id });
|
||||
await interactions.answerQuestions(issue, interaction.id, { answers: [{ questionId: "choice", optionIds: ["yes"] }] },
|
||||
{ userId: "local-board" });
|
||||
const heartbeat = heartbeatService(db);
|
||||
const answerHeartbeat = legacy ? { ...heartbeat, wakeup: (...[agentId, options]: Parameters<typeof heartbeat.wakeup>) => {
|
||||
const contextSnapshot = { ...options?.contextSnapshot };
|
||||
delete contextSnapshot.mutation;
|
||||
return heartbeat.wakeup(agentId, { ...options, contextSnapshot });
|
||||
} } : heartbeat;
|
||||
await questionResponseDeliveryService(db, { heartbeat: answerHeartbeat }).deliver(interaction.id);
|
||||
const [delivery] = await db.select().from(issueQuestionResponseDeliveries)
|
||||
.where(eq(issueQuestionResponseDeliveries.interactionId, interaction.id));
|
||||
expect(delivery).toMatchObject({ status: "fallback_queued", deliveryMode: "wake_fallback" });
|
||||
expect(delivery.targetRunId).toBeTruthy();
|
||||
const [admitted] = await db.update(heartbeatRuns).set({ status: "running", executionStage: "preparing", startedAt: new Date() })
|
||||
.where(eq(heartbeatRuns.id, delivery.targetRunId!)).returning();
|
||||
f.deps.scheduleInteractionContinuationInfrastructureRetryIfEligible.mockImplementation(async run => {
|
||||
await heartbeat.scheduleBoundedRetry(run.id, { now: new Date(), random: () => 0,
|
||||
retryReason: "interaction_continuation_infra_retry", wakeReason: "interaction_continuation_infra_retry", maxAttempts: 2 });
|
||||
return null;
|
||||
});
|
||||
f.deps.scheduleBoundedRetryForRun.mockResolvedValue({ outcome: "not_scheduled",
|
||||
reason: "Wrong retry owner fixture", issueId: issue.id });
|
||||
await f.completion.failRunSetup({ ...f.setup(new ReviewedChatBindingNotReadyError()), run: admitted, runId: admitted.id });
|
||||
expect(f.deps.scheduleBoundedRetryForRun).not.toHaveBeenCalled();
|
||||
expect(f.deps.scheduleInteractionContinuationInfrastructureRetryIfEligible)
|
||||
.toHaveBeenCalledWith(expect.objectContaining({ id: admitted.id }), expect.anything());
|
||||
const children = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, admitted.id));
|
||||
expect(children).toHaveLength(1);
|
||||
expect(children[0]).toMatchObject({ scheduledRetryReason: "interaction_continuation_infra_retry",
|
||||
contextSnapshot: { interactionId: interaction.id, interactionStatus: "answered", mutation: "interaction" } });
|
||||
expect((await db.select().from(issueQuestionResponseDeliveries)
|
||||
.where(eq(issueQuestionResponseDeliveries.id, delivery.id)))[0]).toEqual(delivery);
|
||||
}, 30_000);
|
||||
|
||||
const runnerMismatch = () => new JsonRpcCallError({ code: -32002, message: "The installed Runner does not match the expected revision",
|
||||
data: environmentTaskErrorCodeData(new PluginEnvironmentTaskError("runner_revision_mismatch")) });
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ import { isWorkspaceValidationFailedRun, isWorkspaceValidationFailure, attachPap
|
||||
import { preserveWorkspaceRestoreRecoveryMetadataSql } from "../legacy-workspace-restore-recovery.js";
|
||||
import { cancellationResultJson, controllerLeaseLossCancellation, readRunCancellation } from "../run-cancellation.js";
|
||||
import { CONTROLLER_LEASE_LOST_FAILURE_CODE, LegacyControllerLeaseLostError } from "../legacy-controller-lease.js";
|
||||
import { reviewedChatBindingDenialResultJson } from "../reviewed-chat-binding-diagnostics.js";
|
||||
import { REVIEWED_CHAT_BINDING_NOT_READY, ReviewedChatBindingNotReadyError, reviewedChatBindingDenialResultJson } from "../reviewed-chat-binding-diagnostics.js";
|
||||
import { accountRunCost } from "../run-cost-accounting.js";
|
||||
import { isConversation, settleConversationTurn } from "../agent-conversations.js";
|
||||
import { legacyExecutionNeedsReconciliationWithEvidence } from "../legacy-execution-recovery.js";
|
||||
@@ -1396,6 +1396,9 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC
|
||||
&& run.nativeSessionId === null && run.runnerInstanceId === null ? findEnvironmentProviderUnavailableError(outerErr) : null;
|
||||
const repositoryCatalogUnavailable = !providerDispatchStarted && run.runtimeMode === "legacy"
|
||||
&& run.nativeSessionId === null && run.runnerInstanceId === null ? findRepositoryCatalogUnavailableError(outerErr) : null;
|
||||
const reviewedChatBindingNotReady = !providerDispatchStarted && run.runtimeMode === "legacy"
|
||||
&& run.nativeSessionId === null && run.runnerInstanceId === null && outerErr instanceof ReviewedChatBindingNotReadyError
|
||||
? outerErr : null;
|
||||
const runnerRevisionMismatch = !providerDispatchStarted && run.runtimeMode === "legacy"
|
||||
&& run.nativeSessionId === null && run.runnerInstanceId === null && isRunnerRevisionMismatch(outerErr);
|
||||
const setupFailureErrorCode =
|
||||
@@ -1406,6 +1409,7 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC
|
||||
pluginWorkerUnavailableFailure?.code ??
|
||||
environmentProviderUnavailable?.code ??
|
||||
repositoryCatalogUnavailable?.code ??
|
||||
reviewedChatBindingNotReady?.code ??
|
||||
(runnerRevisionMismatch ? RUNNER_REVISION_MISMATCH_CODE : null) ??
|
||||
workspaceGitScanFailure?.code ??
|
||||
workspaceValidationSetupFailure?.code ??
|
||||
@@ -1580,7 +1584,8 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC
|
||||
// carries the interaction and escalates on exhaustion.
|
||||
const interactionOwnsRetry = (livenessRun.errorCode === PLUGIN_WORKER_UNAVAILABLE_CODE
|
||||
|| livenessRun.errorCode === ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE
|
||||
|| livenessRun.errorCode === REPOSITORY_CATALOG_UNAVAILABLE_CODE)
|
||||
|| livenessRun.errorCode === REPOSITORY_CATALOG_UNAVAILABLE_CODE
|
||||
|| livenessRun.errorCode === REVIEWED_CHAT_BINDING_NOT_READY)
|
||||
&& isResolvedInteractionContinuationWakeContext(livenessRun.contextSnapshot);
|
||||
await (isTransientWorkspacePreparationCode(livenessRun.errorCode) && !interactionOwnsRetry
|
||||
? livenessRun.errorCode === RUNNER_REVISION_MISMATCH_CODE
|
||||
|
||||
@@ -70,7 +70,7 @@ import {
|
||||
} from "@paperclipai/db";
|
||||
import { HttpError } from "../../errors.js";
|
||||
import { isAiConnectionIncompatible } from "../ai-connection-runtime.js";
|
||||
import type { ReviewedChatBindingDenialReason } from "../reviewed-chat-binding-diagnostics.js";
|
||||
import { ReviewedChatBindingNotReadyError, type ReviewedChatBindingDenialReason } from "../reviewed-chat-binding-diagnostics.js";
|
||||
import { filterResolvedGitHubConnectionsForRun } from "../git-credentials.js";
|
||||
import type {
|
||||
AdapterRuntimeMcpAccess,
|
||||
@@ -1768,7 +1768,12 @@ function normalizeInteractionContinuationWakeContext(
|
||||
contextSnapshot: Record<string, unknown>,
|
||||
payload: Record<string, unknown> | null | undefined,
|
||||
) {
|
||||
if (isInteractionResolutionWakePayload(payload)) return;
|
||||
if (isInteractionResolutionWakePayload(payload)) {
|
||||
if (contextSnapshot.interactionKind === "ask_user_questions") {
|
||||
contextSnapshot.mutation = "interaction";
|
||||
}
|
||||
return;
|
||||
}
|
||||
clearInteractionContinuationWakeContext(contextSnapshot);
|
||||
}
|
||||
|
||||
@@ -2239,7 +2244,7 @@ export async function attestReviewedExternalChatRun(input: {
|
||||
if (attemptNumber < 50)
|
||||
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||
}
|
||||
throw new Error("reviewed_chat_execution_binding_not_ready");
|
||||
throw new ReviewedChatBindingNotReadyError();
|
||||
}
|
||||
|
||||
export async function readHeartbeatWakeComments(db: Db, companyId: string, issueId: string | null, commentIds: string[]) {
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { heartbeatRuns, type Db } from "@paperclipai/db";
|
||||
import { INTERACTION_CONTINUATION_INFRA_RETRY_REASON } from "../../modules/run-dispatch/index.js";
|
||||
import { executionFailureRetryCount } from "../execution-recovery-attempt.js";
|
||||
import { REVIEWED_CHAT_BINDING_NOT_READY } from "../reviewed-chat-binding-diagnostics.js";
|
||||
import { nativeSha256 } from "./canonical.js";
|
||||
|
||||
export const EXTERNAL_CHAT_ANSWER_ADMISSION_KEYS = [
|
||||
"source", "issueId", "interactionId", "interactionKind", "interactionStatus",
|
||||
"sourceRunId", "sourceCommentId", "externalChatContinuation",
|
||||
"wakeCommentId", "wakeCommentIds", "commentId",
|
||||
] as const;
|
||||
|
||||
type Run = typeof heartbeatRuns.$inferSelect;
|
||||
type LockMode = "blocking" | "nonblocking" | "read";
|
||||
const record = (value: unknown): Record<string, unknown> =>
|
||||
value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
||||
|
||||
export async function resolveQuestionResponseRetryAdmission(
|
||||
tx: Db,
|
||||
run: Run,
|
||||
lockMode: LockMode,
|
||||
): Promise<Run | null> {
|
||||
const seen = new Set<string>();
|
||||
let current = run;
|
||||
while (current.retryOfRunId) {
|
||||
const context = record(current.contextSnapshot);
|
||||
if (seen.size >= 2 || seen.has(current.id) || current.scopeKind !== "issue" || current.issueId !== context.issueId ||
|
||||
current.issueId !== run.issueId || current.scheduledRetryReason !== INTERACTION_CONTINUATION_INFRA_RETRY_REASON ||
|
||||
context.retryOfRunId !== current.retryOfRunId || context.retryReason !== INTERACTION_CONTINUATION_INFRA_RETRY_REASON ||
|
||||
context.wakeReason !== INTERACTION_CONTINUATION_INFRA_RETRY_REASON ||
|
||||
context.scheduledRetryAttempt !== current.scheduledRetryAttempt ||
|
||||
current.scheduledRetryAttempt === null || current.scheduledRetryAttempt < 1 || current.scheduledRetryAttempt > 2) return null;
|
||||
seen.add(current.id);
|
||||
const query = tx.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, current.retryOfRunId),
|
||||
eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.agentId, run.agentId)));
|
||||
const [parent] = await (lockMode === "read" ? query : lockMode === "nonblocking"
|
||||
? query.for("update", { noWait: true }) : query.for("update"));
|
||||
if (!parent || seen.has(parent.id)) return null;
|
||||
const parentContext = record(parent.contextSnapshot);
|
||||
const recovery = record(record(parent.resultJson).executionRecovery);
|
||||
if (parent.scopeKind !== "issue" || parent.issueId !== run.issueId || parent.status !== "failed" || !parent.finishedAt ||
|
||||
parent.errorCode !== REVIEWED_CHAT_BINDING_NOT_READY || parent.runtimeMode !== "legacy" ||
|
||||
parent.nativeIssueId !== null || parent.nativeSessionId !== null || parent.runnerInstanceId !== null ||
|
||||
parent.processPid !== null || parent.processGroupId !== null || parent.externalRunId !== null ||
|
||||
recovery.kind !== "bootstrap" || recovery.providerWorkStarted !== false ||
|
||||
executionFailureRetryCount(parent) + 1 !== current.scheduledRetryAttempt ||
|
||||
EXTERNAL_CHAT_ANSWER_ADMISSION_KEYS.some(key => nativeSha256(parentContext[key] ?? null) !== nativeSha256(context[key] ?? null))) return null;
|
||||
current = parent;
|
||||
}
|
||||
return record(current.contextSnapshot).wakeReason === "issue_commented" ? current : null;
|
||||
}
|
||||
@@ -28,6 +28,8 @@ import {
|
||||
import { questionResponseDeliveryValues } from "../question-response-delivery.js";
|
||||
import { voiceSessionStore, type VoiceTransaction, currentVoiceCredentialFingerprint } from "../voice/voice-session-store.js";
|
||||
import { nativeSha256 } from "./canonical.js";
|
||||
import { EXTERNAL_CHAT_ANSWER_ADMISSION_KEYS, resolveQuestionResponseRetryAdmission } from "./external-chat-question-response-retry.js";
|
||||
import { INTERACTION_CONTINUATION_INFRA_RETRY_REASON } from "../../modules/run-dispatch/index.js";
|
||||
|
||||
export const EXTERNAL_CHAT_QUESTION_RESPONSE_KEY =
|
||||
"paperclipExternalChatQuestionResponse";
|
||||
@@ -221,7 +223,7 @@ async function resolveQuestionResponseChain(
|
||||
if (
|
||||
context.source !== "issue.interaction.respond" ||
|
||||
context.issueId !== binding.issueId ||
|
||||
context.wakeReason !== "issue_commented" ||
|
||||
(context.wakeReason !== "issue_commented" && context.wakeReason !== INTERACTION_CONTINUATION_INFRA_RETRY_REASON) ||
|
||||
context.interactionKind !== "ask_user_questions" ||
|
||||
context.interactionStatus !== "answered" ||
|
||||
context.externalChatContinuation !== true ||
|
||||
@@ -251,23 +253,14 @@ async function resolveQuestionResponseChain(
|
||||
: runQuery.for("update"));
|
||||
if (!run?.wakeupRequestId) return null;
|
||||
const admitted = record(run.contextSnapshot);
|
||||
for (const key of [
|
||||
"source",
|
||||
"issueId",
|
||||
"interactionId",
|
||||
"interactionKind",
|
||||
"interactionStatus",
|
||||
"sourceRunId",
|
||||
"sourceCommentId",
|
||||
"externalChatContinuation",
|
||||
"wakeCommentId",
|
||||
"wakeCommentIds",
|
||||
]) {
|
||||
for (const key of [...EXTERNAL_CHAT_ANSWER_ADMISSION_KEYS, "wakeReason"]) {
|
||||
if (
|
||||
nativeSha256(admitted[key] ?? null) !== nativeSha256(context[key] ?? null)
|
||||
)
|
||||
return null;
|
||||
}
|
||||
const admission = await resolveQuestionResponseRetryAdmission(tx, run, lockMode);
|
||||
if (!admission?.wakeupRequestId) return null;
|
||||
|
||||
const query = tx
|
||||
.select({
|
||||
@@ -308,10 +301,10 @@ async function resolveQuestionResponseChain(
|
||||
.innerJoin(
|
||||
agentWakeupRequests,
|
||||
and(
|
||||
eq(agentWakeupRequests.id, run.wakeupRequestId),
|
||||
eq(agentWakeupRequests.id, admission.wakeupRequestId),
|
||||
eq(agentWakeupRequests.companyId, issueThreadInteractions.companyId),
|
||||
eq(agentWakeupRequests.agentId, binding.agentId),
|
||||
eq(agentWakeupRequests.runId, binding.runId),
|
||||
eq(agentWakeupRequests.runId, admission.id),
|
||||
),
|
||||
)
|
||||
.where(
|
||||
@@ -420,7 +413,7 @@ async function resolveQuestionResponseChain(
|
||||
!(
|
||||
(delivery.status === "fallback_queued" &&
|
||||
delivery.deliveryMode === "wake_fallback" &&
|
||||
delivery.targetRunId === binding.runId) ||
|
||||
delivery.targetRunId === admission.id) ||
|
||||
(delivery.status === "delivering" &&
|
||||
delivery.targetRunId === null &&
|
||||
delivery.deliveryMode === null)
|
||||
|
||||
@@ -88,7 +88,7 @@ import {
|
||||
isExternalChatWaitAuthorizationContention,
|
||||
resolveExternalChatResponseWaitAuthorization,
|
||||
} from "./chat-attachment-reuse.js";
|
||||
import { attestReviewedExternalChatRun, buildPaperclipWakePayload } from "../heartbeat.js";
|
||||
import { attestReviewedExternalChatRun, buildPaperclipWakePayload, heartbeatService } from "../heartbeat.js";
|
||||
import { voiceCredentialFingerprint } from "../voice/voice-session-store.js";
|
||||
import { nativeSha256 } from "./canonical.js";
|
||||
import { questionResponseDeliveryValues } from "../question-response-delivery.js";
|
||||
@@ -1297,6 +1297,100 @@ describe("native external-chat response wait", () => {
|
||||
},
|
||||
);
|
||||
|
||||
async function answerAttestationRetry(attempts: number) {
|
||||
const native = await seedAnsweredChatTurn();
|
||||
const [admitted] = await db.insert(heartbeatRuns).values({ companyId: native.companyId, agentId: native.agentId,
|
||||
scopeKind: "issue", issueId: native.issueId, status: "running", runtimeMode: "legacy",
|
||||
responsibleUserId: native.userId, wakeupRequestId: native.wakeId, contextSnapshot: native.context }).returning();
|
||||
await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, native.runId));
|
||||
await db.update(agentWakeupRequests).set({ runId: admitted!.id }).where(eq(agentWakeupRequests.id, native.wakeId));
|
||||
await db.update(issueQuestionResponseDeliveries).set({ targetRunId: admitted!.id }).where(eq(issueQuestionResponseDeliveries.id, native.responseDeliveryId));
|
||||
const fixture = { ...native, runId: admitted!.id };
|
||||
await db.update(issues).set({ responsibleUserId: fixture.userId, executionRunId: fixture.runId }).where(eq(issues.id, fixture.issueId));
|
||||
let predecessorId = fixture.runId;
|
||||
let child: typeof heartbeatRuns.$inferSelect | undefined;
|
||||
for (let attempt = 1; attempt <= attempts; attempt += 1) {
|
||||
await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date(), runtimeMode: "legacy",
|
||||
nativeIssueId: null, nativeSessionId: null, runnerInstanceId: null, scopeKind: "issue", issueId: fixture.issueId,
|
||||
responsibleUserId: fixture.userId, errorCode: "reviewed_chat_execution_binding_not_ready",
|
||||
resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } },
|
||||
}).where(eq(heartbeatRuns.id, predecessorId));
|
||||
const scheduled = await heartbeatService(db).scheduleBoundedRetry(predecessorId, {
|
||||
now: new Date(), random: () => 0, retryReason: "interaction_continuation_infra_retry",
|
||||
wakeReason: "interaction_continuation_infra_retry", maxAttempts: 2,
|
||||
});
|
||||
expect(scheduled.outcome).toBe("scheduled");
|
||||
if (scheduled.outcome !== "scheduled") throw new Error("expected the admitted answer retry");
|
||||
child = scheduled.run;
|
||||
predecessorId = child.id;
|
||||
}
|
||||
await db.update(heartbeatRuns).set({ status: "running" }).where(eq(heartbeatRuns.id, child!.id));
|
||||
return { fixture, child: child!, binding: { ...fixture, runId: child!.id }, context: structuredClone(child!.contextSnapshot!) };
|
||||
}
|
||||
|
||||
it.each([1, 2])("attests answered retry %s against its unchanged original user receipt", async (attempts) => {
|
||||
const { fixture, child, binding, context } = await answerAttestationRetry(attempts);
|
||||
const [delivery] = await db.select().from(issueQuestionResponseDeliveries).where(eq(issueQuestionResponseDeliveries.id, fixture.responseDeliveryId));
|
||||
const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, fixture.wakeId));
|
||||
expect(await attestReviewedExternalChatRun({ db, ...binding, contextSnapshot: context })).toBe(true);
|
||||
context.paperclipExternalChatExecutionBound = true;
|
||||
context.paperclipWake = await buildPaperclipWakePayload({ db, companyId: fixture.companyId,
|
||||
agentId: fixture.agentId, runId: child.id, contextSnapshot: context });
|
||||
await db.update(heartbeatRuns).set({ contextSnapshot: context }).where(eq(heartbeatRuns.id, child.id));
|
||||
expect(await resolveExternalChatQuestionResponse(db, binding, context, "read"))
|
||||
.toMatchObject({ interactionIds: [fixture.interactionId], marker: { responseDeliveryId: fixture.responseDeliveryId } });
|
||||
expect(await materializeExternalChatQuestionResponseInput({ db, binding, contextSnapshot: context }))
|
||||
.toEqual([expect.objectContaining({ interactionId: fixture.interactionId })]);
|
||||
expect((await db.select().from(issueQuestionResponseDeliveries).where(eq(issueQuestionResponseDeliveries.id, fixture.responseDeliveryId)))[0]).toEqual(delivery);
|
||||
expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, fixture.wakeId)))[0]).toEqual(wake);
|
||||
});
|
||||
|
||||
it.each(["revoked_identity", "receipt_actor", "delivery_target", "parent_batch", "provider_started", "parent_session", "unrelated_retry", "foreign_parent", "foreign_task"] as const)(
|
||||
"rejects answered retry authority after %s", async (change) => {
|
||||
const { fixture, child, binding, context } = await answerAttestationRetry(1);
|
||||
if (change === "revoked_identity") await db.update(chatIdentityLinks).set({ status: "revoked" }).where(eq(chatIdentityLinks.principalId, fixture.principalId));
|
||||
if (change === "receipt_actor") await db.update(agentWakeupRequests).set({ requestedByActorId: "different-user" }).where(eq(agentWakeupRequests.id, fixture.wakeId));
|
||||
if (change === "delivery_target") await db.update(issueQuestionResponseDeliveries).set({ targetRunId: child.id }).where(eq(issueQuestionResponseDeliveries.id, fixture.responseDeliveryId));
|
||||
if (change === "parent_batch") await db.update(heartbeatRuns).set({ contextSnapshot: { ...fixture.context, wakeCommentIds: [randomUUID()] } }).where(eq(heartbeatRuns.id, fixture.runId));
|
||||
if (change === "provider_started") await db.update(heartbeatRuns).set({ resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: true } } }).where(eq(heartbeatRuns.id, fixture.runId));
|
||||
if (change === "parent_session") await db.update(heartbeatRuns).set({ nativeSessionId: randomUUID() }).where(eq(heartbeatRuns.id, fixture.runId));
|
||||
if (change === "unrelated_retry") await db.update(heartbeatRuns).set({ scheduledRetryReason: "transient_failure" }).where(eq(heartbeatRuns.id, child.id));
|
||||
if (change === "foreign_parent" || change === "foreign_task") {
|
||||
const [company] = change === "foreign_parent" ? await db.insert(companies).values({ name: "Foreign retry", issuePrefix: `F${randomUUID().replaceAll("-", "")}` }).returning() : [];
|
||||
const [actor] = company ? await db.insert(agents).values({ companyId: company.id, name: "Foreign agent", adapterType: "process" }).returning() : [];
|
||||
const companyId = company?.id ?? fixture.companyId;
|
||||
const [other] = await db.insert(issues).values({ companyId, title: "Another task" }).returning();
|
||||
const [foreign] = await db.insert(heartbeatRuns).values({ companyId, agentId: actor?.id ?? fixture.agentId,
|
||||
scopeKind: "issue", issueId: other!.id, finishedAt: new Date(),
|
||||
status: "failed", errorCode: "reviewed_chat_execution_binding_not_ready", contextSnapshot: fixture.context,
|
||||
resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } }).returning();
|
||||
context.retryOfRunId = foreign!.id;
|
||||
await db.update(heartbeatRuns).set({ retryOfRunId: foreign!.id, contextSnapshot: context }).where(eq(heartbeatRuns.id, child.id));
|
||||
}
|
||||
expect(await attestReviewedExternalChatRun({ db, ...binding, contextSnapshot: context })).toBe(false);
|
||||
expect(context.paperclipExternalChatQuestionResponse).toBeUndefined();
|
||||
},
|
||||
);
|
||||
|
||||
it("exhausts answered retries and rejects a forged child beyond their authority bound", async () => {
|
||||
const { fixture, child } = await answerAttestationRetry(2);
|
||||
await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date(),
|
||||
errorCode: "reviewed_chat_execution_binding_not_ready",
|
||||
resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } },
|
||||
}).where(eq(heartbeatRuns.id, child.id));
|
||||
expect(await heartbeatService(db).scheduleBoundedRetry(child.id, { retryReason: "interaction_continuation_infra_retry",
|
||||
wakeReason: "interaction_continuation_infra_retry", maxAttempts: 3 })).toMatchObject({ outcome: "retry_exhausted" });
|
||||
const [wake] = await db.insert(agentWakeupRequests).values({ companyId: fixture.companyId, agentId: fixture.agentId,
|
||||
source: "automation", status: "claimed" }).returning();
|
||||
const context = { ...child.contextSnapshot, retryOfRunId: child.id, scheduledRetryAttempt: 3,
|
||||
executionRetryAccounting: { version: 1, failureRetries: 3, maxTurnContinuations: 0 } };
|
||||
const [forged] = await db.insert(heartbeatRuns).values({ ...child, id: randomUUID(), status: "running",
|
||||
wakeupRequestId: wake!.id, retryOfRunId: child.id, scheduledRetryAttempt: 3, contextSnapshot: context }).returning();
|
||||
await db.update(agentWakeupRequests).set({ runId: forged!.id }).where(eq(agentWakeupRequests.id, wake!.id));
|
||||
await db.update(issues).set({ executionRunId: forged!.id }).where(eq(issues.id, fixture.issueId));
|
||||
expect(await attestReviewedExternalChatRun({ db, ...fixture, runId: forged!.id, contextSnapshot: context })).toBe(false);
|
||||
});
|
||||
|
||||
async function attestAnswer(
|
||||
fixture:
|
||||
| Awaited<ReturnType<typeof seedAnsweredChatTurn>>
|
||||
|
||||
@@ -1212,6 +1212,7 @@ export function questionResponseDeliveryService(
|
||||
interactionId: interaction.id,
|
||||
interactionKind: interaction.kind,
|
||||
interactionStatus: interaction.status,
|
||||
mutation: "interaction",
|
||||
sourceCommentId: continuationSourceCommentId,
|
||||
sourceRunId: interaction.sourceRunId,
|
||||
externalChatContinuation: externalChatBoundary,
|
||||
|
||||
@@ -1,4 +1,13 @@
|
||||
export const REVIEWED_CHAT_BINDING_NOT_AUTHORIZED = "reviewed_chat_execution_binding_not_authorized";
|
||||
export const REVIEWED_CHAT_BINDING_NOT_READY = "reviewed_chat_execution_binding_not_ready";
|
||||
|
||||
export class ReviewedChatBindingNotReadyError extends Error {
|
||||
readonly code = REVIEWED_CHAT_BINDING_NOT_READY;
|
||||
|
||||
constructor() {
|
||||
super(REVIEWED_CHAT_BINDING_NOT_READY);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The first reviewed-chat attestation check that denied a run. Closed labels
|
||||
|
||||
Reference in new issue
Block a user