mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-09 07:10:34 +01:00
fix: retry reviewed chat setup after temporary contention (#19787)
Retry reviewed chat preparation with the original accepted input. Bound retries for authorization proof waits that expire before provider work. Preserve the original answer receipt and queue later wakes separately. Keep plan confirmation semantics by assigning the interaction marker only to question answer delivery and legacy question payloads. Verification: 433 tests across four backend suites, all 17 project-chat browser tests, and all server source/test typecheck layers passed. The plan approval regression was reproduced on the previous head before the fix. All applicable CI passed and exact-head Greptile Apex scored 5/5. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
27170dca49
commit
bf12b2d8ad
16 files changed
+405
-28
No files matched your search
@@ -302,6 +302,25 @@ Messages from the same conversation still coalesce into one run. Messages from
|
||||
different conversations queue separately and run one after the other. Wakes
|
||||
without a conversation keep their normal coalescing.
|
||||
|
||||
If reviewed chat attestation exhausts its short lock or delivery wait before
|
||||
provider dispatch, the run records `reviewed_chat_execution_binding_not_ready`.
|
||||
The existing durable retry scheduler permits at most two further attempts. Each
|
||||
attempt retains the admitted comment batch and checks current execution ownership,
|
||||
provider access and principal permissions again. An answered interaction retains
|
||||
its own bounded continuation retry. A denied authorization, Stop, pause or blocked
|
||||
dependency cannot use this wait as permission to execute. No retry budget resets,
|
||||
new background poller or provider replay are introduced.
|
||||
|
||||
Question-answer delivery records its interaction marker in the wake context.
|
||||
Queue enrichment backfills that marker for older question-answer payloads only.
|
||||
Plan confirmations retain their existing admission and handoff lifecycle.
|
||||
|
||||
An answered retry proves at most two saved pre-dispatch failure ancestors before
|
||||
reading the original user-owned answer receipt. It rechecks that receipt, its
|
||||
original delivery target and current chat permissions. The original receipt and
|
||||
delivery remain unchanged. New comments and control wakes queue separately from
|
||||
a retained chat retry, including while the retry waits for its scheduled time.
|
||||
|
||||
For a truncated external-chat wake, native completion feedback requires the
|
||||
run-bound complete comment-read receipt before accepting a finish report. An
|
||||
unread, partial, stale or foreign receipt returns a corrective
|
||||
|
||||
@@ -19,6 +19,7 @@ import {
|
||||
heartbeatRunEvents,
|
||||
heartbeatRuns,
|
||||
issueRelations,
|
||||
issueComments,
|
||||
issues,
|
||||
projects,
|
||||
} from "@paperclipai/db";
|
||||
@@ -493,7 +494,7 @@ describeEmbeddedPostgres("heartbeat bounded retry scheduling", () => {
|
||||
return { companyId, agentId, issueId, runId, now };
|
||||
}
|
||||
|
||||
it.each(["remote_workspace_prepare_timeout", "reusable_sandbox_resume_failed", "remote_workspace_restore_failed"].flatMap(errorCode =>
|
||||
it.each(["remote_workspace_prepare_timeout", "reusable_sandbox_resume_failed", "remote_workspace_restore_failed", "reviewed_chat_execution_binding_not_ready"].flatMap(errorCode =>
|
||||
(["resume", "paused", "Stop", "dependency"] as const).map(action => [errorCode, action] as const)))("bounds %s bootstrap retries across restart and honors %s", async (errorCode, action) => {
|
||||
const { companyId, agentId, issueId, runId, now } = await seedMaxTurnFixture();
|
||||
await db.update(agents).set({ adapterType: "paperclip_runner", adapterConfig: { provider: "codex", model: "gpt-6.1-sol" } })
|
||||
@@ -535,6 +536,39 @@ describeEmbeddedPostgres("heartbeat bounded retry scheduling", () => {
|
||||
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId))).toHaveLength(3);
|
||||
});
|
||||
|
||||
it("retains the reviewed chat admission on a durable retry without approving the task", async () => {
|
||||
const { companyId, agentId, issueId, runId, now } = await seedMaxTurnFixture({ issueStatus: "in_review" });
|
||||
const wakeCommentIds = [randomUUID(), randomUUID()];
|
||||
const source = "chat:slack";
|
||||
const contextSnapshot = { issueId, taskId: issueId, source, wakeCommentIds, commentId: wakeCommentIds[0] };
|
||||
await db.update(heartbeatRuns).set({
|
||||
errorCode: "reviewed_chat_execution_binding_not_ready", contextSnapshot,
|
||||
resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } },
|
||||
}).where(eq(heartbeatRuns.id, runId));
|
||||
const outcomes = await Promise.all([
|
||||
heartbeat.scheduleBoundedRetry(runId, { now, random: () => 0 }),
|
||||
heartbeatService(db).scheduleBoundedRetry(runId, { now, random: () => 0 }),
|
||||
]);
|
||||
expect(outcomes.every(outcome => outcome.outcome === "scheduled")).toBe(true);
|
||||
const children = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, runId));
|
||||
expect(children).toHaveLength(1);
|
||||
expect(children[0]).toMatchObject({ companyId, agentId, issueId, scopeKind: "issue", status: "scheduled_retry",
|
||||
scheduledRetryAttempt: 1, scheduledRetryReason: "transient_failure", contextSnapshot });
|
||||
expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0])
|
||||
.toMatchObject({ status: "in_review", executionRunId: children[0]!.id });
|
||||
const [laterComment] = await db.insert(issueComments).values({ companyId, issueId,
|
||||
authorType: "user", authorUserId: "responsible-user", body: "A later board follow-up" }).returning();
|
||||
await heartbeat.wakeup(agentId, { source: "on_demand", triggerDetail: "system", reason: "issue_commented",
|
||||
payload: { issueId, commentId: laterComment!.id }, requestedByActorType: "user", requestedByActorId: "responsible-user",
|
||||
contextSnapshot: { issueId, taskId: issueId, source: "issue.comment", wakeCommentId: laterComment!.id, wakeCommentIds: [laterComment!.id] } });
|
||||
const [preserved] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, children[0]!.id));
|
||||
expect(preserved).toEqual(children[0]);
|
||||
const laterWakes = await db.select().from(agentWakeupRequests).where(and(eq(agentWakeupRequests.companyId, companyId),
|
||||
eq(agentWakeupRequests.status, "deferred_issue_execution")));
|
||||
expect(laterWakes).toHaveLength(1);
|
||||
expect(laterWakes[0]).toMatchObject({ runId: null, payload: { issueId, commentId: laterComment!.id } });
|
||||
});
|
||||
|
||||
it("bounds interrupted conversations across restarts and concurrent scheduling", async () => {
|
||||
const { companyId, issueId, runId, now } = await seedMaxTurnFixture();
|
||||
const resultJson = { conversationContinuation: "continue_conversation_v1" };
|
||||
|
||||
@@ -278,6 +278,43 @@ describe.each(["slack", "discord"] as const)("reviewed %s execution binding", (p
|
||||
await temporary?.cleanup();
|
||||
});
|
||||
|
||||
it("names exhausted lock contention without accepting authority, then rechecks access after release", async () => {
|
||||
let release!: () => void;
|
||||
let acquired!: () => void;
|
||||
const held = new Promise<void>((resolve) => { acquired = resolve; });
|
||||
const gate = new Promise<void>((resolve) => { release = resolve; });
|
||||
const lock = db.transaction(async (tx) => {
|
||||
await tx.select().from(issues).where(eq(issues.id, issueId)).for("update");
|
||||
acquired();
|
||||
await gate;
|
||||
});
|
||||
await held;
|
||||
const onDenied = vi.fn();
|
||||
try {
|
||||
await expect(attestReviewedExternalChatRun({
|
||||
db, ...binding, contextSnapshot: context, onDenied,
|
||||
})).rejects.toMatchObject({
|
||||
message: "reviewed_chat_execution_binding_not_ready",
|
||||
code: "reviewed_chat_execution_binding_not_ready",
|
||||
});
|
||||
expect(onDenied).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
release();
|
||||
await lock;
|
||||
}
|
||||
await expect(attest()).resolves.toBe(true);
|
||||
await db.update(companyMemberships).set({ status: "suspended" }).where(eq(companyMemberships.principalId, userId));
|
||||
try {
|
||||
await expect(attest()).resolves.toBe(false);
|
||||
} finally {
|
||||
await db.update(companyMemberships).set({ status: "active" }).where(eq(companyMemberships.principalId, userId));
|
||||
}
|
||||
expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0])
|
||||
.toMatchObject({ status: "in_review", executionRunId: runId, checkoutRunId: null });
|
||||
expect((await db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.id, interactionId)))[0]?.status)
|
||||
.toBe("pending");
|
||||
}, 30_000);
|
||||
|
||||
it("attests existing execution ownership while preserving real pending governance and checkout state", async () => {
|
||||
const [before] = await db
|
||||
.select()
|
||||
|
||||
@@ -83,12 +83,20 @@ export function wakeMustNotJoinActiveRun(
|
||||
): boolean {
|
||||
// A context compaction run does no task work, so ordinary wakes and
|
||||
// compactions never merge, in either arrival order.
|
||||
return interactionReceipt || Boolean(parseObject(activeRunContext).interactionId) ||
|
||||
return hasRetainedExternalChatRetry(activeRunContext) || interactionReceipt || Boolean(parseObject(activeRunContext).interactionId) ||
|
||||
Boolean(readNonEmptyString(parseObject(activeRunContext).contextCompactionRequestId)) ||
|
||||
Boolean(readNonEmptyString(wakeContext.contextCompactionRequestId)) ||
|
||||
wakeChatConversationsConflict(activeRunContext, wakeContext);
|
||||
}
|
||||
|
||||
export function hasRetainedExternalChatRetry(context: unknown): boolean {
|
||||
const active = parseObject(context);
|
||||
const activeSource = readNonEmptyString(active.source);
|
||||
return readNonEmptyString(active.retryOfRunId) !== null &&
|
||||
(activeSource?.startsWith("chat:") === true ||
|
||||
(activeSource === "issue.interaction.respond" && active.externalChatContinuation === true));
|
||||
}
|
||||
|
||||
function isInteractionResolutionWakePayload(payload: Record<string, unknown> | null | undefined): boolean {
|
||||
return readNonEmptyString(payload?.mutation) === "interaction";
|
||||
}
|
||||
|
||||
@@ -28,6 +28,7 @@ export type {
|
||||
RunSummary,
|
||||
} from "./application/types.js";
|
||||
export { WakeQueueApplicationError } from "./application/types.js";
|
||||
export { hasRetainedExternalChatRetry } from "./domain/context.js";
|
||||
export type {
|
||||
IssueSnapshot,
|
||||
RunSnapshot,
|
||||
|
||||
@@ -189,7 +189,7 @@ import type { budgetService } from "../budgets.js";
|
||||
import { isBoardOwnedBudgetBlock } from "../budget-block-ownership.js";
|
||||
import type { instanceSettingsService } from "../instance-settings.js";
|
||||
import type { createRunDispatch } from "../../modules/run-dispatch/index.js";
|
||||
import type { PostCommitEffect as WakeQueuePostCommitEffect, createWakeQueue } from "../../modules/wake-queue/index.js";
|
||||
import { hasRetainedExternalChatRetry, type PostCommitEffect as WakeQueuePostCommitEffect, type createWakeQueue } from "../../modules/wake-queue/index.js";
|
||||
|
||||
type HeartbeatRun = typeof heartbeatRuns.$inferSelect;
|
||||
type HeartbeatRunState = ReturnType<typeof createHeartbeatRunState>;
|
||||
@@ -4566,7 +4566,8 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
// A reviewed chat run attests one conversation, so never fold another
|
||||
// conversation's reply into it; that reply queues its own run instead.
|
||||
const coalescedTargetRun = rawCoalescedTarget &&
|
||||
wakeChatConversationsConflict(rawCoalescedTarget.contextSnapshot, enrichedContextSnapshot)
|
||||
(hasRetainedExternalChatRetry(rawCoalescedTarget.contextSnapshot) ||
|
||||
wakeChatConversationsConflict(rawCoalescedTarget.contextSnapshot, enrichedContextSnapshot))
|
||||
? null
|
||||
: filterZombieCoalesceTarget(rawCoalescedTarget, liveRunExecutions);
|
||||
|
||||
|
||||
@@ -289,6 +289,42 @@ describePostgres("heartbeat retry module database wiring", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("bounds answered chat attestation retries across restarts and escalates their exhaustion", async () => {
|
||||
const { run, agent, companyId } = await fixture();
|
||||
const [wake] = await db.insert(agentWakeupRequests).values({ companyId, agentId: agent.id,
|
||||
runId: run.id, source: "automation", status: "failed" }).returning();
|
||||
const contextSnapshot = { ...run.contextSnapshot, source: "issue.interaction.respond", externalChatContinuation: true,
|
||||
interactionId: randomUUID(), interactionStatus: "answered", interactionKind: "ask_user_questions",
|
||||
mutation: "interaction", wakeReason: "issue_commented", wakeCommentIds: [randomUUID()] };
|
||||
const [failed] = await db.update(heartbeatRuns).set({ wakeupRequestId: wake!.id,
|
||||
errorCode: "reviewed_chat_execution_binding_not_ready", contextSnapshot,
|
||||
}).where(eq(heartbeatRuns.id, run.id)).returning();
|
||||
let predecessor = failed!;
|
||||
for (const attempt of [1, 2]) {
|
||||
const { retries, deps } = boundRetries();
|
||||
expect(await retries.scheduleInteractionContinuationInfrastructureRetryIfEligible(predecessor, agent))
|
||||
.toMatchObject({ outcome: "scheduled", attempt, maxAttempts: 2 });
|
||||
const successors = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, predecessor.id));
|
||||
expect(successors).toHaveLength(1);
|
||||
const successor = successors[0]!;
|
||||
expect(successor).toMatchObject({ status: "scheduled_retry", scheduledRetryAttempt: attempt,
|
||||
scheduledRetryReason: legacy.INTERACTION_CONTINUATION_INFRA_RETRY_REASON,
|
||||
contextSnapshot: { ...contextSnapshot, wakeReason: legacy.INTERACTION_CONTINUATION_INFRA_WAKE_REASON } });
|
||||
expect(deps.escalatePlanApprovalResumeFailureNeedsAttention).not.toHaveBeenCalled();
|
||||
const [next] = await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date(),
|
||||
errorCode: "reviewed_chat_execution_binding_not_ready", resultJson: run.resultJson,
|
||||
}).where(eq(heartbeatRuns.id, successor.id)).returning();
|
||||
predecessor = next!;
|
||||
}
|
||||
const { retries, deps } = boundRetries();
|
||||
expect(await retries.scheduleInteractionContinuationInfrastructureRetryIfEligible(predecessor, agent))
|
||||
.toMatchObject({ outcome: "retry_exhausted", maxAttempts: 2 });
|
||||
expect(deps.escalatePlanApprovalResumeFailureNeedsAttention).toHaveBeenCalledOnce();
|
||||
expect(deps.escalatePlanApprovalResumeFailureNeedsAttention)
|
||||
.toHaveBeenCalledWith(expect.objectContaining({ attempt: 2, maxAttempts: 2, run: predecessor }));
|
||||
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId))).toHaveLength(3);
|
||||
});
|
||||
|
||||
it("keeps the accepted interaction owner and its catalog provider deadline", async () => {
|
||||
const { run, agent } = await fixture();
|
||||
const retryAt = new Date(now.getTime() + 3600_000);
|
||||
|
||||
@@ -28,6 +28,7 @@ import { REUSABLE_SANDBOX_RESUME_FAILED_CODE } from "../reusable-sandbox-resume-
|
||||
import { PLUGIN_WORKER_UNAVAILABLE_CODE } from "../plugin-worker-unavailable-error.js";
|
||||
import { ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE } from "../environment-provider-unavailable-error.js";
|
||||
import { REPOSITORY_CATALOG_UNAVAILABLE_CODE } from "../repository-catalog-unavailable-error.js";
|
||||
import { REVIEWED_CHAT_BINDING_NOT_READY } from "../reviewed-chat-binding-diagnostics.js";
|
||||
import { RUNNER_REVISION_MISMATCH_CODE, RUNNER_REVISION_MISMATCH_RETRY, RUNNER_REVISION_WAIT_RETRY_REASON } from "../runner-revision-mismatch.js";
|
||||
import { CONTROLLER_LEASE_LOST_FAILURE_CODE } from "../legacy-controller-lease.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
@@ -143,7 +144,7 @@ export function isTransientWorkspacePreparationCode(code: string | null | undefi
|
||||
|| code === REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE || code === REUSABLE_SANDBOX_RESUME_FAILED_CODE
|
||||
|| code === REMOTE_WORKSPACE_RESTORE_FAILED_CODE || code === PLUGIN_WORKER_UNAVAILABLE_CODE
|
||||
|| code === RUNNER_REVISION_MISMATCH_CODE || code === ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE
|
||||
|| code === REPOSITORY_CATALOG_UNAVAILABLE_CODE;
|
||||
|| code === REPOSITORY_CATALOG_UNAVAILABLE_CODE || code === REVIEWED_CHAT_BINDING_NOT_READY;
|
||||
}
|
||||
|
||||
export const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS =
|
||||
@@ -359,6 +360,7 @@ function isRetryableInteractionContinuationInfrastructureFailure(
|
||||
run.errorCode === PLUGIN_WORKER_UNAVAILABLE_CODE ||
|
||||
run.errorCode === ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE ||
|
||||
run.errorCode === REPOSITORY_CATALOG_UNAVAILABLE_CODE ||
|
||||
run.errorCode === REVIEWED_CHAT_BINDING_NOT_READY ||
|
||||
run.errorCode === RUNNER_REVISION_MISMATCH_CODE
|
||||
) {
|
||||
return true;
|
||||
|
||||
@@ -16,7 +16,7 @@ import { createHash, randomUUID } from "node:crypto";
|
||||
import { appendHeartbeatRunEvent } from "../heartbeat-run-events.js";
|
||||
import { createNativeRunLogCoverageVerifier, formatNativeRunLogEvent } from "../native-runtime/native-run-log-export.js";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { activityLog, agents, agentWakeupRequests, budgetReservations, caseHandoffs, cases, companies, completionContracts, costEvents, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, heartbeatRunEvents, issueComments, issueRecoveryActions, issues, nativeRunFinalizations } from "@paperclipai/db";
|
||||
import { activityLog, agents, agentWakeupRequests, budgetReservations, caseHandoffs, cases, companies, completionContracts, costEvents, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, heartbeatRunEvents, issueComments, issueQuestionResponseDeliveries, issueRecoveryActions, issues, nativeRunFinalizations } from "@paperclipai/db";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it, onTestFinished, vi } from "vitest";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
|
||||
@@ -30,6 +30,7 @@ import { RemoteWorkspaceOperationError, RemoteWorkspaceOperationTimeoutError, Re
|
||||
import { EnvironmentRunError, environmentRunOrchestrator } from "../environment-run-orchestrator.js";
|
||||
import { ReusableSandboxResumeError } from "../reusable-sandbox-resume-error.js";
|
||||
import { PluginWorkerUnavailableError } from "../plugin-worker-unavailable-error.js";
|
||||
import { ReviewedChatBindingDeniedError, ReviewedChatBindingNotReadyError } from "../reviewed-chat-binding-diagnostics.js";
|
||||
import { environmentTaskErrorCodeData, JsonRpcCallError, PluginEnvironmentTaskError, PluginWorkspaceOperationError, workspaceOperationErrorData } from "@paperclipai/plugin-sdk";
|
||||
import { RUNNER_REVISION_MISMATCH_RETRY } from "../runner-revision-mismatch.js";
|
||||
import { workspaceProviderCallFailure } from "../workspace-provider-failure.js";
|
||||
@@ -46,6 +47,9 @@ import { buildNativeCompletionContract } from "../native-runtime/completion-cont
|
||||
import { readPendingNativeRunStop } from "../native-runtime/native-cancellation-request.js";
|
||||
import { reserveRunBudget } from "../budget-reservations.js";
|
||||
import { hasAcknowledgedNativeReassignmentStopIntent } from "../acknowledged-native-stop.js";
|
||||
import { questionResponseDeliveryService } from "../question-response-delivery.js";
|
||||
import { issueThreadInteractionService } from "../issue-thread-interactions.js";
|
||||
import { heartbeatService } from "../heartbeat.js";
|
||||
|
||||
vi.mock("../live-events.js", () => ({ publishLiveEvent: vi.fn() }));
|
||||
|
||||
@@ -1044,6 +1048,82 @@ describe.skipIf(!support.supported && !process.env.PAPERCLIP_TEST_DATABASE_URL)(
|
||||
} else expect(f.deps.scheduleBoundedRetryForRun).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(["fresh", "started", "untrusted", "denied", "retained", "session", "runner"] as const)("bounds only an owned pre-dispatch chat binding wait: %s", async scenario => {
|
||||
const f = await fixture({ runtimeMode: scenario === "retained" ? "native" : "legacy",
|
||||
...(scenario === "session" ? { nativeSessionId: randomUUID() } : {}),
|
||||
...(scenario === "runner" ? { runnerInstanceId: randomUUID() } : {}) });
|
||||
const error = scenario === "untrusted" ? new Error("reviewed_chat_execution_binding_not_ready")
|
||||
: scenario === "denied" ? new ReviewedChatBindingDeniedError("principal_denied")
|
||||
: new ReviewedChatBindingNotReadyError();
|
||||
f.deps.scheduleBoundedRetryForRun.mockResolvedValueOnce({ outcome: "scheduled" } as never);
|
||||
await f.completion.failRunSetup(f.setup(error, scenario === "started"));
|
||||
const failed = await read(f.run.id);
|
||||
expect(failed).toMatchObject({ status: "failed", errorCode: scenario === "fresh" ? "reviewed_chat_execution_binding_not_ready" : "setup_failed" });
|
||||
if (scenario === "fresh") {
|
||||
expect(failed.resultJson).toMatchObject({ executionRecovery: { kind: "bootstrap", providerWorkStarted: false } });
|
||||
expect(f.deps.scheduleBoundedRetryForRun).toHaveBeenCalledWith(expect.objectContaining({ id: f.run.id }), f.agent);
|
||||
} else expect(f.deps.scheduleBoundedRetryForRun).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps an answered chat interaction on its own bounded continuation retry", async () => {
|
||||
const f = await fixture({ contextSnapshot: { source: "issue.interaction.respond", externalChatContinuation: true,
|
||||
interactionId: "interaction-1", interactionStatus: "answered", interactionKind: "ask_user_questions",
|
||||
mutation: "interaction", wakeReason: "issue_commented" } });
|
||||
await f.completion.failRunSetup(f.setup(new ReviewedChatBindingNotReadyError()));
|
||||
expect(await read(f.run.id)).toMatchObject({ status: "failed", errorCode: "reviewed_chat_execution_binding_not_ready" });
|
||||
expect(f.deps.scheduleInteractionContinuationInfrastructureRetryIfEligible)
|
||||
.toHaveBeenCalledWith(expect.objectContaining({ id: f.run.id }), f.agent);
|
||||
expect(f.deps.scheduleBoundedRetryForRun).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([false, true])("selects the answer retry from the real delivery producer and heartbeat queue (legacy context=%s)", async legacy => {
|
||||
const f = await fixture();
|
||||
await db.update(agents).set({ adapterType: "paperclip_runner", status: "idle",
|
||||
runtimeConfig: { heartbeat: { maxConcurrentRuns: 1 } } }).where(eq(agents.id, f.agent.id));
|
||||
const [issue] = await db.insert(issues).values({ companyId: f.run.companyId, title: "Answer delivery retry",
|
||||
status: "in_progress", assigneeAgentId: f.agent.id, responsibleUserId: "local-board" }).returning();
|
||||
const [questionRun] = await db.insert(heartbeatRuns).values({ companyId: f.run.companyId, agentId: f.agent.id,
|
||||
issueId: issue.id, nativeIssueId: issue.id, runtimeMode: "native", status: "succeeded", finishedAt: new Date(),
|
||||
contextSnapshot: { issueId: issue.id, source: "chat:telegram" } }).returning();
|
||||
const interactions = issueThreadInteractionService(db);
|
||||
const interaction = await interactions.create(issue, { kind: "ask_user_questions", sourceRunId: questionRun.id,
|
||||
continuationPolicy: "wake_assignee", payload: { version: 1, questions: [{ id: "choice", prompt: "Choose",
|
||||
selectionMode: "single", required: true, options: [{ id: "yes", label: "Yes" }] }] } },
|
||||
{ agentId: f.agent.id, runId: questionRun.id });
|
||||
await interactions.answerQuestions(issue, interaction.id, { answers: [{ questionId: "choice", optionIds: ["yes"] }] },
|
||||
{ userId: "local-board" });
|
||||
const heartbeat = heartbeatService(db);
|
||||
const answerHeartbeat = legacy ? { ...heartbeat, wakeup: (...[agentId, options]: Parameters<typeof heartbeat.wakeup>) => {
|
||||
const contextSnapshot = { ...options?.contextSnapshot };
|
||||
delete contextSnapshot.mutation;
|
||||
return heartbeat.wakeup(agentId, { ...options, contextSnapshot });
|
||||
} } : heartbeat;
|
||||
await questionResponseDeliveryService(db, { heartbeat: answerHeartbeat }).deliver(interaction.id);
|
||||
const [delivery] = await db.select().from(issueQuestionResponseDeliveries)
|
||||
.where(eq(issueQuestionResponseDeliveries.interactionId, interaction.id));
|
||||
expect(delivery).toMatchObject({ status: "fallback_queued", deliveryMode: "wake_fallback" });
|
||||
expect(delivery.targetRunId).toBeTruthy();
|
||||
const [admitted] = await db.update(heartbeatRuns).set({ status: "running", executionStage: "preparing", startedAt: new Date() })
|
||||
.where(eq(heartbeatRuns.id, delivery.targetRunId!)).returning();
|
||||
f.deps.scheduleInteractionContinuationInfrastructureRetryIfEligible.mockImplementation(async run => {
|
||||
await heartbeat.scheduleBoundedRetry(run.id, { now: new Date(), random: () => 0,
|
||||
retryReason: "interaction_continuation_infra_retry", wakeReason: "interaction_continuation_infra_retry", maxAttempts: 2 });
|
||||
return null;
|
||||
});
|
||||
f.deps.scheduleBoundedRetryForRun.mockResolvedValue({ outcome: "not_scheduled",
|
||||
reason: "Wrong retry owner fixture", issueId: issue.id });
|
||||
await f.completion.failRunSetup({ ...f.setup(new ReviewedChatBindingNotReadyError()), run: admitted, runId: admitted.id });
|
||||
expect(f.deps.scheduleBoundedRetryForRun).not.toHaveBeenCalled();
|
||||
expect(f.deps.scheduleInteractionContinuationInfrastructureRetryIfEligible)
|
||||
.toHaveBeenCalledWith(expect.objectContaining({ id: admitted.id }), expect.anything());
|
||||
const children = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, admitted.id));
|
||||
expect(children).toHaveLength(1);
|
||||
expect(children[0]).toMatchObject({ scheduledRetryReason: "interaction_continuation_infra_retry",
|
||||
contextSnapshot: { interactionId: interaction.id, interactionStatus: "answered", mutation: "interaction" } });
|
||||
expect((await db.select().from(issueQuestionResponseDeliveries)
|
||||
.where(eq(issueQuestionResponseDeliveries.id, delivery.id)))[0]).toEqual(delivery);
|
||||
}, 30_000);
|
||||
|
||||
const runnerMismatch = () => new JsonRpcCallError({ code: -32002, message: "The installed Runner does not match the expected revision",
|
||||
data: environmentTaskErrorCodeData(new PluginEnvironmentTaskError("runner_revision_mismatch")) });
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ import { isWorkspaceValidationFailedRun, isWorkspaceValidationFailure, attachPap
|
||||
import { preserveWorkspaceRestoreRecoveryMetadataSql } from "../legacy-workspace-restore-recovery.js";
|
||||
import { cancellationResultJson, controllerLeaseLossCancellation, readRunCancellation } from "../run-cancellation.js";
|
||||
import { CONTROLLER_LEASE_LOST_FAILURE_CODE, LegacyControllerLeaseLostError } from "../legacy-controller-lease.js";
|
||||
import { reviewedChatBindingDenialResultJson } from "../reviewed-chat-binding-diagnostics.js";
|
||||
import { REVIEWED_CHAT_BINDING_NOT_READY, ReviewedChatBindingNotReadyError, reviewedChatBindingDenialResultJson } from "../reviewed-chat-binding-diagnostics.js";
|
||||
import { accountRunCost } from "../run-cost-accounting.js";
|
||||
import { isConversation, settleConversationTurn } from "../agent-conversations.js";
|
||||
import { legacyExecutionNeedsReconciliationWithEvidence } from "../legacy-execution-recovery.js";
|
||||
@@ -1396,6 +1396,9 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC
|
||||
&& run.nativeSessionId === null && run.runnerInstanceId === null ? findEnvironmentProviderUnavailableError(outerErr) : null;
|
||||
const repositoryCatalogUnavailable = !providerDispatchStarted && run.runtimeMode === "legacy"
|
||||
&& run.nativeSessionId === null && run.runnerInstanceId === null ? findRepositoryCatalogUnavailableError(outerErr) : null;
|
||||
const reviewedChatBindingNotReady = !providerDispatchStarted && run.runtimeMode === "legacy"
|
||||
&& run.nativeSessionId === null && run.runnerInstanceId === null && outerErr instanceof ReviewedChatBindingNotReadyError
|
||||
? outerErr : null;
|
||||
const runnerRevisionMismatch = !providerDispatchStarted && run.runtimeMode === "legacy"
|
||||
&& run.nativeSessionId === null && run.runnerInstanceId === null && isRunnerRevisionMismatch(outerErr);
|
||||
const setupFailureErrorCode =
|
||||
@@ -1406,6 +1409,7 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC
|
||||
pluginWorkerUnavailableFailure?.code ??
|
||||
environmentProviderUnavailable?.code ??
|
||||
repositoryCatalogUnavailable?.code ??
|
||||
reviewedChatBindingNotReady?.code ??
|
||||
(runnerRevisionMismatch ? RUNNER_REVISION_MISMATCH_CODE : null) ??
|
||||
workspaceGitScanFailure?.code ??
|
||||
workspaceValidationSetupFailure?.code ??
|
||||
@@ -1580,7 +1584,8 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC
|
||||
// carries the interaction and escalates on exhaustion.
|
||||
const interactionOwnsRetry = (livenessRun.errorCode === PLUGIN_WORKER_UNAVAILABLE_CODE
|
||||
|| livenessRun.errorCode === ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE
|
||||
|| livenessRun.errorCode === REPOSITORY_CATALOG_UNAVAILABLE_CODE)
|
||||
|| livenessRun.errorCode === REPOSITORY_CATALOG_UNAVAILABLE_CODE
|
||||
|| livenessRun.errorCode === REVIEWED_CHAT_BINDING_NOT_READY)
|
||||
&& isResolvedInteractionContinuationWakeContext(livenessRun.contextSnapshot);
|
||||
await (isTransientWorkspacePreparationCode(livenessRun.errorCode) && !interactionOwnsRetry
|
||||
? livenessRun.errorCode === RUNNER_REVISION_MISMATCH_CODE
|
||||
|
||||
@@ -70,7 +70,7 @@ import {
|
||||
} from "@paperclipai/db";
|
||||
import { HttpError } from "../../errors.js";
|
||||
import { isAiConnectionIncompatible } from "../ai-connection-runtime.js";
|
||||
import type { ReviewedChatBindingDenialReason } from "../reviewed-chat-binding-diagnostics.js";
|
||||
import { ReviewedChatBindingNotReadyError, type ReviewedChatBindingDenialReason } from "../reviewed-chat-binding-diagnostics.js";
|
||||
import { filterResolvedGitHubConnectionsForRun } from "../git-credentials.js";
|
||||
import type {
|
||||
AdapterRuntimeMcpAccess,
|
||||
@@ -1768,7 +1768,12 @@ function normalizeInteractionContinuationWakeContext(
|
||||
contextSnapshot: Record<string, unknown>,
|
||||
payload: Record<string, unknown> | null | undefined,
|
||||
) {
|
||||
if (isInteractionResolutionWakePayload(payload)) return;
|
||||
if (isInteractionResolutionWakePayload(payload)) {
|
||||
if (contextSnapshot.interactionKind === "ask_user_questions") {
|
||||
contextSnapshot.mutation = "interaction";
|
||||
}
|
||||
return;
|
||||
}
|
||||
clearInteractionContinuationWakeContext(contextSnapshot);
|
||||
}
|
||||
|
||||
@@ -2239,7 +2244,7 @@ export async function attestReviewedExternalChatRun(input: {
|
||||
if (attemptNumber < 50)
|
||||
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||
}
|
||||
throw new Error("reviewed_chat_execution_binding_not_ready");
|
||||
throw new ReviewedChatBindingNotReadyError();
|
||||
}
|
||||
|
||||
export async function readHeartbeatWakeComments(db: Db, companyId: string, issueId: string | null, commentIds: string[]) {
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { heartbeatRuns, type Db } from "@paperclipai/db";
|
||||
import { INTERACTION_CONTINUATION_INFRA_RETRY_REASON } from "../../modules/run-dispatch/index.js";
|
||||
import { executionFailureRetryCount } from "../execution-recovery-attempt.js";
|
||||
import { REVIEWED_CHAT_BINDING_NOT_READY } from "../reviewed-chat-binding-diagnostics.js";
|
||||
import { nativeSha256 } from "./canonical.js";
|
||||
|
||||
export const EXTERNAL_CHAT_ANSWER_ADMISSION_KEYS = [
|
||||
"source", "issueId", "interactionId", "interactionKind", "interactionStatus",
|
||||
"sourceRunId", "sourceCommentId", "externalChatContinuation",
|
||||
"wakeCommentId", "wakeCommentIds", "commentId",
|
||||
] as const;
|
||||
|
||||
type Run = typeof heartbeatRuns.$inferSelect;
|
||||
type LockMode = "blocking" | "nonblocking" | "read";
|
||||
const record = (value: unknown): Record<string, unknown> =>
|
||||
value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
||||
|
||||
export async function resolveQuestionResponseRetryAdmission(
|
||||
tx: Db,
|
||||
run: Run,
|
||||
lockMode: LockMode,
|
||||
): Promise<Run | null> {
|
||||
const seen = new Set<string>();
|
||||
let current = run;
|
||||
while (current.retryOfRunId) {
|
||||
const context = record(current.contextSnapshot);
|
||||
if (seen.size >= 2 || seen.has(current.id) || current.scopeKind !== "issue" || current.issueId !== context.issueId ||
|
||||
current.issueId !== run.issueId || current.scheduledRetryReason !== INTERACTION_CONTINUATION_INFRA_RETRY_REASON ||
|
||||
context.retryOfRunId !== current.retryOfRunId || context.retryReason !== INTERACTION_CONTINUATION_INFRA_RETRY_REASON ||
|
||||
context.wakeReason !== INTERACTION_CONTINUATION_INFRA_RETRY_REASON ||
|
||||
context.scheduledRetryAttempt !== current.scheduledRetryAttempt ||
|
||||
current.scheduledRetryAttempt === null || current.scheduledRetryAttempt < 1 || current.scheduledRetryAttempt > 2) return null;
|
||||
seen.add(current.id);
|
||||
const query = tx.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, current.retryOfRunId),
|
||||
eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.agentId, run.agentId)));
|
||||
const [parent] = await (lockMode === "read" ? query : lockMode === "nonblocking"
|
||||
? query.for("update", { noWait: true }) : query.for("update"));
|
||||
if (!parent || seen.has(parent.id)) return null;
|
||||
const parentContext = record(parent.contextSnapshot);
|
||||
const recovery = record(record(parent.resultJson).executionRecovery);
|
||||
if (parent.scopeKind !== "issue" || parent.issueId !== run.issueId || parent.status !== "failed" || !parent.finishedAt ||
|
||||
parent.errorCode !== REVIEWED_CHAT_BINDING_NOT_READY || parent.runtimeMode !== "legacy" ||
|
||||
parent.nativeIssueId !== null || parent.nativeSessionId !== null || parent.runnerInstanceId !== null ||
|
||||
parent.processPid !== null || parent.processGroupId !== null || parent.externalRunId !== null ||
|
||||
recovery.kind !== "bootstrap" || recovery.providerWorkStarted !== false ||
|
||||
executionFailureRetryCount(parent) + 1 !== current.scheduledRetryAttempt ||
|
||||
EXTERNAL_CHAT_ANSWER_ADMISSION_KEYS.some(key => nativeSha256(parentContext[key] ?? null) !== nativeSha256(context[key] ?? null))) return null;
|
||||
current = parent;
|
||||
}
|
||||
return record(current.contextSnapshot).wakeReason === "issue_commented" ? current : null;
|
||||
}
|
||||
@@ -28,6 +28,8 @@ import {
|
||||
import { questionResponseDeliveryValues } from "../question-response-delivery.js";
|
||||
import { voiceSessionStore, type VoiceTransaction, currentVoiceCredentialFingerprint } from "../voice/voice-session-store.js";
|
||||
import { nativeSha256 } from "./canonical.js";
|
||||
import { EXTERNAL_CHAT_ANSWER_ADMISSION_KEYS, resolveQuestionResponseRetryAdmission } from "./external-chat-question-response-retry.js";
|
||||
import { INTERACTION_CONTINUATION_INFRA_RETRY_REASON } from "../../modules/run-dispatch/index.js";
|
||||
|
||||
export const EXTERNAL_CHAT_QUESTION_RESPONSE_KEY =
|
||||
"paperclipExternalChatQuestionResponse";
|
||||
@@ -221,7 +223,7 @@ async function resolveQuestionResponseChain(
|
||||
if (
|
||||
context.source !== "issue.interaction.respond" ||
|
||||
context.issueId !== binding.issueId ||
|
||||
context.wakeReason !== "issue_commented" ||
|
||||
(context.wakeReason !== "issue_commented" && context.wakeReason !== INTERACTION_CONTINUATION_INFRA_RETRY_REASON) ||
|
||||
context.interactionKind !== "ask_user_questions" ||
|
||||
context.interactionStatus !== "answered" ||
|
||||
context.externalChatContinuation !== true ||
|
||||
@@ -251,23 +253,14 @@ async function resolveQuestionResponseChain(
|
||||
: runQuery.for("update"));
|
||||
if (!run?.wakeupRequestId) return null;
|
||||
const admitted = record(run.contextSnapshot);
|
||||
for (const key of [
|
||||
"source",
|
||||
"issueId",
|
||||
"interactionId",
|
||||
"interactionKind",
|
||||
"interactionStatus",
|
||||
"sourceRunId",
|
||||
"sourceCommentId",
|
||||
"externalChatContinuation",
|
||||
"wakeCommentId",
|
||||
"wakeCommentIds",
|
||||
]) {
|
||||
for (const key of [...EXTERNAL_CHAT_ANSWER_ADMISSION_KEYS, "wakeReason"]) {
|
||||
if (
|
||||
nativeSha256(admitted[key] ?? null) !== nativeSha256(context[key] ?? null)
|
||||
)
|
||||
return null;
|
||||
}
|
||||
const admission = await resolveQuestionResponseRetryAdmission(tx, run, lockMode);
|
||||
if (!admission?.wakeupRequestId) return null;
|
||||
|
||||
const query = tx
|
||||
.select({
|
||||
@@ -308,10 +301,10 @@ async function resolveQuestionResponseChain(
|
||||
.innerJoin(
|
||||
agentWakeupRequests,
|
||||
and(
|
||||
eq(agentWakeupRequests.id, run.wakeupRequestId),
|
||||
eq(agentWakeupRequests.id, admission.wakeupRequestId),
|
||||
eq(agentWakeupRequests.companyId, issueThreadInteractions.companyId),
|
||||
eq(agentWakeupRequests.agentId, binding.agentId),
|
||||
eq(agentWakeupRequests.runId, binding.runId),
|
||||
eq(agentWakeupRequests.runId, admission.id),
|
||||
),
|
||||
)
|
||||
.where(
|
||||
@@ -420,7 +413,7 @@ async function resolveQuestionResponseChain(
|
||||
!(
|
||||
(delivery.status === "fallback_queued" &&
|
||||
delivery.deliveryMode === "wake_fallback" &&
|
||||
delivery.targetRunId === binding.runId) ||
|
||||
delivery.targetRunId === admission.id) ||
|
||||
(delivery.status === "delivering" &&
|
||||
delivery.targetRunId === null &&
|
||||
delivery.deliveryMode === null)
|
||||
|
||||
@@ -88,7 +88,7 @@ import {
|
||||
isExternalChatWaitAuthorizationContention,
|
||||
resolveExternalChatResponseWaitAuthorization,
|
||||
} from "./chat-attachment-reuse.js";
|
||||
import { attestReviewedExternalChatRun, buildPaperclipWakePayload } from "../heartbeat.js";
|
||||
import { attestReviewedExternalChatRun, buildPaperclipWakePayload, heartbeatService } from "../heartbeat.js";
|
||||
import { voiceCredentialFingerprint } from "../voice/voice-session-store.js";
|
||||
import { nativeSha256 } from "./canonical.js";
|
||||
import { questionResponseDeliveryValues } from "../question-response-delivery.js";
|
||||
@@ -1297,6 +1297,100 @@ describe("native external-chat response wait", () => {
|
||||
},
|
||||
);
|
||||
|
||||
async function answerAttestationRetry(attempts: number) {
|
||||
const native = await seedAnsweredChatTurn();
|
||||
const [admitted] = await db.insert(heartbeatRuns).values({ companyId: native.companyId, agentId: native.agentId,
|
||||
scopeKind: "issue", issueId: native.issueId, status: "running", runtimeMode: "legacy",
|
||||
responsibleUserId: native.userId, wakeupRequestId: native.wakeId, contextSnapshot: native.context }).returning();
|
||||
await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, native.runId));
|
||||
await db.update(agentWakeupRequests).set({ runId: admitted!.id }).where(eq(agentWakeupRequests.id, native.wakeId));
|
||||
await db.update(issueQuestionResponseDeliveries).set({ targetRunId: admitted!.id }).where(eq(issueQuestionResponseDeliveries.id, native.responseDeliveryId));
|
||||
const fixture = { ...native, runId: admitted!.id };
|
||||
await db.update(issues).set({ responsibleUserId: fixture.userId, executionRunId: fixture.runId }).where(eq(issues.id, fixture.issueId));
|
||||
let predecessorId = fixture.runId;
|
||||
let child: typeof heartbeatRuns.$inferSelect | undefined;
|
||||
for (let attempt = 1; attempt <= attempts; attempt += 1) {
|
||||
await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date(), runtimeMode: "legacy",
|
||||
nativeIssueId: null, nativeSessionId: null, runnerInstanceId: null, scopeKind: "issue", issueId: fixture.issueId,
|
||||
responsibleUserId: fixture.userId, errorCode: "reviewed_chat_execution_binding_not_ready",
|
||||
resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } },
|
||||
}).where(eq(heartbeatRuns.id, predecessorId));
|
||||
const scheduled = await heartbeatService(db).scheduleBoundedRetry(predecessorId, {
|
||||
now: new Date(), random: () => 0, retryReason: "interaction_continuation_infra_retry",
|
||||
wakeReason: "interaction_continuation_infra_retry", maxAttempts: 2,
|
||||
});
|
||||
expect(scheduled.outcome).toBe("scheduled");
|
||||
if (scheduled.outcome !== "scheduled") throw new Error("expected the admitted answer retry");
|
||||
child = scheduled.run;
|
||||
predecessorId = child.id;
|
||||
}
|
||||
await db.update(heartbeatRuns).set({ status: "running" }).where(eq(heartbeatRuns.id, child!.id));
|
||||
return { fixture, child: child!, binding: { ...fixture, runId: child!.id }, context: structuredClone(child!.contextSnapshot!) };
|
||||
}
|
||||
|
||||
it.each([1, 2])("attests answered retry %s against its unchanged original user receipt", async (attempts) => {
|
||||
const { fixture, child, binding, context } = await answerAttestationRetry(attempts);
|
||||
const [delivery] = await db.select().from(issueQuestionResponseDeliveries).where(eq(issueQuestionResponseDeliveries.id, fixture.responseDeliveryId));
|
||||
const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, fixture.wakeId));
|
||||
expect(await attestReviewedExternalChatRun({ db, ...binding, contextSnapshot: context })).toBe(true);
|
||||
context.paperclipExternalChatExecutionBound = true;
|
||||
context.paperclipWake = await buildPaperclipWakePayload({ db, companyId: fixture.companyId,
|
||||
agentId: fixture.agentId, runId: child.id, contextSnapshot: context });
|
||||
await db.update(heartbeatRuns).set({ contextSnapshot: context }).where(eq(heartbeatRuns.id, child.id));
|
||||
expect(await resolveExternalChatQuestionResponse(db, binding, context, "read"))
|
||||
.toMatchObject({ interactionIds: [fixture.interactionId], marker: { responseDeliveryId: fixture.responseDeliveryId } });
|
||||
expect(await materializeExternalChatQuestionResponseInput({ db, binding, contextSnapshot: context }))
|
||||
.toEqual([expect.objectContaining({ interactionId: fixture.interactionId })]);
|
||||
expect((await db.select().from(issueQuestionResponseDeliveries).where(eq(issueQuestionResponseDeliveries.id, fixture.responseDeliveryId)))[0]).toEqual(delivery);
|
||||
expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, fixture.wakeId)))[0]).toEqual(wake);
|
||||
});
|
||||
|
||||
it.each(["revoked_identity", "receipt_actor", "delivery_target", "parent_batch", "provider_started", "parent_session", "unrelated_retry", "foreign_parent", "foreign_task"] as const)(
|
||||
"rejects answered retry authority after %s", async (change) => {
|
||||
const { fixture, child, binding, context } = await answerAttestationRetry(1);
|
||||
if (change === "revoked_identity") await db.update(chatIdentityLinks).set({ status: "revoked" }).where(eq(chatIdentityLinks.principalId, fixture.principalId));
|
||||
if (change === "receipt_actor") await db.update(agentWakeupRequests).set({ requestedByActorId: "different-user" }).where(eq(agentWakeupRequests.id, fixture.wakeId));
|
||||
if (change === "delivery_target") await db.update(issueQuestionResponseDeliveries).set({ targetRunId: child.id }).where(eq(issueQuestionResponseDeliveries.id, fixture.responseDeliveryId));
|
||||
if (change === "parent_batch") await db.update(heartbeatRuns).set({ contextSnapshot: { ...fixture.context, wakeCommentIds: [randomUUID()] } }).where(eq(heartbeatRuns.id, fixture.runId));
|
||||
if (change === "provider_started") await db.update(heartbeatRuns).set({ resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: true } } }).where(eq(heartbeatRuns.id, fixture.runId));
|
||||
if (change === "parent_session") await db.update(heartbeatRuns).set({ nativeSessionId: randomUUID() }).where(eq(heartbeatRuns.id, fixture.runId));
|
||||
if (change === "unrelated_retry") await db.update(heartbeatRuns).set({ scheduledRetryReason: "transient_failure" }).where(eq(heartbeatRuns.id, child.id));
|
||||
if (change === "foreign_parent" || change === "foreign_task") {
|
||||
const [company] = change === "foreign_parent" ? await db.insert(companies).values({ name: "Foreign retry", issuePrefix: `F${randomUUID().replaceAll("-", "")}` }).returning() : [];
|
||||
const [actor] = company ? await db.insert(agents).values({ companyId: company.id, name: "Foreign agent", adapterType: "process" }).returning() : [];
|
||||
const companyId = company?.id ?? fixture.companyId;
|
||||
const [other] = await db.insert(issues).values({ companyId, title: "Another task" }).returning();
|
||||
const [foreign] = await db.insert(heartbeatRuns).values({ companyId, agentId: actor?.id ?? fixture.agentId,
|
||||
scopeKind: "issue", issueId: other!.id, finishedAt: new Date(),
|
||||
status: "failed", errorCode: "reviewed_chat_execution_binding_not_ready", contextSnapshot: fixture.context,
|
||||
resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } }).returning();
|
||||
context.retryOfRunId = foreign!.id;
|
||||
await db.update(heartbeatRuns).set({ retryOfRunId: foreign!.id, contextSnapshot: context }).where(eq(heartbeatRuns.id, child.id));
|
||||
}
|
||||
expect(await attestReviewedExternalChatRun({ db, ...binding, contextSnapshot: context })).toBe(false);
|
||||
expect(context.paperclipExternalChatQuestionResponse).toBeUndefined();
|
||||
},
|
||||
);
|
||||
|
||||
it("exhausts answered retries and rejects a forged child beyond their authority bound", async () => {
|
||||
const { fixture, child } = await answerAttestationRetry(2);
|
||||
await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date(),
|
||||
errorCode: "reviewed_chat_execution_binding_not_ready",
|
||||
resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } },
|
||||
}).where(eq(heartbeatRuns.id, child.id));
|
||||
expect(await heartbeatService(db).scheduleBoundedRetry(child.id, { retryReason: "interaction_continuation_infra_retry",
|
||||
wakeReason: "interaction_continuation_infra_retry", maxAttempts: 3 })).toMatchObject({ outcome: "retry_exhausted" });
|
||||
const [wake] = await db.insert(agentWakeupRequests).values({ companyId: fixture.companyId, agentId: fixture.agentId,
|
||||
source: "automation", status: "claimed" }).returning();
|
||||
const context = { ...child.contextSnapshot, retryOfRunId: child.id, scheduledRetryAttempt: 3,
|
||||
executionRetryAccounting: { version: 1, failureRetries: 3, maxTurnContinuations: 0 } };
|
||||
const [forged] = await db.insert(heartbeatRuns).values({ ...child, id: randomUUID(), status: "running",
|
||||
wakeupRequestId: wake!.id, retryOfRunId: child.id, scheduledRetryAttempt: 3, contextSnapshot: context }).returning();
|
||||
await db.update(agentWakeupRequests).set({ runId: forged!.id }).where(eq(agentWakeupRequests.id, wake!.id));
|
||||
await db.update(issues).set({ executionRunId: forged!.id }).where(eq(issues.id, fixture.issueId));
|
||||
expect(await attestReviewedExternalChatRun({ db, ...fixture, runId: forged!.id, contextSnapshot: context })).toBe(false);
|
||||
});
|
||||
|
||||
async function attestAnswer(
|
||||
fixture:
|
||||
| Awaited<ReturnType<typeof seedAnsweredChatTurn>>
|
||||
|
||||
@@ -1212,6 +1212,7 @@ export function questionResponseDeliveryService(
|
||||
interactionId: interaction.id,
|
||||
interactionKind: interaction.kind,
|
||||
interactionStatus: interaction.status,
|
||||
mutation: "interaction",
|
||||
sourceCommentId: continuationSourceCommentId,
|
||||
sourceRunId: interaction.sourceRunId,
|
||||
externalChatContinuation: externalChatBoundary,
|
||||
|
||||
@@ -1,4 +1,13 @@
|
||||
export const REVIEWED_CHAT_BINDING_NOT_AUTHORIZED = "reviewed_chat_execution_binding_not_authorized";
|
||||
export const REVIEWED_CHAT_BINDING_NOT_READY = "reviewed_chat_execution_binding_not_ready";
|
||||
|
||||
export class ReviewedChatBindingNotReadyError extends Error {
|
||||
readonly code = REVIEWED_CHAT_BINDING_NOT_READY;
|
||||
|
||||
constructor() {
|
||||
super(REVIEWED_CHAT_BINDING_NOT_READY);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The first reviewed-chat attestation check that denied a run. Closed labels
|
||||
|
||||
Reference in new issue
Block a user