From bb19d6741e346edbfbaa7611953add4451afa48a Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Wed, 18 Nov 2026 15:13:29 -0800 Subject: [PATCH] Handle incomplete PR delivery during reassessment Treat unchanged unmerged PRs as unfinished delivery before status commit. Reuse the bounded continuation and recovery policy so reassessment does not repeatedly query GitHub through status-race retries. Lock source and destination tasks before work-product updates, including human updates. Re-read the product under those locks so restoring an archived PR cannot bypass the completion snapshot. Add four reassessment regressions and a real PostgreSQL lock regression. The five cases reproduced the previous failures. Server typecheck and six focused lifecycle/work-product suites pass after the fixes. Co-Authored-By: Paperclip --- doc/architecture/native-status-arbitration.md | 8 +++- .../native-status-arbiter-corpus.test.ts | 45 +++++++++++++++++++ server/src/__tests__/work-products.test.ts | 10 +++-- .../native-completion-feedback.test.ts | 36 ++++++++++++++- .../native-status-reassessment.ts | 35 ++++++++++++++- server/src/services/work-products.ts | 22 +++++++-- 6 files changed, 145 insertions(+), 11 deletions(-) diff --git a/doc/architecture/native-status-arbitration.md b/doc/architecture/native-status-arbitration.md index d5d761c4ec3..37ab7a925b0 100644 --- a/doc/architecture/native-status-arbitration.md +++ b/doc/architecture/native-status-arbitration.md @@ -139,9 +139,13 @@ uses its existing bounded incomplete-work continuation. The agent can fix review or CI failures, use the existing monitor for an external wait, or report a concrete blocker. This creates no generic human completion approval. +Reassessment uses the same bounded incomplete-delivery continuation as initial +finalization. A stable unmerged PR does not create a status-race retry deadline. The transactional commit also checks these outputs, including reassessment -paths. Provider reads occur before the transaction, with at most twenty PRs -and a shared ten-second abort signal for GitHub requests. The locked transaction requires merge proof +paths. Human and agent work-product updates lock their source and destination +tasks in a stable order before changing status or identity. Provider reads occur +before the transaction, with at most twenty PRs and a shared ten-second abort +signal for GitHub requests. The locked transaction requires merge proof for every current deliverable, so a changed output requires a new assessment. There is no new polling worker or schema. Existing saved contracts remain valid; new contracts include the delivery guidance and existing unfinished results use diff --git a/server/src/__tests__/native-status-arbiter-corpus.test.ts b/server/src/__tests__/native-status-arbiter-corpus.test.ts index c97879ebee9..92944206db9 100644 --- a/server/src/__tests__/native-status-arbiter-corpus.test.ts +++ b/server/src/__tests__/native-status-arbiter-corpus.test.ts @@ -2920,6 +2920,51 @@ describe("P6-31 Section 18.13 executable status-authority corpus", () => { expect(await db.select().from(statusDecisions).where(eq(statusDecisions.issueId, seeded.issueId))).toHaveLength(2); }); + it.each([ + { trigger: "retired review", correctiveUsed: false }, + { trigger: "new evidence", correctiveUsed: false }, + { trigger: "retired review", correctiveUsed: true }, + { trigger: "new evidence", correctiveUsed: true }, + ])("settles unmerged delivery from $trigger without retry polling (corrective used=$correctiveUsed)", async ({ trigger, correctiveUsed }) => { + const seeded = await seedAutomaticReview(); + const [originalAssessment] = await db.select().from(workAssessments).where(eq(workAssessments.id, seeded.assessmentId)); + await db.update(workAssessments).set({ createdAt: new Date(Date.now() - 2000), + assessmentJson: { ...originalAssessment.assessmentJson, allCriteriaSatisfied: false } }).where(eq(workAssessments.id, seeded.assessmentId)); + await db.insert(issueWorkProducts).values({ companyId, issueId: seeded.issueId, type: "pull_request", + provider: "github", title: "Closed implementation PR", status: "closed", url: "https://github.com/example/project/pull/42", + createdAt: new Date(Date.now() - 1000), updatedAt: new Date(Date.now() - 1000) }); + if (trigger === "retired review") await dismissAutomaticCompletionReviews(db, seeded.issueId); + else await db.update(issueThreadInteractions).set({ status: "cancelled", result: { version: 1, outcome: "withdrawn" } }) + .where(eq(issueThreadInteractions.id, seeded.interaction.id)); + if (correctiveUsed) { + const [wake] = await db.insert(agentWakeupRequests).values({ companyId, agentId, source: "automation", + triggerDetail: "system", reason: "issue_status_changed", status: "consumed", + requestedByActorType: "system", requestedByActorId: "native-status-committer", + payload: { continuationIdempotencyKey: "native-completion-incomplete" } }).returning(); + await db.update(heartbeatRuns).set({ wakeupRequestId: wake.id }).where(eq(heartbeatRuns.id, seeded.runId)); + } + const lookup = vi.fn().mockResolvedValue("open"); + const resolve = vi.spyOn(pullRequestMerge, "createPullRequestMergeStateResolver").mockReturnValue(lookup); + try { + const work = nativeStatusReassessmentWork(db), scope = selectedScope(seeded); + const [pending] = await work.load(scope); + expect(pending).toBeDefined(); + expect(await work.check(pending)).toBeNull(); + const [latest] = await db.select().from(statusDecisions).where(eq(statusDecisions.issueId, seeded.issueId)) + .orderBy(statusDecisions.createdAt, statusDecisions.id).then(rows => rows.filter(row => row.id !== seeded.decision.id)); + const effects = latest.decisionJson.effects as Array<{ kind: string; summary?: string; nextAction?: string }>; + expect(latest.toStatus).not.toBe("done"); + expect(effects.some(effect => effect.kind === "enqueue_continuation")).toBe(!correctiveUsed); + expect(effects.some(effect => effect.kind === "record_finalization_error")).toBe(correctiveUsed); + expect(effects.some(effect => (effect.summary ?? effect.nextAction ?? "").includes("not been verified as merged"))).toBe(true); + const assessments = await db.select().from(workAssessments).where(eq(workAssessments.runId, seeded.runId)); + expect(await work.load(scope)).toEqual([]); + expect(await work.check(pending)).toBeNull(); + expect(await db.select().from(workAssessments).where(eq(workAssessments.runId, seeded.runId))).toHaveLength(assessments.length); + expect(lookup).toHaveBeenCalledTimes(1); + } finally { resolve.mockRestore(); } + }); + it("respects a saved retry and live lease, then reloads the released owner", async () => { const seeded = await seedAutomaticReview(); await dismissAutomaticCompletionReviews(db, seeded.issueId); diff --git a/server/src/__tests__/work-products.test.ts b/server/src/__tests__/work-products.test.ts index cef395409fd..ede8420ddd9 100644 --- a/server/src/__tests__/work-products.test.ts +++ b/server/src/__tests__/work-products.test.ts @@ -1,3 +1,4 @@ +import { issues } from "@paperclipai/db"; import { PgDialect } from "drizzle-orm/pg-core"; import type { SQL } from "drizzle-orm"; import { describe, expect, it, vi } from "vitest"; @@ -202,8 +203,11 @@ describe("workProductService", () => { it("uses a transaction when promoting an existing work product to primary", async () => { const existingRow = createWorkProductRow({ isPrimary: false }); - const selectWhere = vi.fn(async () => [existingRow]); - const selectFrom = vi.fn(() => ({ where: selectWhere })); + const selectFrom = vi.fn((table: unknown) => { + const rows = table === issues ? [{ id: "issue-1" }] : [existingRow]; + const query = Promise.resolve(rows); + return Object.assign(query, { where: vi.fn(() => query), orderBy: vi.fn(() => query), for: vi.fn(() => query) }); + }); const txSelect = vi.fn(() => ({ from: selectFrom })); const updateReturning = vi @@ -231,7 +235,7 @@ describe("workProductService", () => { }); expect(transaction).toHaveBeenCalledTimes(1); - expect(txSelect).toHaveBeenCalledTimes(1); + expect(txSelect).toHaveBeenCalled(); expect(txUpdate).toHaveBeenCalledTimes(2); expect(result?.reviewState).toBe("ready_for_review"); }); diff --git a/server/src/services/native-runtime/native-completion-feedback.test.ts b/server/src/services/native-runtime/native-completion-feedback.test.ts index ed1dff34f60..8081ed60dd7 100644 --- a/server/src/services/native-runtime/native-completion-feedback.test.ts +++ b/server/src/services/native-runtime/native-completion-feedback.test.ts @@ -1,6 +1,6 @@ import { createServer } from "node:http"; import { createHash, randomUUID } from "node:crypto"; -import { eq } from "drizzle-orm"; +import { eq, sql } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { agents, approvals, companies, createDb, documentRevisions, executionWorkspaces, executionWorkspaceRepositories, heartbeatRuns, issueExecutionDecisions, issues, issueApprovals, issueThreadInteractions, issueTreeHolds, issueDocuments, issueWorkProducts } from "@paperclipai/db"; import { startRecoveryPostgres } from "../../__tests__/helpers/recovery-postgres.js"; @@ -8,7 +8,8 @@ import type { PrpStructuredRunResult } from "../../vendor/paperclip-runner/index import { documentService } from "../documents.js"; import * as pullRequestMerge from "../github-pull-request-merge.js"; import { validateNativeDeliverableEvidence } from "./native-deliverable-feedback.js"; -import { verifyNativePullRequestCompletion } from "./native-pull-request-completion.js"; +import { nativePullRequestCompletionReferences, verifyNativePullRequestCompletion } from "./native-pull-request-completion.js"; +import { workProductService } from "../work-products.js"; import { nativeCompletionFeedback } from "./native-completion-feedback.js"; import { executionStageApprovedByRun } from "./native-run-finalizer.js"; import { PaperclipRunnerToolAuthority } from "./paperclip-runner-tool-authority.js"; @@ -151,6 +152,37 @@ describe("native final-response feedback", () => { } finally { resolve.mockRestore(); } }); + it("serializes human restoration of an archived PR with the completion task lock", async () => { + const value = await fixture(); + const [product] = await db.insert(issueWorkProducts).values({ companyId: value.companyId, issueId: value.issueId, + type: "pull_request", provider: "github", title: "Archived delivery", status: "archived", + url: "https://github.com/example/project/pull/42" }).returning(); + let acquired!: (pid: number) => void; + const locked = new Promise(resolve => { acquired = resolve; }); + let release!: () => void; + const unlocked = new Promise(resolve => { release = resolve; }); + const completion = db.transaction(async tx => { + await tx.select({ id: issues.id }).from(issues).where(eq(issues.id, value.issueId)).for("update"); + expect(await nativePullRequestCompletionReferences(tx as unknown as typeof db, { ...value, lockSources: true }, done)).toEqual([]); + const [owner] = await tx.execute(sql`select pg_backend_pid() as pid`) as unknown as Array<{ pid: number }>; + acquired(owner.pid); + await unlocked; + }); + const ownerPid = await locked; + let restored = false; + const restoration = workProductService(db).update(product.id, { status: "active", url: "https://github.com/example/project/pull/43" }, null) + .then(row => { restored = true; return row; }); + try { + await vi.waitFor(async () => { + const [wait] = await db.execute(sql`select exists(select 1 from pg_stat_activity activity + where ${ownerPid} = any(pg_blocking_pids(activity.pid))) as blocked`) as unknown as Array<{ blocked: boolean }>; + expect(restored).toBe(false); + expect(wait.blocked).toBe(true); + }, { timeout: 1500 }); + } finally { release(); await completion; await restoration; } + expect(restored).toBe(true); + }); + it("aborts a stalled provider body without holding the task lock or completing the task", async () => { const value = await fixture(); let observed!: () => void; diff --git a/server/src/services/native-runtime/native-status-reassessment.ts b/server/src/services/native-runtime/native-status-reassessment.ts index b2dca3eaa56..b711766893c 100644 --- a/server/src/services/native-runtime/native-status-reassessment.ts +++ b/server/src/services/native-runtime/native-status-reassessment.ts @@ -1,8 +1,9 @@ +import { NativeDeliverableEvidenceError, validateNativeDeliverableEvidence } from "./native-deliverable-feedback.js"; import { readRequiredQualityReview } from "../required-quality-review.js"; import { completionReviewStageEntry } from "../issue-execution-policy.js"; import { hasNativeWorkspaceFinalizationOwner } from "./native-workspace-finalization-recovery-state.js"; import { and, desc, eq, gt, or } from "drizzle-orm"; -import { completionContracts, heartbeatRuns, issueWorkProducts, issues, nativeRunResults, +import { agentWakeupRequests, completionContracts, heartbeatRuns, issueWorkProducts, issues, nativeRunResults, statusDecisions, workAssessments, workspaceOperations, type Db } from "@paperclipai/db"; import { issueService } from "../issues.js"; import { decisionHasRetiredAutomaticReview } from "./automatic-completion-reviews.js"; @@ -246,6 +247,38 @@ export async function reassessNativeStatusFromSource( decision = supersedeStaleOwnerCompletion(decision, { ...currentIssue, status: row.issueStatus }, row.agentId, row.runId); } decision = gateDoneOnConfiguredReview(decision, reviewStage, row.agentId); + if (decision.statusAction === "done" && currentIssue) { + const run = currentRun ?? await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, row.runId), + eq(heartbeatRuns.companyId, row.companyId), eq(heartbeatRuns.nativeIssueId, row.issueId))) + .limit(1).then(rows => rows[0]); + if (!run) return { kind: "deferred", retry: true }; + const continuation = record(run.contextSnapshot?.executionContinuation); + const objective = typeof continuation.objective === "string" ? continuation.objective + : [currentIssue.title, currentIssue.description].filter(Boolean).join("\n"); + try { + await validateNativeDeliverableEvidence(db, { companyId: row.companyId, issueId: row.issueId, + runId: row.runId, objective, semanticToolReceipts: run.resultJson?.semanticToolReceipts }, + record(resultRow.resultJson).result); + } catch (error) { + if (!(error instanceof NativeDeliverableEvidenceError)) throw error; + const [wake] = run.wakeupRequestId ? await db.select({ payload: agentWakeupRequests.payload }).from(agentWakeupRequests) + .where(and(eq(agentWakeupRequests.id, run.wakeupRequestId), eq(agentWakeupRequests.companyId, row.companyId))) : []; + const dependency = readiness ?? await issueService(db).getDependencyReadiness(row.issueId, db); + const gate = governanceGate ?? await pendingNativeGovernance({ db, companyId: row.companyId, issueId: row.issueId, + runId: row.runId, executionState: record(currentIssue.executionState) }); + const incomplete = resolveNativeFinalizerStatus({ + assessment: { ...reassessment, hasBlockingRemainingWork: true, continuation: null }, + terminalState: "succeeded", workspaceFinalizeStatus: "succeeded", governanceGate: gate, + hasPendingChildCompletion, hasUnresolvedIssueBlockers: dependency.unresolvedBlockerCount > 0, + allowIncompleteContinuation: record(wake?.payload).continuationIdempotencyKey !== "native-completion-incomplete", + priorIssueStatus: row.issueStatus as NativeAuthoritativeIssueStatus, agentId: row.agentId, + reviewOwnerUserId: currentIssue.responsibleUserId ?? currentIssue.createdByUserId, + }); + decision = { ...incomplete, effects: incomplete.effects.map(effect => effect.kind === "enqueue_continuation" + ? { ...effect, summary: error.message } : effect.kind === "record_finalization_error" + ? { ...effect, nextAction: error.message } : effect) }; + } + } let committed: Awaited>; try { committed = await commitNativeStatusDecision({ diff --git a/server/src/services/work-products.ts b/server/src/services/work-products.ts index e4373a34081..107023b6cf0 100644 --- a/server/src/services/work-products.ts +++ b/server/src/services/work-products.ts @@ -1,9 +1,9 @@ import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; import { assertAgentInteraction } from "./agent-interaction-policy.js"; import { notifyApprovalQueueWork } from "./approval-work-notifications.js"; -import { and, desc, eq, inArray } from "drizzle-orm"; +import { and, desc, eq, inArray, or } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; -import { executionWorkspaces, heartbeatRunEvents, issueWorkProducts, workspaceRuntimeServices } from "@paperclipai/db"; +import { executionWorkspaces, heartbeatRunEvents, issues, issueWorkProducts, workspaceRuntimeServices } from "@paperclipai/db"; import type { IssueWorkProduct } from "@paperclipai/shared"; import { unprocessable } from "../errors.js"; import { insertRowsInChunks } from "./batch-insert.js"; @@ -23,6 +23,14 @@ import type { ImportIssueWorkProductRow } from "./import-write-types.js"; type IssueWorkProductRow = typeof issueWorkProducts.$inferSelect; type WorkProductTransaction = Parameters[0]>[0]; +async function lockWorkProductIssues(tx: WorkProductTransaction, scopes: Array<{ companyId: string; issueId: string }>) { + const unique = [...new Map(scopes.map(scope => [`${scope.companyId}:${scope.issueId}`, scope])).values()]; + const rows = await tx.select({ id: issues.id }).from(issues).where(or(...unique.map(scope => + and(eq(issues.id, scope.issueId), eq(issues.companyId, scope.companyId))))) + .orderBy(issues.id).for("update"); + if (rows.length !== unique.length) throw unprocessable("Work product tasks must exist in their bound companies."); +} + async function validateExecutionWorkspace( tx: WorkProductTransaction, companyId: string, @@ -311,10 +319,18 @@ export function workProductService( update: async (id: string, patch: Partial, actorAgentId?: string | null) => { const row = await db.transaction(async (tx) => { + const [location] = await tx.select({ companyId: issueWorkProducts.companyId, issueId: issueWorkProducts.issueId }) + .from(issueWorkProducts).where(eq(issueWorkProducts.id, id)); + if (!location) return null; + await lockWorkProductIssues(tx, [location, { + companyId: patch.companyId ?? location.companyId, issueId: patch.issueId ?? location.issueId, + }]); const existing = await tx .select() .from(issueWorkProducts) - .where(eq(issueWorkProducts.id, id)) + .where(and(eq(issueWorkProducts.id, id), eq(issueWorkProducts.companyId, location.companyId), + eq(issueWorkProducts.issueId, location.issueId))) + .for("update") .then((rows) => rows[0] ?? null); if (!existing) return null; await assertAgentInteraction(tx, { companyId: existing.companyId, actorAgentId, operation: "edit", issueId: existing.issueId,