diff --git a/doc/architecture/native-status-arbitration.md b/doc/architecture/native-status-arbitration.md index d5d761c4ec3..37ab7a925b0 100644 --- a/doc/architecture/native-status-arbitration.md +++ b/doc/architecture/native-status-arbitration.md @@ -139,9 +139,13 @@ uses its existing bounded incomplete-work continuation. The agent can fix review or CI failures, use the existing monitor for an external wait, or report a concrete blocker. This creates no generic human completion approval. +Reassessment uses the same bounded incomplete-delivery continuation as initial +finalization. A stable unmerged PR does not create a status-race retry deadline. The transactional commit also checks these outputs, including reassessment -paths. Provider reads occur before the transaction, with at most twenty PRs -and a shared ten-second abort signal for GitHub requests. The locked transaction requires merge proof +paths. Human and agent work-product updates lock their source and destination +tasks in a stable order before changing status or identity. Provider reads occur +before the transaction, with at most twenty PRs and a shared ten-second abort +signal for GitHub requests. The locked transaction requires merge proof for every current deliverable, so a changed output requires a new assessment. There is no new polling worker or schema. Existing saved contracts remain valid; new contracts include the delivery guidance and existing unfinished results use diff --git a/server/src/__tests__/native-status-arbiter-corpus.test.ts b/server/src/__tests__/native-status-arbiter-corpus.test.ts index c97879ebee9..92944206db9 100644 --- a/server/src/__tests__/native-status-arbiter-corpus.test.ts +++ b/server/src/__tests__/native-status-arbiter-corpus.test.ts @@ -2920,6 +2920,51 @@ describe("P6-31 Section 18.13 executable status-authority corpus", () => { expect(await db.select().from(statusDecisions).where(eq(statusDecisions.issueId, seeded.issueId))).toHaveLength(2); }); + it.each([ + { trigger: "retired review", correctiveUsed: false }, + { trigger: "new evidence", correctiveUsed: false }, + { trigger: "retired review", correctiveUsed: true }, + { trigger: "new evidence", correctiveUsed: true }, + ])("settles unmerged delivery from $trigger without retry polling (corrective used=$correctiveUsed)", async ({ trigger, correctiveUsed }) => { + const seeded = await seedAutomaticReview(); + const [originalAssessment] = await db.select().from(workAssessments).where(eq(workAssessments.id, seeded.assessmentId)); + await db.update(workAssessments).set({ createdAt: new Date(Date.now() - 2000), + assessmentJson: { ...originalAssessment.assessmentJson, allCriteriaSatisfied: false } }).where(eq(workAssessments.id, seeded.assessmentId)); + await db.insert(issueWorkProducts).values({ companyId, issueId: seeded.issueId, type: "pull_request", + provider: "github", title: "Closed implementation PR", status: "closed", url: "https://github.com/example/project/pull/42", + createdAt: new Date(Date.now() - 1000), updatedAt: new Date(Date.now() - 1000) }); + if (trigger === "retired review") await dismissAutomaticCompletionReviews(db, seeded.issueId); + else await db.update(issueThreadInteractions).set({ status: "cancelled", result: { version: 1, outcome: "withdrawn" } }) + .where(eq(issueThreadInteractions.id, seeded.interaction.id)); + if (correctiveUsed) { + const [wake] = await db.insert(agentWakeupRequests).values({ companyId, agentId, source: "automation", + triggerDetail: "system", reason: "issue_status_changed", status: "consumed", + requestedByActorType: "system", requestedByActorId: "native-status-committer", + payload: { continuationIdempotencyKey: "native-completion-incomplete" } }).returning(); + await db.update(heartbeatRuns).set({ wakeupRequestId: wake.id }).where(eq(heartbeatRuns.id, seeded.runId)); + } + const lookup = vi.fn().mockResolvedValue("open"); + const resolve = vi.spyOn(pullRequestMerge, "createPullRequestMergeStateResolver").mockReturnValue(lookup); + try { + const work = nativeStatusReassessmentWork(db), scope = selectedScope(seeded); + const [pending] = await work.load(scope); + expect(pending).toBeDefined(); + expect(await work.check(pending)).toBeNull(); + const [latest] = await db.select().from(statusDecisions).where(eq(statusDecisions.issueId, seeded.issueId)) + .orderBy(statusDecisions.createdAt, statusDecisions.id).then(rows => rows.filter(row => row.id !== seeded.decision.id)); + const effects = latest.decisionJson.effects as Array<{ kind: string; summary?: string; nextAction?: string }>; + expect(latest.toStatus).not.toBe("done"); + expect(effects.some(effect => effect.kind === "enqueue_continuation")).toBe(!correctiveUsed); + expect(effects.some(effect => effect.kind === "record_finalization_error")).toBe(correctiveUsed); + expect(effects.some(effect => (effect.summary ?? effect.nextAction ?? "").includes("not been verified as merged"))).toBe(true); + const assessments = await db.select().from(workAssessments).where(eq(workAssessments.runId, seeded.runId)); + expect(await work.load(scope)).toEqual([]); + expect(await work.check(pending)).toBeNull(); + expect(await db.select().from(workAssessments).where(eq(workAssessments.runId, seeded.runId))).toHaveLength(assessments.length); + expect(lookup).toHaveBeenCalledTimes(1); + } finally { resolve.mockRestore(); } + }); + it("respects a saved retry and live lease, then reloads the released owner", async () => { const seeded = await seedAutomaticReview(); await dismissAutomaticCompletionReviews(db, seeded.issueId); diff --git a/server/src/__tests__/work-products.test.ts b/server/src/__tests__/work-products.test.ts index cef395409fd..ede8420ddd9 100644 --- a/server/src/__tests__/work-products.test.ts +++ b/server/src/__tests__/work-products.test.ts @@ -1,3 +1,4 @@ +import { issues } from "@paperclipai/db"; import { PgDialect } from "drizzle-orm/pg-core"; import type { SQL } from "drizzle-orm"; import { describe, expect, it, vi } from "vitest"; @@ -202,8 +203,11 @@ describe("workProductService", () => { it("uses a transaction when promoting an existing work product to primary", async () => { const existingRow = createWorkProductRow({ isPrimary: false }); - const selectWhere = vi.fn(async () => [existingRow]); - const selectFrom = vi.fn(() => ({ where: selectWhere })); + const selectFrom = vi.fn((table: unknown) => { + const rows = table === issues ? [{ id: "issue-1" }] : [existingRow]; + const query = Promise.resolve(rows); + return Object.assign(query, { where: vi.fn(() => query), orderBy: vi.fn(() => query), for: vi.fn(() => query) }); + }); const txSelect = vi.fn(() => ({ from: selectFrom })); const updateReturning = vi @@ -231,7 +235,7 @@ describe("workProductService", () => { }); expect(transaction).toHaveBeenCalledTimes(1); - expect(txSelect).toHaveBeenCalledTimes(1); + expect(txSelect).toHaveBeenCalled(); expect(txUpdate).toHaveBeenCalledTimes(2); expect(result?.reviewState).toBe("ready_for_review"); }); diff --git a/server/src/services/native-runtime/native-completion-feedback.test.ts b/server/src/services/native-runtime/native-completion-feedback.test.ts index ed1dff34f60..8081ed60dd7 100644 --- a/server/src/services/native-runtime/native-completion-feedback.test.ts +++ b/server/src/services/native-runtime/native-completion-feedback.test.ts @@ -1,6 +1,6 @@ import { createServer } from "node:http"; import { createHash, randomUUID } from "node:crypto"; -import { eq } from "drizzle-orm"; +import { eq, sql } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { agents, approvals, companies, createDb, documentRevisions, executionWorkspaces, executionWorkspaceRepositories, heartbeatRuns, issueExecutionDecisions, issues, issueApprovals, issueThreadInteractions, issueTreeHolds, issueDocuments, issueWorkProducts } from "@paperclipai/db"; import { startRecoveryPostgres } from "../../__tests__/helpers/recovery-postgres.js"; @@ -8,7 +8,8 @@ import type { PrpStructuredRunResult } from "../../vendor/paperclip-runner/index import { documentService } from "../documents.js"; import * as pullRequestMerge from "../github-pull-request-merge.js"; import { validateNativeDeliverableEvidence } from "./native-deliverable-feedback.js"; -import { verifyNativePullRequestCompletion } from "./native-pull-request-completion.js"; +import { nativePullRequestCompletionReferences, verifyNativePullRequestCompletion } from "./native-pull-request-completion.js"; +import { workProductService } from "../work-products.js"; import { nativeCompletionFeedback } from "./native-completion-feedback.js"; import { executionStageApprovedByRun } from "./native-run-finalizer.js"; import { PaperclipRunnerToolAuthority } from "./paperclip-runner-tool-authority.js"; @@ -151,6 +152,37 @@ describe("native final-response feedback", () => { } finally { resolve.mockRestore(); } }); + it("serializes human restoration of an archived PR with the completion task lock", async () => { + const value = await fixture(); + const [product] = await db.insert(issueWorkProducts).values({ companyId: value.companyId, issueId: value.issueId, + type: "pull_request", provider: "github", title: "Archived delivery", status: "archived", + url: "https://github.com/example/project/pull/42" }).returning(); + let acquired!: (pid: number) => void; + const locked = new Promise(resolve => { acquired = resolve; }); + let release!: () => void; + const unlocked = new Promise(resolve => { release = resolve; }); + const completion = db.transaction(async tx => { + await tx.select({ id: issues.id }).from(issues).where(eq(issues.id, value.issueId)).for("update"); + expect(await nativePullRequestCompletionReferences(tx as unknown as typeof db, { ...value, lockSources: true }, done)).toEqual([]); + const [owner] = await tx.execute(sql`select pg_backend_pid() as pid`) as unknown as Array<{ pid: number }>; + acquired(owner.pid); + await unlocked; + }); + const ownerPid = await locked; + let restored = false; + const restoration = workProductService(db).update(product.id, { status: "active", url: "https://github.com/example/project/pull/43" }, null) + .then(row => { restored = true; return row; }); + try { + await vi.waitFor(async () => { + const [wait] = await db.execute(sql`select exists(select 1 from pg_stat_activity activity + where ${ownerPid} = any(pg_blocking_pids(activity.pid))) as blocked`) as unknown as Array<{ blocked: boolean }>; + expect(restored).toBe(false); + expect(wait.blocked).toBe(true); + }, { timeout: 1500 }); + } finally { release(); await completion; await restoration; } + expect(restored).toBe(true); + }); + it("aborts a stalled provider body without holding the task lock or completing the task", async () => { const value = await fixture(); let observed!: () => void; diff --git a/server/src/services/native-runtime/native-status-reassessment.ts b/server/src/services/native-runtime/native-status-reassessment.ts index b2dca3eaa56..b711766893c 100644 --- a/server/src/services/native-runtime/native-status-reassessment.ts +++ b/server/src/services/native-runtime/native-status-reassessment.ts @@ -1,8 +1,9 @@ +import { NativeDeliverableEvidenceError, validateNativeDeliverableEvidence } from "./native-deliverable-feedback.js"; import { readRequiredQualityReview } from "../required-quality-review.js"; import { completionReviewStageEntry } from "../issue-execution-policy.js"; import { hasNativeWorkspaceFinalizationOwner } from "./native-workspace-finalization-recovery-state.js"; import { and, desc, eq, gt, or } from "drizzle-orm"; -import { completionContracts, heartbeatRuns, issueWorkProducts, issues, nativeRunResults, +import { agentWakeupRequests, completionContracts, heartbeatRuns, issueWorkProducts, issues, nativeRunResults, statusDecisions, workAssessments, workspaceOperations, type Db } from "@paperclipai/db"; import { issueService } from "../issues.js"; import { decisionHasRetiredAutomaticReview } from "./automatic-completion-reviews.js"; @@ -246,6 +247,38 @@ export async function reassessNativeStatusFromSource( decision = supersedeStaleOwnerCompletion(decision, { ...currentIssue, status: row.issueStatus }, row.agentId, row.runId); } decision = gateDoneOnConfiguredReview(decision, reviewStage, row.agentId); + if (decision.statusAction === "done" && currentIssue) { + const run = currentRun ?? await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, row.runId), + eq(heartbeatRuns.companyId, row.companyId), eq(heartbeatRuns.nativeIssueId, row.issueId))) + .limit(1).then(rows => rows[0]); + if (!run) return { kind: "deferred", retry: true }; + const continuation = record(run.contextSnapshot?.executionContinuation); + const objective = typeof continuation.objective === "string" ? continuation.objective + : [currentIssue.title, currentIssue.description].filter(Boolean).join("\n"); + try { + await validateNativeDeliverableEvidence(db, { companyId: row.companyId, issueId: row.issueId, + runId: row.runId, objective, semanticToolReceipts: run.resultJson?.semanticToolReceipts }, + record(resultRow.resultJson).result); + } catch (error) { + if (!(error instanceof NativeDeliverableEvidenceError)) throw error; + const [wake] = run.wakeupRequestId ? await db.select({ payload: agentWakeupRequests.payload }).from(agentWakeupRequests) + .where(and(eq(agentWakeupRequests.id, run.wakeupRequestId), eq(agentWakeupRequests.companyId, row.companyId))) : []; + const dependency = readiness ?? await issueService(db).getDependencyReadiness(row.issueId, db); + const gate = governanceGate ?? await pendingNativeGovernance({ db, companyId: row.companyId, issueId: row.issueId, + runId: row.runId, executionState: record(currentIssue.executionState) }); + const incomplete = resolveNativeFinalizerStatus({ + assessment: { ...reassessment, hasBlockingRemainingWork: true, continuation: null }, + terminalState: "succeeded", workspaceFinalizeStatus: "succeeded", governanceGate: gate, + hasPendingChildCompletion, hasUnresolvedIssueBlockers: dependency.unresolvedBlockerCount > 0, + allowIncompleteContinuation: record(wake?.payload).continuationIdempotencyKey !== "native-completion-incomplete", + priorIssueStatus: row.issueStatus as NativeAuthoritativeIssueStatus, agentId: row.agentId, + reviewOwnerUserId: currentIssue.responsibleUserId ?? currentIssue.createdByUserId, + }); + decision = { ...incomplete, effects: incomplete.effects.map(effect => effect.kind === "enqueue_continuation" + ? { ...effect, summary: error.message } : effect.kind === "record_finalization_error" + ? { ...effect, nextAction: error.message } : effect) }; + } + } let committed: Awaited>; try { committed = await commitNativeStatusDecision({ diff --git a/server/src/services/work-products.ts b/server/src/services/work-products.ts index e4373a34081..107023b6cf0 100644 --- a/server/src/services/work-products.ts +++ b/server/src/services/work-products.ts @@ -1,9 +1,9 @@ import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; import { assertAgentInteraction } from "./agent-interaction-policy.js"; import { notifyApprovalQueueWork } from "./approval-work-notifications.js"; -import { and, desc, eq, inArray } from "drizzle-orm"; +import { and, desc, eq, inArray, or } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; -import { executionWorkspaces, heartbeatRunEvents, issueWorkProducts, workspaceRuntimeServices } from "@paperclipai/db"; +import { executionWorkspaces, heartbeatRunEvents, issues, issueWorkProducts, workspaceRuntimeServices } from "@paperclipai/db"; import type { IssueWorkProduct } from "@paperclipai/shared"; import { unprocessable } from "../errors.js"; import { insertRowsInChunks } from "./batch-insert.js"; @@ -23,6 +23,14 @@ import type { ImportIssueWorkProductRow } from "./import-write-types.js"; type IssueWorkProductRow = typeof issueWorkProducts.$inferSelect; type WorkProductTransaction = Parameters[0]>[0]; +async function lockWorkProductIssues(tx: WorkProductTransaction, scopes: Array<{ companyId: string; issueId: string }>) { + const unique = [...new Map(scopes.map(scope => [`${scope.companyId}:${scope.issueId}`, scope])).values()]; + const rows = await tx.select({ id: issues.id }).from(issues).where(or(...unique.map(scope => + and(eq(issues.id, scope.issueId), eq(issues.companyId, scope.companyId))))) + .orderBy(issues.id).for("update"); + if (rows.length !== unique.length) throw unprocessable("Work product tasks must exist in their bound companies."); +} + async function validateExecutionWorkspace( tx: WorkProductTransaction, companyId: string, @@ -311,10 +319,18 @@ export function workProductService( update: async (id: string, patch: Partial, actorAgentId?: string | null) => { const row = await db.transaction(async (tx) => { + const [location] = await tx.select({ companyId: issueWorkProducts.companyId, issueId: issueWorkProducts.issueId }) + .from(issueWorkProducts).where(eq(issueWorkProducts.id, id)); + if (!location) return null; + await lockWorkProductIssues(tx, [location, { + companyId: patch.companyId ?? location.companyId, issueId: patch.issueId ?? location.issueId, + }]); const existing = await tx .select() .from(issueWorkProducts) - .where(eq(issueWorkProducts.id, id)) + .where(and(eq(issueWorkProducts.id, id), eq(issueWorkProducts.companyId, location.companyId), + eq(issueWorkProducts.issueId, location.issueId))) + .for("update") .then((rows) => rows[0] ?? null); if (!existing) return null; await assertAgentInteraction(tx, { companyId: existing.companyId, actorAgentId, operation: "edit", issueId: existing.issueId,