diff --git a/server/src/__tests__/native-result-commit-idle.test.ts b/server/src/__tests__/native-result-commit-idle.test.ts new file mode 100644 index 00000000000..b65533dddd5 --- /dev/null +++ b/server/src/__tests__/native-result-commit-idle.test.ts @@ -0,0 +1,242 @@ +import { randomUUID } from "node:crypto"; +import { setTimeout as delay } from "node:timers/promises"; +import { eq } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { agents, agentWakeupRequests, companies, completionContracts, createDb, heartbeatRuns, issueRecoveryActions, issues, + nativeRunFinalizations, nativeRunResults, statusDecisions, workspaceOperations, type Db } from "@paperclipai/db"; +import { subscribeDatabaseWork } from "@paperclipai/db/work-signals"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { notifyNativeFinalizationWork } from "../services/native-finalization-notifications.js"; +import { notifyNativeResultCommit } from "../services/native-result-commit-notifications.js"; +import { nativeStatusReassessmentWork } from "../services/native-runtime/native-status-reassessment-work.js"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; +import { buildIssueMonitorTriggeredPatch, normalizeIssueExecutionPolicy } from "../services/issue-execution-policy.js"; +import { nativeResultCommitWork } from "../services/native-runtime/native-result-commit-work.js"; +import { finalizeNativeRun } from "../services/native-runtime/native-run-finalizer.js"; +import * as monitorDelivery from "../services/native-runtime/native-monitor-delivery.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; +import { PaperclipRunnerToolAuthority } from "../services/native-runtime/paperclip-runner-tool-authority.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; + +const support = await getEmbeddedPostgresTestSupport(); +describe.skipIf(!support.supported)("native result commit of a recorded unfinalizable result", () => { + let temporary: Awaited>; + let db: Db; + let coordinator: ReturnType | undefined; + const errors: unknown[] = []; + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("native-result-commit-idle-"); + db = createDb(temporary.connectionString); + }, 60_000); + afterEach(async () => { + await coordinator?.stop(); coordinator = undefined; passes.length = 0; + vi.useRealTimers(); + expect(errors.splice(0)).toEqual([]); + }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); }); + + type Shape = "in_progress" | "blocked"; + const syncReference = { schema: "paperclip.native-workspace-sync/v3", state: "finalized", descriptorSha256: "a".repeat(64), + baselineSha256: "b".repeat(64), finalHostSha256: null, workspaceId: "workspace", leaseId: "lease", + providerLeaseId: "provider-lease", remoteCwd: "/workspace" }; + async function seed(shape: Shape = "in_progress") { + const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID(); + const contractId = randomUUID(), resultId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Idle commits", issuePrefix: `I${companyId.slice(0, 7)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Worker", adapterType: "paperclip_runner", status: "active" }); + await db.insert(issues).values({ id: issueId, companyId, title: "Saved check", status: "in_progress", assigneeAgentId: agentId }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, nativeIssueId: issueId, + runtimeMode: "native", status: "running", contextSnapshot: { issueId } }); + await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId)); + const nextCheckAt = new Date(Date.now() + 60_000).toISOString(); + await new PaperclipRunnerToolAuthority(db, { companyId, agentId, issueId, runId }).execute({ + tool: "set_task_monitor", callId: randomUUID(), arguments: { idempotencyKey: "saved-monitor", + monitor: { nextCheckAt, notes: "Check the saved result." } } }); + await db.insert(completionContracts).values({ id: contractId, companyId, issueId, revision: 1, + schemaVersion: "paperclip.completion-contract.v1", policyVersion: "test", risk: "standard", + completionAuthority: "server_arbiter", incompleteCriteriaPolicy: "preserve_non_terminal", + contractJson: { revision: "test", criteria: [] }, canonicalSha256: randomUUID(), + createdByActorType: "system", createdByActorId: "test" }); + const accepted = { result: { schema: "paperclip.run_result.v1", reportedWorkDisposition: "yielded", + summary: "Wait for the saved check.", completionClaim: { contractRevision: "test", objectiveSatisfied: false, + criteria: [], remainingWork: [{ description: "Check again", blocksCompletion: true }] }, + continuation: { kind: "monitor", summary: "Wait for the check", idempotencyKey: "provider-selected" }, + evidence: [], verification: [], attentionRequests: [], artifacts: [] }, + terminal: { schema: "paperclip.prp.terminal.v1", runTerminalState: "succeeded", turnTerminalState: "completed", reportedWorkDisposition: "yielded" } }; + await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date(Date.now() - 3_600_000), + completionContractId: contractId, ...(shape === "blocked" ? { errorCode: "native_workspace_sync_out_retry_exhausted", + runnerProfileJson: { nativeWorkspaceSync: syncReference } } : {}) }).where(eq(heartbeatRuns.id, runId)); + await db.insert(nativeRunResults).values({ id: resultId, companyId, issueId, runId, completionContractId: contractId, + serverFingerprint: randomUUID(), canonicalSha256: randomUUID(), schemaStatus: "accepted", resultJson: accepted }); + await db.insert(nativeRunFinalizations).values({ companyId, issueId, runId, phase: "retryable_failure", resultId, + attempt: shape === "blocked" ? 1 : 0 }); + await db.insert(workspaceOperations).values({ companyId, issueId, heartbeatRunId: runId, + phase: "workspace_finalize", status: "succeeded" }); + const [task] = await db.select().from(issues).where(eq(issues.id, issueId)); + await db.update(issues).set({ ...buildIssueMonitorTriggeredPatch({ issue: task!, + policy: normalizeIssueExecutionPolicy(task!.executionPolicy), triggeredAt: new Date() }), executionRunId: null, checkoutRunId: null, + ...(shape === "blocked" ? { status: "blocked" } : {}), + }).where(eq(issues.id, issueId)); + return { companyId, agentId, issueId, runId, nextCheckAt }; + } + + const passes: string[] = []; + const counted: typeof finalizeNativeRun = input => { passes.push(input.runId); return finalizeNativeRun(input); }; + async function start() { + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, canReconcile: () => true, + onError: error => errors.push(error) }); + const commits = nativeResultCommitWork(db, counted).register(coordinator, () => true); + const reassessments = nativeStatusReassessmentWork(db).register(coordinator, () => true); + await Promise.all([commits.ready, reassessments.ready]); + } + async function advance(ms: number) { + const until = Date.now() + ms; + while (Date.now() < until) { + await vi.advanceTimersByTimeAsync(Math.min(1_000, until - Date.now())); await delay(1); + while (idleWorkSnapshot().active > 0) await delay(5); + } + } + async function observe(work: () => Promise) { + const statements: string[] = [], signals: string[] = [], prior = db.$client.options.debug; + db.$client.options.debug = (_connection, query) => { statements.push(query); }; + const unsubscribe = Object.values(DELIVERY_QUEUES).map(queue => + subscribeDatabaseWork(db, queue, event => { if (event === "settled") signals.push(queue); })); + const before = passes.length; + try { await work(); } finally { + db.$client.options.debug = prior; + for (const stop of unsubscribe) stop(); + } + return { statements: statements.length, signals: signals.length, passes: passes.length - before }; + } + const coordinatorRow = async (runId: string) => + (await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)))[0]!; + + async function recordLostMonitor(shape: Shape = "in_progress") { + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true }); + const f = await seed(shape); + const first = await observe(async () => { await start(); await advance(60_000); }); + expect(first.passes).toBe(1); + const recorded = await coordinatorRow(f.runId); + expect(recorded).toMatchObject({ phase: "retryable_failure", failureCode: "monitor_wait_authority_lost" }); + const decisions = await db.select().from(statusDecisions).where(eq(statusDecisions.runId, f.runId)); + expect(decisions.map(decision => decision.reasonCode)).toEqual(["monitor_wait_authority_lost"]); + expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId))) + .toMatchObject([{ status: "active", kind: "active_run_watchdog" }]); + return { ...f, recorded }; + } + + it.each(["in_progress", "blocked"])("stays quiet for half an hour once a %s task's lost monitor authority is recorded", async shape => { + const f = await recordLostMonitor(shape); + const runBefore = (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0]!; + const idle = await observe(() => advance(30 * 60_000)); + expect(idle).toEqual({ statements: 0, signals: 0, passes: 0 }); + const runAfter = (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0]!; + expect(runAfter.updatedAt).toEqual(runBefore.updatedAt); + expect(await coordinatorRow(f.runId)).toMatchObject({ phase: "retryable_failure", failureCode: "monitor_wait_authority_lost", + decisionId: f.recorded.decisionId, updatedAt: f.recorded.updatedAt }); + expect(await db.select().from(statusDecisions).where(eq(statusDecisions.runId, f.runId))).toHaveLength(1); + expect((await db.select().from(issues).where(eq(issues.id, f.issueId)))[0]!.status).toBe(shape); + }, 300_000); + + it("ignores repeated hints for a recorded result without finalizing it again", async () => { + const f = await recordLostMonitor(); + const hinted = await observe(async () => { + for (let index = 0; index < 3; index += 1) { + await db.transaction(async tx => { await notifyNativeFinalizationWork(tx, { companyId: f.companyId, runId: f.runId }); }); + await advance(5_000); + } + }); + expect(hinted.passes).toBe(0); + expect(await coordinatorRow(f.runId)).toMatchObject({ updatedAt: f.recorded.updatedAt }); + }, 300_000); + + it("still commits promptly once a failed workspace save is followed by a successful one", async () => { + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true }); + const f = await seed(); + await finalizeNativeRun({ db, runId: f.runId, workspaceFinalizeStatus: "failed", projectRunStatus: true }); + expect(await coordinatorRow(f.runId)).toMatchObject({ phase: "retryable_failure", failureCode: "workspace_finalization_failed" }); + await start(); + await advance(60_000); + expect(passes.filter(runId => runId === f.runId)).toHaveLength(1); + expect(await coordinatorRow(f.runId)).toMatchObject({ phase: "retryable_failure", failureCode: "monitor_wait_authority_lost" }); + const idle = await observe(() => advance(30 * 60_000)); + expect(idle).toEqual({ statements: 0, signals: 0, passes: 0 }); + }, 300_000); + + it("still finalizes a new accepted result for the same run", async () => { + const f = await recordLostMonitor(); + const [result] = await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, f.runId)); + const resultId = randomUUID(); + await db.transaction(async tx => { + await notifyNativeResultCommit(tx, { companyId: f.companyId, runId: f.runId }); + await tx.insert(nativeRunResults).values({ ...result!, id: resultId, serverFingerprint: randomUUID(), canonicalSha256: randomUUID() }); + await tx.update(nativeRunFinalizations).set({ resultId }).where(eq(nativeRunFinalizations.runId, f.runId)); + }); + await advance(5_000); + expect(passes.filter(runId => runId === f.runId)).toHaveLength(2); + expect(await coordinatorRow(f.runId)).toMatchObject({ resultId, failureCode: "monitor_wait_authority_lost" }); + }, 300_000); + + it("settles a recorded result promptly once its monitor wake reaches a successor run", async () => { + const f = await recordLostMonitor(); + await advance(10 * 60_000); + const successorId = randomUUID(); + await db.insert(heartbeatRuns).values({ id: successorId, companyId: f.companyId, agentId: f.agentId, status: "running", + contextSnapshot: { issueId: f.issueId }, createdAt: new Date() }); + await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agentId, source: "automation", + reason: "issue_monitor_due", status: "claimed", runId: successorId, + idempotencyKey: `issue-monitor:${f.issueId}:${f.nextCheckAt}`, payload: { issueId: f.issueId, nextCheckAt: f.nextCheckAt } }); + await db.transaction(async tx => { + await notifyNativeResultCommit(tx, { companyId: f.companyId, issueId: f.issueId }); + await tx.update(issues).set({ executionRunId: successorId, checkoutRunId: successorId, updatedAt: new Date() }) + .where(eq(issues.id, f.issueId)); + }); + await advance(5_000); + expect(passes.filter(runId => runId === f.runId)).toHaveLength(2); + expect(await coordinatorRow(f.runId)).toMatchObject({ phase: "committed" }); + expect((await db.select().from(statusDecisions).where(eq(statusDecisions.runId, f.runId))).map(row => row.reasonCode)) + .toContain("scheduled_monitor_dispatched"); + const idle = await observe(() => advance(30 * 60_000)); + expect(idle).toEqual({ statements: 0, signals: 0, passes: 0 }); + }, 300_000); + + it("reassesses a recorded result once the board changes its task", async () => { + const f = await recordLostMonitor("blocked"); + await advance(10 * 60_000); + await db.transaction(async tx => { + await notifyNativeResultCommit(tx, { companyId: f.companyId, issueId: f.issueId }); + await tx.update(issues).set({ assigneeAgentId: null, updatedAt: new Date() }).where(eq(issues.id, f.issueId)); + }); + await advance(5_000); + expect(passes.filter(runId => runId === f.runId)).toHaveLength(2); + const idle = await observe(() => advance(30 * 60_000)); + expect(idle).toEqual({ statements: 0, signals: 0, passes: 0 }); + }, 300_000); + + it("settles a monitor wake that reaches a successor run during finalization", async () => { + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true }); + const f = await seed(); + const successorId = randomUUID(); + const original = monitorDelivery.readNativeMonitorDelivery; + const race = vi.spyOn(monitorDelivery, "readNativeMonitorDelivery").mockImplementationOnce(async (...args) => { + const before = await original(...args); + await db.insert(heartbeatRuns).values({ id: successorId, companyId: f.companyId, agentId: f.agentId, status: "running", + contextSnapshot: { issueId: f.issueId }, createdAt: new Date(Date.now() - 1_000) }); + await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agentId, source: "automation", + reason: "issue_monitor_due", status: "claimed", runId: successorId, + idempotencyKey: `issue-monitor:${f.issueId}:${f.nextCheckAt}`, payload: { issueId: f.issueId, nextCheckAt: f.nextCheckAt } }); + await db.update(issues).set({ executionRunId: successorId, checkoutRunId: successorId, updatedAt: new Date(Date.now() - 1_000) }) + .where(eq(issues.id, f.issueId)); + return before; + }); + try { + await start(); + await advance(60_000); + } finally { race.mockRestore(); } + expect(await coordinatorRow(f.runId)).toMatchObject({ phase: "committed" }); + expect((await db.select().from(statusDecisions).where(eq(statusDecisions.runId, f.runId))).map(row => row.reasonCode)) + .toEqual(["scheduled_monitor_dispatched"]); + const idle = await observe(() => advance(30 * 60_000)); + expect(idle).toEqual({ statements: 0, signals: 0, passes: 0 }); + }, 300_000); +}); diff --git a/server/src/services/native-runtime/native-result-commit-work.ts b/server/src/services/native-runtime/native-result-commit-work.ts index 0a221e1c53e..a39de93647c 100644 --- a/server/src/services/native-runtime/native-result-commit-work.ts +++ b/server/src/services/native-runtime/native-result-commit-work.ts @@ -1,5 +1,6 @@ -import { and, asc, eq, exists, gt, not, notInArray } from "drizzle-orm"; -import { heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults, workspaceOperations, type Db } from "@paperclipai/db"; +import { and, asc, eq, exists, gt, not, notInArray, sql, type SQL } from "drizzle-orm"; +import { heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults, statusDecisions, workAssessments, workspaceOperations, + type Db } from "@paperclipai/db"; import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js"; import { DELIVERY_QUEUES } from "../delivery-work-notifications.js"; import type { NativeResultCommitScope } from "../native-result-commit-notifications.js"; @@ -11,7 +12,25 @@ import { readNativeWorkspaceSyncReference } from "./native-workspace-sync.js"; type Entry = { id: string; companyId: string; issueId: string; dueAt: number }; - +function recordedFinalizationErrorCondition(db: Db): SQL { + return and(eq(nativeRunFinalizations.phase, "retryable_failure"), + sql`${nativeRunFinalizations.failureCode} is distinct from 'workspace_finalization_failed'`, + exists(db.select({ id: statusDecisions.id }).from(statusDecisions) + .innerJoin(workAssessments, and(eq(workAssessments.id, statusDecisions.assessmentId), + eq(workAssessments.companyId, statusDecisions.companyId), eq(workAssessments.issueId, statusDecisions.issueId))) + .where(and(eq(statusDecisions.id, nativeRunFinalizations.decisionId), + eq(statusDecisions.companyId, nativeRunFinalizations.companyId), eq(statusDecisions.issueId, nativeRunFinalizations.issueId), + eq(statusDecisions.assessmentId, nativeRunFinalizations.assessmentId), + eq(workAssessments.runId, nativeRunFinalizations.runId), eq(workAssessments.resultId, nativeRunFinalizations.resultId), + sql`${statusDecisions.decisionJson}->'effects' @> jsonb_build_array(jsonb_build_object( + 'kind', 'record_finalization_error', 'cause', ${nativeRunFinalizations.failureCode}::text))`))), + sql`${issues.updatedAt} <= ${nativeRunFinalizations.updatedAt}`, + sql`not exists (select 1 from heartbeat_runs successor + where successor.company_id = ${nativeRunFinalizations.companyId} and successor.id <> ${nativeRunFinalizations.runId} + and (successor.context_snapshot->>'issueId' = ${nativeRunFinalizations.issueId}::text + or successor.issue_id = ${nativeRunFinalizations.issueId}) + and successor.created_at > ${nativeRunFinalizations.updatedAt})`)!; +} export function nativeResultCommitWork(db: Db, finalize = finalizeNativeRun) { function candidates(scope?: NativeResultCommitScope, after?: string) { @@ -28,6 +47,7 @@ export function nativeResultCommitWork(db: Db, finalize = finalizeNativeRun) { .where(and(eq(heartbeatRuns.runtimeMode, "native"), nativeRunnerOwnershipNotHeldCondition(), notInArray(nativeRunFinalizations.phase, ["committed", "terminal_failure"]), not(pendingNativeStatusEffectsCondition(db)), + not(recordedFinalizationErrorCondition(db)), exists(db.select({ id: workspaceOperations.id }).from(workspaceOperations).where(and( eq(workspaceOperations.companyId, nativeRunFinalizations.companyId), eq(workspaceOperations.issueId, nativeRunFinalizations.issueId), eq(workspaceOperations.heartbeatRunId, nativeRunFinalizations.runId), diff --git a/server/src/services/native-runtime/status-decision-committer.ts b/server/src/services/native-runtime/status-decision-committer.ts index 999a52c1d13..37877b261d7 100644 --- a/server/src/services/native-runtime/status-decision-committer.ts +++ b/server/src/services/native-runtime/status-decision-committer.ts @@ -1918,6 +1918,10 @@ export async function commitNativeStatusDecision(input: { input.decision.statusAction !== "preserve" || input.decision.toStatus !== issue.status || input.decision.effects.length !== 0) throw new NativeStatusRaceError(); } + if (reasonCode === "monitor_wait_authority_lost" && issue.executionRunId !== input.runId && + issue.checkoutRunId !== input.runId && await readNativeMonitorDelivery(tx as unknown as Db, input)) { + throw new NativeStatusRaceError(); + } if (reasonCode === "scheduled_monitor_superseded") { const expected = input.requireSupersededMonitorWait; let actual: Awaited> = null;