diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 22c12055bbf..e3f8e6a1e48 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -802,6 +802,8 @@ Normal issue-thread feedback can still wake the current assignee according to th Restart recovery revisits accepted deferred messages through the latest issue run, including historical mention queues for another agent. Ordinary messages for a former owner retain their assignment handoff. Native sources require their matching committed finalization, settled environment leases, and released finalizer and copyback ownership before recovery clears old issue claims. A successful retained warm release can satisfy the lease check. Released local bookkeeping without a provider allocation and a matching SSH bookkeeping release need no provider cleanup receipt. Plugin releases record their successful acknowledgement. Pending or failed cleanup and durable owners still hold work. A successful fenced cleanup completion clears its in-flight claim in the same database write, so a restart before retry bookkeeping cannot strand saved input. The normal release checks preserve cleanup holds, pauses, and operator Stops. Recovery checks for a competing turn under the issue lock before it promotes saved input. +A user can send messages after a run's final turn took its input but before the run settles. A native run can finish its result and status decision before its environment cleanup completes. The messages remain saved while that cleanup holds the source. Releasing or retaining the run's environment lease, or releasing its workspace finalization owner, reconsiders the saved messages through this recovery path. They are delivered once. A task that finalization marked `done` is reopened for them. The release and admission gates above still apply. + Repeated eligibility notifications must not move an unchecked, ready deferred receipt behind receipts already reconsidered. Paginated reconstruction preserves pending ready receipts' dispatch priority while refreshing their current state. Receipts whose wait deadline has passed join the ready queue on reconstruction. Future wait grace periods, claimed-work deadlines, infrastructure retry backoffs, and current admission checks remain effective. A paused recipient does not fail an accepted deferred request. Release and periodic recovery preserve its pending receipt and may process an eligible request behind it. Resuming the agent allows normal admission to reconsider the saved input. Missing, foreign, terminated, and unapproved recipients retain their rejection rules. Explicit task holds and operator Stops still prevent unauthorized execution. This does not replay historical failed requests. diff --git a/server/src/__tests__/deferred-execution-receipt-work.test.ts b/server/src/__tests__/deferred-execution-receipt-work.test.ts index 1130f3301f8..e9f51ba3986 100644 --- a/server/src/__tests__/deferred-execution-receipt-work.test.ts +++ b/server/src/__tests__/deferred-execution-receipt-work.test.ts @@ -15,7 +15,7 @@ import { issueTreeControlService } from "../services/issue-tree-control.js"; import { randomUUID } from "node:crypto"; import { and, eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; -import { agents, agentWakeupRequests, authUsers, chatActions, chatEndpoints, toolApplications, toolConnections, companies, companyMemberships, costEvents, createDb, projects, reconcileDatabaseWork, heartbeatRuns, issueComments, issues } from "@paperclipai/db"; +import { activityLog, agents, agentWakeupRequests, authUsers, chatActions, environmentLeases, nativeRunFinalizations, chatEndpoints, toolApplications, toolConnections, companies, companyMemberships, costEvents, createDb, projects, reconcileDatabaseWork, heartbeatRuns, issueComments, issues } from "@paperclipai/db"; import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; @@ -24,6 +24,7 @@ import { registerDeferredReceiptWork } from "../services/heartbeat/deferred-rece import { heartbeatService } from "../services/heartbeat.js"; import { requestTaskRetry } from "../services/task-retry.js"; import { issueService } from "../services/issues.js"; +import { environmentService } from "../services/environments.js"; import { idleWorkSnapshot } from "../services/task-admission.js"; vi.setConfig({ testTimeout: 90_000 }); @@ -1014,4 +1015,61 @@ describe("deferred receipt durable dispatch", () => { }); await quiet(); expect(dispatchNative).not.toHaveBeenCalled(); }); + it("delivers messages saved behind a finishing native run once, when its environment cleanup settles", async () => { + // Remote native finalization commits Done and projects the run terminal + // without the in-process release. The provider environment then settles + // (harness backup, warm retention). Messages the user sent after the final + // turn took its input wait on that cleanup and must then run exactly once. + const f = await fixture(); + await db.update(issues).set({ status: "done", completedAt: new Date() }).where(eq(issues.id, f.issue.id)); + await db.update(heartbeatRuns).set({ status: "succeeded", errorCode: null, resultJson: {}, runtimeMode: "native", + nativeIssueId: f.issue.id, finishedAt: new Date() }).where(eq(heartbeatRuns.id, f.source.id)); + await db.insert(nativeRunFinalizations).values({ companyId: f.companyId, issueId: f.issue.id, runId: f.source.id, phase: "committed" }); + const [lease] = await db.insert(environmentLeases).values({ companyId: f.companyId, heartbeatRunId: f.source.id, + issueId: f.issue.id, provider: "daytona", providerLeaseId: "sandbox-finalizing", status: "active", + leasePolicy: "reuse_by_environment" }).returning(); + const comments = []; + for (const body of ["Also mention saved searches.", "Also mention it works on mobile."]) { + comments.push((await db.insert(issueComments).values({ companyId: f.companyId, issueId: f.issue.id, authorType: "user", + authorUserId: f.actorId, body }).returning())[0]); + } + const commentIds = comments.map(comment => comment.id); + const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id, + source: "automation", triggerDetail: "system", reason: "issue_execution_deferred", status: "deferred_issue_execution", + requestedByActorType: "user", requestedByActorId: f.actorId, + payload: { issueId: f.issue.id, commentId: commentIds[1], mutation: "comment", _paperclipWakeContext: { + issueId: f.issue.id, taskId: f.issue.id, commentId: commentIds[1], wakeCommentId: commentIds[1], + wakeCommentIds: commentIds, wakeReason: "issue_commented", source: "issue.comment" } } }).returning(); + const readWake = async () => (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id)))[0]; + const issueRuns = () => db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${f.issue.id}`); + + // The receipt is checked while the source environment is still active. + await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready; + await waitForReal(async () => { + await Promise.allSettled([...passes]); + expect(coordinator.nextWakeAt()).toBeNull(); expect(idleWorkSnapshot().active).toBe(0); + }, { timeout: 10_000 }); + expect(await readWake()).toMatchObject({ status: "deferred_issue_execution", runId: null }); + expect(await issueRuns()).toHaveLength(1); + + await environmentService(db).releaseLease(lease.id, "retained", { cleanupStatus: "success" }); + await waitForReal(async () => { + await Promise.allSettled([...passes]); + expect((await readWake()).runId).toBeTruthy(); + }, { timeout: 10_000 }); + // Exactly one turn carries the saved messages, in order, under the + // original user's authority, and the completed task is reopened for it. + const commentRuns = async () => (await issueRuns()).filter(run => run.contextSnapshot?.wakeReason === "issue_commented"); + const [successor, ...duplicates] = await commentRuns(); + expect(duplicates).toEqual([]); + expect(successor.contextSnapshot?.wakeCommentIds).toEqual(commentIds); + expect((await readWake()).runId).toBe(successor.id); + expect(await db.select().from(activityLog).where(and(eq(activityLog.entityId, f.issue.id), + sql`${activityLog.details}->>'reopened' = 'true'`))).toHaveLength(1); + await finish(f.issue.id); + // A later settlement signal cannot deliver the same messages again. + await environmentService(db).releaseLease(lease.id, "retained", { cleanupStatus: "success" }); + await quiet(); + expect(await commentRuns()).toHaveLength(1); + }); }); diff --git a/server/src/services/environments.ts b/server/src/services/environments.ts index ded4efccdc9..785668c8d04 100644 --- a/server/src/services/environments.ts +++ b/server/src/services/environments.ts @@ -4,6 +4,7 @@ import { notifyNativeEnvironmentSettlement } from "./native-environment-settleme import { notifyRetainedNativeSessionCleanup } from "./retained-native-session-cleanup-notifications.js"; import { notifyChatInboundWakeNotices } from "./chat-inbound-wake-notice-notifications.js"; import { notifyEnvironmentCleanup } from "./environment-cleanup-notifications.js"; +import { notifyDeferredExecutionWork } from "./deferred-execution-notifications.js"; import { withoutCleanupVerification } from "./cleanup-verification-proof.js"; import { randomUUID } from "node:crypto"; @@ -1729,6 +1730,9 @@ export function environmentService(db: Db) { await notifyRetainedNativeSessionCleanup(tx, { companyId: previous.companyId, runId: previous.heartbeatRunId, }); + // Messages saved behind a finished run wait for its cleanup to + // settle. Reconsider them through normal deferred admission. + await notifyDeferredExecutionWork(tx, { companyId: previous.companyId }); } if (previous.creationOperationId && previous.status === "pending_cleanup" && options?.cleanupStatus === "success" && status !== "pending_cleanup") { await tx.update(sandboxCreationOperations).set({ state: "resolved", updatedAt: now }) @@ -1868,6 +1872,7 @@ export function environmentService(db: Db) { if (run) { await notifyEnvironmentCleanup(tx, { companyId: run.companyId, runId: heartbeatRunId }); await notifyRetainedNativeSessionCleanup(tx, { companyId: run.companyId, runId: heartbeatRunId }); + await notifyDeferredExecutionWork(tx, { companyId: run.companyId }); } return tx.update(environmentLeases) .set({ diff --git a/server/src/services/native-runtime/native-workspace-finalization-ownership.ts b/server/src/services/native-runtime/native-workspace-finalization-ownership.ts index 62415c69f3e..3dd2316b76b 100644 --- a/server/src/services/native-runtime/native-workspace-finalization-ownership.ts +++ b/server/src/services/native-runtime/native-workspace-finalization-ownership.ts @@ -2,6 +2,7 @@ import { notifyNativeResultCommit } from "../native-result-commit-notifications. import { notifyNativeWorkspaceFinalization, releaseNativeWorkspaceFinalization } from "../native-workspace-finalization-notifications.js"; import { notifyNativeStatusReassessment } from "../native-status-reassessment-notifications.js"; import { notifyQueuedRunDispatch } from "../run-dispatch-notifications.js"; +import { notifyDeferredExecutionWork } from "../deferred-execution-notifications.js"; import { beginRunPhase } from "../execution-preparation-timing.js"; import { logActivity } from "../activity-log.js"; import { issueRecoveryActionService } from "../issue-recovery-actions.js"; @@ -304,6 +305,8 @@ export async function withNativeWorkspaceFinalizationOwnership( await notifyNativeWorkspaceFinalization(tx, { companyId: input.companyId, runId: input.runId }); await notifyNativeStatusReassessment(tx, { companyId: input.companyId, runId: input.runId }); await notifyQueuedRunDispatch(tx, { companyId: input.companyId }); + // Saved messages also wait on this owner after the run finished. + await notifyDeferredExecutionWork(tx, { companyId: input.companyId }); await tx.update(heartbeatRuns).set({ runnerProfileJson: sql`coalesce(${heartbeatRuns.runnerProfileJson}, '{}'::jsonb) - ${OWNER_KEY}`, }).where(owned);