diff --git a/server/src/__tests__/agent-instruction-copy-retry-backoff.test.ts b/server/src/__tests__/agent-instruction-copy-retry-backoff.test.ts index ea54b58b6bb..72384a54d0a 100644 --- a/server/src/__tests__/agent-instruction-copy-retry-backoff.test.ts +++ b/server/src/__tests__/agent-instruction-copy-retry-backoff.test.ts @@ -4,7 +4,8 @@ import path from "node:path"; import { createHash, randomUUID } from "node:crypto"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { eq } from "drizzle-orm"; -import { agents, agentInstructionWorkingCopies, authUsers, companies, createDb, heartbeatRuns } from "@paperclipai/db"; +import { spawnSync } from "node:child_process"; +import { agents, agentInstructionWorkingCopies, authUsers, companies, createDb, environmentLeases, heartbeatRuns, subscribeDatabaseWork } from "@paperclipai/db"; import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { meterDatabaseStatements } from "./helpers/database-statement-meter.js"; import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; @@ -17,6 +18,10 @@ import { INSTRUCTION_COPY_RETRY_MS, } from "../services/agent-instruction-working-copies.js"; import { appendHeartbeatRunEvent } from "../services/heartbeat-run-events.js"; +import { recordNativeLocalProcessStopInTransaction } from "../services/native-local-process-stop.js"; +import { notifyInstructionCopyStopEvidence } from "../services/instruction-collection-notifications.js"; +import { environmentService } from "../services/environments.js"; +import { remoteTerminationReceipt } from "../services/remote-execution-termination.js"; import { idleWorkSnapshot } from "../services/task-admission.js"; const external = resolveExternalTestDatabaseUrl("PAPERCLIP_INSTRUCTION_COPY_TEST_DATABASE_URL"); @@ -237,4 +242,79 @@ describe("instruction working copies that cannot make progress", () => { expect(row.nextAttemptAt!.getTime()).toBeGreaterThan(Date.now()); } finally { await worker.stop(); } }, 120_000); + it("collects a copy on the next pass once its local stop is recorded, long after its run finished", async () => { + const [runId] = await unprovenStops(1, 30 * DAY); + clock(); const worker = await start("collection"); + try { + expect((await copy(runId!)).nextAttemptAt!.getTime() - Date.now()).toBeGreaterThan(5 * HOUR); + const deadPid = spawnSync(process.execPath, ["-e", ""]).pid!; + const [run] = await db.update(heartbeatRuns).set({ processPid: deadPid }).where(eq(heartbeatRuns.id, runId!)).returning(); + expect(await db.transaction(tx => recordNativeLocalProcessStopInTransaction(tx as unknown as typeof db, run!))).toBe(true); + await advance(10_000, () => Date.now() + 1000); + expect(await copy(runId!)).toMatchObject({ state: "unchanged", nextAttemptAt: null }); + } finally { await worker.stop(); } + }, 120_000); + + it("collects a remote copy on the next pass once its lease records the termination receipt", async () => { + const [runId] = await runs(1, 30 * DAY); + const environmentId = randomUUID(); + await db.insert(agentInstructionWorkingCopies).values({ runId: runId!, companyId, agentId, responsibleUserId: userId, entryFile: "AGENTS.md", + baseHash: CONTENT_HASH, localRoot: path.dirname(entryPath(runId!)), executionRoot: "/workspace/instructions", + location: `remote:${environmentId}`, state: "prepared" }); + const [lease] = await db.insert(environmentLeases).values({ companyId, heartbeatRunId: runId!, provider: "fake-sandbox", + providerLeaseId: `sandbox-${runId}`, status: "active" }).returning(); + clock(); const worker = await start("collection"); + try { + expect(await copy(runId!)).toMatchObject({ state: "pending_collection", errorCode: "INSTRUCTION_STOP_UNCONFIRMED" }); + expect((await copy(runId!)).nextAttemptAt!.getTime() - Date.now()).toBeGreaterThan(5 * HOUR); + await environmentService(db).releaseLease(lease!.id, "released", { cleanupStatus: "success", + remoteExecutionTermination: remoteTerminationReceipt(lease!, { providerLeaseId: lease!.providerLeaseId, state: "stopped" }) }); + await advance(10_000, () => Date.now() + 1000); + expect(await copy(runId!)).toMatchObject({ state: "unavailable", errorCode: "INSTRUCTION_COLLECTION_UNAVAILABLE", nextAttemptAt: null }); + } finally { await worker.stop(); } + }, 120_000); + + it("sends a run-scoped hint only when a waiting copy is brought forward, and writes nothing otherwise", async () => { + const [waiting] = await unprovenStops(1, 30 * DAY); + const [none] = await runs(1, DAY); + const due = new Date(Date.now() + 6 * HOUR); + await db.update(agentInstructionWorkingCopies).set({ state: "pending_collection", nextAttemptAt: due }).where(eq(agentInstructionWorkingCopies.runId, waiting!)); + const hints: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.instructionCollection, (event, scoped) => { + if (event === "settled") hints.push(...(scoped ?? ["*"])); + }); + try { + meter.startLog(); + await db.transaction(tx => notifyInstructionCopyStopEvidence(tx, { companyId, runId: none! })); + const quiet = meter.stopLog(); + expect(quiet.filter(entry => !/^(begin|commit)/i.test(entry.sql.trim())).map(entry => entry.sql.trim().split(/\s/)[0])).toEqual(["select"]); + expect(hints).toEqual([]); + await db.transaction(tx => notifyInstructionCopyStopEvidence(tx, { companyId, runId: waiting! })); + await vi.waitFor(() => expect(hints).toEqual([JSON.stringify({ companyId, runId: waiting })])); + expect((await copy(waiting!)).nextAttemptAt!.getTime()).toBeLessThanOrEqual(Date.now()); + } finally { unsubscribe(); } + }, 60_000); + it("collects a copy whose stop is recorded between the worker's evidence read and its backoff write", async () => { + const [runId] = await unprovenStops(1, 30 * DAY); + await db.update(agentInstructionWorkingCopies).set({ state: "pending_collection", nextAttemptAt: new Date(Date.now() - 1000) }) + .where(eq(agentInstructionWorkingCopies.runId, runId!)); + const deadPid = spawnSync(process.execPath, ["-e", ""]).pid!; + const [run] = await db.update(heartbeatRuns).set({ processPid: deadPid }).where(eq(heartbeatRuns.id, runId!)).returning(); + const transaction = db.transaction.bind(db); + let recorded = false; + const transactions = vi.spyOn(db, "transaction").mockImplementation((async (callback, options) => { + if (!recorded) { + recorded = true; + expect(await transaction(tx => recordNativeLocalProcessStopInTransaction(tx as unknown as typeof db, run!))).toBe(true); + } + return transaction(callback, options); + }) as typeof db.transaction); + clock(); const worker = await start("collection"); + try { + transactions.mockRestore(); + expect(recorded).toBe(true); + await advance(10_000, () => Date.now() + 1000); + expect(await copy(runId!)).toMatchObject({ state: "unchanged", nextAttemptAt: null }); + } finally { transactions.mockRestore(); await worker.stop(); } + }, 120_000); }); diff --git a/server/src/services/agent-instruction-working-copies.ts b/server/src/services/agent-instruction-working-copies.ts index 7f2d98b98b6..f588c06284f 100644 --- a/server/src/services/agent-instruction-working-copies.ts +++ b/server/src/services/agent-instruction-working-copies.ts @@ -483,13 +483,14 @@ export function agentInstructionWorkingCopyService(db: Db, options: { environmen : { nextAttemptAt: evidence === "directory_unavailable" ? retryAt : new Date(Date.now() + INSTRUCTION_COPY_RETRY_MS) }, true, sql`EXISTS (SELECT 1 FROM ${heartbeatRuns} WHERE ${heartbeatRuns.id} = ${copies.runId} AND ${heartbeatRuns.companyId} = ${copies.companyId} - AND ${heartbeatRuns.status} IN ('succeeded', 'failed', 'cancelled', 'timed_out', 'interrupted'))`); + AND ${heartbeatRuns.status} IN ('succeeded', 'failed', 'cancelled', 'timed_out', 'interrupted')) + AND date_trunc('milliseconds', ${copies.updatedAt}) = ${current.copy.updatedAt.toISOString()}::timestamptz`); const settled = claimed && evidence !== null ? await collectRecovered(claimed, evidence, retryAt) : claimed; const remaining = settled === undefined || settled === null ? (await readCollection(entry))?.copy : collectionCandidate(settled) ? settled : null; if (!remaining) return null; const next = { ...entry, dueAt: remaining.nextAttemptAt?.getTime() ?? 0 }; - if (next.dueAt <= Date.now()) throw new Error("Instruction collection remains unresolved without a future retry"); + if (next.dueAt <= Date.now() && claimed) throw new Error("Instruction collection remains unresolved without a future retry"); return next; } async function recoverStopped() { diff --git a/server/src/services/environment-cleanup-notifications.ts b/server/src/services/environment-cleanup-notifications.ts index 6beca498981..06b634454a6 100644 --- a/server/src/services/environment-cleanup-notifications.ts +++ b/server/src/services/environment-cleanup-notifications.ts @@ -5,6 +5,7 @@ import { notifyNativeEnvironmentSettlement, type NativeEnvironmentSettlementScop import type { Db } from "@paperclipai/db"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyQueuedRunDispatch } from "./run-dispatch-notifications.js"; +import { notifyInstructionCopyStopEvidence } from "./instruction-collection-notifications.js"; export type EnvironmentCleanupWorkScope = NativeEnvironmentSettlementScope & { leaseId?: string }; @@ -32,6 +33,7 @@ export async function notifyEnvironmentCleanup(transaction: object, scope?: Envi await notifyQueuedRunDispatch(transaction, scope); // A settled run lease can make that run's abandoned native owner releasable. if (scope?.runId) { + await notifyInstructionCopyStopEvidence(transaction, { companyId: scope.companyId, runId: scope.runId }); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.executionFinalization); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.executionReplacement); } diff --git a/server/src/services/instruction-collection-notifications.ts b/server/src/services/instruction-collection-notifications.ts index d0c19355c00..74c607f8359 100644 --- a/server/src/services/instruction-collection-notifications.ts +++ b/server/src/services/instruction-collection-notifications.ts @@ -1,4 +1,7 @@ +import { sql } from "drizzle-orm"; +import type { Db } from "@paperclipai/db"; import { notifySkillVersionRetention } from "./skill-version-retention-notifications.js"; +import { notifyDirectoryCleanup } from "./directory-cleanup-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; export type InstructionCollectionScope = { companyId: string; runId?: string }; @@ -7,3 +10,23 @@ export async function notifyInstructionCollection(transaction: object, scope?: I if (scope) await notifySkillVersionRetention(transaction, scope.companyId); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.instructionCollection, scope ? JSON.stringify(scope) : "*"); } + +const waitingCopy = (scope: { companyId: string; runId: string }) => sql`company_id = ${scope.companyId} and run_id = ${scope.runId} and ( + (not (coalesce(receipt, '{}'::jsonb) ? 'retainedByRunId') and ( + (attempts < 3 and state in ('prepared', 'pending_collection', 'unchanged_turn', 'warm_saved')) + or (state = 'unavailable' and process_stopped_at is null and location like 'remote:%' + and receipt->>'schema' = 'paperclip.agent-files.v1'))) + or (state in ('saved', 'unchanged', 'resolved', 'unavailable') and process_stopped_at is not null + and receipt->>'schema' = 'paperclip.agent-files.v1' and coalesce(receipt->>'retainedByRunId', '') = '' + and (receipt->>'cleanupPending' = 'true' or (receipt->>'cleanupPending' is distinct from 'false' and receipt ? 'baseline'))))`; + +export async function notifyInstructionCopyStopEvidence(transaction: object, scope: { companyId: string; runId: string }) { + if (typeof (transaction as Db).execute !== "function") return; + const found = await (transaction as Db).execute(sql`select 1 from agent_instruction_working_copies where ${waitingCopy(scope)}`); + if (!Array.isArray(found) || found.length === 0) return; + await notifyInstructionCollection(transaction, scope); + await notifyDirectoryCleanup(transaction, scope); + await (transaction as Db).execute(sql`update agent_instruction_working_copies + set next_attempt_at = least(coalesce(next_attempt_at, now()), now()), updated_at = clock_timestamp() + where ${waitingCopy(scope)}`); +} diff --git a/server/src/services/native-local-process-stop.ts b/server/src/services/native-local-process-stop.ts index 7c7a5e39546..8b1b7c867bc 100644 --- a/server/src/services/native-local-process-stop.ts +++ b/server/src/services/native-local-process-stop.ts @@ -1,4 +1,5 @@ import { notifyLiveRunRecovery } from "./live-run-recovery-notifications.js"; +import { notifyInstructionCopyStopEvidence } from "./instruction-collection-notifications.js"; import { and, desc, eq, gt, inArray, isNotNull, isNull, or, sql } from "drizzle-orm"; import { environmentLeases, heartbeatRunEvents, heartbeatRuns, type Db } from "@paperclipai/db"; import { notifyRetainedNativeSessionCleanup } from "./retained-native-session-cleanup-notifications.js"; @@ -87,6 +88,7 @@ async function appendLocalProcessStop(db: Db, run: { id: string; companyId: stri message: "Recovery verified the local process stopped before clearing its identity.", payload: { processPid: run.processPid, processGroupId: run.processGroupId, processStartedAt: run.processStartedAt }, }); + await notifyInstructionCopyStopEvidence(db, { companyId: run.companyId, runId: run.id }); } /** Transaction form for callers that clear the process identity: a stopped @@ -151,6 +153,7 @@ export async function recordUnidentifiedLocalRunnerStop(db: Db, run: Pick { await notifyRetainedNativeSessionCleanup(tx, { companyId: lease.companyId, runId: lease.heartbeatRunId! }); + await notifyInstructionCopyStopEvidence(tx, { companyId: lease.companyId, runId: lease.heartbeatRunId! }); return tx.update(environmentLeases).set({ status: "released", cleanupStatus: "success", failureReason: "workspace_source_unavailable", releasedAt: now, updatedAt: now, metadata: sql`(${environmentLeases.metadata} - 'sandboxStopAndRetain' - 'nativeWorkspaceExportResume' - 'sandboxStopAndRetainReceipt') || ${JSON.stringify({