diff --git a/server/src/__tests__/agent-instruction-copy-retry-backoff.test.ts b/server/src/__tests__/agent-instruction-copy-retry-backoff.test.ts new file mode 100644 index 00000000000..ea54b58b6bb --- /dev/null +++ b/server/src/__tests__/agent-instruction-copy-retry-backoff.test.ts @@ -0,0 +1,240 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { createHash, randomUUID } from "node:crypto"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { eq } from "drizzle-orm"; +import { agents, agentInstructionWorkingCopies, authUsers, companies, createDb, heartbeatRuns } from "@paperclipai/db"; +import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { meterDatabaseStatements } from "./helpers/database-statement-meter.js"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { DELIVERY_QUEUES, subscribeDeliveryWork } from "../services/delivery-work-notifications.js"; +import { + agentInstructionWorkingCopyService, + instructionCopyRetryAt, + INSTRUCTION_COPY_MAX_RETRY_MS, + INSTRUCTION_COPY_RETRY_MS, +} from "../services/agent-instruction-working-copies.js"; +import { appendHeartbeatRunEvent } from "../services/heartbeat-run-events.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; + +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_INSTRUCTION_COPY_TEST_DATABASE_URL"); +const AGENT_FILES = "paperclip.agent-files.v1"; +const MINUTE = 60_000, HOUR = 60 * MINUTE, DAY = 24 * HOUR; +const CONTENT = "# Instructions\n"; +const CONTENT_HASH = createHash("sha256").update(CONTENT).digest("hex"); + +describe("instruction working copies that cannot make progress", () => { + let database: Awaited>; + let db: ReturnType; + let meter: ReturnType; + let home: string; + let companyId: string, agentId: string, userId: string; + const previousHome = process.env.PAPERCLIP_HOME; + + beforeAll(async () => { + home = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "instruction-copy-backoff-"))); + process.env.PAPERCLIP_HOME = home; + if (!external) database = await startEmbeddedPostgresTestDatabase("instruction-copy-backoff-db-"); + const url = external ?? database.connectionString; + db = createDb(url, { idleTimeoutSeconds: 0, maxLifetimeSeconds: 20 * 24 * 3600, connectTimeoutSeconds: 3600 }); + meter = meterDatabaseStatements(db, url); + }, 90_000); + afterAll(async () => { + meter?.uninstall(); + if (previousHome === undefined) delete process.env.PAPERCLIP_HOME; else process.env.PAPERCLIP_HOME = previousHome; + await db?.$client.end({ timeout: 1 }); + await database?.cleanup(); + if (home) await fs.rm(home, { recursive: true, force: true }); + }); + beforeEach(async () => { + companyId = randomUUID(); agentId = randomUUID(); userId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Backoff", issuePrefix: randomUUID().slice(0, 8) }); + await db.insert(authUsers).values({ id: userId, name: "Editor", email: `${userId}@example.test`, createdAt: new Date(), updatedAt: new Date() }); + await db.insert(agents).values({ id: agentId, companyId, name: "Target", adapterConfig: { instructionsBundleMode: "managed" } }); + }); + afterEach(async () => { + vi.useRealTimers(); vi.restoreAllMocks(); + await db.delete(agentInstructionWorkingCopies).where(eq(agentInstructionWorkingCopies.companyId, companyId)); + }); + + const clock = () => vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true }); + const coordinator = () => createDeliveryWorkCoordinator({ owner: db, canRun: () => true, canReconcile: () => true, onError: () => {} }); + + async function runs(count: number, finishedAgo: number, runtimeMode = "native") { + const ids = Array.from({ length: count }, () => randomUUID()); + const finishedAt = new Date(Date.now() - finishedAgo); + await db.insert(heartbeatRuns).values(ids.map(id => ({ id, companyId, agentId, status: "failed", runtimeMode, + invocationSource: "on_demand", responsibleUserId: userId, startedAt: finishedAt, finishedAt }))); + return ids; + } + const entryPath = (runId: string) => path.join(home, "work", runId, ".paperclip-runtime", `instruction-edits-${runId}`, "instructions", "AGENTS.md"); + async function unprovenStops(count: number, finishedAgo: number) { + const ids = await runs(count, finishedAgo); + await db.insert(agentInstructionWorkingCopies).values(ids.map(runId => ({ runId, companyId, agentId, responsibleUserId: userId, + entryFile: "AGENTS.md", baseHash: CONTENT_HASH, localRoot: path.dirname(entryPath(runId)), executionRoot: path.dirname(entryPath(runId)), + location: "local", state: "prepared" }))); + for (const runId of ids) { + await fs.mkdir(path.dirname(entryPath(runId)), { recursive: true }); + await fs.writeFile(entryPath(runId), CONTENT); + } + return ids; + } + async function stuckCleanups(count: number, options: { lockRoot?: boolean } = {}) { + const ids = await runs(count, 0, "legacy"); + if (options.lockRoot !== false) for (const runId of ids) await fs.mkdir(path.join(home, "agents", agentId, "file-sync", "runs", runId, "live"), { recursive: true }); + await db.insert(agentInstructionWorkingCopies).values(ids.map(runId => { + const localRoot = path.join(home, "agents", agentId, "file-sync", "runs", runId, "live"); + return { runId, companyId, agentId, responsibleUserId: userId, entryFile: "AGENTS.md", baseHash: CONTENT_HASH, + localRoot, executionRoot: `/workspace/.paperclip-runtime/agent-files/${agentId}/${runId}`, + location: `remote:${randomUUID()}`, state: "saved", processStoppedAt: new Date(), receipt: { schema: AGENT_FILES, state: "saved", + cleanupPending: true, cleanup: { leaseId: randomUUID(), remoteCwd: "/workspace" }, materializationRunId: runId } }; + })); + return ids; + } + async function copy(runId: string) { + const [row] = await db.select().from(agentInstructionWorkingCopies).where(eq(agentInstructionWorkingCopies.runId, runId)); + return row!; + } + async function settle() { + let last = -1; + for (let quiet = 0, turns = 0; quiet < 40 && turns < 20_000; turns++) { + await new Promise(resolve => setImmediate(resolve)); + const current = meter.read().statements; + quiet = current === last && idleWorkSnapshot().active === 0 ? quiet + 1 : 0; + last = current; + } + } + async function advance(ms: number, wakeAt: () => number | null) { + const before = meter.read(); + meter.startLog(); + const end = Date.now() + ms; + while (Date.now() < end) { + const next = wakeAt(); + const quietUntil = next === null || next > end ? end : Math.max(Date.now(), next - 1000); + if (quietUntil > Date.now()) await vi.advanceTimersByTimeAsync(quietUntil - Date.now()); + if (Date.now() >= end) break; + for (let step = 0; step < 3; step++) { await vi.advanceTimersByTimeAsync(1000); await settle(); } + } + await settle(); + const log = meter.stopLog(); + const after = meter.read(); + return { statements: after.statements - before.statements, transactions: after.transactions - before.transactions, + workSignals: log.filter(entry => entry.sql.includes("pg_current_xact_id")).length, + stopChecks: log.filter(entry => entry.sql.includes(`from "heartbeat_run_events"`)).length, + cleanupClaims: log.filter(entry => /^update "agent_instruction_working_copies" set "next_attempt_at" = \$1, "updated_at" = \$2 where/.test(entry.sql)).length }; + } + async function start(...queues: Array<"collection" | "cleanup">) { + const worker = coordinator(); + const service = agentInstructionWorkingCopyService(db); + if (queues.includes("collection")) await service.registerStoppedCollectionWork(worker, () => true, async () => true).ready; + if (queues.includes("cleanup")) await service.registerDirectoryCleanupWork(worker, () => true, async () => true).ready; + await settle(); + return worker; + } + + it("names a backoff that grows with the wait and stops growing at its cap", () => { + const now = Date.parse("2026-11-14T00:00:00Z"); + expect(INSTRUCTION_COPY_RETRY_MS).toBe(30_000); + expect(INSTRUCTION_COPY_MAX_RETRY_MS).toBe(6 * HOUR); + const delay = (waited: number) => instructionCopyRetryAt(new Date(now - waited), now).getTime() - now; + expect([0, 30 * MINUTE, 2 * HOUR, 12 * HOUR, 30 * DAY].map(delay)).toEqual([30_000, 15 * MINUTE, HOUR, 6 * HOUR, 6 * HOUR]); + }); + + it("re-reads an old unproven stop on the capped backoff with five statements and no work signal", async () => { + const [runId] = await unprovenStops(1, 2 * DAY); + clock(); const worker = await start("collection"); + try { + expect(await copy(runId!)).toMatchObject({ state: "pending_collection", errorCode: "INSTRUCTION_STOP_UNCONFIRMED", attempts: 0 }); + expect((await copy(runId!)).nextAttemptAt!.getTime() - Date.now()).toBeGreaterThan(6 * HOUR - MINUTE); + const window = await advance(12 * HOUR + MINUTE, worker.nextWakeAt); + expect(window.stopChecks).toBe(2); + expect(window.statements).toBe(10); + expect(window.transactions).toBe(2); + expect(window.workSignals).toBe(0); + expect(await copy(runId!)).toMatchObject({ state: "pending_collection", attempts: 0, processStoppedAt: null }); + } finally { await worker.stop(); } + }, 120_000); + + it("keeps re-checking a copy owned by a live older server and collects it once that server records the stop", async () => { + const [runId] = await unprovenStops(1, 5 * MINUTE); + clock(); const worker = await start("collection"); + try { + const early = await advance(10 * MINUTE, worker.nextWakeAt); + expect(early.stopChecks).toBeGreaterThanOrEqual(2); + expect(await copy(runId!)).toMatchObject({ state: "pending_collection", processStoppedAt: null }); + await appendHeartbeatRunEvent(db, { companyId, agentId, runId: runId!, eventType: "native.local_process_stopped", stream: "system" }); + await advance(10 * MINUTE, worker.nextWakeAt); + expect(await copy(runId!)).toMatchObject({ state: "unchanged", nextAttemptAt: null }); + expect((await copy(runId!)).processStoppedAt).not.toBeNull(); + } finally { await worker.stop(); } + }, 120_000); + + it("collects a copy that waited for weeks within one capped backoff of its stop being recorded", async () => { + const [runId] = await unprovenStops(1, 30 * DAY); + clock(); const worker = await start("collection"); + try { + await advance(HOUR, worker.nextWakeAt); + await appendHeartbeatRunEvent(db, { companyId, agentId, runId: runId!, eventType: "native.local_process_stopped", stream: "system" }); + await advance(INSTRUCTION_COPY_MAX_RETRY_MS, worker.nextWakeAt); + expect(await copy(runId!)).toMatchObject({ state: "unchanged", nextAttemptAt: null }); + } finally { await worker.stop(); } + }, 120_000); + + it("drains a backlog of old unproven stops in bounded batches without work signals", async () => { + await unprovenStops(300, 30 * DAY); + const signals: string[] = []; + const unsubscribe = [DELIVERY_QUEUES.instructionCollection, DELIVERY_QUEUES.strandedAgentStatus, DELIVERY_QUEUES.skillVersionRetention] + .map(queue => subscribeDeliveryWork(db, queue, () => signals.push(queue))); + clock(); const worker = coordinator(); + try { + const before = meter.read(); + await agentInstructionWorkingCopyService(db).registerStoppedCollectionWork(worker, () => true, async () => true).ready; + const drain = await advance(60_000, () => Date.now() + 1000); + const rows = await db.select({ next: agentInstructionWorkingCopies.nextAttemptAt }).from(agentInstructionWorkingCopies) + .where(eq(agentInstructionWorkingCopies.companyId, companyId)); + expect(rows.every(row => row.next!.getTime() - Date.now() > 5 * HOUR)).toBe(true); + expect(meter.read().transactions - before.transactions).toBeLessThanOrEqual(300); + expect(drain.stopChecks).toBeLessThanOrEqual(300); + expect(drain.workSignals).toBe(0); + expect(signals).toEqual([]); + expect((await advance(HOUR, worker.nextWakeAt)).statements).toBe(0); + } finally { unsubscribe.forEach(stop => stop()); await worker.stop(); } + }, 120_000); + + it("still signals when a write moves the deadline earlier or changes queue membership", async () => { + const [earlier, missing] = await unprovenStops(2, 2 * DAY); + clock(); const worker = await start("collection"); + await worker.stop(); + const service = agentInstructionWorkingCopyService(db); + const signals: string[] = []; + const unsubscribe = subscribeDeliveryWork(db, DELIVERY_QUEUES.instructionCollection, () => signals.push("collection")); + try { + await fs.rm(entryPath(earlier!)); await fs.mkdir(entryPath(earlier!)); + await service.collectStopped({ companyId, runId: earlier! }); + expect(await copy(earlier!)).toMatchObject({ state: "pending_collection", attempts: 1, errorCode: "INSTRUCTION_COLLECTION_FAILED" }); + expect((await copy(earlier!)).nextAttemptAt!.getTime() - Date.now()).toBeLessThanOrEqual(INSTRUCTION_COPY_RETRY_MS); + await vi.waitFor(() => expect(signals).toHaveLength(1)); + await fs.rm(entryPath(missing!)); + await service.collectStopped({ companyId, runId: missing! }); + expect(await copy(missing!)).toMatchObject({ state: "unavailable", errorCode: "INSTRUCTION_FILE_MISSING" }); + await vi.waitFor(() => expect(signals).toHaveLength(2)); + } finally { unsubscribe(); } + }, 60_000); + + it.each([true, false])("backs off a cleanup that cannot finish and keeps the saved copy (cleanup throws: %s)", async throws => { + const [runId] = await stuckCleanups(1, { lockRoot: !throws }); + clock(); const worker = await start("cleanup"); + try { + const window = await advance(6 * HOUR, worker.nextWakeAt); + expect(window.cleanupClaims).toBeGreaterThan(0); + expect(window.workSignals).toBe(0); + expect(window.cleanupClaims).toBeLessThanOrEqual(32); + expect((await advance(6 * HOUR, worker.nextWakeAt)).cleanupClaims).toBeLessThanOrEqual(4); + const row = await copy(runId!); + expect(row).toMatchObject({ state: "saved", receipt: expect.objectContaining({ cleanupPending: true }) }); + expect(row.nextAttemptAt!.getTime()).toBeGreaterThan(Date.now()); + } finally { await worker.stop(); } + }, 120_000); +}); diff --git a/server/src/services/agent-instruction-working-copies.ts b/server/src/services/agent-instruction-working-copies.ts index a3161d2e347..7f2d98b98b6 100644 --- a/server/src/services/agent-instruction-working-copies.ts +++ b/server/src/services/agent-instruction-working-copies.ts @@ -35,6 +35,15 @@ const hash = (bytes: Uint8Array) => createHash("sha256").update(bytes).digest("h const quote = (value: string) => `'${value.replaceAll("'", `'"'"'`)}'`; const completed = new Set(["saved", "unchanged", "resolved"]); const MAX_COLLECTION_ATTEMPTS = 3; +export const INSTRUCTION_COPY_RETRY_MS = 30_000; +export const INSTRUCTION_COPY_MAX_RETRY_MS = 6 * 60 * 60_000; +const settledStates = new Set(["saved", "unchanged", "resolved", "unavailable", "conflict"]); + +export function instructionCopyRetryAt(waitingSince: Date, now = Date.now()) { + const waited = Math.max(0, now - waitingSince.getTime()); + return new Date(now + Math.min(INSTRUCTION_COPY_MAX_RETRY_MS, Math.max(INSTRUCTION_COPY_RETRY_MS, Math.floor(waited / 2)))); +} + // Only orchestration can register a live target; tools and HTTP callers cannot // supply one. These handles are optional: durable captured bytes survive restart. const liveTargets = new Map(); @@ -100,20 +109,25 @@ export function agentInstructionWorkingCopyService(db: Db, options: { environmen return row ?? null; } async function tryPatch(row: Copy, values: Partial, requireDue = false, condition?: SQL) { - const signalsCollection = collectionCandidate(row) || collectionCandidate({ ...row, ...values }); - const signalsCleanup = directoryCleanupEntry(row) !== null || directoryCleanupEntry({ ...row, ...values }) !== null; + const next = { ...row, ...values } as Copy; + const signalsCollection = collectionCandidate(row) || collectionCandidate(next); + const signalsCleanup = directoryCleanupEntry(row) !== null || directoryCleanupEntry(next) !== null; const signalsCommit = row.state === "pending_commit" || values.state === "pending_commit"; + const where = and( + scope(row.companyId, row.runId), eq(copies.state, row.state), eq(copies.attempts, row.attempts), eq(copies.baseHash, row.baseHash), eq(copies.localRoot, row.localRoot), + sql`${copies.receipt}->>'retainedByRunId' IS NOT DISTINCT FROM ${typeof row.receipt?.retainedByRunId === "string" ? row.receipt.retainedByRunId : null}`, + requireDue ? or(isNull(copies.nextAttemptAt), lte(copies.nextAttemptAt, new Date())) : undefined, condition, + ); + if (onlyDefers(row, next)) { + return db.transaction(async tx => (await tx.update(copies).set({ ...values, updatedAt: new Date() }).where(where).returning())[0]); + } const write = async (tx: Db) => { await notifyStrandedAgentStatus(tx, { companyId: row.companyId, agentId: row.agentId }); if (signalsCommit) await notifyInstructionCommit(tx, { companyId: row.companyId, runId: row.runId }); if (signalsCleanup) await notifyDirectoryCleanup(tx, { companyId: row.companyId, runId: row.runId }); if (signalsCollection) await notifyInstructionCollection(tx, { companyId: row.companyId, runId: row.runId }); - const [updated] = await tx.update(copies).set({ ...values, updatedAt: new Date() }).where(and( - scope(row.companyId, row.runId), eq(copies.state, row.state), eq(copies.attempts, row.attempts), eq(copies.baseHash, row.baseHash), eq(copies.localRoot, row.localRoot), - sql`${copies.receipt}->>'retainedByRunId' IS NOT DISTINCT FROM ${typeof row.receipt?.retainedByRunId === "string" ? row.receipt.retainedByRunId : null}`, - requireDue ? or(isNull(copies.nextAttemptAt), lte(copies.nextAttemptAt, new Date())) : undefined, condition, - )).returning(); + const [updated] = await tx.update(copies).set({ ...values, updatedAt: new Date() }).where(where).returning(); return updated; }; return db.transaction(tx => write(tx as unknown as Db)); @@ -128,15 +142,12 @@ export function agentInstructionWorkingCopyService(db: Db, options: { environmen return { type: "agent", companyId: row.companyId, agentId: row.agentId, runId: row.runId, onBehalfOfUserId: row.responsibleUserId }; } const directories = agentDirectoryWorkingCopyService(db, get, patch, options.environmentRuntime); - async function recoverDirectoryCleanup(row: Copy, cleanup: () => Promise) { - // Reserve a later retry before I/O, including lock waits. A blocked owner - // must not occupy every sweep or prevent other copies from being reclaimed. + async function recoverDirectoryCleanup(row: Copy, retryAt: Date, cleanup: () => Promise) { try { - await patch(row, { nextAttemptAt: new Date(Date.now() + 30_000) }); await cleanup(); } catch (error) { - try { await patch(row, { nextAttemptAt: new Date(Date.now() + 30_000) }); } + try { await patch(row, { nextAttemptAt: retryAt }); } catch (retryError) { logger.warn({ err: retryError, runId: row.runId }, "Agent file cleanup retry could not be scheduled"); } @@ -380,6 +391,19 @@ export function agentInstructionWorkingCopyService(db: Db, options: { environmen || row.state === "preparing" && isAgentDirectoryCopy(row)) || row.state === "unavailable" && !row.processStoppedAt && row.location.startsWith("remote:") && isAgentDirectoryCopy(row)); } + function strandsAgent(row: Copy) { + return !settledStates.has(row.state) || !row.processStoppedAt || row.nextAttemptAt !== null + || Object.hasOwn(row.receipt ?? {}, "baseline") || row.receipt?.cleanupPending === true; + } + function onlyDefers(row: Copy, next: Copy) { + const due = (copy: Copy) => copy.nextAttemptAt?.getTime() ?? 0; + const deferred = (before: boolean, after: boolean) => before === after && (!after || due(next) >= due(row)); + return deferred(collectionCandidate(row), collectionCandidate(next)) + && deferred(directoryCleanupEntry(row) !== null, directoryCleanupEntry(next) !== null) + && deferred(commitEntry(row) !== null, commitEntry(next) !== null) + && (row.state === "pending_commit") === (next.state === "pending_commit") + && strandsAgent(row) === strandsAgent(next); + } function collectionPredicate() { return and(sql`NOT (coalesce(${copies.receipt}, '{}'::jsonb) ? 'retainedByRunId') AND ( (${copies.attempts} < 3 AND (${copies.state} IN ('prepared', 'pending_collection', 'unchanged_turn', 'warm_saved') @@ -388,39 +412,41 @@ export function agentInstructionWorkingCopyService(db: Db, options: { environmen AND ${copies.location} LIKE 'remote:%' AND ${copies.receipt}->>'schema' = 'paperclip.agent-files.v1'))`, inArray(heartbeatRuns.status, ["succeeded", "failed", "cancelled", "timed_out", "interrupted"])); } - async function collectRecovered(row: Copy, runtimeMode: string | null) { - if (row.state === "unavailable" && isAgentDirectoryCopy(row)) { - await recoverDirectoryCleanup(row, () => directories.recoverUnavailable(row)); - } else if (row.state === "preparing" && isAgentDirectoryCopy(row)) { - - + type StopEvidence = "directory_unavailable" | "directory_preparing" | "local" | "remote" | null; + async function stopEvidence(row: Copy, runtimeMode: string | null): Promise { + if (row.state === "unavailable" && isAgentDirectoryCopy(row)) return "directory_unavailable"; + if (row.state === "preparing" && isAgentDirectoryCopy(row)) return "directory_preparing"; + if (row.location === "local") return row.processStoppedAt || runtimeMode === "native" && + await hasNativeLocalProcessStop(db, row.companyId, row.runId) ? "local" : null; + return await remoteExecutionHasStopped(db, row.companyId, row.runId) ? "remote" : null; + } + function awaitingStop(row: Copy, waitingSince: Date): Partial { + return { ...(row.state === "prepared" ? { state: "pending_collection", errorCode: "INSTRUCTION_STOP_UNCONFIRMED", + errorMessage: "The provider's stop has not been confirmed. Instruction collection is pending; no save is claimed." } : {}), + nextAttemptAt: instructionCopyRetryAt(waitingSince) }; + } + async function collectRecovered(row: Copy, evidence: Exclude, retryAt: Date): Promise { + if (evidence === "directory_unavailable") { + await recoverDirectoryCleanup(row, retryAt, () => directories.recoverUnavailable(row)); + } else if (evidence === "directory_preparing") { await directories.release(await patch(row, { state: "unavailable", processStoppedAt: new Date(), errorCode: "AGENT_FILES_PREPARE_FAILED", errorMessage: "Agent-file preparation was interrupted. The temporary copy was discarded.", nextAttemptAt: null })); - } else if (row.location === "local" && (row.processStoppedAt || runtimeMode === "native" && - await hasNativeLocalProcessStop(db, row.companyId, row.runId))) { + } else if (evidence === "local") { const result = await collectStopped({ companyId: row.companyId, runId: row.runId }); if (result && isAgentDirectoryCopy(result) && completed.has(result.state)) await directories.release(result); - } else if (row.location !== "local" && await remoteExecutionHasStopped(db, row.companyId, row.runId)) { - if (row.receipt?.warm === true) { - const unavailable = row.receipt?.warm === true - ? await patch(row, { state: "unavailable", errorCode: "AGENT_FILES_FINAL_COLLECTION_UNAVAILABLE", - errorMessage: "The last completed checkpoint remains saved. Files changed afterwards could not be collected after remote termination.", nextAttemptAt: null }) - : await reportUnavailable(row.companyId, row.runId); - if (unavailable && isAgentDirectoryCopy(unavailable)) { + } else if (row.receipt?.warm === true) { + const unavailable = await patch(row, { state: "unavailable", errorCode: "AGENT_FILES_FINAL_COLLECTION_UNAVAILABLE", + errorMessage: "The last completed checkpoint remains saved. Files changed afterwards could not be collected after remote termination.", nextAttemptAt: null }); + if (isAgentDirectoryCopy(unavailable)) { await directories.release(await patch(unavailable, { processStoppedAt: unavailable.processStoppedAt ?? new Date() })); } - } else { - if (isAgentDirectoryCopy(row)) await directories.recoverStoppedRemote(row); - else await reportUnavailable(row.companyId, row.runId); + } else if (isAgentDirectoryCopy(row)) { + const result = await directories.recoverStoppedRemote(row); + if (result && collectionCandidate(result) && !result.processStoppedAt && (result.nextAttemptAt?.getTime() ?? 0) < retryAt.getTime()) { + return patch(result, { nextAttemptAt: retryAt }); } - } else if (row.state === "prepared") { - await patch(row, { state: "pending_collection", errorCode: "INSTRUCTION_STOP_UNCONFIRMED", - errorMessage: "The provider's stop has not been confirmed. Instruction collection is pending; no save is claimed.", nextAttemptAt: new Date(Date.now() + 30_000) }); - } else if (["warm_saved", "pending_collection", "unchanged_turn"].includes(row.state)) { - - - await patch(row, { nextAttemptAt: new Date(Date.now() + 30_000) }); - } + } else await reportUnavailable(row.companyId, row.runId); + return undefined; } type CollectionEntry = { id: string; companyId: string; dueAt: number }; async function loadCollection(scope: InstructionCollectionScope | undefined, admitted: () => boolean) { @@ -439,7 +465,8 @@ export function agentInstructionWorkingCopyService(db: Db, options: { environmen throw new Error("Instruction collection reconstruction stopped"); } async function readCollection(entry: CollectionEntry) { - const [current] = await db.select({ copy: copies, runtimeMode: heartbeatRuns.runtimeMode }).from(copies) + const [current] = await db.select({ copy: copies, runtimeMode: heartbeatRuns.runtimeMode, + waitingSince: sql`(extract(epoch from coalesce(${heartbeatRuns.finishedAt}, ${copies.createdAt})) * 1000)::float8`.mapWith(value => new Date(Number(value))) }).from(copies) .innerJoin(heartbeatRuns, and(eq(heartbeatRuns.companyId, copies.companyId), eq(heartbeatRuns.id, copies.runId))) .where(and(collectionPredicate(), scope(entry.companyId, entry.id))); return current; @@ -450,14 +477,18 @@ export function agentInstructionWorkingCopyService(db: Db, options: { environmen const dueAt = current.copy.nextAttemptAt?.getTime() ?? 0; if (dueAt > Date.now()) return { ...entry, dueAt }; if (!admitted()) throw new Error("Instruction collection recovery stopped"); - const claimed = await tryPatch(current.copy, { nextAttemptAt: new Date(Date.now() + 30_000) }, true, + const evidence = await stopEvidence(current.copy, current.runtimeMode); + const retryAt = instructionCopyRetryAt(current.waitingSince); + const claimed = await tryPatch(current.copy, evidence === null ? awaitingStop(current.copy, current.waitingSince) + : { nextAttemptAt: evidence === "directory_unavailable" ? retryAt : new Date(Date.now() + INSTRUCTION_COPY_RETRY_MS) }, true, sql`EXISTS (SELECT 1 FROM ${heartbeatRuns} WHERE ${heartbeatRuns.id} = ${copies.runId} AND ${heartbeatRuns.companyId} = ${copies.companyId} AND ${heartbeatRuns.status} IN ('succeeded', 'failed', 'cancelled', 'timed_out', 'interrupted'))`); - if (claimed) await collectRecovered(claimed, current.runtimeMode); - const remaining = await readCollection(entry); + const settled = claimed && evidence !== null ? await collectRecovered(claimed, evidence, retryAt) : claimed; + const remaining = settled === undefined || settled === null ? (await readCollection(entry))?.copy + : collectionCandidate(settled) ? settled : null; if (!remaining) return null; - const next = { ...entry, dueAt: remaining.copy.nextAttemptAt?.getTime() ?? 0 }; + const next = { ...entry, dueAt: remaining.nextAttemptAt?.getTime() ?? 0 }; if (next.dueAt <= Date.now()) throw new Error("Instruction collection remains unresolved without a future retry"); return next; } @@ -522,9 +553,21 @@ export function agentInstructionWorkingCopyService(db: Db, options: { environmen const current = directoryCleanupEntry(row); if (!row || !current || current.dueAt > Date.now()) return current; if (!admitted()) throw new Error("Directory cleanup recovery stopped"); - const claimed = await tryPatch(row, { nextAttemptAt: new Date(Date.now() + 30_000) }, true); - if (claimed) await directories.release(claimed); - const remaining = directoryCleanupEntry(await get(entry.companyId, entry.id)); + const retryAt = instructionCopyRetryAt(row.processStoppedAt ?? row.updatedAt); + const claimed = await tryPatch(row, { nextAttemptAt: new Date(Date.now() + INSTRUCTION_COPY_RETRY_MS) }, true); + const deferCleanup = async () => { + const latest = await get(entry.companyId, entry.id); + const pending = directoryCleanupEntry(latest); + return claimed && latest && pending && pending.dueAt < retryAt.getTime() ? patch(latest, { nextAttemptAt: retryAt }) : latest; + }; + if (claimed) { + try { await directories.release(claimed); } + catch (error) { + await deferCleanup(); + throw error; + } + } + const remaining = directoryCleanupEntry(await deferCleanup()); if (remaining && remaining.dueAt <= Date.now()) throw new Error("Directory cleanup remains unresolved without a future retry"); return remaining; }