mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-16 03:31:44 +01:00
Poll asynchronous workspace provider preparation (#20865)
## Thinking Path > - Paperclip starts agent runs through a workspace provider when a project uses remote storage. > - The provider must prepare the workspace before the agent starts. > - One long worker call ties the checkout to its transport. > - Some checkouts take longer than the transport allows. > - This change lets an opted-in provider accept work and report its deadline. > - Core waits for completion through short status calls and stops at the provider deadline. ## Linked Issues or Issue Description **What happened?** A workspace provider can lose its reply when preparation takes longer than a single transport request. This stops run bootstrap before agent execution. **Expected behavior** Core should wait for a provider-confirmed workspace through short requests. The provider's deadline must bound the wait. **Steps to reproduce** Use a workspace provider that takes longer to prepare a checkout than its transport can remain open. Start a run that requires a new checkout. ## What Changed - Add an optional workspace preparation method to the plugin SDK. - Validate pending receipts and keep the original binding checks on completed replies. - Poll only during active preparation. Retry a lost status reply within the advertised deadline. - Preserve the original operation for providers that do not offer asynchronous preparation. - Reject deadline offers that exceed the existing lifecycle lock budget. Honor accepted offers without shortening them. - Checkout verification uses the existing synchronous RPC with a 10-second deadline and a 12-second transport timeout. [Core verification PR #20876](https://github.com/paperclipai/paperclip/pull/20876) ## Verification - All 63 workspace SDK and Core integration tests passed with temporary local PostgreSQL. The tests cover lost status replies, deadline expiry, oversized offers, and synchronous verification with an async-capable worker. - All 58 runner SDK input, evidence, and public type tests passed with the regenerated source pins. The local server typecheck passed. ## Risks - This change activates only when the provider advertises the method. Existing providers keep their current behavior. - Polling uses one-second intervals during active preparation. It stops at completion, failure, or the provider deadline. - No database migration is required. ## Model Used - OpenAI GPT-6 Codex with local code execution and tests. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
e601f9d2fc
commit
5dfbb04b85
11 files changed
+233
-27
No files matched your search
@@ -1,6 +1,6 @@
|
||||
import { createPluginWorkspaceFixture } from "./helpers/plugin-workspaces.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import {
|
||||
createDb,
|
||||
@@ -76,11 +76,19 @@ const support = await getEmbeddedPostgresTestSupport();
|
||||
issueId: f.issueId, status: "running", contextSnapshot: { taskRetryRequestId: retryIntentId },
|
||||
}).returning();
|
||||
const fresh = { mode: "fresh_isolated_workspace", retryIntentId, sourceRunId: f.runId, retainedSourceIds: [] } as const;
|
||||
const workers: WorkspaceWorkers = { ...f.workers, call: async (...args) => {
|
||||
const call = vi.fn<WorkspaceWorkers["call"]>(async (...args) => {
|
||||
const result = await f.workers.call(...args) as { kind: string };
|
||||
if (state === "unconfirmed") throw new Error("checkout incomplete");
|
||||
return { ...result, created: false };
|
||||
} };
|
||||
});
|
||||
const workers: WorkspaceWorkers = {
|
||||
getWorker: (pluginId) => {
|
||||
const worker = f.workers.getWorker(pluginId);
|
||||
return worker && { ...worker, supportedMethods: ["workspaceOperation", "workspacePrepare"] };
|
||||
},
|
||||
call,
|
||||
};
|
||||
const verificationStarted = Date.now();
|
||||
const result = await verifyRetryWorkspace(db, workers, retry, f.issueId, null,
|
||||
{ ...fresh, retainedSourceIds: [] });
|
||||
const verified = state === "ready" || state === "lost_reply";
|
||||
@@ -88,11 +96,73 @@ const support = await getEmbeddedPostgresTestSupport();
|
||||
? { mode: "reuse_issue_workspace", retryIntentId, sourceRunId: f.runId, workspaceId, cwd: null }
|
||||
: fresh);
|
||||
expect(f.calls.slice(1)).toHaveLength(state === "live_writer" || state === "unsupported" ? 0 : 1);
|
||||
if (f.calls.length > 1) expect(f.calls[1]).toMatchObject({ executionWorkspaceId: workspaceId,
|
||||
runId: retry.id, operation: { kind: "prepare", verifyExisting: true } });
|
||||
if (f.calls.length > 1) {
|
||||
expect(call).toHaveBeenCalledExactlyOnceWith(f.pluginId, "workspaceOperation", expect.objectContaining({
|
||||
executionWorkspaceId: workspaceId, runId: retry.id, operation: expect.objectContaining({ verifyExisting: true }),
|
||||
}), 12_000);
|
||||
const deadline = Date.parse(f.calls[1].deadline);
|
||||
expect(deadline).toBeGreaterThanOrEqual(verificationStarted + 10_000);
|
||||
expect(deadline).toBeLessThanOrEqual(Date.now() + 10_000);
|
||||
}
|
||||
const [retained] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, workspaceId));
|
||||
expect(retained.providerRef).toBe(`checkout:${workspaceId}`);
|
||||
});
|
||||
|
||||
it("polls bounded requests using the provider's advertised preparation deadline", async () => {
|
||||
const f = await fixture();
|
||||
const deadline = new Date(Date.now() + 30_000).toISOString();
|
||||
const phases: string[] = [];
|
||||
const workers = {
|
||||
getWorker: () => ({ supportedMethods: ["workspaceOperation", "workspacePrepare"] }),
|
||||
call: async (pluginId: string, method: string, params: any, timeout: number) => {
|
||||
expect(method).toBe("workspacePrepare");
|
||||
expect(timeout).toBeLessThanOrEqual(15_000);
|
||||
phases.push(params.phase);
|
||||
if (params.phase === "submit") return { kind: "pending", deadline };
|
||||
expect(params.deadline).toBe(deadline);
|
||||
if (phases.length === 2) throw new Error("lost status reply");
|
||||
const { phase: _phase, ...original } = params;
|
||||
return f.workers.call(pluginId, "workspaceOperation", original);
|
||||
},
|
||||
} as unknown as WorkspaceWorkers;
|
||||
expect(await preparePluginWorkspace(db, workers, f.input)).toBeTruthy();
|
||||
expect(phases).toEqual(["submit", "poll", "poll"]);
|
||||
expect(f.calls).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("stops preparation at the provider deadline before issuing a poll", async () => {
|
||||
const f = await fixture();
|
||||
const workers = {
|
||||
getWorker: () => ({ supportedMethods: ["workspaceOperation", "workspacePrepare"] }),
|
||||
call: async (_pluginId: string, method: string, params: { phase: string }) => {
|
||||
expect(method).toBe("workspacePrepare");
|
||||
expect(params.phase).toBe("submit");
|
||||
return { kind: "pending", deadline: new Date(Date.now() - 1).toISOString() };
|
||||
},
|
||||
} as unknown as WorkspaceWorkers;
|
||||
await expect(preparePluginWorkspace(db, workers, f.input)).rejects.toMatchObject({
|
||||
kind: "provider", reason: "deadline_exceeded", retryable: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects an oversized provider deadline before it can retain the workspace lifecycle lock", async () => {
|
||||
const f = await fixture();
|
||||
const workers = {
|
||||
getWorker: () => ({ supportedMethods: ["workspaceOperation", "workspacePrepare"] }),
|
||||
call: async (_pluginId: string, method: string, params: { phase: string }) => {
|
||||
expect(method).toBe("workspacePrepare");
|
||||
expect(params.phase).toBe("submit");
|
||||
return { kind: "pending", deadline: "2099-01-01T00:00:00Z" };
|
||||
},
|
||||
} as unknown as WorkspaceWorkers;
|
||||
await expect(preparePluginWorkspace(db, workers, f.input)).rejects.toMatchObject({
|
||||
kind: "provider", reason: "unsupported_operation", retryable: false,
|
||||
});
|
||||
const [task] = await db.select().from(issues).where(eq(issues.id, f.issueId));
|
||||
expect(task?.executionWorkspaceId).toBeTruthy();
|
||||
expect(await preparePluginWorkspace(db, f.workers, { ...f.input, existingWorkspaceId: task!.executionWorkspaceId })).toBeTruthy();
|
||||
});
|
||||
|
||||
it.each([
|
||||
["missing", undefined],
|
||||
["stopped", { status: "stopped", supportedMethods: [] }],
|
||||
|
||||
@@ -1285,7 +1285,7 @@ export function createPluginWorkerHandle(
|
||||
if (method === "resourceLifecycle" && params.resourceType === "project" && readNonEmptyString(params.resourceId)) {
|
||||
return { companyId: directCompanyId, projectGit: { projectId: params.resourceId as string } };
|
||||
}
|
||||
if (method === "workspaceOperation" && isRecord(params.operation)) {
|
||||
if ((method === "workspaceOperation" || method === "workspacePrepare") && isRecord(params.operation)) {
|
||||
const operation = params.operation;
|
||||
const source = operation.kind === "prepare" ? operation.source : operation.binding;
|
||||
const workspaceId = isRecord(source) ? readNonEmptyString(operation.kind === "prepare" ? source.id : source.projectWorkspaceId) : null;
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
import { setTimeout as delay } from "node:timers/promises";
|
||||
import { parseWorkspacePrepareResult, parseWorkspaceProviderResult, type PluginWorkspaceParams } from "@paperclipai/plugin-sdk";
|
||||
import type { PluginWorkerManager } from "./plugin-worker-manager.js";
|
||||
import { WorkspaceProviderFailureError, workspaceProviderCallFailure } from "./workspace-provider-failure.js";
|
||||
|
||||
type ProviderRequest = Omit<PluginWorkspaceParams, "deadline">;
|
||||
type WorkspaceWorker = Pick<PluginWorkerManager, "call" | "getWorker">;
|
||||
const legacyWorkspaceOperationMs = 300_000;
|
||||
const legacyRpcGraceMs = 10_000;
|
||||
|
||||
export async function invokeWorkspaceProvider(workers: WorkspaceWorker, params: ProviderRequest) {
|
||||
const verifyExisting = params.operation.kind === "prepare" && params.operation.verifyExisting;
|
||||
if (!verifyExisting && params.operation.kind === "prepare" && workers.getWorker(params.pluginId)?.supportedMethods?.includes("workspacePrepare")) {
|
||||
const result = await prepareWorkspaceWithPolling(workers, params);
|
||||
if (result) return result;
|
||||
}
|
||||
const timeout = verifyExisting ? 10_000 : params.operation.kind === "finalize" ? 30_000 : legacyWorkspaceOperationMs;
|
||||
const request = { ...params, deadline: new Date(Date.now() + timeout).toISOString() };
|
||||
return parseWorkspaceProviderResult(request, await workers.call(
|
||||
params.pluginId, "workspaceOperation", request, timeout + (verifyExisting ? 2_000 : legacyRpcGraceMs),
|
||||
));
|
||||
}
|
||||
|
||||
async function prepareWorkspaceWithPolling(workers: Pick<PluginWorkerManager, "call">, params: ProviderRequest) {
|
||||
const rpcTimeout = 15_000;
|
||||
const submit = { ...params, phase: "submit" as const, deadline: new Date(Date.now() + rpcTimeout).toISOString() };
|
||||
const receipt = parseWorkspacePrepareResult(submit, await workers.call(params.pluginId, "workspacePrepare", submit, rpcTimeout));
|
||||
if (receipt.kind === "unsupported") return null;
|
||||
if (receipt.kind === "prepared") return receipt;
|
||||
const offeredMs = Date.parse(receipt.deadline) - Date.now();
|
||||
if (offeredMs > legacyWorkspaceOperationMs + legacyRpcGraceMs)
|
||||
throw new WorkspaceProviderFailureError({
|
||||
kind: "provider", reason: "unsupported_operation", retryable: false,
|
||||
message: "Workspace provider offered a deadline outside the supported lifecycle budget",
|
||||
});
|
||||
const expiresAt = performance.now() + offeredMs;
|
||||
const remaining = () => {
|
||||
const ms = Math.ceil(expiresAt - performance.now());
|
||||
if (ms <= 0) throw new WorkspaceProviderFailureError({
|
||||
kind: "provider", reason: "deadline_exceeded", retryable: true,
|
||||
message: "Workspace storage operation expired; retry the same workspace",
|
||||
});
|
||||
return ms;
|
||||
};
|
||||
const poll = { ...params, phase: "poll" as const, deadline: receipt.deadline };
|
||||
for (;;) {
|
||||
await delay(Math.min(1000, remaining()));
|
||||
const timeout = Math.min(rpcTimeout, remaining());
|
||||
let response: unknown;
|
||||
try {
|
||||
response = await workers.call(params.pluginId, "workspacePrepare", poll, timeout);
|
||||
} catch (error) {
|
||||
const failure = workspaceProviderCallFailure(error);
|
||||
if (failure.kind === "transport" || failure.kind === "timeout") continue;
|
||||
throw failure;
|
||||
}
|
||||
remaining();
|
||||
const status = parseWorkspacePrepareResult(poll, response);
|
||||
if (status.kind === "prepared") return status;
|
||||
if (status.kind === "unsupported" || status.deadline !== receipt.deadline)
|
||||
throw new Error("Workspace provider changed the preparation deadline");
|
||||
}
|
||||
}
|
||||
@@ -17,16 +17,17 @@ import {
|
||||
type WorkspaceProviderBinding,
|
||||
} from "@paperclipai/shared";
|
||||
import {
|
||||
parseWorkspaceProviderResult,
|
||||
workspaceProviderOperationSchema,
|
||||
type PluginWorkspaceOperation,
|
||||
type PluginWorkspaceRecoveryPullRequest,
|
||||
type PluginWorkspaceParams,
|
||||
type PluginWorkspaceResult,
|
||||
type PluginFreshWorkspace,
|
||||
} from "@paperclipai/plugin-sdk";
|
||||
import { invokeWorkspaceProvider } from "./plugin-workspace-preparation.js";
|
||||
import { notifyTerminalWorkspaceWork } from "./terminal-workspace-work.js";
|
||||
import { withExecutionWorkspaceLifecycleLock } from "./execution-workspace-lifecycle-lock.js";
|
||||
import { workspaceProviderCallFailure } from "./workspace-provider-failure.js";
|
||||
import { WorkspaceProviderFailureError, workspaceProviderCallFailure } from "./workspace-provider-failure.js";
|
||||
import {
|
||||
assertTaskWorkspaceExecutionAccess,
|
||||
assertTaskWorkspaceSourceProjectAccess,
|
||||
@@ -87,7 +88,7 @@ export async function callWorkspaceProvider(
|
||||
operation.fresh = await stageProviderFreshWorkspace({ db, companyId: workspace.companyId,
|
||||
issueId: workspace.sourceIssueId, ...operation.fresh, existing: operation.fresh });
|
||||
}
|
||||
const params: PluginWorkspaceParams = {
|
||||
const params: Omit<PluginWorkspaceParams, "deadline"> = {
|
||||
pluginId,
|
||||
companyId: workspace.companyId,
|
||||
projectId: workspace.projectId,
|
||||
@@ -102,22 +103,17 @@ export async function callWorkspaceProvider(
|
||||
)
|
||||
.digest("hex")),
|
||||
generation: operationGeneration,
|
||||
deadline: new Date(Date.now() + (verifyExisting ? 10_000 : operation.kind === "finalize" ? 30_000 : 300_000)).toISOString(),
|
||||
runId: input.runId ?? null,
|
||||
agentId: input.agentId ?? null,
|
||||
operation,
|
||||
};
|
||||
let value: unknown;
|
||||
let result: PluginWorkspaceResult;
|
||||
try {
|
||||
value = await workers!.call(
|
||||
pluginId,
|
||||
"workspaceOperation",
|
||||
params,
|
||||
verifyExisting ? 12_000 : operation.kind === "finalize" ? 40_000 : 310_000,
|
||||
);
|
||||
result = await invokeWorkspaceProvider(workers!, params);
|
||||
} catch (error) {
|
||||
// A worker that never received the call, a lost reply and a provider-reported
|
||||
// failure need different recovery; only a closed provider reason crosses over.
|
||||
if (error instanceof WorkspaceProviderFailureError) throw error;
|
||||
throw workspaceProviderCallFailure(error);
|
||||
}
|
||||
// Recheck enablement and ownership after an external effect, including an ambiguous retry.
|
||||
@@ -139,7 +135,6 @@ export async function callWorkspaceProvider(
|
||||
) {
|
||||
throw conflict("Workspace changed during storage preparation");
|
||||
}
|
||||
const result = parseWorkspaceProviderResult(params, value);
|
||||
if (result.kind === "prepared" && result.binding.pluginId !== pluginId)
|
||||
throw conflict("Workspace provider ownership changed");
|
||||
return result;
|
||||
|
||||
Reference in new issue
Block a user