mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-16 11:59:26 +01:00
Verify plugin workspaces on run leases and release settled tasks
Acquire the logical placement before verifying a retained checkout and reuse it through preparation. Persist definite refusal before release and preserve the existing workspace and session choices on fresh fallback and restart. Always invoke provider release after task settlement. Reuse saved stop proof during cleanup retries and preserve completed run outcomes. Add database regressions for verification placement, restart, refusal, transport failure, and release replay. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
14 files changed
+539
-262
No files matched your search
@@ -204,12 +204,16 @@ A storage provider can declare `workspaceProvider.workspaceVerificationVersion:
|
||||
Before creating a fresh workspace for an admitted Retry, Core may send the
|
||||
original task-owned workspace's `prepare` request with `verifyExisting: true`.
|
||||
Core sends verification through `workspaceOperation` with a 10-second
|
||||
deadline and a 12-second RPC timeout.
|
||||
deadline and a 12-second RPC timeout. Drivers with task placement receive the
|
||||
selected run's `taskLease` before Core describes the Runner task. Use that lease
|
||||
to select the same execution placement for verification and later preparation.
|
||||
The provider must verify a completed checkout without creating or fetching
|
||||
one and return `prepared` with `created: false`. Core checks prior writer
|
||||
settlement and continues normal preparation on the retained workspace after
|
||||
verification. A missing or unconfirmed checkout leaves the fresh retry
|
||||
fallback in place. Local workspaces use the same ownership and writer
|
||||
verification. Return the workspace failure reason `binding_mismatch` for a
|
||||
definitively missing or invalid checkout. Core can then release the verification
|
||||
lease and use the authorized fresh retry fallback. Transport failures and
|
||||
timeouts keep the current checkout decision retryable. Local workspaces use the same ownership and writer
|
||||
checks with Core's existing local Git verification.
|
||||
|
||||
A storage provider can declare `workspaceProvider.finalizationVersion: 1`.
|
||||
|
||||
@@ -416,7 +416,7 @@ databaseSuite("Core plugin Runner task lifecycle", () => {
|
||||
return;
|
||||
}
|
||||
const next = await acquire;
|
||||
expect(operations).toEqual(["stop", "status", "complete", "environmentAcquireLease"]);
|
||||
expect(operations).toEqual(["stop", "status", "complete", "environmentReleaseLease", "environmentAcquireLease"]);
|
||||
expect(next.lease.id).not.toBe(oldLeaseId);
|
||||
await expect(resolvePluginTaskResume(db, { companyId, leaseId: next.lease.id, priorIdentity })).resolves.toBe("old-task");
|
||||
await expect(runtime.task({ companyId, leaseId: next.lease.id, operation: { kind: "submit", projectIds: [], bootstrapTicket: "fixture", runner } })).rejects.toThrow("identity mismatch");
|
||||
|
||||
@@ -23,6 +23,7 @@ const support=external ? {supported:true} : await getEmbeddedPostgresTestSupport
|
||||
const companyId=randomUUID(),agentId=randomUUID(),issueId=randomUUID(),environmentId=randomUUID(),pluginId=randomUUID();
|
||||
let runId=randomUUID(),allocation=0,stopped=false,outage=false,releaseFails=false,unavailable=false;
|
||||
const calls:string[]=[],dispatched:Array<[string,string]>=[],failures=new Set<string>();
|
||||
const releaseMetadata: unknown[] = [];
|
||||
let projectIds=[...initialProjects],providerExpiry=initialExpiry;
|
||||
const taskExpiries=new Map<string,string|undefined>(),acquireRequests:any[]=[];
|
||||
let resumeGate:Promise<void>|undefined,releaseResume:(()=>void)|undefined,resumes=0;
|
||||
@@ -47,7 +48,7 @@ const support=external ? {supported:true} : await getEmbeddedPostgresTestSupport
|
||||
}
|
||||
if(method==="environmentResumeLease") {if(outage) throw Error("unreachable");
|
||||
if(resumeGate){if(++resumes===2)releaseResume!();await resumeGate;}return {providerLeaseId:stopped||unavailable?null:p.providerLeaseId,expiresAt:taskExpiries.get(p.providerLeaseId),metadata:{runId:p.runId}};}
|
||||
if(method==="environmentReleaseLease") {if(releaseFails) throw Error("release unavailable");return {providerLeaseId:p.providerLeaseId,state:"stopped"};}
|
||||
if(method==="environmentReleaseLease") {releaseMetadata.push(p.leaseMetadata);if(releaseFails) throw Error("release unavailable");return {providerLeaseId:p.providerLeaseId,state:"stopped"};}
|
||||
if(p.operation.kind==="status") return {kind:"status",taskId:p.taskId,phase:stopped?"completed":"running",executionStopped:stopped,sessionStateAvailable:stopped,preparationStopped:stopped};
|
||||
if(p.operation.kind==="read_state") return {kind:"state",taskId:p.taskId,state:{...p.operation.identity,lifecycle:"ready"}};
|
||||
if(p.operation.kind==="stop") stopped=true;
|
||||
@@ -66,11 +67,11 @@ const support=external ? {supported:true} : await getEmbeddedPostgresTestSupport
|
||||
launch:{runnerVersion:"fixture",runnerDigest:`sha256:${"a".repeat(64)}`,maxOutboxBytes:262144,p0ReserveBytes:65536,lifecyclePolicy:{mode:"warm",idleTimeoutMs:300000}}}});
|
||||
await runtime().releaseRunLeases(runId,"released",undefined,"keep_running");
|
||||
await db.update(heartbeatRuns).set({status:"succeeded"}).where(eq(heartbeatRuns.id,runId));
|
||||
return {companyId,first,identity,calls,dispatched,runtime,acquire,acquireRequests,
|
||||
return {companyId,first,identity,calls,dispatched,runtime,acquire,acquireRequests,releaseMetadata,
|
||||
fail(kind:string){failures.add(kind);},unavailable(){unavailable=true;},
|
||||
setExpiry(next?:string){providerExpiry=next;},
|
||||
async nextRun(){runId=randomUUID();await db.insert(heartbeatRuns).values({id:runId,companyId,agentId,nativeIssueId:issueId,invocationSource:"manual",status:"running"});return runId;},
|
||||
setProjects(next:string[]){projectIds=next;},stopped(){stopped=true;},failRelease(){releaseFails=true;},outage(){outage=true;},
|
||||
setProjects(next:string[]){projectIds=next;},stopped(){stopped=true;},failRelease(){releaseFails=true;},restoreRelease(){releaseFails=false;},outage(){outage=true;},
|
||||
raceResume(){resumeGate=new Promise<void>(resolve=>{releaseResume=resolve;});}};
|
||||
}
|
||||
it("hands a retained process to a new run after controller reconstruction",async()=>{
|
||||
@@ -118,7 +119,7 @@ const support=external ? {supported:true} : await getEmbeddedPostgresTestSupport
|
||||
if(prepared) await db.update(environmentLeases).set({metadata:sql`${environmentLeases.metadata} || ${JSON.stringify({runnerTaskStartup:{contextId:"settled-startup",deadlineAtMs:1,runId:f.identity.runId}})}::jsonb`}).where(eq(environmentLeases.id,f.first.lease.id));
|
||||
const calls=f.calls.length,dispatched=f.dispatched.length;
|
||||
const next=await f.acquire();
|
||||
expect(f.calls.slice(calls)).toEqual(["environmentResumeLease","status","complete","environmentAcquireLease"]);
|
||||
expect(f.calls.slice(calls)).toEqual(["environmentResumeLease","status","complete","environmentReleaseLease","environmentAcquireLease"]);
|
||||
expect(f.dispatched.slice(dispatched)).toEqual([["status",f.first.lease.providerLeaseId],["complete",f.first.lease.providerLeaseId]]);
|
||||
expect(next.lease.providerLeaseId).not.toBe(f.first.lease.providerLeaseId);
|
||||
const [old]=await db.select().from(environmentLeases).where(eq(environmentLeases.id,f.first.lease.id));
|
||||
@@ -129,7 +130,7 @@ const support=external ? {supported:true} : await getEmbeddedPostgresTestSupport
|
||||
const f=await fixture();f.unavailable();await f.nextRun();
|
||||
const calls=f.calls.length;
|
||||
await f.acquire();
|
||||
expect(f.calls.slice(calls)).toEqual(["environmentResumeLease","status","stop","status","complete","environmentAcquireLease"]);
|
||||
expect(f.calls.slice(calls)).toEqual(["environmentResumeLease","status","stop","status","complete","environmentReleaseLease","environmentAcquireLease"]);
|
||||
});
|
||||
it.each([["status",false],["complete",false],["stop",true]] as const)("keeps a stopped retained task for cleanup retry when %s fails",async(kind,running)=>{
|
||||
const f=await fixture();if(running) f.unavailable(); else f.stopped();await f.nextRun();
|
||||
@@ -143,6 +144,29 @@ const support=external ? {supported:true} : await getEmbeddedPostgresTestSupport
|
||||
await expect(f.runtime().retryPendingSandboxTeardown({environment:null,lease:pending})).resolves.toEqual({providerLeaseId:f.first.lease.providerLeaseId,state:"stopped"});
|
||||
expect(f.calls.slice(calls)).toEqual(["stop","status","complete","environmentReleaseLease"]);
|
||||
});
|
||||
it("retries lease release after completed task settlement without repeating settlement", async () => {
|
||||
const f = await fixture();
|
||||
f.stopped();
|
||||
f.failRelease();
|
||||
await f.nextRun();
|
||||
const before = f.calls.length;
|
||||
await expect(f.acquire()).rejects.toThrow("release unavailable");
|
||||
expect(f.calls.slice(before)).toEqual(["environmentResumeLease", "status", "complete", "environmentReleaseLease"]);
|
||||
const pending = (await environmentService(db).getLeaseById(f.first.lease.id))!;
|
||||
expect(pending).toMatchObject({
|
||||
status: "pending_cleanup", cleanupStatus: "failed",
|
||||
metadata: { runnerTaskDispatched: false, runnerTaskExecutionStopped: true },
|
||||
});
|
||||
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.identity.runId));
|
||||
expect(run.status).toBe("succeeded");
|
||||
f.restoreRelease();
|
||||
const retry = f.calls.length;
|
||||
await expect(f.runtime().retryPendingSandboxTeardown({ environment: null, lease: pending }))
|
||||
.resolves.toEqual({ providerLeaseId: f.first.lease.providerLeaseId, state: "stopped" });
|
||||
expect(f.calls.slice(retry)).toEqual(["environmentReleaseLease"]);
|
||||
expect(f.releaseMetadata).toHaveLength(2);
|
||||
for (const metadata of f.releaseMetadata) expect(metadata).toMatchObject({ runnerTaskDispatched: false, runnerTaskExecutionStopped: true });
|
||||
});
|
||||
it("does not allocate a replacement when the retained computer cannot be inspected",async()=>{
|
||||
const f=await fixture();f.outage();await f.nextRun();
|
||||
await expect(f.acquire()).rejects.toThrow("unreachable");
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { JsonRpcCallError, PLUGIN_RPC_ERROR_CODES, PluginWorkspaceOperationError, workspaceOperationErrorData } from "@paperclipai/plugin-sdk";
|
||||
import { isPluginWorkspaceReplay } from "../services/plugin-workspace-replay-context.js";
|
||||
import { createPluginWorkspaceFixture } from "./helpers/plugin-workspaces.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
@@ -51,7 +52,7 @@ const support = await getEmbeddedPostgresTestSupport();
|
||||
});
|
||||
const fixture = (companyId?: string) =>
|
||||
createPluginWorkspaceFixture(db, companyId);
|
||||
it.each(["ready", "lost_reply", "unconfirmed", "live_writer", "unsupported"] as const)(
|
||||
it.each(["ready", "lost_reply", "unconfirmed", "transport", "worker_lookup", "live_writer", "older_retained", "unsupported"] as const)(
|
||||
"verifies the same checkout before choosing a fresh retry (%s)", async (state) => {
|
||||
const f = await fixture();
|
||||
const [plugin] = await db.select().from(plugins).where(eq(plugins.id, f.pluginId));
|
||||
@@ -73,6 +74,15 @@ const support = await getEmbeddedPostgresTestSupport();
|
||||
status: state === "live_writer" ? "active" : "released",
|
||||
releasedAt: state === "live_writer" ? null : new Date(),
|
||||
});
|
||||
if (state === "older_retained") {
|
||||
const [older] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.input.agentId,
|
||||
issueId: f.issueId, status: "succeeded", finishedAt: new Date() }).returning();
|
||||
await db.insert(environmentLeases).values({
|
||||
companyId: f.companyId, issueId: f.issueId, heartbeatRunId: older.id,
|
||||
provider: "plugin:test:tasks", executionWorkspaceId: workspaceId, leasePolicy: "reuse_by_environment",
|
||||
status: "retained", releasedAt: null,
|
||||
});
|
||||
}
|
||||
const retryIntentId = randomUUID();
|
||||
const [retry] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.input.agentId,
|
||||
issueId: f.issueId, status: "running", contextSnapshot: { taskRetryRequestId: retryIntentId },
|
||||
@@ -80,24 +90,34 @@ const support = await getEmbeddedPostgresTestSupport();
|
||||
const fresh = { mode: "fresh_isolated_workspace", retryIntentId, sourceRunId: f.runId, retainedSourceIds: [] } as const;
|
||||
const call = vi.fn<WorkspaceWorkers["call"]>(async (...args) => {
|
||||
const result = await f.workers.call(...args) as { kind: string };
|
||||
if (state === "unconfirmed") throw new Error("checkout incomplete");
|
||||
if (state === "transport") throw new Error("connection closed");
|
||||
if (state === "unconfirmed") throw new JsonRpcCallError({
|
||||
code: PLUGIN_RPC_ERROR_CODES.WORKER_ERROR,
|
||||
message: "checkout incomplete",
|
||||
data: workspaceOperationErrorData(new PluginWorkspaceOperationError("binding_mismatch")),
|
||||
});
|
||||
return { ...result, created: false };
|
||||
});
|
||||
const workers: WorkspaceWorkers = {
|
||||
getWorker: (pluginId) => {
|
||||
if (state === "worker_lookup") throw new Error("worker lookup timed out");
|
||||
const worker = f.workers.getWorker(pluginId);
|
||||
return worker && { ...worker, supportedMethods: ["workspaceOperation", "workspacePrepare"] };
|
||||
},
|
||||
call,
|
||||
};
|
||||
const verificationStarted = Date.now();
|
||||
const result = await verifyRetryWorkspace(db, workers, retry, f.issueId, null,
|
||||
{ ...fresh, retainedSourceIds: [] });
|
||||
const result = verifyRetryWorkspace(db, workers, retry, f.issueId, null, fresh);
|
||||
if (state === "transport" || state === "worker_lookup") {
|
||||
if (state === "transport") await expect(result).rejects.toMatchObject({ kind: "transport", retryable: true });
|
||||
else await expect(result).rejects.toThrow("worker lookup timed out");
|
||||
return;
|
||||
}
|
||||
const verified = state === "ready" || state === "lost_reply";
|
||||
expect(result).toEqual(verified
|
||||
expect(await result).toEqual(verified
|
||||
? { mode: "reuse_issue_workspace", retryIntentId, sourceRunId: f.runId, workspaceId, cwd: null }
|
||||
: fresh);
|
||||
expect(f.calls.slice(1)).toHaveLength(state === "live_writer" || state === "unsupported" ? 0 : 1);
|
||||
expect(f.calls.slice(1)).toHaveLength(["live_writer", "older_retained", "unsupported"].includes(state) ? 0 : 1);
|
||||
if (f.calls.length > 1) {
|
||||
expect(call).toHaveBeenCalledExactlyOnceWith(f.pluginId, "workspaceOperation", expect.objectContaining({
|
||||
executionWorkspaceId: workspaceId, runId: retry.id, operation: expect.objectContaining({ verifyExisting: true }),
|
||||
|
||||
@@ -317,6 +317,7 @@ export function environmentRunOrchestrator(
|
||||
| { kind: "plugin_task"; leaseId: string; preparation?: true };
|
||||
/** A new controller must stop the previous plugin task before allocating a replacement. */
|
||||
recoverPluginTask?: boolean;
|
||||
replacementWorkspaceCreation?: string;
|
||||
}): Promise<EnvironmentAcquisitionResult> {
|
||||
// Step 1: Resolve environment
|
||||
const selectedEnvironment = await resolveEnvironment({
|
||||
@@ -370,6 +371,7 @@ export function environmentRunOrchestrator(
|
||||
executionWorkspaceSettings: input.executionWorkspaceSettings,
|
||||
runnerTaskProjectIds: input.runnerTaskProjectIds,
|
||||
runnerTaskResumeSessionId: input.runnerTaskResumeSessionId,
|
||||
replacementWorkspaceCreation: input.replacementWorkspaceCreation,
|
||||
adapterType: input.adapterType ?? null,
|
||||
warmRunnerIdleTimeoutMs,
|
||||
runnerReconnectGraceMs,
|
||||
|
||||
@@ -484,6 +484,7 @@ export interface EnvironmentDriverAcquireInput {
|
||||
executionWorkspaceSettings: IssueExecutionWorkspaceSettings | null;
|
||||
runnerTaskProjectIds?: readonly string[];
|
||||
runnerTaskResumeSessionId?: string | null;
|
||||
replacementWorkspaceCreation?: string;
|
||||
/**
|
||||
* The harness/adapter type for this run (the agent's adapter). Drivers that
|
||||
* materialize a per-run sandbox use it to select the runtime image so a single
|
||||
@@ -3473,27 +3474,43 @@ function createPluginEnvironmentDriver(
|
||||
};
|
||||
}
|
||||
|
||||
async function releasePluginRunLease(input: EnvironmentDriverReleaseInput, executionStopObserved = false) {
|
||||
await db.update(environmentLeases).set({ metadata: sql`coalesce(${environmentLeases.metadata}, '{}'::jsonb) || '{"pluginLeaseCleanupMethod":"release"}'::jsonb` }).where(and(eq(environmentLeases.id, input.lease.id), eq(environmentLeases.companyId, input.lease.companyId)));
|
||||
async function cleanupPluginRunLease(input: { environment: Environment | null; lease: EnvironmentLease },
|
||||
executionStopObserved = false) {
|
||||
const settled = await settleDispatchedEnvironmentTask(db, workerManager, input.lease, { executionStopObserved });
|
||||
const [current] = await db.select().from(environmentLeases).where(and(eq(environmentLeases.id, input.lease.id),
|
||||
eq(environmentLeases.companyId, input.lease.companyId)));
|
||||
if (!current || current.providerLeaseId !== input.lease.providerLeaseId) throw new Error("Plugin lease cleanup ownership changed");
|
||||
const lease = toEnvironmentLeaseSnapshot(current);
|
||||
const { plugin, driverKey, driverConfig } = await resolvePluginDriverForRelease({ ...input, lease });
|
||||
const environmentId = input.environment?.id ?? readString(lease.metadata?.environmentId);
|
||||
if (!environmentId) throw new Error("Plugin lease cleanup has no saved environment identity");
|
||||
const receipt = await workerManager.call(plugin.id, lease.metadata?.pluginLeaseCleanupMethod === "destroy" ? "environmentDestroyLease" : "environmentReleaseLease", {
|
||||
driverKey, companyId: lease.companyId, environmentId, issueId: lease.issueId,
|
||||
config: driverConfig, providerLeaseId: lease.providerLeaseId, leaseMetadata: lease.metadata ?? undefined,
|
||||
});
|
||||
const taskStopped = settled !== false ||
|
||||
(input.lease.metadata?.runnerTaskLease === true && input.lease.metadata?.runnerTaskDispatched !== true && !input.lease.metadata?.runnerTaskStartup && !input.lease.metadata?.runnerTaskStartupCleanup);
|
||||
if (settled !== "completed") {
|
||||
const { plugin, driverKey, driverConfig } = await resolvePluginDriverForRelease(input);
|
||||
await workerManager.call(plugin.id, "environmentReleaseLease", {
|
||||
driverKey,
|
||||
companyId: input.lease.companyId,
|
||||
environmentId: input.environment.id,
|
||||
issueId: input.lease.issueId,
|
||||
config: driverConfig,
|
||||
providerLeaseId: input.lease.providerLeaseId,
|
||||
leaseMetadata: input.lease.metadata ?? undefined,
|
||||
});
|
||||
(lease.metadata?.runnerTaskLease === true && lease.metadata?.runnerTaskDispatched !== true && !lease.metadata?.runnerTaskStartup && !lease.metadata?.runnerTaskStartupCleanup);
|
||||
return remoteTerminationReceipt(lease, receipt) ? receipt
|
||||
: taskStopped ? { providerLeaseId: lease.providerLeaseId, state: "stopped" } : undefined;
|
||||
}
|
||||
|
||||
async function releasePluginRunLease(input: EnvironmentDriverReleaseInput, executionStopObserved = false) {
|
||||
const scope = and(eq(environmentLeases.id, input.lease.id), eq(environmentLeases.companyId, input.lease.companyId));
|
||||
await db.update(environmentLeases).set({ metadata: sql`coalesce(${environmentLeases.metadata}, '{}'::jsonb) || '{"pluginLeaseCleanupMethod":"release"}'::jsonb` }).where(scope);
|
||||
let receipt: Awaited<ReturnType<typeof cleanupPluginRunLease>>;
|
||||
try {
|
||||
receipt = await cleanupPluginRunLease(input, executionStopObserved);
|
||||
} catch (error) {
|
||||
await writeEnvironmentCleanup(db, tx => tx.update(environmentLeases).set({ status: "pending_cleanup", cleanupStatus: "failed",
|
||||
failureReason: "release_cleanup_failed", releasedAt: new Date(), updatedAt: new Date(),
|
||||
}).where(and(scope, inArray(environmentLeases.status, ["active", "retained"]),
|
||||
eq(environmentLeases.providerLeaseId, input.lease.providerLeaseId!))),
|
||||
{ companyId: input.lease.companyId, runId: input.lease.heartbeatRunId ?? undefined });
|
||||
throw error;
|
||||
}
|
||||
return await environmentsSvc.releaseLease(input.lease.id, input.status, {
|
||||
cleanupStatus: "success",
|
||||
...(taskStopped ? { remoteExecutionTermination: remoteTerminationReceipt(input.lease,
|
||||
{ providerLeaseId: input.lease.providerLeaseId, state: "stopped" }) } : {}),
|
||||
remoteExecutionTermination: remoteTerminationReceipt(input.lease, receipt),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -3574,7 +3591,9 @@ function createPluginEnvironmentDriver(
|
||||
companyId: input.companyId, environmentId: input.environment.id, heartbeatRunId: input.heartbeatRunId,
|
||||
issueId: input.issueId, executionWorkspaceId: input.executionWorkspaceId,
|
||||
provider: `plugin:${parsed.config.pluginKey}:${parsed.config.driverKey}`, pluginId: plugin.id, driverKey: parsed.config.driverKey,
|
||||
config: {}, timeoutMs: rpcTimeoutMs, requestKey: fingerprint,
|
||||
config: {}, timeoutMs: rpcTimeoutMs,
|
||||
requestKey: input.replacementWorkspaceCreation && input.replacementWorkspaceCreation === input.executionWorkspaceId
|
||||
? `${fingerprint}:${input.executionWorkspaceId}` : fingerprint,
|
||||
metadata: { driver: "plugin", pluginId: plugin.id, pluginKey: parsed.config.pluginKey, driverKey: parsed.config.driverKey,
|
||||
...(input.purpose ? { purpose: input.purpose, loginCommandKey: input.loginCommandKey } : {}) },
|
||||
}, identity => workerManager.call(plugin.id, "environmentAcquireLease", {
|
||||
@@ -3694,18 +3713,7 @@ function createPluginEnvironmentDriver(
|
||||
} catch { return false; }
|
||||
},
|
||||
|
||||
async retryPendingSandboxTeardown(input) {
|
||||
const taskStopped = await settleDispatchedEnvironmentTask(db, workerManager, input.lease) ||
|
||||
(input.lease.metadata?.runnerTaskLease === true && input.lease.metadata?.runnerTaskDispatched !== true && !input.lease.metadata?.runnerTaskStartup && !input.lease.metadata?.runnerTaskStartupCleanup);
|
||||
const { plugin, driverKey, driverConfig } = await resolvePluginDriverForRelease(input);
|
||||
const environmentId = input.environment?.id ?? readString(input.lease.metadata?.environmentId);
|
||||
if (!environmentId) throw new Error("Plugin lease cleanup has no saved environment identity");
|
||||
const receipt = await workerManager.call(plugin.id, input.lease.metadata?.pluginLeaseCleanupMethod === "destroy" ? "environmentDestroyLease" : "environmentReleaseLease", {
|
||||
driverKey, companyId: input.lease.companyId, environmentId, issueId: input.lease.issueId,
|
||||
config: driverConfig, providerLeaseId: input.lease.providerLeaseId, leaseMetadata: input.lease.metadata ?? undefined,
|
||||
});
|
||||
return taskStopped ? { providerLeaseId: input.lease.providerLeaseId, state: "stopped" } : receipt;
|
||||
},
|
||||
retryPendingSandboxTeardown: cleanupPluginRunLease,
|
||||
|
||||
async realizeWorkspace(input) {
|
||||
const { plugin, pluginKey, driverKey, driverConfig } = await resolvePluginDriverForRelease({
|
||||
@@ -4045,6 +4053,7 @@ export function environmentRuntimeService(
|
||||
executionWorkspaceSettings?: IssueExecutionWorkspaceSettings | null;
|
||||
runnerTaskProjectIds?: readonly string[];
|
||||
runnerTaskResumeSessionId?: string | null;
|
||||
replacementWorkspaceCreation?: string;
|
||||
/** The agent's adapter type for this run (mixed-harness environments). */
|
||||
adapterType?: string | null;
|
||||
resolveSnapshotIdentity?: boolean;
|
||||
@@ -4091,6 +4100,7 @@ export function environmentRuntimeService(
|
||||
executionWorkspaceSettings: input.executionWorkspaceSettings ?? null,
|
||||
runnerTaskProjectIds: input.runnerTaskProjectIds,
|
||||
runnerTaskResumeSessionId: input.runnerTaskResumeSessionId,
|
||||
replacementWorkspaceCreation: input.replacementWorkspaceCreation,
|
||||
purpose: input.purpose,
|
||||
loginCommandKey: input.loginCommandKey,
|
||||
adapterType: input.adapterType ?? null,
|
||||
|
||||
@@ -565,13 +565,15 @@ export async function executeEnvironmentTask(db: Db, workers: Pick<PluginWorkerM
|
||||
const receipt=remoteTerminationReceipt(owner,{providerLeaseId:taskId,state:"stopped"});
|
||||
if(!receipt) continue;
|
||||
await writeEnvironmentCleanup(db, tx => tx.update(environmentLeases).set({
|
||||
...(owner.status === "retained" ? {status:"released" as const,releasedAt:new Date(),cleanupStatus:"success" as const}:{}),
|
||||
...(owner.id !== lease.id && owner.status === "retained" ? {status:"released" as const,releasedAt:new Date(),cleanupStatus:"success" as const}:{}),
|
||||
metadata:sql`coalesce(${environmentLeases.metadata}, '{}'::jsonb) || ${JSON.stringify({remoteExecutionTermination:receipt,runnerTaskExecutionStopped:true})}::jsonb`,
|
||||
}).where(and(eq(environmentLeases.id,owner.id),eq(environmentLeases.providerLeaseId,taskId))), { companyId: owner.companyId, runId: owner.heartbeatRunId ?? undefined });
|
||||
}
|
||||
}
|
||||
await db.update(environmentLeases).set({
|
||||
metadata: sql`coalesce(${environmentLeases.metadata}, '{}'::jsonb) || '{"runnerTaskDispatched":false}'::jsonb`,
|
||||
metadata: sql`(case when ${environmentLeases.metadata}->>'runnerTaskPreparationStopped' = 'true'
|
||||
then coalesce(${environmentLeases.metadata}, '{}'::jsonb) - 'runnerTaskStartup'
|
||||
else coalesce(${environmentLeases.metadata}, '{}'::jsonb) end) || '{"runnerTaskDispatched":false}'::jsonb`,
|
||||
}).where(and(eq(environmentLeases.id, lease.id), eq(environmentLeases.companyId, input.companyId),
|
||||
eq(environmentLeases.providerLeaseId, taskId), sql`${environmentLeases.metadata}->>'runnerTaskExecutionStopped' = 'true'`));
|
||||
}
|
||||
@@ -597,8 +599,11 @@ export async function settleDispatchedEnvironmentTask(db: Db, workers: PluginWor
|
||||
metadata: sql`coalesce(${environmentLeases.metadata}, '{}'::jsonb) || '{"runnerTaskRetiring":true}'::jsonb`,
|
||||
}).where(and(eq(environmentLeases.id, lease.id), eq(environmentLeases.companyId, lease.companyId)))
|
||||
.returning({ metadata: environmentLeases.metadata, heartbeatRunId: environmentLeases.heartbeatRunId, taskId: environmentLeases.providerLeaseId });
|
||||
const startup = current?.metadata?.runnerTaskStartup;
|
||||
const startupCleanup = current?.metadata?.runnerTaskStartupCleanup as { requiresPreparationStop?: boolean } | undefined;
|
||||
const startup = current?.metadata?.runnerTaskStartup;
|
||||
if (startupCleanup && current?.metadata?.runnerTaskExecutionStopped === true && current.metadata.runnerTaskDispatched === false &&
|
||||
!current.metadata.runnerTaskAdmissionRejection && !startup &&
|
||||
(!startupCleanup.requiresPreparationStop || current.metadata.runnerTaskPreparationStopped === true)) return "completed";
|
||||
if (current?.metadata?.runnerTaskDispatched !== true && !startup && !startupCleanup) {
|
||||
if (current?.metadata) await settlePluginTaskCredentialHandoff(db, {
|
||||
companyId: lease.companyId, leaseId: lease.id, expectedTaskId: current.taskId ?? undefined,
|
||||
|
||||
@@ -20,6 +20,7 @@ import { taskRetryRunAncestry } from "./recovery-run-ancestry.js";
|
||||
import { parseNativeExecutionInput } from "../vendor/paperclip-runner/index.js";
|
||||
import { nativeExecutionUsesTaskWorkspace } from "./heartbeat/external-provider.js";
|
||||
import { preparePluginWorkspace, type WorkspaceWorkers } from "./plugin-workspaces.js";
|
||||
import { findWorkspaceProviderFailure } from "./workspace-provider-failure.js";
|
||||
import { readFreshContinuationReadiness } from "./fresh-continuation-fallback.js";
|
||||
|
||||
export type ContinuationReadiness =
|
||||
@@ -522,6 +523,9 @@ export async function verifyRetryWorkspace(
|
||||
issueId: string,
|
||||
responsibleUserId: string | null,
|
||||
readiness: ContinuationReadiness,
|
||||
options?: {
|
||||
selectCandidate?: (workspace: typeof executionWorkspaces.$inferSelect) => Promise<boolean>;
|
||||
},
|
||||
): Promise<ContinuationReadiness> {
|
||||
if (readiness.mode !== "fresh_isolated_workspace" ||
|
||||
run.contextSnapshot?.taskRetryRequestId !== readiness.retryIntentId) return readiness;
|
||||
@@ -541,7 +545,7 @@ export async function verifyRetryWorkspace(
|
||||
companyId: run.companyId, issueId, agentId: run.agentId, successorRunId: run.id,
|
||||
source, cause: null, evidence: null,
|
||||
sourceRetryIntentId: source.contextSnapshot?.taskRetryRequestId,
|
||||
verifyProviderWorkspace: workers ? async workspace => {
|
||||
verifyProviderWorkspace: options?.selectCandidate ?? (workers ? async workspace => {
|
||||
try {
|
||||
return Boolean(await preparePluginWorkspace(db, workers, {
|
||||
companyId: run.companyId, issueId, agentId: run.agentId, runId: run.id, responsibleUserId,
|
||||
@@ -549,15 +553,22 @@ export async function verifyRetryWorkspace(
|
||||
existingWorkspaceId: workspace.id, mode: workspace.mode, strategy: workspace.strategyType,
|
||||
verifyExisting: true,
|
||||
}));
|
||||
} catch {
|
||||
} catch (error) {
|
||||
const failure = findWorkspaceProviderFailure(error);
|
||||
if (failure?.reason !== "binding_mismatch" && failure?.reason !== "unsupported_operation") throw error;
|
||||
return false;
|
||||
}
|
||||
} : undefined,
|
||||
} : undefined),
|
||||
});
|
||||
if (!retained) return readiness;
|
||||
const reuse: ContinuationReadiness = { mode: "reuse_issue_workspace", retryIntentId: readiness.retryIntentId,
|
||||
sourceRunId: source.id, workspaceId: retained.id, cwd: retained.cwd };
|
||||
await persistActivity(db, { companyId: run.companyId, actorType: "system", actorId: "task-recovery",
|
||||
action: "issue.retry_workspace_reused", entityType: "issue", entityId: issueId, runId: run.id, details: reuse });
|
||||
if (!options?.selectCandidate) await recordVerifiedRetryWorkspace(db, run, issueId, reuse);
|
||||
return reuse;
|
||||
}
|
||||
|
||||
export async function recordVerifiedRetryWorkspace(db: Db, run: { companyId: string; id: string }, issueId: string,
|
||||
readiness: Extract<ContinuationReadiness, { mode: "reuse_issue_workspace" }>) {
|
||||
await persistActivity(db, { companyId: run.companyId, actorType: "system", actorId: "task-recovery",
|
||||
action: "issue.retry_workspace_reused", entityType: "issue", entityId: issueId, runId: run.id, details: readiness });
|
||||
}
|
||||
+184
-150
@@ -34,7 +34,7 @@ import { registerCancelledWorkspacePreservationWork } from "./native-runtime/nat
|
||||
import { readUpgradeReadiness } from "./heartbeat/upgrade-readiness.js";
|
||||
import { DELIVERY_QUEUES } from "./delivery-work-notifications.js";
|
||||
import { writeExecutionControl } from "./execution-control-notifications.js";
|
||||
import { readFreshWorkspaceReadiness, verifyRetryWorkspace } from "./fresh-workspace-continuation.js";
|
||||
import { readFreshWorkspaceReadiness, verifyRetryWorkspace, type ContinuationReadiness } from "./fresh-workspace-continuation.js";
|
||||
import { settleFailedNativeRecoveryPreparation } from "./native-runtime/native-recovery-preparation-failure.js";
|
||||
import { promptInjectionPreventionService } from "./prompt-injection-prevention.js";
|
||||
import { createHeartbeatAgentStatus } from "./heartbeat/agent-status.js";
|
||||
@@ -49,7 +49,7 @@ import { createHeartbeatInstructionCopyRelease, createHeartbeatInstructionWorkin
|
||||
import { AgentDirectoryPreparationDeferral, deferRunForBusyAgentDirectory, prepareWithAgentDirectoryDeferral } from "./heartbeat/agent-directory-busy.js";
|
||||
import { preparePredecessorRetirementFallback } from "./heartbeat/predecessor-retirement-fallback.js";
|
||||
import { prepareHeartbeatRuntimeServiceIntents, startHeartbeatRuntimeServices, recordHeartbeatAdapterRuntimeServices } from "./heartbeat/runtime-services.js";
|
||||
import { bindHeartbeatRunnerHomeMode, prepareHeartbeatExecutionConfiguration } from "./heartbeat/execution-configuration.js";
|
||||
import { bindHeartbeatRunnerHomeMode, buildHeartbeatWorkspaceConfiguration, prepareHeartbeatExecutionConfiguration } from "./heartbeat/execution-configuration.js";
|
||||
import { adoptsLiveRemoteRunner } from "./heartbeat/live-runner-adoption.js";
|
||||
import { createRunLogWriterAuthority } from "./heartbeat/run-log-authority.js";
|
||||
import { watchShutdownHandoff } from "./heartbeat/shutdown-handoff.js";
|
||||
@@ -60,7 +60,7 @@ import { prepareHeartbeatSession, resolveHeartbeatAdapterSession } from "./heart
|
||||
import { prepareHeartbeatTaskContext } from "./heartbeat/task-context.js";
|
||||
import { selectHeartbeatEnvironment } from "./heartbeat/environment-selection.js";
|
||||
import { reconcileMuseNativeTerminalRun } from "./native-runtime/muse-terminal-reconciliation.js";
|
||||
import { prepareHeartbeatWorkspace } from "./heartbeat/workspace-preparation.js";
|
||||
import { prepareHeartbeatWorkspace, RetryWorkspaceRefused } from "./heartbeat/workspace-preparation.js";
|
||||
import { nativeRecoveryClaimRelinquished, ownNativeExecutionLeaseLapsed, releaseLapsedNativeRestartClaim } from "./native-runtime/native-execution-lease.js";
|
||||
import { executeHeartbeatRuntime, NativeSessionResumeScheduledError, NativeWorkspaceFinalizeScheduledError } from "./heartbeat/runtime-execution.js";
|
||||
import { preserveNativeFailedWorkspace } from "./native-runtime/native-failed-workspace-recovery.js";
|
||||
@@ -2565,10 +2565,20 @@ export function heartbeatService(
|
||||
// Initialization has persisted the active context, including an explicit
|
||||
// absence of identity inherited from an automatic continuation.
|
||||
responsibleUserId = identityContext.responsibleUserId;
|
||||
if (issueId && freshWorkspace?.mode === "fresh_isolated_workspace") {
|
||||
const fallback = freshWorkspace;
|
||||
freshWorkspace = await timePreparation("workspace_verification", () =>
|
||||
verifyRetryWorkspace(db, options.pluginWorkerManager, run, issueId, responsibleUserId, fallback));
|
||||
const freshRetryFallback = freshWorkspace?.mode === "fresh_isolated_workspace" ? freshWorkspace : undefined;
|
||||
let retryWorkspaceCandidate: Extract<ContinuationReadiness, { mode: "reuse_issue_workspace" }> | undefined;
|
||||
if (issueId && freshRetryFallback && !externalProvider &&
|
||||
parseObject(context.retryWorkspaceRefusal).intentId !== freshRetryFallback.retryIntentId) {
|
||||
let verifyManagedWorkspace = false;
|
||||
const retained = await timePreparation("workspace_verification", () =>
|
||||
verifyRetryWorkspace(db, options.pluginWorkerManager, run, issueId, responsibleUserId, freshRetryFallback,
|
||||
agent.adapterType === "paperclip_runner" && options.pluginWorkerManager
|
||||
? { selectCandidate: async () => { verifyManagedWorkspace = true; return true; } } : undefined));
|
||||
if (retained.mode === "reuse_issue_workspace") {
|
||||
freshWorkspace = retained;
|
||||
if (verifyManagedWorkspace)
|
||||
retryWorkspaceCandidate = retained;
|
||||
}
|
||||
}
|
||||
run = {
|
||||
...run,
|
||||
@@ -2753,29 +2763,33 @@ export function heartbeatService(
|
||||
legacyUseProjectWorkspace:
|
||||
issueAssigneeOverrides?.useProjectWorkspace ?? null,
|
||||
});
|
||||
const requestedExecutionWorkspaceMode = freshWorkspace?.mode === "fresh_isolated_workspace"
|
||||
? "isolated_workspace" : trustPreset.kind === "low_trust_review" &&
|
||||
resolvedExecutionWorkspaceMode === "shared_workspace"
|
||||
? "isolated_workspace"
|
||||
: resolvedExecutionWorkspaceMode;
|
||||
const issueRef = issueContext
|
||||
? {
|
||||
id: issueContext.id,
|
||||
identifier: issueContext.identifier,
|
||||
title: issueContext.title,
|
||||
status: issueContext.status,
|
||||
priority: issueContext.priority,
|
||||
workMode: issueContext.workMode,
|
||||
conversationAgentId: issueContext.conversationAgentId,
|
||||
conversationUserId: issueContext.conversationUserId,
|
||||
reviewPolicy: issueContext.reviewPolicy,
|
||||
description: issueContext.description,
|
||||
projectId: issueContext.projectId,
|
||||
projectWorkspaceId: issueContext.projectWorkspaceId,
|
||||
executionWorkspaceId: freshWorkspace?.mode === "fresh_isolated_workspace" ? null : issueContext.executionWorkspaceId,
|
||||
executionWorkspacePreference: freshWorkspace?.mode === "fresh_isolated_workspace" ? "create_new" : issueContext.executionWorkspacePreference,
|
||||
}
|
||||
: null;
|
||||
const workspaceInputs = () => {
|
||||
const requestedExecutionWorkspaceMode = freshWorkspace?.mode === "fresh_isolated_workspace"
|
||||
? "isolated_workspace" : trustPreset.kind === "low_trust_review" &&
|
||||
resolvedExecutionWorkspaceMode === "shared_workspace"
|
||||
? "isolated_workspace"
|
||||
: resolvedExecutionWorkspaceMode;
|
||||
const issueRef = issueContext
|
||||
? {
|
||||
id: issueContext.id,
|
||||
identifier: issueContext.identifier,
|
||||
title: issueContext.title,
|
||||
status: issueContext.status,
|
||||
priority: issueContext.priority,
|
||||
workMode: issueContext.workMode,
|
||||
conversationAgentId: issueContext.conversationAgentId,
|
||||
conversationUserId: issueContext.conversationUserId,
|
||||
reviewPolicy: issueContext.reviewPolicy,
|
||||
description: issueContext.description,
|
||||
projectId: issueContext.projectId,
|
||||
projectWorkspaceId: issueContext.projectWorkspaceId,
|
||||
executionWorkspaceId: freshWorkspace?.mode === "fresh_isolated_workspace" ? null : issueContext.executionWorkspaceId,
|
||||
executionWorkspacePreference: freshWorkspace?.mode === "fresh_isolated_workspace" ? "create_new" : issueContext.executionWorkspacePreference,
|
||||
}
|
||||
: null;
|
||||
return { requestedExecutionWorkspaceMode, issueRef };
|
||||
};
|
||||
let { requestedExecutionWorkspaceMode, issueRef } = workspaceInputs();
|
||||
const storedLedgerScope = parseObject(parseObject(run.usageJson).ledgerScope);
|
||||
const runLedgerScope = Object.keys(storedLedgerScope).length > 0
|
||||
? storedLedgerScope
|
||||
@@ -2791,18 +2805,144 @@ export function heartbeatService(
|
||||
policy: { trustPreset, experimentalInstanceSettings, recovering: Boolean(runOptions.nativeRestartRecovery || runOptions.nativeLeaseOwner) },
|
||||
services: { issuesSvc, getPinnedSkillTestContext },
|
||||
}));
|
||||
const sourceWorkspaceConfig = buildExecutionWorkspaceAdapterConfig({
|
||||
agentConfig: config,
|
||||
projectPolicy: projectExecutionWorkspacePolicy,
|
||||
issueSettings: issueExecutionWorkspaceSettings,
|
||||
mode: requestedExecutionWorkspaceMode,
|
||||
legacyUseProjectWorkspace:
|
||||
issueAssigneeOverrides?.useProjectWorkspace ?? null,
|
||||
adapterConfigOverrides: {
|
||||
...Object.fromEntries(Object.entries(issueAssigneeOverrides?.adapterConfig ?? {}).filter(([key]) => requestedAiBinding?.mode !== "router" || !["provider", "acpxAgent", "model", "modelReasoningEffort", "reasoningEffort", "effort", "variant"].includes(key))),
|
||||
...(requestedAiBinding?.mode === "router" ? parseObject(parseObject(context.aiRouterSelection).runtimeConfig) : {}),
|
||||
const selectWorkspace = async () => {
|
||||
const sourceWorkspaceConfig = buildExecutionWorkspaceAdapterConfig({
|
||||
agentConfig: config,
|
||||
projectPolicy: projectExecutionWorkspacePolicy,
|
||||
issueSettings: issueExecutionWorkspaceSettings,
|
||||
mode: requestedExecutionWorkspaceMode,
|
||||
legacyUseProjectWorkspace:
|
||||
issueAssigneeOverrides?.useProjectWorkspace ?? null,
|
||||
adapterConfigOverrides: {
|
||||
...Object.fromEntries(Object.entries(issueAssigneeOverrides?.adapterConfig ?? {}).filter(([key]) => requestedAiBinding?.mode !== "router" || !["provider", "acpxAgent", "model", "modelReasoningEffort", "reasoningEffort", "effort", "variant"].includes(key))),
|
||||
...(requestedAiBinding?.mode === "router" ? parseObject(parseObject(context.aiRouterSelection).runtimeConfig) : {}),
|
||||
},
|
||||
});
|
||||
const selection = await timePreparation("environment_selection", () => selectHeartbeatEnvironment(db, {
|
||||
run, agent,
|
||||
task: { issueRef, issueContext, issueId, context, executionProjectId, workspaceProjectContext, boundSourceWorkspace,
|
||||
freshWorkspace, hasSelectedWorkspaceSource: Boolean(selectedWorkspaceSource), taskWorkspaceIntent: taskWorkspaceIntentRow?.intent ?? null, responsibleUserId },
|
||||
policy: {
|
||||
trustPreset, requestedExecutionWorkspaceMode, isolatedWorkspacesEnabled: runtimeWorkspaceSelectionEnabled,
|
||||
projectExecutionWorkspacePolicy, issueExecutionWorkspaceSettings,
|
||||
},
|
||||
config: { config, sourceWorkspaceConfig, issueAssigneeOverrides },
|
||||
services: { executionWorkspacesSvc, environmentsSvc, instanceSettings, findSharedWorkspaceHolder, findExecutionWorkspaceHolder },
|
||||
}));
|
||||
return { ...selection, sourceWorkspaceConfig };
|
||||
};
|
||||
let workspaceSelection = await selectWorkspace();
|
||||
const executionConfiguration = await timePreparation("execution_configuration", () => prepareHeartbeatExecutionConfiguration(db, {
|
||||
run, agent,
|
||||
task: { issueId, context, responsibleUserId, acceptedPlanContinuationWake, acceptedPlanWakeRoutingDecision, taskProjectContext, routineEnvContext, pinnedSkillTest,
|
||||
adoptingLiveRunner: adoptsLiveRemoteRunner({ run, restartRecovery: runOptions.nativeRestartRecovery, leaseOwner: runOptions.nativeLeaseOwner }) },
|
||||
config: { sourceWorkspaceConfig: workspaceSelection.sourceWorkspaceConfig, aiBinding },
|
||||
policy: { trustPreset, useIsolatedTaskDirectory: workspaceSelection.useIsolatedTaskDirectory,
|
||||
explicitTaskDirectory: workspaceSelection.explicitTaskDirectory, requestedExecutionWorkspaceMode },
|
||||
environment: workspaceSelection,
|
||||
services: { secretsSvc, companySkills, appendRunEvent, finalizeAiConnectionBusyDeferral },
|
||||
onFailureSecretsResolved: (readSecrets) => { readFailureReportSecrets = () => [...readSecrets(), ...identityRedactor.values]; },
|
||||
onManagedAiRuntimeAcquired: (runtime) => { managedAiRuntime = runtime; },
|
||||
onIdentityRedactorPrepared: (redactor) => { identityRedactor = redactor; },
|
||||
}));
|
||||
if (!executionConfiguration.prepared) return;
|
||||
let { mergedConfig, configSnapshot } = executionConfiguration;
|
||||
const { runScopedMentionedSkillKeys, githubSelection, useHostGitHub,
|
||||
resolvedConfig, configuredTaskEnvironment, secretKeys, secretManifest,
|
||||
runtimeSkillEntries, nativeRunnerPreparationSpans, agentIdentity, observability,
|
||||
} = executionConfiguration;
|
||||
let { runtimeConfig } = executionConfiguration;
|
||||
const resolvedFailureSecrets = readFailureReportSecrets();
|
||||
readFailureReportSecrets = () => [
|
||||
...resolvedFailureSecrets,
|
||||
...identityRedactor.values,
|
||||
...collectRunFailureSecretValues(runtimeConfig.env, secretKeys),
|
||||
];
|
||||
const prepareSession = () => timePreparation("session", () => prepareHeartbeatSession(db, {
|
||||
run, agent,
|
||||
task: { context, issueRef, issueContext, taskProjectContext, workspaceProjectContext, wakeCommentId, acceptedPlanContinuationWake, acceptedPlanWakeRoutingDecision },
|
||||
session: { taskSession, taskSessionDecodedParams, explicitResumeSessionParams, explicitResumeSessionDisplayId, sessionCodec },
|
||||
config: {
|
||||
runtimeConfig, agentIdentity, issueAssigneeOverrides, originalAiIssueOverrides,
|
||||
managedAiRuntime, aiBinding, requestedAiBinding, secretManifest, runtimeSkillEntries, routineEnvContext,
|
||||
},
|
||||
});
|
||||
workspace: {
|
||||
...workspaceSelection, requestedExecutionWorkspaceMode, projectExecutionWorkspacePolicy, issueExecutionWorkspaceSettings,
|
||||
},
|
||||
environment: workspaceSelection,
|
||||
getLatestAgentConfigRevision,
|
||||
}));
|
||||
let preparedSession = await prepareSession();
|
||||
const prepareWorkspace = () => timePreparation("workspace", () => prepareHeartbeatWorkspace(db, {
|
||||
run, agent, controllerLease, nativeRunnerPreparationSpans, signal: executionControl.controller.signal,
|
||||
task: {
|
||||
issueRef,
|
||||
issueId,
|
||||
wasIssueAssignee: issueContext?.assigneeAgentId === agent.id,
|
||||
context,
|
||||
responsibleUserId,
|
||||
previousSessionParams: preparedSession.previousSessionParams,
|
||||
retryWorkspaceCandidate,
|
||||
workspaceSelection: taskWorkspaceIntentRow?.intent ?? null,
|
||||
freshWorkspace,
|
||||
},
|
||||
policy: {
|
||||
trustPreset,
|
||||
isolatedWorkspacesEnabled: runtimeWorkspaceSelectionEnabled,
|
||||
effectiveExecutionWorkspaceMode: workspaceSelection.effectiveExecutionWorkspaceMode,
|
||||
requestedExecutionWorkspaceMode,
|
||||
useIsolatedTaskDirectory: workspaceSelection.useIsolatedTaskDirectory,
|
||||
nativeChatWorkspaceScope: workspaceSelection.nativeChatWorkspaceScope,
|
||||
projectExecutionWorkspacePolicy,
|
||||
issueExecutionWorkspaceSettings,
|
||||
environmentExecutionWorkspaceSettings,
|
||||
executionPolicy: workspaceSelection.executionPolicy,
|
||||
},
|
||||
environment: workspaceSelection,
|
||||
config: {
|
||||
mergedConfig,
|
||||
configSnapshot,
|
||||
secretManifest,
|
||||
managedAiCredentialIdentity: managedAiRuntime?.identity,
|
||||
runnerTaskResumeSessionId: run.nativeSessionId ?? preparedSession.taskResumeSessionId,
|
||||
},
|
||||
reuse: {
|
||||
...workspaceSelection,
|
||||
runOptions,
|
||||
},
|
||||
services: {
|
||||
workspaceWorkers: options.pluginWorkerManager,
|
||||
envOrchestrator,
|
||||
executionWorkspacesSvc,
|
||||
workspaceOperationsSvc,
|
||||
resolveWorkspaceForRun,
|
||||
resolveReusedGitWorkspaceAnchor,
|
||||
appendRunEvent,
|
||||
},
|
||||
}));
|
||||
let preparedWorkspace;
|
||||
try {
|
||||
preparedWorkspace = await prepareWorkspace();
|
||||
} catch (error) {
|
||||
if (!(error instanceof RetryWorkspaceRefused) || !freshRetryFallback) throw error;
|
||||
retryWorkspaceCandidate = undefined;
|
||||
freshWorkspace = freshRetryFallback;
|
||||
({ requestedExecutionWorkspaceMode, issueRef } = workspaceInputs());
|
||||
const selected = await selectWorkspace();
|
||||
if (selected.selectedEnvironmentId !== workspaceSelection.selectedEnvironmentId)
|
||||
throw new Error("Retry environment changed during workspace verification");
|
||||
workspaceSelection = selected;
|
||||
({ mergedConfig, configSnapshot } = buildHeartbeatWorkspaceConfiguration(
|
||||
{ ...selected.sourceWorkspaceConfig, homeMode: mergedConfig.homeMode },
|
||||
{ trustPreset, useIsolatedTaskDirectory: selected.useIsolatedTaskDirectory,
|
||||
explicitTaskDirectory: selected.explicitTaskDirectory, requestedExecutionWorkspaceMode },
|
||||
selected.selectedEnvironmentId));
|
||||
runtimeConfig = { ...runtimeConfig };
|
||||
if (Object.hasOwn(mergedConfig, "workspaceStrategy")) runtimeConfig.workspaceStrategy = mergedConfig.workspaceStrategy;
|
||||
else delete runtimeConfig.workspaceStrategy;
|
||||
preparedSession = await prepareSession();
|
||||
preparedWorkspace = await prepareWorkspace();
|
||||
}
|
||||
const {
|
||||
persistedRunnerProfile,
|
||||
persistedNativeExecutionInput,
|
||||
@@ -2825,58 +2965,9 @@ export function heartbeatService(
|
||||
materializeNativeChatWorktree,
|
||||
explicitTaskDirectory,
|
||||
useIsolatedTaskDirectory,
|
||||
} = await timePreparation("environment_selection", () => selectHeartbeatEnvironment(db, {
|
||||
run, agent,
|
||||
task: { issueRef, issueContext, issueId, context, executionProjectId, workspaceProjectContext, boundSourceWorkspace,
|
||||
freshWorkspace, hasSelectedWorkspaceSource: Boolean(selectedWorkspaceSource), taskWorkspaceIntent: taskWorkspaceIntentRow?.intent ?? null, responsibleUserId },
|
||||
policy: {
|
||||
trustPreset, requestedExecutionWorkspaceMode, isolatedWorkspacesEnabled: runtimeWorkspaceSelectionEnabled,
|
||||
projectExecutionWorkspacePolicy, issueExecutionWorkspaceSettings,
|
||||
},
|
||||
config: { config, sourceWorkspaceConfig, issueAssigneeOverrides },
|
||||
services: { executionWorkspacesSvc, environmentsSvc, instanceSettings, findSharedWorkspaceHolder, findExecutionWorkspaceHolder },
|
||||
}));
|
||||
sourceWorkspaceConfig,
|
||||
} = workspaceSelection;
|
||||
let nativeChatExpectedCwd = selectedNativeChatExpectedCwd;
|
||||
const executionConfiguration = await timePreparation("execution_configuration", () => prepareHeartbeatExecutionConfiguration(db, {
|
||||
run, agent,
|
||||
task: { issueId, context, responsibleUserId, acceptedPlanContinuationWake, acceptedPlanWakeRoutingDecision, taskProjectContext, routineEnvContext, pinnedSkillTest,
|
||||
adoptingLiveRunner: adoptsLiveRemoteRunner({ run, restartRecovery: runOptions.nativeRestartRecovery, leaseOwner: runOptions.nativeLeaseOwner }) },
|
||||
config: { sourceWorkspaceConfig, aiBinding },
|
||||
policy: { trustPreset, useIsolatedTaskDirectory, explicitTaskDirectory, requestedExecutionWorkspaceMode },
|
||||
environment: { selectedEnvironmentId, selectedEnvironmentForConfig, resolvedInstanceSettings, persistedNativeExecutionInput },
|
||||
services: { secretsSvc, companySkills, appendRunEvent, finalizeAiConnectionBusyDeferral },
|
||||
onFailureSecretsResolved: (readSecrets) => { readFailureReportSecrets = () => [...readSecrets(), ...identityRedactor.values]; },
|
||||
onManagedAiRuntimeAcquired: (runtime) => { managedAiRuntime = runtime; },
|
||||
onIdentityRedactorPrepared: (redactor) => { identityRedactor = redactor; },
|
||||
}));
|
||||
if (!executionConfiguration.prepared) return;
|
||||
const {
|
||||
mergedConfig, configSnapshot, runScopedMentionedSkillKeys, githubSelection, useHostGitHub,
|
||||
resolvedConfig, configuredTaskEnvironment, secretKeys, secretManifest,
|
||||
runtimeSkillEntries, nativeRunnerPreparationSpans, agentIdentity, observability,
|
||||
} = executionConfiguration;
|
||||
let { runtimeConfig } = executionConfiguration;
|
||||
const resolvedFailureSecrets = readFailureReportSecrets();
|
||||
readFailureReportSecrets = () => [
|
||||
...resolvedFailureSecrets,
|
||||
...identityRedactor.values,
|
||||
...collectRunFailureSecretValues(runtimeConfig.env, secretKeys),
|
||||
];
|
||||
const preparedSession = await timePreparation("session", () => prepareHeartbeatSession(db, {
|
||||
run, agent,
|
||||
task: { context, issueRef, issueContext, taskProjectContext, workspaceProjectContext, wakeCommentId, acceptedPlanContinuationWake, acceptedPlanWakeRoutingDecision },
|
||||
session: { taskSession, taskSessionDecodedParams, explicitResumeSessionParams, explicitResumeSessionDisplayId, sessionCodec },
|
||||
config: {
|
||||
runtimeConfig, agentIdentity, issueAssigneeOverrides, originalAiIssueOverrides,
|
||||
managedAiRuntime, aiBinding, requestedAiBinding, secretManifest, runtimeSkillEntries, routineEnvContext,
|
||||
},
|
||||
workspace: {
|
||||
requestedExecutionWorkspaceMode, effectiveExecutionWorkspaceMode, projectExecutionWorkspacePolicy,
|
||||
issueExecutionWorkspaceSettings, executionPolicy, requestedReusableExecutionWorkspaceConfig, reusableExistingExecutionWorkspace,
|
||||
},
|
||||
environment: { environmentResolution, selectedEnvironmentId, selectedEnvironmentForConfig },
|
||||
getLatestAgentConfigRevision,
|
||||
}));
|
||||
const { sessionConfigMetadata, configuredModel, resetTaskSession, sessionResetReason, taskSessionForRun, getFreshSessionHandoff, previousSessionParams } = preparedSession;
|
||||
const {
|
||||
resolvedWorkspace,
|
||||
@@ -2898,64 +2989,7 @@ export function heartbeatService(
|
||||
workspaceRealization,
|
||||
executionTarget,
|
||||
workspaceRestoreSource,
|
||||
} = await timePreparation("workspace", () => prepareHeartbeatWorkspace(db, {
|
||||
run, agent, controllerLease, nativeRunnerPreparationSpans, signal: executionControl.controller.signal,
|
||||
task: {
|
||||
issueRef,
|
||||
issueId,
|
||||
wasIssueAssignee: issueContext?.assigneeAgentId === agent.id,
|
||||
context,
|
||||
responsibleUserId,
|
||||
previousSessionParams,
|
||||
workspaceSelection: taskWorkspaceIntentRow?.intent ?? null,
|
||||
freshWorkspace,
|
||||
},
|
||||
policy: {
|
||||
trustPreset,
|
||||
isolatedWorkspacesEnabled: runtimeWorkspaceSelectionEnabled,
|
||||
effectiveExecutionWorkspaceMode,
|
||||
requestedExecutionWorkspaceMode,
|
||||
useIsolatedTaskDirectory,
|
||||
nativeChatWorkspaceScope,
|
||||
projectExecutionWorkspacePolicy,
|
||||
issueExecutionWorkspaceSettings,
|
||||
environmentExecutionWorkspaceSettings,
|
||||
executionPolicy,
|
||||
},
|
||||
environment: {
|
||||
selectedEnvironmentId,
|
||||
localEnvironment,
|
||||
selectedEnvironmentForConfig,
|
||||
environmentResolution,
|
||||
resolvedInstanceSettings,
|
||||
},
|
||||
config: {
|
||||
mergedConfig,
|
||||
configSnapshot,
|
||||
secretManifest,
|
||||
managedAiCredentialIdentity: managedAiRuntime?.identity,
|
||||
runnerTaskResumeSessionId: run.nativeSessionId ?? preparedSession.taskResumeSessionId,
|
||||
},
|
||||
reuse: {
|
||||
requestedShouldReuseExisting,
|
||||
existingExecutionWorkspace,
|
||||
reusableExistingExecutionWorkspace,
|
||||
workspaceReuseRequest,
|
||||
nativeRecoveryExecutionWorkspaceId,
|
||||
persistedNativeExecutionInput,
|
||||
isDotRun,
|
||||
runOptions,
|
||||
},
|
||||
services: {
|
||||
workspaceWorkers: options.pluginWorkerManager,
|
||||
envOrchestrator,
|
||||
executionWorkspacesSvc,
|
||||
workspaceOperationsSvc,
|
||||
resolveWorkspaceForRun,
|
||||
resolveReusedGitWorkspaceAnchor,
|
||||
appendRunEvent,
|
||||
},
|
||||
}));
|
||||
} = preparedWorkspace;
|
||||
const preparedInstructions = instructionWorkingCopy = createHeartbeatInstructionWorkingCopy({
|
||||
observabilityCompatibilityKey: observability?.compatibilityKey,
|
||||
db, instructionCopies, readRun: () => run!, agent, context,
|
||||
|
||||
@@ -94,6 +94,23 @@ export interface HeartbeatExecutionConfigurationInput {
|
||||
onIdentityRedactorPrepared: (redactor: ReturnType<typeof createAgentIdentityRedactor>) => void;
|
||||
}
|
||||
|
||||
export function buildHeartbeatWorkspaceConfiguration(
|
||||
sourceWorkspaceConfig: Record<string, unknown>,
|
||||
{ useIsolatedTaskDirectory, explicitTaskDirectory, requestedExecutionWorkspaceMode }: HeartbeatExecutionConfigurationInput["policy"],
|
||||
selectedEnvironmentId: string | null,
|
||||
) {
|
||||
const forceDirectoryStrategy = useIsolatedTaskDirectory || (explicitTaskDirectory && (
|
||||
requestedExecutionWorkspaceMode === "agent_default" || Object.keys(parseObject(sourceWorkspaceConfig.workspaceStrategy)).length > 0
|
||||
));
|
||||
const mergedConfig: Record<string, unknown> = {
|
||||
...sourceWorkspaceConfig,
|
||||
// The base below is already task-owned. Keep directory transport while
|
||||
// preserving isolated mode and the mandatory sandbox preflight.
|
||||
...(forceDirectoryStrategy ? { workspaceStrategy: { type: "project_primary" } } : {}),
|
||||
};
|
||||
return { mergedConfig, configSnapshot: buildExecutionWorkspaceConfigSnapshot(mergedConfig, selectedEnvironmentId) };
|
||||
}
|
||||
|
||||
/** Assembles effective run configuration; the executor retains cleanup ownership. */
|
||||
export async function prepareHeartbeatExecutionConfiguration(db: Db, input: HeartbeatExecutionConfigurationInput) {
|
||||
const { run, agent, onFailureSecretsResolved, onManagedAiRuntimeAcquired, onIdentityRedactorPrepared } = input;
|
||||
@@ -137,22 +154,10 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hear
|
||||
// preparation step can fail. Subsequent provider/identity values extend it.
|
||||
onFailureSecretsResolved(() => []);
|
||||
const aiBinding = preconfiguredHome ? undefined : input.config.aiBinding;
|
||||
const { trustPreset, useIsolatedTaskDirectory, explicitTaskDirectory, requestedExecutionWorkspaceMode } = input.policy;
|
||||
const { trustPreset } = input.policy;
|
||||
const { secretsSvc, companySkills, appendRunEvent, finalizeAiConnectionBusyDeferral } = input.services;
|
||||
let managedAiRuntime: ManagedAiRuntime | undefined;
|
||||
const forceDirectoryStrategy = useIsolatedTaskDirectory || (explicitTaskDirectory && (
|
||||
requestedExecutionWorkspaceMode === "agent_default" || Object.keys(parseObject(sourceWorkspaceConfig.workspaceStrategy)).length > 0
|
||||
));
|
||||
const mergedConfig = {
|
||||
...sourceWorkspaceConfig,
|
||||
// The base below is already task-owned. Keep directory transport while
|
||||
// preserving isolated mode and the mandatory sandbox preflight.
|
||||
...(forceDirectoryStrategy ? { workspaceStrategy: { type: "project_primary" } } : {}),
|
||||
};
|
||||
const configSnapshot = buildExecutionWorkspaceConfigSnapshot(
|
||||
mergedConfig,
|
||||
selectedEnvironmentId,
|
||||
);
|
||||
const { mergedConfig, configSnapshot } = buildHeartbeatWorkspaceConfiguration(sourceWorkspaceConfig, input.policy, selectedEnvironmentId);
|
||||
const executionRunConfig =
|
||||
stripWorkspaceRuntimeFromExecutionRunConfig(mergedConfig);
|
||||
const skillActor: AuthorizationActor = { type: "agent", agentId: agent.id, companyId: agent.companyId,
|
||||
|
||||
@@ -1797,7 +1797,8 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
|
||||
// path uses the provider and configuration recorded on the lease first;
|
||||
// the environment is lifecycle context and only a legacy fallback.
|
||||
const isOrphanEphemeralLease = lease.leasePolicy === "ephemeral";
|
||||
const useRecordedTeardown = isOrphanEphemeralLease || !environment || hasStopOnlyCleanup(lease);
|
||||
const useRecordedTeardown = isOrphanEphemeralLease || !environment || hasStopOnlyCleanup(lease) ||
|
||||
(metadata.driver === "plugin" && metadata.pluginLeaseCleanupMethod === "release");
|
||||
|
||||
// Do not consume a finite cleanup attempt while the provider plugin is
|
||||
// briefly unavailable. A plugin worker restart, a plugin reload, or a
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { preparePluginWorkspace } from "../plugin-workspaces.js";
|
||||
import { execFile as execFileCallback } from "node:child_process";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { mkdtemp, mkdir, readFile, readdir, rm, writeFile, access, rename, symlink } from "node:fs/promises";
|
||||
@@ -23,7 +24,7 @@ import { instanceSettingsService } from "../instance-settings.js";
|
||||
import { workspaceOperationService } from "../workspace-operations.js";
|
||||
import { nativeWorkspaceSyncInternals, prepareNativeWorkspaceSync } from "../native-runtime/native-workspace-sync.js";
|
||||
import { createHeartbeatWorkspaceResolver, prepareProjectRepositoryWorkspaces, resolveExecutionWorkspaceReuseRequestForIssue } from "./workspaces.js";
|
||||
import { prepareHeartbeatWorkspace, type HeartbeatWorkspacePreparationInput } from "./workspace-preparation.js";
|
||||
import { RetryWorkspaceRefused, prepareHeartbeatWorkspace, type HeartbeatWorkspacePreparationInput } from "./workspace-preparation.js";
|
||||
import { buildNativeExecutionInput } from "../native-runtime/native-execution-input.js";
|
||||
import { nativeRuntimeContextFixture } from "../native-runtime/runtime-context.test-fixture.js";
|
||||
import * as managedRepositoryCheckout from "../managed-repository-checkout.js";
|
||||
@@ -965,15 +966,18 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace
|
||||
{ fromDefault: false, mode: "isolated_workspace" as const, fresh: false, portable: true },
|
||||
{ fromDefault: false, mode: "isolated_workspace" as const, fresh: false, portable: true, replace: true },
|
||||
{ fromDefault: false, mode: "isolated_workspace" as const, fresh: false, portable: true, replaceAfterPrepare: true },
|
||||
])("prepares managed storage with the selected Launcher (fromDefault: $fromDefault, mode: $mode, fresh: $fresh, portable: $portable, replace: $replace, replaceAfterPrepare: $replaceAfterPrepare)", async ({ fromDefault, mode: defaultMode, fresh, portable, replace, replaceAfterPrepare }) => {
|
||||
{ fromDefault: false, mode: "isolated_workspace" as const, fresh: false, portable: true, verify: true },
|
||||
{ fromDefault: false, mode: "isolated_workspace" as const, fresh: false, portable: true, verify: true, refuse: true },
|
||||
{ fromDefault: false, mode: "isolated_workspace" as const, fresh: false, portable: true, verify: true, restart: true },
|
||||
])("prepares managed storage with the selected Launcher (fromDefault: $fromDefault, mode: $mode, fresh: $fresh, portable: $portable, replace: $replace, replaceAfterPrepare: $replaceAfterPrepare, verify: $verify, refuse: $refuse, restart: $restart)", async ({ fromDefault, mode: defaultMode, fresh, portable, replace, replaceAfterPrepare, verify, refuse, restart }) => {
|
||||
const { input, source, issue } = await fixture(true);
|
||||
if (portable) input.agent.adapterType = "paperclip_runner";
|
||||
const pluginKey = `test.managed-storage-${fromDefault}-${defaultMode}-${fresh}-${portable}${replace ? "-replacement" : replaceAfterPrepare ? "-late-replacement" : ""}`;
|
||||
const pluginKey = `test.managed-storage-${fromDefault}-${defaultMode}-${fresh}-${portable}${replace ? "-replacement" : replaceAfterPrepare ? "-late-replacement" : refuse ? "-refuse" : restart ? "-restart" : verify ? "-verify" : ""}`;
|
||||
const [plugin] = await db.insert(plugins).values({ pluginKey, packageName: "test-storage", version: "1.0.0", apiVersion: 1, status: "ready", categories: ["workspace"],
|
||||
manifestJson: { id: pluginKey, apiVersion: 1, version: "1.0.0", displayName: "Storage", description: "Storage", author: "Test", categories: ["workspace"],
|
||||
workspaceProvider: { freshWorkspaceVersion: 1, finalizationVersion: 1, modes: ["isolated_workspace", "operator_branch"], strategies: ["git_worktree"], ...(fromDefault ? { defaultForProjects: { mode: defaultMode, strategy: "git_worktree" as const } } : {}) }, capabilities: ["workspaces.manage", "events.subscribe", "environment.drivers.register"],
|
||||
workspaceProvider: { freshWorkspaceVersion: 1, finalizationVersion: 1, workspaceVerificationVersion: 1, modes: ["isolated_workspace", "operator_branch"], strategies: ["git_worktree"], ...(fromDefault ? { defaultForProjects: { mode: defaultMode, strategy: "git_worktree" as const } } : {}) }, capabilities: ["workspaces.manage", "events.subscribe", "environment.drivers.register"],
|
||||
environmentDrivers: [{ driverKey: "tasks", displayName: "Tasks", supportsTasks: true, configSchema: { type: "object" },
|
||||
...(portable ? { sessionHandoffVersion: 1 as const } : {}) }],
|
||||
...(portable ? { sessionHandoffVersion: 1 as const, freshWorkspaceVersion: 1 as const } : {}) }],
|
||||
resourceLifecycle: ["project"], entrypoints: { worker: "./worker.js" } } }).returning();
|
||||
const [environment] = await db.insert(environments).values({ name: `Storage tasks ${pluginKey}`, driver: "plugin", status: "active",
|
||||
config: { pluginKey, driverKey: "tasks", driverConfig: {} } }).returning();
|
||||
@@ -1001,15 +1005,20 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace
|
||||
const prepareRequests: Array<{ operation: Parameters<typeof callWorkspaceProvider>[2]["operation"]; operationId: string }> = [];
|
||||
const finalizationIds: string[] = [];
|
||||
let executionWorkspaceId: string | undefined;
|
||||
input.services.workspaceWorkers = { getWorker: () => ({ supportedMethods: [
|
||||
"workspaceOperation", "environmentTask", ...(portable ? ["workspacePrepare"] : []),
|
||||
let loseVerificationReply = restart === true;
|
||||
input.services.workspaceWorkers = { isRunning: () => true, getWorker: () => ({ supportedMethods: [
|
||||
"workspaceOperation", "environmentTask", "environmentReleaseLease", ...(portable ? ["workspacePrepare"] : []),
|
||||
] }), call: async (_: string, method: string, params: any) => {
|
||||
if (method === "environmentAcquireLease") {
|
||||
expect(replace).toBe(true);
|
||||
expect(replace || restart).toBe(true);
|
||||
expect(params.executionWorkspaceId).toBe(executionWorkspaceId);
|
||||
sequence.push("replace");
|
||||
sequence.push(restart ? "allocate" : "replace");
|
||||
return { providerLeaseId: readyTaskId };
|
||||
}
|
||||
if (method === "environmentReleaseLease") {
|
||||
sequence.push("release");
|
||||
return { providerLeaseId: params.providerLeaseId, state: "stopped" };
|
||||
}
|
||||
if (method === "environmentTask") {
|
||||
if (replace && params.operation.kind !== "describe") {
|
||||
expect(params.taskId).toBe(providerLeaseId);
|
||||
@@ -1023,28 +1032,49 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace
|
||||
sequence.push("placement");
|
||||
return { kind: "description", taskId: readyTaskId, runner: { version: "test", digest: `sha256:${"a".repeat(64)}` } };
|
||||
}
|
||||
expect(params.taskLease).toEqual(portable ? taskLease : undefined);
|
||||
expect(params.taskLease).toEqual(portable && (!verify || executionWorkspaceId) ? taskLease : undefined);
|
||||
if (params.operation.kind === "finalize") {
|
||||
sequence.push("finalize");
|
||||
finalizationIds.push(params.operationId);
|
||||
return { kind: "finalized", headSha: "a".repeat(40), branchName: "test-WSP-1" };
|
||||
}
|
||||
if (portable) {
|
||||
if (params.operation.verifyExisting) {
|
||||
expect(method).toBe("workspaceOperation");
|
||||
sequence.push("workspace verify");
|
||||
if (loseVerificationReply) {
|
||||
loseVerificationReply = false;
|
||||
throw new Error("workspace verification reply lost");
|
||||
}
|
||||
} else if (portable && (!verify || executionWorkspaceId)) {
|
||||
expect(method).toBe("workspacePrepare");
|
||||
expect(params.phase).toBe("submit");
|
||||
expect(params.executionWorkspaceId).toBe(executionWorkspaceId);
|
||||
prepareRequests.push(params);
|
||||
}
|
||||
sequence.push("workspace prepare");
|
||||
if (!params.operation.verifyExisting) sequence.push("workspace prepare");
|
||||
if (replaceAfterPrepare && prepareRequests.length === 2) throw new Error("lost workspace preparation reply");
|
||||
return { kind: "prepared", binding: { pluginId: plugin.id, projectId: issue.projectId, executionWorkspaceId: params.executionWorkspaceId,
|
||||
projectWorkspaceId: issue.projectWorkspaceId, sourceRevision: params.operation.source.revision,
|
||||
workspaceRef: replaceAfterPrepare && prepareRequests.length === 3 ? "provider:replacement-checkout" : "provider:checkout" },
|
||||
branchName: params.operation.branchName, baseSha: "a".repeat(40), created: true, branchCreatedByRuntime: true };
|
||||
branchName: params.operation.branchName, baseSha: "a".repeat(40), created: refuse || !params.operation.verifyExisting, branchCreatedByRuntime: true };
|
||||
} } as any;
|
||||
input.services.envOrchestrator = environmentRunOrchestrator(db, {
|
||||
pluginWorkerManager: input.services.workspaceWorkers as never,
|
||||
});
|
||||
if (verify) {
|
||||
const prepared = await preparePluginWorkspace(db, input.services.workspaceWorkers, {
|
||||
companyId: input.agent.companyId, projectId: issue.projectId, projectWorkspaceId: issue.projectWorkspaceId,
|
||||
issueId: issue.id, runId: input.run.id, agentId: input.agent.id, responsibleUserId: null,
|
||||
existingWorkspaceId: null, mode: "isolated_workspace", strategy: "git_worktree", branchName: "test-WSP-1",
|
||||
});
|
||||
const workspace = (await input.services.executionWorkspacesSvc.getStoredById(input.agent.companyId, prepared!.id))!;
|
||||
input.task.issueRef = { ...input.task.issueRef!, executionWorkspaceId: workspace.id };
|
||||
input.reuse.existingExecutionWorkspace = input.reuse.reusableExistingExecutionWorkspace = workspace;
|
||||
input.task.freshWorkspace = input.task.retryWorkspaceCandidate = { mode: "reuse_issue_workspace",
|
||||
retryIntentId: randomUUID(), sourceRunId: randomUUID(), workspaceId: workspace.id, cwd: workspace.cwd };
|
||||
sequence.length = 0;
|
||||
prepareRequests.length = 0;
|
||||
}
|
||||
const acquire = input.services.envOrchestrator.acquireForRun;
|
||||
vi.spyOn(input.services.envOrchestrator, "acquireForRun").mockImplementation(async args => {
|
||||
sequence.push("acquire");
|
||||
@@ -1054,6 +1084,7 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace
|
||||
}
|
||||
expect(args.persistedExecutionWorkspace).toMatchObject({ mode: defaultMode, id: expect.any(String) });
|
||||
executionWorkspaceId = args.persistedExecutionWorkspace!.id;
|
||||
if (restart) return acquire(args);
|
||||
const [lease] = await db.insert(environmentLeases).values({
|
||||
companyId: args.companyId, environmentId: environment.id, heartbeatRunId: args.heartbeatRunId, issueId: args.issueId,
|
||||
executionWorkspaceId: args.persistedExecutionWorkspace!.id, provider: `plugin:${pluginKey}:tasks`, providerLeaseId,
|
||||
@@ -1089,7 +1120,50 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace
|
||||
}
|
||||
throw new Error("reached execution handoff");
|
||||
});
|
||||
const portableSequence = ["acquire", ...(replace ? ["replace"] : []), "placement", "workspace prepare"];
|
||||
if (restart) {
|
||||
await expect(prepareHeartbeatWorkspace(db, input)).rejects.toMatchObject({ kind: "transport", retryable: true });
|
||||
expect(sequence).toEqual(["acquire", "allocate", "workspace verify"]);
|
||||
const [lease] = await leases(input.run.id);
|
||||
expect(lease).toMatchObject({ providerLeaseId, status: "active" });
|
||||
expect((await readRun(input.run.id)).contextSnapshot?.retryWorkspaceRefusal).toBeUndefined();
|
||||
input.reuse.runOptions.nativeRestartRecovery = {
|
||||
kind: "bootstrap_incomplete", runId: input.run.id, leaseOwner: "restarted-controller",
|
||||
controllerGeneration: 2, providerAttempt: 1, restartKind: "hard", recoveryRequestId: null,
|
||||
};
|
||||
await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow("reached execution handoff");
|
||||
expect(sequence).toEqual(["acquire", "allocate", "workspace verify", "acquire", "workspace verify", "placement", "workspace prepare"]);
|
||||
expect((await leases(input.run.id)).map(item => item.id)).toEqual([lease.id]);
|
||||
await db.update(environmentLeases).set({ metadata: { ...lease.metadata, runnerTaskDispatched: true } })
|
||||
.where(eq(environmentLeases.id, lease.id));
|
||||
await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow("Workspace verification lease already started execution");
|
||||
expect(sequence.slice(-2)).toEqual(["workspace prepare", "acquire"]);
|
||||
return;
|
||||
}
|
||||
if (refuse) {
|
||||
const candidate = input.task.retryWorkspaceCandidate!;
|
||||
await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow(RetryWorkspaceRefused);
|
||||
expect(sequence).toEqual(["acquire", "workspace verify", "release"]);
|
||||
const persisted = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.run.id)).then(([row]) => row);
|
||||
expect(persisted.contextSnapshot?.retryWorkspaceRefusal).toMatchObject({
|
||||
intentId: candidate.retryIntentId, workspaceId: candidate.workspaceId, leaseId: expect.any(String),
|
||||
});
|
||||
expect((await leases(input.run.id))[0]).toMatchObject({ status: "released", cleanupStatus: "success" });
|
||||
const next = new Error("fresh acquisition reached");
|
||||
vi.mocked(input.services.envOrchestrator.acquireForRun).mockRejectedValue(next);
|
||||
await expect(prepareHeartbeatWorkspace(db, {
|
||||
...input, run: persisted,
|
||||
reuse: { ...input.reuse, existingExecutionWorkspace: null, reusableExistingExecutionWorkspace: null },
|
||||
task: { ...input.task, context: persisted.contextSnapshot!, retryWorkspaceCandidate: undefined,
|
||||
issueRef: { ...input.task.issueRef!, executionWorkspaceId: null },
|
||||
freshWorkspace: { mode: "fresh_isolated_workspace", retryIntentId: candidate.retryIntentId,
|
||||
sourceRunId: candidate.sourceRunId, retainedSourceIds: [] } },
|
||||
})).rejects.toBe(next);
|
||||
expect(input.services.envOrchestrator.acquireForRun).toHaveBeenLastCalledWith(expect.objectContaining({
|
||||
replacementWorkspaceCreation: expect.any(String), persistedExecutionWorkspace: expect.objectContaining({ id: expect.not.stringMatching(candidate.workspaceId) }),
|
||||
}));
|
||||
return;
|
||||
}
|
||||
const portableSequence = ["acquire", ...(verify ? ["workspace verify"] : []), ...(replace ? ["replace"] : []), "placement", "workspace prepare"];
|
||||
if (replaceAfterPrepare) {
|
||||
const { executionTarget } = await prepareHeartbeatWorkspace(db, input);
|
||||
if (executionTarget?.kind !== "remote" || executionTarget.transport !== "sandbox" || !executionTarget.runnerTask)
|
||||
|
||||
@@ -5,7 +5,8 @@ import { assertEphemeralWorkspaceExecution, readRemoteWorkspaceAuthority, Remote
|
||||
import { WorkspaceBusyDeferral } from "./retries.js";
|
||||
import { remoteWorkspaceHoldRefusesStart } from "../remote-workspace-writer-hold.js";
|
||||
import { WORKSPACE_BUSY_RETRY_REASON } from "../../modules/run-dispatch/index.js";
|
||||
import { readFreshWorkspaceReadiness } from "../fresh-workspace-continuation.js";
|
||||
import { findWorkspaceProviderFailure } from "../workspace-provider-failure.js";
|
||||
import { readFreshWorkspaceReadiness, recordVerifiedRetryWorkspace, type ContinuationReadiness } from "../fresh-workspace-continuation.js";
|
||||
import { assertNativeSessionPreparationAuthority } from "../native-runtime/native-execution-lease.js";
|
||||
import { NativeExecutionContext } from "../native-runtime/native-execution-context.js";
|
||||
import { resolveRemoteWorkspaceSource } from "./remote-workspace-source.js";
|
||||
@@ -39,9 +40,9 @@ import { resolvePaperclipInstanceRoot } from "../../home-paths.js";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { CONTROLLER_PROVISION_COPY_ONLY, workspaceRequiresCopyOnlyProvisioning } from "../workspace-host-provision.js";
|
||||
import { and, asc, eq } from "drizzle-orm";
|
||||
import { and, asc, eq, sql } from "drizzle-orm";
|
||||
import { type ExecutionWorkspace } from "@paperclipai/shared";
|
||||
import { executionWorkspaces as executionWorkspacesTable, heartbeatRuns, issues, projectWorkspaces } from "@paperclipai/db";
|
||||
import { environmentLeases, executionWorkspaces as executionWorkspacesTable, heartbeatRuns, issues, projectWorkspaces } from "@paperclipai/db";
|
||||
import { getStartupTracer } from "../../instrumentation.js";
|
||||
import { createHostDuplexObservabilityRecorder } from "../duplex-observability-recorder.js";
|
||||
import { incrementToolRuntimeMetricCounter } from "../tool-runtime-metrics.js";
|
||||
@@ -103,6 +104,8 @@ type WorkspaceMetadataInput = Parameters<typeof buildEffectiveRunWorkspaceConfig
|
||||
/** Exact-lease failures that prove the retained sandbox can no longer be used. */
|
||||
const RETAINED_SANDBOX_GONE = new RegExp(`^${NATIVE_REMOTE_RECOVERY_LEASE_MISMATCH}: stage=environment_(?:acquire|resume) field=(?:lease|lease_status|lease_expired|provider_lease_id)$`);
|
||||
|
||||
export class RetryWorkspaceRefused extends Error {}
|
||||
|
||||
/** Prepares the workspace and environment; dispatch and terminal lease release stay with the caller. */
|
||||
export interface HeartbeatWorkspacePreparationInput {
|
||||
signal?: AbortSignal;
|
||||
@@ -121,6 +124,7 @@ export interface HeartbeatWorkspacePreparationInput {
|
||||
responsibleUserId: string | null;
|
||||
previousSessionParams: Record<string, unknown> | null;
|
||||
freshWorkspace?: import("../fresh-workspace-continuation.js").ContinuationReadiness | null;
|
||||
retryWorkspaceCandidate?: Extract<ContinuationReadiness, { mode: "reuse_issue_workspace" }>;
|
||||
workspaceSelection?: import("@paperclipai/shared").TaskWorkspaceIntent | null;
|
||||
};
|
||||
policy: {
|
||||
@@ -161,7 +165,7 @@ export interface HeartbeatWorkspacePreparationInput {
|
||||
};
|
||||
services: {
|
||||
workspaceWorkers?: WorkspaceWorkers;
|
||||
envOrchestrator: Pick<ReturnType<typeof environmentRunOrchestrator>, "resolveEnvironment" | "acquireForRun" | "confirmWorkspaceTaskPlacement" | "realizeForRun">;
|
||||
envOrchestrator: Pick<ReturnType<typeof environmentRunOrchestrator>, "resolveEnvironment" | "acquireForRun" | "confirmWorkspaceTaskPlacement" | "realizeForRun" | "releaseForRun">;
|
||||
executionWorkspacesSvc: Pick<ReturnType<typeof executionWorkspaceService>, "create" | "update" | "getStoredById" | "bindTaskWorkspace" | "prepareTaskRepositoriesForAdmission" | "assertTaskRepositoryPublicationAllowed" | "prepareConfiguredRepositoryCopy" | "assertTaskRepositoryRecoveryBinding">;
|
||||
workspaceOperationsSvc: Pick<ReturnType<typeof workspaceOperationService>, "createRecorder">;
|
||||
resolveWorkspaceForRun: WorkspaceResolver["resolveWorkspaceForRun"];
|
||||
@@ -580,6 +584,29 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
|
||||
throw new Error(allowlistDecision.reason);
|
||||
}
|
||||
};
|
||||
const refused = parseObject(context.retryWorkspaceRefusal);
|
||||
const refusedWorkspaceId = fresh && input.task.freshWorkspace?.mode === "fresh_isolated_workspace" &&
|
||||
refused.intentId === input.task.freshWorkspace.retryIntentId && typeof refused.workspaceId === "string"
|
||||
? refused.workspaceId : undefined;
|
||||
const priorVerificationLease = refusedWorkspaceId && selectedEnvironmentForConfig && typeof refused.leaseId === "string"
|
||||
? (await db.select().from(environmentLeases).where(and(
|
||||
eq(environmentLeases.id, refused.leaseId), eq(environmentLeases.companyId, agent.companyId), eq(environmentLeases.heartbeatRunId, run.id),
|
||||
eq(environmentLeases.environmentId, selectedEnvironmentForConfig.id), eq(environmentLeases.executionWorkspaceId, refusedWorkspaceId),
|
||||
)))[0] : undefined;
|
||||
if (refusedWorkspaceId && typeof refused.leaseId === "string" && !priorVerificationLease)
|
||||
throw new Error("Workspace verification lease changed");
|
||||
const releaseVerificationLease = async (leaseId: string) => {
|
||||
await controllerLease.assertOwned();
|
||||
await envOrchestrator.releaseForRun({ companyId: agent.companyId, agentId: agent.id, heartbeatRunId: run.id });
|
||||
const [lease] = await db.select({ status: environmentLeases.status, releasedAt: environmentLeases.releasedAt,
|
||||
cleanupStatus: environmentLeases.cleanupStatus }).from(environmentLeases).where(and(
|
||||
eq(environmentLeases.id, leaseId), eq(environmentLeases.companyId, agent.companyId), eq(environmentLeases.heartbeatRunId, run.id)));
|
||||
if (!lease?.releasedAt || !["released", "expired"].includes(lease.status) || lease.cleanupStatus !== "success")
|
||||
throw new Error("Workspace verification lease cleanup is pending");
|
||||
};
|
||||
if (priorVerificationLease && (!priorVerificationLease.releasedAt || priorVerificationLease.cleanupStatus !== "success"))
|
||||
await releaseVerificationLease(priorVerificationLease.id);
|
||||
const retryCandidate = input.task.retryWorkspaceCandidate;
|
||||
const environmentAcquisitionInput = (
|
||||
persistedExecutionWorkspace: Parameters<typeof envOrchestrator.acquireForRun>[0]["persistedExecutionWorkspace"],
|
||||
projectId: string | null,
|
||||
@@ -596,46 +623,64 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
|
||||
runnerTaskProjectIds: [...new Set([projectId, ...resolvedWorkspace.runnerTaskProjectIds ?? [],
|
||||
...(additionalWorkspaces ?? []).map(workspace => workspace.projectId)]
|
||||
.filter((id): id is string => Boolean(id)))],
|
||||
recoverPluginTask: !portablePreparationLeaseId &&
|
||||
recoverPluginTask: !priorVerificationLease && !portablePreparationLeaseId &&
|
||||
!(retryCandidate && restartRecovery?.kind === "bootstrap_incomplete") &&
|
||||
(restartRecovery?.kind === "resume_dead_runner" || restartRecovery?.kind === "bootstrap_incomplete"),
|
||||
issueId: issueId ?? null,
|
||||
heartbeatRunId: run.id,
|
||||
agentId: agent.id,
|
||||
persistedExecutionWorkspace,
|
||||
...(priorVerificationLease && persistedExecutionWorkspace && persistedExecutionWorkspace.id !== refusedWorkspaceId
|
||||
? { replacementWorkspaceCreation: persistedExecutionWorkspace.id } : {}),
|
||||
executionWorkspaceSettings: environmentExecutionWorkspaceSettings,
|
||||
});
|
||||
const pluginRecovery = restartRecovery?.kind === "reattach_remote_runner" && restartRecovery.remote.kind === "plugin_task"
|
||||
? restartRecovery.remote : portablePreparationLeaseId
|
||||
? restartRecovery.remote : portablePreparationLeaseId && portablePreparationLeaseId !== priorVerificationLease?.id
|
||||
? { kind: "plugin_task" as const, leaseId: portablePreparationLeaseId, preparation: true as const } : undefined;
|
||||
let workspaceAcquisition: Awaited<ReturnType<typeof envOrchestrator.acquireForRun>> | undefined;
|
||||
let workspaceAcquireStartedAtMs: number | undefined;
|
||||
let ensureTaskWorkspace: ((taskId: string, assertOwned: () => void) => Promise<void> | undefined) | undefined;
|
||||
const pluginPrepared = controllerOnly || remoteAuthority ? null : await preparePluginWorkspace(db, input.services.workspaceWorkers, {
|
||||
const acquireTaskLease: NonNullable<Parameters<typeof preparePluginWorkspace>[2]["acquireTaskLease"]> = async workspace => {
|
||||
if (workspaceAcquisition) {
|
||||
const { environment, lease } = workspaceAcquisition;
|
||||
if (lease.executionWorkspaceId !== workspace.id || typeof lease.metadata?.driverKey !== "string" || !lease.providerLeaseId)
|
||||
throw new Error("Workspace placement changed");
|
||||
return { leaseId: lease.id, taskLease: { environmentId: environment.id, driverKey: lease.metadata.driverKey, providerLeaseId: lease.providerLeaseId } };
|
||||
}
|
||||
if (!selectedEnvironmentForConfig) throw new Error("Workspace placement is unavailable");
|
||||
assertAllowedEnvironment(selectedEnvironmentForConfig);
|
||||
await controllerLease.assertOwned();
|
||||
workspaceAcquireStartedAtMs = Date.now();
|
||||
if (restartRecovery?.kind === "reattach_remote_runner" && restartRecovery.runId !== run.id)
|
||||
throw remoteRecoveryLeaseMismatch("workspace_preparation", "run_id");
|
||||
const acquisition = await timeWorkspace("workspace_environment_acquire", () => envOrchestrator.acquireForRun({
|
||||
...environmentAcquisitionInput(workspace, resolvedWorkspace.projectId, resolvedWorkspace.additionalWorkspaces),
|
||||
...(pluginRecovery ? { reattachRemoteLease: pluginRecovery } : {}),
|
||||
}));
|
||||
await controllerLease.assertOwned();
|
||||
if (retryCandidate) {
|
||||
const metadata = acquisition.lease.metadata;
|
||||
if (metadata?.runnerTaskDispatched === true || metadata?.runnerTaskIdentity ||
|
||||
metadata?.runnerTaskStartup || metadata?.runnerTaskStartupCleanup || metadata?.runnerTaskRetiring === true)
|
||||
throw new Error("Workspace verification lease already started execution");
|
||||
workspaceAcquisition = acquisition;
|
||||
return acquireTaskLease(workspace);
|
||||
}
|
||||
const placement = await envOrchestrator.confirmWorkspaceTaskPlacement(acquisition, {
|
||||
companyId: agent.companyId, environmentId: selectedEnvironmentForConfig.id, workspaceId: workspace.id, signal: input.signal,
|
||||
});
|
||||
await controllerLease.assertOwned();
|
||||
workspaceAcquisition = placement.acquisition;
|
||||
return { leaseId: placement.acquisition.lease.id, taskLease: placement.taskLease };
|
||||
};
|
||||
const pluginPrepareInput: Parameters<typeof preparePluginWorkspace>[2] = {
|
||||
companyId: agent.companyId, projectId: resolvedWorkspace.projectId, projectWorkspaceId: resolvedWorkspace.workspaceId,
|
||||
issueId, runId: run.id, agentId: agent.id, responsibleUserId,
|
||||
existingWorkspaceId: providerFresh ? providerFresh.previousWorkspaceId : reusableExistingExecutionWorkspace?.id ?? issueRef?.executionWorkspaceId ?? null,
|
||||
...(providerFresh ? { fresh: providerFresh, expectedBindingRevision: previousProviderBindingRevision } : {}),
|
||||
selectedProviderPluginId: projectExecutionWorkspacePolicy?.storageProviderPluginId ?? null,
|
||||
environment: selectedEnvironmentForConfig,
|
||||
acquireTaskLease: async workspace => {
|
||||
if (!selectedEnvironmentForConfig) throw new Error("Workspace Launcher placement is unavailable");
|
||||
assertAllowedEnvironment(selectedEnvironmentForConfig);
|
||||
await controllerLease.assertOwned();
|
||||
workspaceAcquireStartedAtMs = Date.now();
|
||||
if (restartRecovery?.kind === "reattach_remote_runner" && restartRecovery.runId !== run.id)
|
||||
throw remoteRecoveryLeaseMismatch("workspace_preparation", "run_id");
|
||||
const acquisition = await timeWorkspace("workspace_environment_acquire", () => envOrchestrator.acquireForRun({
|
||||
...environmentAcquisitionInput(workspace, resolvedWorkspace.projectId, resolvedWorkspace.additionalWorkspaces),
|
||||
...(pluginRecovery ? { reattachRemoteLease: pluginRecovery } : {}),
|
||||
}));
|
||||
await controllerLease.assertOwned();
|
||||
const placement = await envOrchestrator.confirmWorkspaceTaskPlacement(acquisition, {
|
||||
companyId: agent.companyId, environmentId: selectedEnvironmentForConfig.id, workspaceId: workspace.id, signal: input.signal,
|
||||
});
|
||||
await controllerLease.assertOwned();
|
||||
workspaceAcquisition = placement.acquisition;
|
||||
return { leaseId: placement.acquisition.lease.id, taskLease: placement.taskLease };
|
||||
},
|
||||
acquireTaskLease,
|
||||
onTaskPrepared: ensure => { ensureTaskWorkspace = ensure; },
|
||||
mode: requestedExecutionWorkspaceMode,
|
||||
strategy: resolveEffectiveWorkspaceStrategyType(requestedExecutionWorkspaceMode, mergedConfig),
|
||||
@@ -647,7 +692,45 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
|
||||
})) : null,
|
||||
baseRef: typeof parseObject(mergedConfig.workspaceStrategy).baseRef === "string"
|
||||
? String(parseObject(mergedConfig.workspaceStrategy).baseRef) : null,
|
||||
});
|
||||
};
|
||||
if (retryCandidate && !controllerOnly && !remoteAuthority) {
|
||||
const verified = await timeWorkspace("workspace_verification", () => preparePluginWorkspace(db, input.services.workspaceWorkers, {
|
||||
...pluginPrepareInput,
|
||||
existingWorkspaceId: retryCandidate.workspaceId,
|
||||
verifyExisting: true,
|
||||
assertOwned: () => input.signal?.throwIfAborted(),
|
||||
onTaskPrepared: undefined,
|
||||
})).catch(error => {
|
||||
const failure = findWorkspaceProviderFailure(error);
|
||||
if (failure?.reason !== "binding_mismatch" && failure?.reason !== "unsupported_operation") throw error;
|
||||
return null;
|
||||
});
|
||||
await controllerLease.assertOwned();
|
||||
if (!verified) {
|
||||
const refusal = { intentId: retryCandidate.retryIntentId, workspaceId: retryCandidate.workspaceId,
|
||||
leaseId: workspaceAcquisition?.lease.id ?? null };
|
||||
const [saved] = await db.update(heartbeatRuns).set({ contextSnapshot: sql`coalesce(${heartbeatRuns.contextSnapshot}, '{}'::jsonb)
|
||||
|| ${JSON.stringify({ retryWorkspaceRefusal: refusal })}::jsonb` })
|
||||
.where(and(eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.companyId, agent.companyId), eq(heartbeatRuns.status, "running")))
|
||||
.returning({ id: heartbeatRuns.id });
|
||||
if (!saved) throw new Error("Workspace verification lost run ownership");
|
||||
context.retryWorkspaceRefusal = refusal;
|
||||
if (workspaceAcquisition) await releaseVerificationLease(workspaceAcquisition.lease.id);
|
||||
throw new RetryWorkspaceRefused();
|
||||
}
|
||||
await recordVerifiedRetryWorkspace(db, run, issueId!, retryCandidate);
|
||||
if (workspaceAcquisition && selectedEnvironmentForConfig) {
|
||||
const providerLeaseId = workspaceAcquisition.lease.providerLeaseId;
|
||||
const placement = await envOrchestrator.confirmWorkspaceTaskPlacement(workspaceAcquisition, {
|
||||
companyId: agent.companyId, environmentId: selectedEnvironmentForConfig.id, workspaceId: retryCandidate.workspaceId, signal: input.signal,
|
||||
});
|
||||
if (placement.acquisition.lease.providerLeaseId !== providerLeaseId)
|
||||
throw new Error("Workspace placement changed after verification");
|
||||
workspaceAcquisition = placement.acquisition;
|
||||
}
|
||||
}
|
||||
const pluginPrepared = controllerOnly || remoteAuthority ? null : await preparePluginWorkspace(db, input.services.workspaceWorkers, pluginPrepareInput);
|
||||
|
||||
const freshTaskReservationId = providerFresh && pluginTaskWorkspace && !pluginPrepared && issueId
|
||||
? await reserveFreshPluginTaskWorkspace({ db, companyId: agent.companyId, issueId, runId: run.id, agentId: agent.id,
|
||||
fresh: providerFresh, responsibleUserId, expectedBindingRevision: previousProviderBindingRevision, owner: taskWorkspaceOwner!,
|
||||
|
||||
@@ -86,6 +86,7 @@ export async function preparePluginWorkspace(
|
||||
fresh?: PluginFreshWorkspace;
|
||||
expectedBindingRevision?: number;
|
||||
verifyExisting?: true;
|
||||
assertOwned?: () => void;
|
||||
acquireTaskLease?: (workspace: Pick<ExecutionWorkspace, "id" | "mode">) => Promise<{ leaseId: string; taskLease: TaskPlacement }>;
|
||||
onTaskPrepared?: (ensure: EnsureTaskWorkspace) => void;
|
||||
},
|
||||
@@ -165,6 +166,7 @@ export async function preparePluginWorkspace(
|
||||
const selectedProvider = await requireWorkspaceProviderWorker(db, workers, input.companyId, pluginId, true);
|
||||
if (input.environment) assertProjectStorageEnvironment(selectedProvider, input.environment);
|
||||
const declaration = selectedProvider.manifestJson.workspaceProvider!;
|
||||
if (input.verifyExisting && declaration.workspaceVerificationVersion !== 1) return null;
|
||||
if (input.fresh && declaration.freshWorkspaceVersion !== 1)
|
||||
throw conflict("A fresh attempt is unavailable for this workspace storage provider. The existing workspace is retained.", { reason: "fresh_workspace_storage_unsupported" });
|
||||
if (
|
||||
@@ -348,7 +350,7 @@ export async function preparePluginWorkspace(
|
||||
driver.driverKey === driverKey && driver.sessionHandoffVersion === 1)
|
||||
? await input.acquireTaskLease?.({ id: workspace.id, mode: mode as ExecutionWorkspace["mode"] }) : undefined;
|
||||
const prepare = (workspace: Workspace, taskLease?: TaskPlacement, replacement?: { leaseId: string; assertOwned: () => void }) => withExecutionWorkspaceLifecycleLock(db, workspace.id, async (tx) => {
|
||||
replacement?.assertOwned();
|
||||
(replacement?.assertOwned ?? input.assertOwned)?.();
|
||||
const [current] = await tx
|
||||
.select()
|
||||
.from(executionWorkspaces)
|
||||
@@ -401,7 +403,7 @@ export async function preparePluginWorkspace(
|
||||
runId: input.runId,
|
||||
agentId: input.agentId,
|
||||
taskLease,
|
||||
assertOwned: replacement?.assertOwned,
|
||||
assertOwned: replacement?.assertOwned ?? input.assertOwned,
|
||||
operation: {
|
||||
kind: "prepare",
|
||||
source: sourceInput,
|
||||
@@ -430,8 +432,9 @@ export async function preparePluginWorkspace(
|
||||
(saved && result.binding.workspaceRef !== saved.workspaceRef) ||
|
||||
(workspace.branchName !== null && result.branchName !== workspace.branchName) ||
|
||||
(typeof savedBase === "string" && result.baseSha !== savedBase))) {
|
||||
throw conflict("The existing checkout could not be verified");
|
||||
return null;
|
||||
}
|
||||
(replacement?.assertOwned ?? input.assertOwned)?.();
|
||||
await authorize();
|
||||
const [currentProject] = await tx
|
||||
.select()
|
||||
@@ -485,12 +488,13 @@ export async function preparePluginWorkspace(
|
||||
return updated!;
|
||||
});
|
||||
const prepared = await prepare(workspace, placement?.taskLease);
|
||||
if (placement && input.onTaskPrepared) {
|
||||
if (prepared && placement && input.onTaskPrepared) {
|
||||
let preparedTaskId = placement.taskLease.providerLeaseId;
|
||||
let preparedWorkspace = prepared;
|
||||
input.onTaskPrepared((taskId, assertOwned) => {
|
||||
if (taskId === preparedTaskId) return;
|
||||
return prepare(preparedWorkspace, { ...placement.taskLease, providerLeaseId: taskId }, { leaseId: placement.leaseId, assertOwned }).then(updated => {
|
||||
if (!updated) throw conflict("Workspace preparation is incomplete");
|
||||
preparedWorkspace = updated;
|
||||
preparedTaskId = taskId;
|
||||
});
|
||||
|
||||
Reference in new issue
Block a user