Reconcile Pi prerequisites with the integrated runtime while holding admission

This commit is contained in:
Dotta committed 2026-10-08 15:01:33 -05:00
1 parent 20ce062701
commit f620e7157b
313 files changed
+13086 -2187

No files matched your search

@@ -6,6 +6,7 @@ import { createRequire } from "node:module";
import childProcess, { execFileSync } from "node:child_process";
import { prepareAdapterExecutionTargetRuntime } from "@paperclipai/adapter-utils/execution-target";
import { captureDirectorySnapshot } from "@paperclipai/adapter-utils/workspace-restore-merge";
import { prepareSandboxManagedRuntime } from "@paperclipai/adapter-utils/sandbox-managed-runtime";
import { agentDirectoryBaselineDigest, agentDirectoryProbeProgram, observeLocalAgentDirectory, probeAgentDirectory, retireAgentDirectoryTransferScratch, agentDirectoryTransferCleanupProgram } from "../services/agent-directory-probe.js";
describe("stable live agent directory observation", () => {
@@ -98,6 +99,51 @@ describe("stable live agent directory observation", () => {
expect(probeAgentDirectory(remote).digest).toBe(agentDirectoryBaselineDigest(baseline));
} finally { await runtime?.cleanupWorkspaceSnapshot?.(); fs.rmSync(remote, { recursive: true, force: true }); }
});
it("keeps restart fallback transfer scratch outside the native memory directory", async () => {
const remoteCwd = `${root}-remote`;
const agentHome = join(remoteCwd, ".paperclip-runtime", "agent-files", "agent", "original-run");
const scratch = join(remoteCwd, ".paperclip-runtime", "paperclip-runner", "agent-file-transfers", "agent", "original-run");
fs.mkdirSync(agentHome, { recursive: true });
fs.cpSync(root, agentHome, { recursive: true });
const target = { kind: "remote" as const, transport: "sandbox" as const, remoteCwd,
// A newly bound controller target may have no native sync hooks. Exercise
// the production command/base64 fallback rather than a transport mock.
runner: { execute: async (input: Parameters<import("@paperclipai/adapter-utils/command-managed-runtime").CommandManagedRuntimeRunner["execute"]>[0]) => ({
stdout: execFileSync(input.command, input.args ?? [], { cwd: input.cwd, input: input.stdin, encoding: "utf8",
env: { PATH: process.env.PATH, ...input.env }, timeout: input.timeoutMs, maxBuffer: 32 * 1024 * 1024 }),
stderr: "", exitCode: 0, signal: null, timedOut: false, pid: null, startedAt: new Date().toISOString(),
}) } };
let runtime: Awaited<ReturnType<typeof prepareAdapterExecutionTargetRuntime>> | undefined;
try {
const baseline = await captureDirectorySnapshot(root);
runtime = await prepareAdapterExecutionTargetRuntime({ target, runId: "restart-transfer", adapterKey: "agent-files",
workspaceLocalDir: root, workspaceRemoteDir: agentHome, runtimeRootDir: scratch,
syncWorkspace: true, workspaceInboundMode: "adopt_remote", workspaceBaseline: baseline,
workspaceGitSnapshot: null, workspaceFileMode: "all", workspaceExclude: [".paperclip-runtime", ".paperclip-runtime/**"] });
const memory = "0123456789abcdef0123456789abcdef\n";
fs.writeFileSync(join(agentHome, "notes", "memory.txt"), memory);
await runtime.restoreWorkspace();
expect(fs.readFileSync(join(root, "notes", "memory.txt"), "utf8")).toBe(memory);
// A transfer must not introduce a reserved path that the product's own
// full agent-directory probe rejects, including after controller recovery.
expect(() => probeAgentDirectory(agentHome)).not.toThrow();
expect(fs.existsSync(join(agentHome, ".paperclip-runtime"))).toBe(false);
expect(runtime.runtimeRootDir).toBe(scratch);
expect(fs.readdirSync(scratch)).toEqual([]);
} finally { await runtime?.cleanupWorkspaceSnapshot?.(); fs.rmSync(remoteCwd, { recursive: true, force: true }); }
});
it.each(["outside", "reserved-root", "relative", "traversal", "trailing-slash", "nul"])("rejects %s transfer scratch before filesystem work", async kind => {
const reserved = join(root, ".paperclip-runtime");
const invalid = {
outside: `${root}-foreign/scratch`, "reserved-root": reserved, relative: ".paperclip-runtime/scratch",
traversal: `${reserved}/../../scratch`, "trailing-slash": `${reserved}/scratch/`, nul: `${reserved}/scratch\0`,
}[kind]!;
await expect(prepareSandboxManagedRuntime({
spec: { transport: "sandbox", provider: "fixture", sandboxId: "owned", remoteCwd: root, apiKey: null },
adapterKey: "agent-files", client: {} as never, workspaceLocalDir: root, runtimeRootDir: invalid,
})).rejects.toThrow("Transfer scratch must remain within the lease runtime tree");
expect(fs.existsSync(reserved)).toBe(false);
});
it.each(["file", "extra-directory", "symlink", "race"])("fails closed on %s in transfer-owned scratch without deleting contents", kind => {
const parent = join(root, ".paperclip-runtime"), scratch = join(parent, "agent-files");
fs.mkdirSync(scratch, { recursive: true });
@@ -225,6 +225,8 @@ describe("persistent agent directories", () => {
await copies.recoverStopped();
expect((await copies.get(companyId, secondId))?.processStoppedAt).toBeNull();
await stop(secondId);
// Make the deferred maintenance retry due after the new owner stop proof.
await db.update(agentInstructionWorkingCopies).set({ nextAttemptAt: null }).where(eq(agentInstructionWorkingCopies.runId, secondId));
await copies.recoverStopped();
}
expect(await copies.get(companyId, secondId)).toMatchObject({ state: "saved", receipt: { cleanupPending: false } });
@@ -556,7 +558,8 @@ describe("persistent agent directories", () => {
}
expect(execute).toHaveBeenCalledTimes(2);
expect(execute).toHaveBeenLastCalledWith(expect.objectContaining({ lease: expect.objectContaining({ id: leaseId, providerLeaseId: "original-sandbox" }),
command: "rm", args: ["-rf", "--", executionRoot], bypassSession: true }));
command: "rm", args: ["-rf", "--", executionRoot,
path.posix.join(remoteCwd, ".paperclip-runtime", "paperclip-runner", "agent-file-transfers", agentId, copy.runId)], bypassSession: true }));
await copies.recoverCaptured();
expect(execute).toHaveBeenCalledTimes(2);
await copies.release(companyId, copy.runId);
@@ -856,6 +859,9 @@ describe("persistent agent directories", () => {
await fs.mkdir(path.join(root, "build"));
await fs.writeFile(path.join(root, "build", "personal.txt"), "cache-like names are still agent files");
const first = await prepare();
const firstScratch = path.join(remoteCwd, ".paperclip-runtime", "paperclip-runner", "agent-file-transfers", agentId, first.runId);
expect(await fs.readdir(firstScratch)).toEqual([]);
await expect(fs.stat(path.join(first.executionRoot, ".paperclip-runtime"))).rejects.toMatchObject({ code: "ENOENT" });
expect(await fs.readFile(path.join(first.executionRoot, "build", "personal.txt"), "utf8")).toBe("cache-like names are still agent files");
await fs.mkdir(path.join(first.executionRoot, "notes"));
await fs.writeFile(path.join(first.executionRoot, "notes", "bytes.bin"), Buffer.from([0, 128, 255]));
@@ -865,6 +871,7 @@ describe("persistent agent directories", () => {
expect((await execFile("git", ["-C", remoteCwd, "status", "--porcelain", "--untracked-files=all"])).stdout).toBe("?? task-only.txt\n");
expect((await copies.collectStopped({ companyId, runId: first.runId, target: executionTarget }))?.state).toBe("saved");
await copies.release(companyId, first.runId);
await expect(fs.stat(firstScratch)).rejects.toMatchObject({ code: "ENOENT" });
expect((await copies.get(companyId, first.runId))?.receipt?.baseline).toBeUndefined();
await expect(fs.stat(first.localRoot)).rejects.toMatchObject({ code: "ENOENT" });
await fs.rm(remoteCwd, { recursive: true });
@@ -1188,8 +1195,8 @@ describe("persistent agent directories", () => {
const execute = vi.fn(), restoreWorkspace = vi.fn(), cleanupWorkspaceSnapshot = vi.fn().mockResolvedValue(undefined);
const executionTarget = { kind: "remote" as const, transport: "sandbox" as const, environmentId, leaseId, remoteCwd, runner: { execute } };
const shell = vi.spyOn(executionTargetTools, "runAdapterExecutionTargetShellCommand").mockResolvedValue({ exitCode: 0, signal: null, timedOut: false, stdout: "", stderr: "" });
const stage = vi.spyOn(executionTargetTools, "prepareAdapterExecutionTargetRuntime").mockImplementation(async () => ({ target: executionTarget,
workspaceRemoteDir: path.posix.join(remoteCwd, ".paperclip-runtime", "agent-files", agentId, runId), runtimeRootDir: null,
const stage = vi.spyOn(executionTargetTools, "prepareAdapterExecutionTargetRuntime").mockImplementation(async options => ({ target: executionTarget,
workspaceRemoteDir: path.posix.join(remoteCwd, ".paperclip-runtime", "agent-files", agentId, runId), runtimeRootDir: options.runtimeRootDir ?? null,
assetDirs: {}, additionalSourceDirs: {}, additionalSourceFailures: [], workspaceSyncSnapshot: null, restoreWorkspace, cleanupWorkspaceSnapshot }));
try {
const copy = (await copies.prepare({ ...target(), runId, cwd: home, target: executionTarget }))!;
@@ -300,6 +300,10 @@ describe("registered run instruction copies", () => {
expect((await revisions.readCurrent(target(), board()))?.content).toBe(initial);
expect((await copies.list(companyId, agentId, board()))[0]).toMatchObject({ state: "pending_collection", content: null });
await appendHeartbeatRunEvent(db, { ...target(), runId: copy.runId, eventType: "native.local_process_stopped", stream: "system" });
// Recovery scheduled a retry while stop authority was missing. Make that
// retry due before asking the restarted controller to collect the copy.
await db.update(agentInstructionWorkingCopies).set({ nextAttemptAt: new Date(0) })
.where(eq(agentInstructionWorkingCopies.runId, copy.runId));
await copies.recoverStopped();
expect((await revisions.readCurrent(target(), board()))?.content).toBe("edit before controller restart");
});
@@ -1091,12 +1091,23 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
const companyId = randomUUID();
const assignedAgentId = randomUUID();
const replacementAgentId = randomUUID();
await db.insert(companies).values({
id: companyId,
name: `Chat Test ${companyId.slice(0, 8)}`,
issuePrefix: `C${companyId.replace(/-/g, "").toUpperCase()}`,
requireBoardApprovalForNewAgents: false,
});
let inserted = false;
// Retired fixtures retain company rows. Retry only the short-prefix unique
// conflict; every other database error must still fail the test immediately.
for (let attempt = 0; attempt < companyPrefixAttempts; attempt += 1) {
const [company] = await db.insert(companies).values({
id: companyId,
name: `Chat Test ${companyId.slice(0, 8)}`,
issuePrefix: nextIssuePrefix(),
requireBoardApprovalForNewAgents: false,
}).onConflictDoNothing({ target: companies.issuePrefix }).returning({ id: companies.id });
if (company) {
inserted = true;
fixtureCompanies.add(companyId);
break;
}
}
if (!inserted) throw new Error(`Could not allocate a chat fixture company prefix after ${companyPrefixAttempts} attempts`);
const now = new Date();
await db
.insert(authUsers)
@@ -1614,6 +1625,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
await service.processPendingPublications();
const providerRuntime = fakeRuntime.endpoints.get(endpointId);
if (providerRuntime) providerRuntime.posts.length = 0;
return setupFollowUpMessageId;
}
async function configuredSlackEndpoint(
@@ -28229,8 +28241,18 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
}),
trigger: "direct_message",
});
await qualifySetupRoundTrip(service, endpoint.id, userId);
const setupMessageId = await qualifySetupRoundTrip(service, endpoint.id, userId);
await service.test(endpoint.id, "owner-user");
if (provider === "telegram") {
// Setup dispatches receipt cleanup asynchronously. Finish it before
// measuring reaction removals owned by this fixture's working run.
await service.processPendingReceiptReactions();
await waitForProcessedReceiptRemoval(endpoint.id, {
threadId: thread.thread.id,
messageId: setupMessageId,
emoji: "eyes",
});
}
const [conversation] = await service.listConversations(endpoint.id);
const runId = randomUUID();
await db.insert(heartbeatRuns).values({
@@ -62562,7 +62584,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
},
);
it("orders reversed GitHub edit and delete callbacks behind their durable root", async () => {
it.each([false, true])("orders reversed GitHub edit and delete callbacks behind their durable root (root processed before replay: %s)", async (rootProcessedFirst) => {
const fixture = await seedCompany();
const deferred: Array<() => void | Promise<void>> = [];
const { callbacks, endpoint, service, wakeup, webhookSecret } =
@@ -62665,8 +62687,28 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
// One root-conversation drain plus one durable-ingress callback per HTTP
// request is queued. The duplicate delivery callback becomes a no-op.
expect(deferred).toHaveLength(4);
if (rootProcessedFirst) {
await deferred.shift()?.();
await vi.waitFor(async () => {
const [root] = await db
.select({ state: chatDeliveries.state })
.from(chatDeliveries)
.where(eq(chatDeliveries.endpointId, endpoint.id));
expect(root.state).toBe("processed");
});
}
await drainDeferred();
// The scheduler callbacks start background promises. Wait for durable
// replay admission before draining the conversation work it schedules.
await vi.waitFor(async () => {
const deliveries = await db
.select({ id: chatDeliveries.id })
.from(chatDeliveries)
.where(eq(chatDeliveries.endpointId, endpoint.id));
expect(deliveries).toHaveLength(3);
});
await vi.waitFor(async () => {
await drainDeferred();
const deliveries = await db
.select({
eventKind: chatDeliveries.eventKind,
@@ -12,6 +12,7 @@ const mockUpdateExecutionWorkspace = vi.hoisted(() => vi.fn());
const mockLogActivity = vi.hoisted(() => vi.fn());
const mockLoggerInfo = vi.hoisted(() => vi.fn());
const mockGetEnvironment = vi.hoisted(() => vi.fn());
const mockGetLease = vi.hoisted(() => vi.fn());
vi.mock("../services/environment-execution-target.js", () => ({
resolveEnvironmentExecutionTarget: mockResolveEnvironmentExecutionTarget,
@@ -30,6 +31,7 @@ vi.mock("../services/environments.js", () => ({
environmentService: vi.fn(() => ({
ensureLocalEnvironment: vi.fn(),
getById: mockGetEnvironment,
getLeaseById: mockGetLease,
acquireLease: vi.fn(),
releaseLease: vi.fn(),
updateLeaseMetadata: mockUpdateLeaseMetadata,
@@ -815,3 +817,56 @@ describe("admitted native lifecycle recovery", () => {
expect(environment.config.reuseLease).toBe(false);
});
});
describe("live remote runner lease reattachment", () => {
const input = {
companyId: "company-1", selectedEnvironmentId: "env-1", localEnvironmentId: "local",
adapterType: "paperclip_runner", admittedLifecycleMode: "per_turn" as const,
issueId: "task-1", heartbeatRunId: "run-1", agentId: "agent-1",
persistedExecutionWorkspace: { id: "ew-1", mode: "shared_workspace" as const },
executionWorkspaceSettings: null,
reattachRemoteLease: { leaseId: "lease-1", providerLeaseId: "original-sandbox", remoteCwd: "/remote/workspace" },
};
const originalLease = () => makeLease({
provider: "daytona", providerLeaseId: "original-sandbox", issueId: "task-1", executionWorkspaceId: "ew-1",
metadata: { agentId: "agent-1", remoteCwd: "/remote/workspace",
sandboxLeaseAcquisition: { outcome: "created", providerLeaseId: "original-sandbox" } },
});
beforeEach(() => {
vi.clearAllMocks();
mockGetEnvironment.mockResolvedValue({ ...makeEnvironment("sandbox"), config: { provider: "daytona", reuseLease: false } });
mockGetLease.mockResolvedValue(originalLease());
});
it("reattaches the original active ephemeral lease without acquiring or resuming a sandbox", async () => {
const savedLease = originalLease();
mockGetLease.mockResolvedValue(savedLease);
const runtime = makeMockRuntime();
const result = await environmentRunOrchestrator({} as never, { environmentRuntime: runtime }).acquireForRun(input);
expect(mockGetLease).toHaveBeenCalledWith("lease-1");
expect(runtime.acquireRunLease).not.toHaveBeenCalled();
expect(result.lease).toEqual(savedLease);
expect(result.leaseContext.executionWorkspaceId).toBe("ew-1");
expect(result.environment.config.reuseLease).toBe(false);
});
it.each([
{ companyId: "foreign" }, { environmentId: "foreign" }, { heartbeatRunId: "another-run" },
{ issueId: "another-task" }, { executionWorkspaceId: "another-workspace" },
{ providerLeaseId: "replacement" }, { provider: "another-provider" },
{ status: "released" }, { expiresAt: new Date(0) }, { releasedAt: new Date() }, { cleanupStatus: "pending" },
{ metadata: { agentId: "another-agent", remoteCwd: "/remote/workspace" } },
{ metadata: { agentId: "agent-1", remoteCwd: "/other/workspace" } },
] as Partial<EnvironmentLease>[])("fails closed before provider acquisition for a mismatched lease %j", async (override) => {
mockGetLease.mockResolvedValue({ ...originalLease(), ...override });
const runtime = makeMockRuntime();
await expect(environmentRunOrchestrator({} as never, { environmentRuntime: runtime }).acquireForRun(input))
.rejects.toThrow("native_remote_recovery_lease_mismatch");
expect(runtime.acquireRunLease).not.toHaveBeenCalled();
});
it("never replaces a missing original lease", async () => {
mockGetLease.mockResolvedValue(null);
const runtime = makeMockRuntime();
await expect(environmentRunOrchestrator({} as never, { environmentRuntime: runtime }).acquireForRun(input))
.rejects.toThrow("native_remote_recovery_lease_mismatch");
expect(runtime.acquireRunLease).not.toHaveBeenCalled();
});
});
@@ -6679,13 +6679,24 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
});
});
it("destroys scoped reusable plugin-backed sandbox leases", async () => {
it.each([
{ status: "active", scope: "workspace" },
{ status: "failed", scope: "workspace" },
{ status: "failed", scope: "issue" },
] as const)("destroys $status reusable plugin-backed sandbox leases scoped to an $scope", async ({ status, scope }) => {
const { pluginId, companyId, runId, executionWorkspaceId, reusableLease } =
await seedReusablePluginSandboxLease();
await db
.update(heartbeatRuns)
.set({ status: "succeeded" })
.set({ status: status === "failed" ? "failed" : "succeeded" })
.where(eq(heartbeatRuns.id, runId));
if (status === "failed") await environmentService(db).releaseLease(reusableLease.id, "failed");
const issueId = randomUUID();
if (scope === "issue") {
await db.insert(issues).values({ id: issueId, companyId, title: "Failed reusable lease cleanup", status: "done", priority: "medium" });
await db.update(environmentLeases).set({ issueId }).where(eq(environmentLeases.id, reusableLease.id));
}
const selection = scope === "issue" ? { issueId } : { executionWorkspaceId };
const workerManager = {
isRunning: vi.fn((id: string) => id === pluginId),
@@ -6699,9 +6710,12 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
} as unknown as PluginWorkerManager;
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
// A known issue/workspace id does not authorize another company's teardown.
expect(await runtimeWithPlugin.destroyReusableSandboxLeases({ companyId: randomUUID(), ...selection })).toEqual([]);
expect(workerManager.call).not.toHaveBeenCalled();
const destroyed = await runtimeWithPlugin.destroyReusableSandboxLeases({
companyId,
executionWorkspaceId,
...selection,
failureReason: "execution_workspace_closed",
});
@@ -6724,9 +6738,10 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
});
});
it("does not destroy a scoped reusable lease while its run is active", async () => {
it.each(["active", "failed"] as const)("does not destroy a %s scoped reusable lease while its run is active", async (status) => {
const { pluginId, companyId, executionWorkspaceId, reusableLease } =
await seedReusablePluginSandboxLease();
if (status === "failed") await environmentService(db).releaseLease(reusableLease.id, "failed");
const workerManager = {
isRunning: vi.fn((id: string) => id === pluginId),
call: vi.fn(async () => undefined),
@@ -6752,9 +6767,106 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
expect(workerManager.call).not.toHaveBeenCalled();
await expect(
environmentService(db).getLeaseById(reusableLease.id),
).resolves.toMatchObject({ status: "active" });
).resolves.toMatchObject({ status });
});
it.each(["issue", "workspace", "environment"] as const)(
"durably claims failed reusable cleanup before %s teardown and recovers a provider failure",
async (scope) => {
const seeded = await seedReusablePluginSandboxLease();
await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, seeded.runId));
await environmentService(db).releaseLease(seeded.reusableLease.id, "failed");
const issueId = randomUUID();
if (scope === "issue") {
await db.insert(issues).values({ id: issueId, companyId: seeded.companyId, title: "Cleanup crash window", status: "done" });
await db.update(environmentLeases).set({ issueId }).where(eq(environmentLeases.id, seeded.reusableLease.id));
}
let enterProvider!: () => void;
let finishProvider!: () => void;
const entered = new Promise<void>((resolve) => { enterProvider = resolve; });
const finish = new Promise<void>((resolve) => { finishProvider = resolve; });
let unavailable = true;
const call = vi.fn(async (_id: string, method: string) => {
if (method !== "environmentDestroyLease") throw new Error(`Unexpected method ${method}`);
if (unavailable) {
enterProvider();
await finish;
throw new Error("Provider response lost");
}
return { providerLeaseId: seeded.reusableLease.providerLeaseId, state: "destroyed" };
});
const makeRuntime = () => environmentRuntimeService(db, { pluginWorkerManager: {
isRunning: () => true, call,
getWorker: () => ({ supportedMethods: ["environmentDestroyLease"] }),
} as unknown as PluginWorkerManager });
const runtime = makeRuntime();
const close = (service: ReturnType<typeof makeRuntime>) => scope === "environment"
? service.destroyReusableSandboxLeasesForEnvironment({ environmentId: seeded.environment.id })
: service.destroyReusableSandboxLeases({ companyId: seeded.companyId,
...(scope === "issue" ? { issueId } : { executionWorkspaceId: seeded.executionWorkspaceId }) });
const closing = close(runtime);
try {
await entered;
// This independently persisted row is what survives controller loss
// during the provider call. No provider receipt has arrived yet.
expect(await environmentService(db).getLeaseById(seeded.reusableLease.id)).toMatchObject({
status: "pending_cleanup", cleanupStatus: "failed", companyId: seeded.companyId,
providerLeaseId: seeded.reusableLease.providerLeaseId,
metadata: { pendingCleanupAttemptId: expect.any(String), pendingCleanupInFlight: true,
pendingCleanupLeaseExpiresAtMs: expect.any(Number) },
});
await close(makeRuntime());
expect(await heartbeatService(db, { environmentRuntime: makeRuntime() }).sweepPendingCleanupLeases({ backoffMs: 0 }))
.toEqual({ swept: 0, destroyed: 0, capped: 0 });
expect(call).toHaveBeenCalledOnce();
} finally {
finishProvider();
await closing;
}
expect(await environmentService(db).getLeaseById(seeded.reusableLease.id)).toMatchObject({
status: "pending_cleanup", metadata: { pendingCleanupInFlight: false },
});
unavailable = false;
expect(await heartbeatService(db, { environmentRuntime: makeRuntime() }).sweepPendingCleanupLeases({ backoffMs: 0 }))
.toEqual({ swept: 1, destroyed: 1, capped: 0 });
expect(call).toHaveBeenCalledTimes(2);
expect(await environmentService(db).getLeaseById(seeded.reusableLease.id)).toMatchObject({
status: "expired", cleanupStatus: "success", metadata: { remoteExecutionTermination: { state: "destroyed" } },
});
},
);
it.each(["queued", "scheduled_retry", "running"] as const)(
"fences scoped cleanup when its holding run becomes %s after selection", async (status) => {
const seeded = await seedReusablePluginSandboxLease();
await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, seeded.runId));
await environmentService(db).releaseLease(seeded.reusableLease.id, "failed");
const originalService = environmentsModule.environmentService;
const serviceSpy = vi.spyOn(environmentsModule, "environmentService").mockImplementation((client) => {
const service = originalService(client);
return { ...service, getById: async (id: string) => {
const environment = await service.getById(id);
if (id === seeded.environment.id) {
await db.update(heartbeatRuns).set({ status }).where(eq(heartbeatRuns.id, seeded.runId));
}
return environment;
} };
});
const call = vi.fn(async () => undefined);
const runtime = environmentRuntimeService(db, { pluginWorkerManager: {
isRunning: () => true, call, getWorker: () => ({ supportedMethods: ["environmentDestroyLease"] }),
} as unknown as PluginWorkerManager });
try {
expect(await runtime.destroyReusableSandboxLeases({ companyId: seeded.companyId,
executionWorkspaceId: seeded.executionWorkspaceId })).toEqual([]);
expect(call).not.toHaveBeenCalled();
expect(await originalService(db).getLeaseById(seeded.reusableLease.id)).toMatchObject({ status: "failed" });
} finally {
serviceSpy.mockRestore();
}
},
);
it("destroys reusable plugin-backed sandbox leases scoped to an environment", async () => {
const { pluginId, runId, reusableLease } = await seedReusablePluginSandboxLease();
// The holding run is finished, so the reservation is stale and destroyable.
@@ -6796,9 +6908,51 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
});
});
it("keeps a reusable lease held by an in-flight run out of the environment-scoped destroy", async () => {
it("destroys a failed reusable sandbox after successful release before deleting its environment", async () => {
const { pluginId, companyId, runId, environment, reusableLease } = await seedReusablePluginSandboxLease();
const workerManager = {
isRunning: vi.fn((id: string) => id === pluginId),
call: vi.fn(async (_pluginId: string, method: string) => {
if (method === "environmentReleaseLease") {
return { providerLeaseId: reusableLease.providerLeaseId, state: "stopped" };
}
if (method === "environmentDestroyLease") {
// The provider call must already have a durable cleanup reference.
expect(await environmentService(db).getLeaseById(reusableLease.id)).toMatchObject({
status: "pending_cleanup", environmentId: environment.id,
});
return { providerLeaseId: reusableLease.providerLeaseId, state: "destroyed" };
}
throw new Error(`Unexpected plugin method: ${method}`);
}),
getWorker: vi.fn(() => ({ supportedMethods: ["environmentReleaseLease", "environmentDestroyLease"] })),
} as unknown as PluginWorkerManager;
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, runId));
const released = await runtimeWithPlugin.releaseRunLeases(runId, "failed", undefined, undefined, true);
expect(released[0]?.lease).toMatchObject({
id: reusableLease.id, status: "failed", cleanupStatus: "success",
metadata: { remoteExecutionTermination: { state: "stopped" } },
});
const result = await runtimeWithPlugin.destroyReusableSandboxLeasesForEnvironment({
environmentId: environment.id, failureReason: "environment_deleted",
});
expect(result).toEqual({ destroyed: 1, failed: 0, skippedLiveRun: 0 });
expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentDestroyLease",
expect.objectContaining({ companyId, environmentId: environment.id, providerLeaseId: reusableLease.providerLeaseId }),
31234);
await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({
status: "expired", cleanupStatus: "success",
metadata: { remoteExecutionTermination: { state: "destroyed" } },
});
expect((await environmentService(db).removeIfDeletable(environment.id))?.id).toBe(environment.id);
});
it.each(["active", "failed"] as const)("keeps a %s reusable lease held by an in-flight run out of the environment-scoped destroy", async (status) => {
const { pluginId, reusableLease } = await seedReusablePluginSandboxLease();
// seedEnvironment leaves the holding run in `running` status.
if (status === "failed") await environmentService(db).releaseLease(reusableLease.id, "failed");
const workerManager = {
isRunning: vi.fn((id: string) => id === pluginId),
@@ -6816,7 +6970,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
expect(workerManager.call).not.toHaveBeenCalled();
// The lease keeps its reusable status, so the delete guard still blocks.
await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({
status: "active",
status,
leasePolicy: "reuse_by_environment",
});
});
@@ -636,6 +636,18 @@ describeEmbeddedPostgres("environmentService leases", () => {
expect(releasedImpact?.reusableSandboxLeaseCount).toBe(1);
expect(await svc.removeIfDeletable(environmentId)).toBeNull();
// A failed run can release its reusable sandbox successfully without
// destroying it. The provider handle still needs environment-scoped cleanup.
await svc.releaseLease(lease.id, "failed", {
failureReason: "adapter_or_run_failure",
cleanupStatus: "success",
});
const failedImpact = await svc.getDeleteBlastRadius(environmentId);
expect.soft(failedImpact?.canDelete).toBe(false);
expect.soft(failedImpact?.deleteBlockedReasons).toContain("reusable_sandbox_lease");
expect.soft(failedImpact?.reusableSandboxLeaseCount).toBe(1);
expect(await svc.removeIfDeletable(environmentId)).toBeNull();
const rows = await db.select().from(environments).where(eq(environments.id, environmentId));
expect(rows).toHaveLength(1);
const storedLease = await svc.getLeaseById(lease.id);
@@ -134,50 +134,65 @@ describe("workspace Git scan route load regression", () => {
...unavailableMethods(),
};
const app = createLoadApp(db, companyId, service);
const pendingResponses = Array.from({ length: 500 }, (_, index) => request(app)
// Model one busy API server, not 500 independent ephemeral listeners whose
// accept/close races can reset clients before they reach the scheduler.
const server = app.listen(0, "127.0.0.1");
await new Promise<void>(resolve => server.once("listening", resolve));
const pendingResponses = Array.from({ length: 500 }, (_, index) => request(server)
.get(`/api/issues/issue-${index}/file-resources/list`)
.set("x-test-actor", `actor-${index % 73}`)
.query({ mode: "changed" })
.then((response) => response));
// Observe every client rejection immediately, including on assertion failure.
const settledResponses = Promise.allSettled(pendingResponses);
try {
await vi.waitFor(
() => expect(scheduler.snapshot().totals.singleFlightJoins).toBe(498),
{ timeout: 15_000, interval: 20 },
);
const loadedSnapshot = scheduler.snapshot();
expect(loadedSnapshot).toMatchObject({ activeCount: 2, queuedCount: 0, inFlightCount: 2 });
await vi.waitFor(
() => expect(scheduler.snapshot().totals.singleFlightJoins).toBe(498),
{ timeout: 15_000, interval: 20 },
);
const loadedSnapshot = scheduler.snapshot();
expect(loadedSnapshot).toMatchObject({ activeCount: 2, queuedCount: 0, inFlightCount: 2 });
const healthLatencies: number[] = [];
for (let index = 0; index < 25; index += 1) {
const startedAt = performance.now();
const response = await request(server).get("/api/health");
healthLatencies.push(performance.now() - startedAt);
expect(response.status).toBe(200);
}
healthLatencies.sort((left, right) => left - right);
const healthP99Ms = healthLatencies[Math.ceil(healthLatencies.length * 0.99) - 1]!;
expect(healthP99Ms).toBeLessThan(250);
const healthLatencies: number[] = [];
for (let index = 0; index < 25; index += 1) {
const startedAt = performance.now();
const response = await request(app).get("/api/health");
healthLatencies.push(performance.now() - startedAt);
expect(response.status).toBe(200);
releaseScans();
const responses = (await settledResponses).map(result => {
if (result.status === "rejected") throw result.reason;
return result.value;
});
const outcomeCounts = responses.reduce<Record<number, number>>((counts, response) => {
counts[response.status] = (counts[response.status] ?? 0) + 1;
return counts;
}, {});
expect(outcomeCounts).toEqual({ 200: 500 });
expect({ runnerCalls, peakActive }).toEqual({ runnerCalls: 2, peakActive: 2 });
expect(scheduler.snapshot()).toMatchObject({ activeCount: 0, queuedCount: 0, inFlightCount: 0 });
console.info("workspace Git scan regression metrics", {
requests: responses.length,
repositories: roots.length,
underlyingScans: runnerCalls,
peakActive,
peakQueued: loadedSnapshot.queuedCount,
outcomes: outcomeCounts,
healthP99Ms: Math.round(healthP99Ms * 100) / 100,
unreapedChildCount: 0,
});
} finally {
releaseScans();
server.closeAllConnections();
await settledResponses;
await new Promise<void>((resolve, reject) => server.close(error => error ? reject(error) : resolve()));
}
healthLatencies.sort((left, right) => left - right);
const healthP99Ms = healthLatencies[Math.ceil(healthLatencies.length * 0.99) - 1]!;
expect(healthP99Ms).toBeLessThan(250);
releaseScans();
const responses = await Promise.all(pendingResponses);
const outcomeCounts = responses.reduce<Record<number, number>>((counts, response) => {
counts[response.status] = (counts[response.status] ?? 0) + 1;
return counts;
}, {});
expect(outcomeCounts).toEqual({ 200: 500 });
expect({ runnerCalls, peakActive }).toEqual({ runnerCalls: 2, peakActive: 2 });
expect(scheduler.snapshot()).toMatchObject({ activeCount: 0, queuedCount: 0, inFlightCount: 0 });
console.info("workspace Git scan regression metrics", {
requests: responses.length,
repositories: roots.length,
underlyingScans: runnerCalls,
peakActive,
peakQueued: loadedSnapshot.queuedCount,
outcomes: outcomeCounts,
healthP99Ms: Math.round(healthP99Ms * 100) / 100,
unreapedChildCount: 0,
});
}, 60_000);
it.each([
@@ -1,5 +1,5 @@
import { ensureNativeCompletionContract } from "../services/native-runtime/completion-contracts.js";
import { readNativeCursorPlanWait, hasCommittedNativeCursorPlanWait } from "../services/native-runtime/native-cursor-plan-wait.js";
import { readNativePlanWait, hasCommittedNativePlanWait } from "../services/native-runtime/native-plan-wait.js";
import { nativeSha256 } from "../services/native-runtime/canonical.js";
import { buildQuestionResponseDeliveryEnvelope } from "../services/question-response-delivery.js";
import * as executionContinuation from "../services/execution-continuation.js";
@@ -7930,8 +7930,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
});
expect(next?.contextSnapshot?.wakeCommentIds).toEqual([pending!.id, go!.id]);
expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, deferred!.id)))[0]).toMatchObject({ status: "coalesced", runId: next!.id });
await heartbeat.drainActiveRunExecutions();
expect((await heartbeat.getRun(next!.id))?.status).not.toBe("running");
await vi.waitFor(async () => expect((await heartbeat.getRun(next!.id))?.status).not.toBe("running"));
});
it.each(["dedicated deferred donor", "non-coalescing recipient", "persistent agent conversation"] as const)(
@@ -14537,226 +14536,6 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
expect(recoveryAction.ownerType).toBe("board");
});
describe("accepted Cursor planning boundaries", () => {
const nativeImplementation = mockExecutePaperclipNativeSession.getMockImplementation();
beforeEach(() => {
mockExecutePaperclipNativeSession.mockImplementation(async () => { throw new Error("Accepted-plan tests must never execute a provider"); });
});
afterEach(() => { mockExecutePaperclipNativeSession.mockImplementation(nativeImplementation!); });
async function seedAcceptedCursorPlanWait(semanticFinish = false, sourceSequenceOffset = 0) {
const f = await seedStrandedIssueFixture({ status: "in_progress", runStatus: "succeeded", livenessState: "advanced" });
const instance = randomUUID(), interactionId = randomUUID();
const contractInput = { db, companyId: f.companyId,
issue: { id: f.issueId, title: "Review the plan before further work", description: "Explicit completion required" }, actorId: "test" };
const { row: persistedContract, contract } = await ensureNativeCompletionContract(contractInput);
const reused = await ensureNativeCompletionContract(contractInput);
expect(reused.row.id).toBe(persistedContract.id);
expect(reused.contract).toEqual(contract);
const contractId = persistedContract.id, contractSha = persistedContract.canonicalSha256;
// Production binds policy/schema as well as the body; a body-only hash is not a valid contract receipt.
expect(contractSha).not.toBe(nativeSha256(contract));
const model = "gpt-5.6-luna[context=272k,reasoning=medium,fast=false]";
await db.update(agents).set({ adapterType: "paperclip_runner", adapterConfig: { provider: "acpx", acpxAgent: "cursor", model, acpxSessionMode: "plan", acpxPermissionMode: "approve-all" } }).where(eq(agents.id, f.agentId));
await db.update(agentWakeupRequests).set({ status: "completed" }).where(eq(agentWakeupRequests.id, f.wakeupRequestId));
await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: f.issueId, nativeSessionId: f.runId,
completionContractId: contractId, completionContractSha256: contractSha, runnerInstanceId: instance,
runnerProfileJson: { nativeExecutionInput: { binding: { companyId: f.companyId, issueId: f.issueId, runId: f.runId, agentId: f.agentId }, provider: { kind: "acpx", agent: "cursor", model, cursorMode: "plan", permissionMode: "approve-all",
profile: paperclipRunner.resolveQualifiedAcpxProfile("cursor", model) }, session: { normalizedSessionId: f.runId },
completionContract: { id: contractId, sha256: contractSha, contract } } } }).where(eq(heartbeatRuns.id, f.runId));
const planId = `plan-${"a".repeat(64)}`;
const questionSet = { schema: "paperclip.question_set.v1", title: "Native plan", description: "Exact accepted revision", questions: [{ id: planId, prompt: "Proceed?", required: true,
answerMode: "single_select", options: [{ id: "accept", label: "Accept" }, { id: "reject", label: "Reject" }, { id: "cancel", label: "Cancel" }] }] };
const [interaction] = await db.insert(issueThreadInteractions).values({ id: interactionId, companyId: f.companyId, issueId: f.issueId, sourceRunId: f.runId,
createdByAgentId: f.agentId, kind: "ask_user_questions", status: "answered", continuationPolicy: "none",
idempotencyKey: `paperclip-runner-question:${f.runId}:request`, resolvedByUserId: "responsible-user", resolvedAt: new Date(Date.now() - 1000),
payload: { version: 1, questions: [{ id: planId, prompt: "Proceed?", selectionMode: "single", required: true, allowOther: false, options: [{ id: "accept", label: "Accept" }, { id: "reject", label: "Reject" }, { id: "cancel", label: "Cancel" }] }], runtimeRequestId: "request", questionSet: questionSet as never }, result: { version: 1, answers: [{ questionId: planId, optionIds: ["accept"] }] } }).returning();
const answer = buildQuestionResponseDeliveryEnvelope(interaction as never);
const [delivery] = await db.insert(issueQuestionResponseDeliveries).values({ companyId: f.companyId, issueId: f.issueId, interactionId,
sourceRunId: f.runId, targetRunId: f.runId, correlationId: randomUUID(), status: "delivered", deliveryMode: "steered", acknowledgedAt: new Date(), payloadSha256: nativeSha256(answer) }).returning();
const port = new PaperclipControlPlanePort(db, { companyId: f.companyId, issueId: f.issueId, runId: f.runId, agentId: f.agentId,
sessionId: f.runId, completionContractId: contractId, completionContractSha256: contractSha, sourceInstanceId: instance, controlPlaneSourceInstanceId: `control-${f.runId}` });
const events = [
{ eventType: "runtime_request.created", payload: { request: { schema: "paperclip.runtime_request.v2", requestKind: "runtime", requestId: "request", type: "input", status: "pending", turnId: "turn", itemId: "native-plan-tool", prompt: "Review plan",
origin: { adapter: "acpx-runtime-sidecar", provider: "cursor", method: "cursor/create_plan" }, input: questionSet } } },
{ eventType: "tool.execution.started", payload: { schema: "paperclip.tool.execution.v1", executionId: "native-plan-tool", transport: "builtin", operation: "execute", status: "running", readOnly: false, namespace: null, name: "Create Plan", target: null, inputUpdated: true, output: null, outputBytes: 0, outputTruncated: false, outputDigest: null, progress: null, exitCode: null, durationMs: null } },
{ eventType: "runtime_request.resolved", payload: { requestId: "request", turnId: "turn", action: "submit", response: answer.response } },
{ eventType: "tool.execution.completed", payload: { schema: "paperclip.tool.execution.v1", executionId: "native-plan-tool", transport: "builtin", operation: "execute", status: "completed", readOnly: false, namespace: null, name: "Create Plan", target: null, inputUpdated: false, output: null, outputBytes: 0, outputTruncated: false, outputDigest: null, progress: null, exitCode: null, durationMs: null } },
{ eventType: "turn.completed", payload: { status: "completed", error: null } },
];
for (const [index, event] of events.entries()) await port.appendEvent({ schema: "paperclip.prp.event.v1", sourceEventId: `${instance}:${sourceSequenceOffset + index + 1}`, sourceSeq: sourceSequenceOffset + index + 1,
sourceInstanceId: instance, sourceKind: "runner", runId: f.runId, normalizedSessionId: f.runId, turnId: "turn", schemaVersion: 1, priority: 0,
emittedAt: new Date().toISOString(), ...event } as paperclipRunner.PrpEvent);
const proof = await readNativeCursorPlanWait(db, f);
expect(proof).not.toBeNull();
const result = semanticFinish ? { ...proof!.result, reportedWorkDisposition: "done" as const, summary: "Explicit semantic finish wins", continuation: undefined,
completionClaim: { contractRevision: "1", objectiveSatisfied: true, criteria: [{ criterionId: "objective", status: "satisfied" as const, evidenceRefs: [] }], remainingWork: [] } } : proof!.result;
await port.completeRun({ result, turnId: "turn", terminal: { schema: "paperclip.prp.terminal.v1", runTerminalState: "succeeded", turnTerminalState: "completed", reportedWorkDisposition: result.reportedWorkDisposition } });
return { ...f, interactionId, deliveryId: delivery!.id, proof: proof! };
}
it("keeps an accepted Cursor plan passive across restart, future configuration changes, and paused recovery", async () => {
const f = await seedAcceptedCursorPlanWait();
await finalizeNativeRun({ db, runId: f.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
await finalizeNativeRun({ db, runId: f.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
expect(await hasCommittedNativeCursorPlanWait(db, f)).toBe(true);
expect(await db.select().from(statusDecisions).where(eq(statusDecisions.runId, f.runId))).toEqual([expect.objectContaining({ reasonCode: "native_plan_accepted_waiting_for_continuation", toStatus: "in_progress" })]);
expect(await db.select().from(statusDecisionEffects).where(eq(statusDecisionEffects.issueId, f.issueId))).toEqual([expect.objectContaining({ effectKind: "issue_status_projection", targetType: "issue", deliveryState: "delivered" })]);
expect(await db.select().from(issueComments).where(eq(issueComments.createdByRunId, f.runId))).toEqual([expect.objectContaining({ body: expect.stringContaining("next message") })]);
const current = paperclipRunner.resolveQualifiedAcpxProfile("cursor", "gpt-5.6-luna[context=272k,reasoning=medium,fast=false]");
const resolver = vi.spyOn(paperclipRunner, "resolveQualifiedAcpxProfile").mockReturnValue({ ...current,
agentProfileVersion: current.agentProfileVersion + 1, commandDigest: `sha256:${"b".repeat(64)}` });
try {
expect(await readNativeCursorPlanWait(db, f)).toBeNull(); // Old profile cannot create a new wait.
for (const status of ["idle", "paused"] as const) {
await db.update(agents).set({ status, adapterConfig: { provider: "acpx", acpxAgent: "cursor",
model: "future-model", acpxSessionMode: "agent", acpxPermissionMode: "approve-reads" } }).where(eq(agents.id, f.agentId));
expect(await hasCommittedNativeCursorPlanWait(db, f)).toBe(true);
const recovered = await heartbeatService(db).reconcileStrandedAssignedIssues();
expect(recovered.continuationRequeued).toBe(0); expect(recovered.escalated).toBe(0);
}
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId));
const changed = structuredClone(run!.runnerProfileJson!);
(changed.nativeExecutionInput as { provider: { profile: { commandDigest: string } } }).provider.profile.commandDigest = "tampered-original-profile";
await db.update(heartbeatRuns).set({ runnerProfileJson: changed }).where(eq(heartbeatRuns.id, f.runId));
expect(await hasCommittedNativeCursorPlanWait(db, f)).toBe(false);
await db.update(heartbeatRuns).set({ runnerProfileJson: run!.runnerProfileJson }).where(eq(heartbeatRuns.id, f.runId));
expect(await hasCommittedNativeCursorPlanWait(db, f)).toBe(true);
const [tool] = await db.select().from(heartbeatRunEvents).where(and(eq(heartbeatRunEvents.runId, f.runId), eq(heartbeatRunEvents.eventType, "tool.execution.completed")));
const changedTool = structuredClone(tool!.payload!) as { prpEvent: { payload: { name: string } } };
changedTool.prpEvent.payload.name = "mutated committed tool evidence";
await db.update(heartbeatRunEvents).set({ payload: changedTool, sourcePayloadSha256: nativeSha256(changedTool.prpEvent) }).where(eq(heartbeatRunEvents.id, tool!.id));
expect(await hasCommittedNativeCursorPlanWait(db, f)).toBe(false);
await db.update(heartbeatRunEvents).set({ payload: tool!.payload, sourcePayloadSha256: tool!.sourcePayloadSha256 }).where(eq(heartbeatRunEvents.id, tool!.id));
expect(await hasCommittedNativeCursorPlanWait(db, f)).toBe(true);
} finally { resolver.mockRestore(); }
expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.companyId))).toHaveLength(1);
expect(mockAdapterExecute).not.toHaveBeenCalled(); expect(mockExecutePaperclipNativeSession).not.toHaveBeenCalled();
});
it("retains an exact historical Cursor6 committed wait across more than 1000 ignored progress rows without allowing new unbound admission", async () => {
const progressCount = 1002;
const f = await seedAcceptedCursorPlanWait(false, progressCount);
await finalizeNativeRun({ db, runId: f.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId));
const profile = structuredClone(run!.runnerProfileJson!) as any;
Object.assign(profile.nativeExecutionInput.provider.profile, { agentProfileVersion: 6, commandDigest: "sha256:377dcea64a727ce799cc112458d4b40ba4bc6574cd6c6f7233b6efd5917a6c4b" });
await db.update(heartbeatRuns).set({ runnerProfileJson: profile }).where(eq(heartbeatRuns.id, f.runId));
await db.delete(heartbeatRunEvents).where(and(eq(heartbeatRunEvents.runId, f.runId), inArray(heartbeatRunEvents.eventType, ["tool.execution.started", "tool.execution.completed"])));
// Reconstruct the persisted Cursor6 receipt format using its unchanged
// source facts. This historical format had no tool lifecycle binding.
const rows = await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, f.runId));
const event = (kind: string) => (rows.find(r => r.eventType === kind)!.payload as any).prpEvent;
const [contract] = await db.select().from(completionContracts).where(eq(completionContracts.id, run!.completionContractId!));
const [interaction] = await db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.id, f.interactionId));
const [delivery] = await db.select().from(issueQuestionResponseDeliveries).where(eq(issueQuestionResponseDeliveries.id, f.deliveryId));
const legacy = { ...f.proof.source }; delete legacy.toolExecutionId; delete legacy.toolLifecycleSha256;
legacy.authoritySha256 = nativeSha256({ admission: profile.nativeExecutionInput, contract, created: event("runtime_request.created"), resolved: event("runtime_request.resolved"), terminal: event("turn.completed"), interaction, delivery, resolvedAt: interaction!.resolvedAt!.toISOString(), acknowledgedAt: delivery!.acknowledgedAt!.toISOString() });
const [decision] = await db.select().from(statusDecisions).where(eq(statusDecisions.runId, f.runId));
await db.update(statusDecisions).set({ decisionJson: { ...decision!.decisionJson, cursorPlanWait: legacy } }).where(eq(statusDecisions.id, decision!.id));
expect(await readNativeCursorPlanWait(db, f)).toBeNull();
expect(await hasCommittedNativeCursorPlanWait(db, f)).toBe(true);
// Fill the reserved source prefix with genuine durable progress rows. Shift
// only DB ordering keys; the original PRP facts and committed receipt stay
// byte-identical. Cursor6 never queried these rows, even above its budget.
await db.update(heartbeatRunEvents).set({ seq: sql`${heartbeatRunEvents.seq} + ${progressCount}` }).where(eq(heartbeatRunEvents.runId, f.runId));
const progress = Array.from({ length: progressCount }, (_, index) => {
const seq = index + 1;
const prpEvent = { ...event("runtime_request.created"), sourceEventId: `${run!.runnerInstanceId}:${seq}`, sourceSeq: seq, eventType: "tool.execution.progressed",
payload: { schema: "paperclip.tool.execution.v1", executionId: "earlier-tool", transport: "builtin", operation: "read", status: "running" } };
return { companyId: f.companyId, runId: f.runId, agentId: f.agentId, seq, eventType: prpEvent.eventType, payload: { prpEvent }, sourceInstanceId: run!.runnerInstanceId,
sourceEventId: prpEvent.sourceEventId, sourceSeq: seq, sourcePayloadSha256: nativeSha256(prpEvent), protocolSchemaVersion: 1 };
});
await db.insert(heartbeatRunEvents).values(progress);
expect(await hasCommittedNativeCursorPlanWait(db, f)).toBe(true);
const [unchangedDecision] = await db.select().from(statusDecisions).where(eq(statusDecisions.id, decision!.id));
expect(unchangedDecision!.decisionJson!.cursorPlanWait).toEqual(legacy);
// Completed rows belonged to the old query. A later completion must still
// invalidate the normal-terminal boundary rather than being filtered away.
const completion = { ...event("turn.completed"), sourceEventId: `${run!.runnerInstanceId}:99999`, sourceSeq: 99999, eventType: "tool.execution.completed", payload: { ...progress[0]!.payload.prpEvent.payload, status: "completed" } };
const [extra] = await db.insert(heartbeatRunEvents).values({ ...progress[0]!, seq: 99999, eventType: completion.eventType, payload: { prpEvent: completion }, sourceEventId: completion.sourceEventId, sourceSeq: 99999, sourcePayloadSha256: nativeSha256(completion) }).returning();
expect(await hasCommittedNativeCursorPlanWait(db, f)).toBe(false);
await db.delete(heartbeatRunEvents).where(eq(heartbeatRunEvents.id, extra!.id));
expect(await hasCommittedNativeCursorPlanWait(db, f)).toBe(true);
profile.nativeExecutionInput.provider.profile.commandDigest = "tampered-history";
await db.update(heartbeatRuns).set({ runnerProfileJson: profile }).where(eq(heartbeatRuns.id, f.runId));
expect(await hasCommittedNativeCursorPlanWait(db, f)).toBe(false);
});
it("admits a later ordinary user message after an accepted Cursor plan without changing Plan mode or replaying its run", async () => {
const f = await seedAcceptedCursorPlanWait();
await finalizeNativeRun({ db, runId: f.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
// Occupy the single dispatch slot: this checks actual user-wake admission
// without executing any provider or fabricating a second semantic result.
const occupied = randomUUID();
await db.update(agents).set({ runtimeConfig: { heartbeat: { wakeOnDemand: true, maxConcurrentRuns: 1 } } }).where(eq(agents.id, f.agentId));
await db.insert(heartbeatRuns).values({ id: occupied, companyId: f.companyId, agentId: f.agentId, status: "running", startedAt: new Date(), contextSnapshot: {} });
const [comment] = await db.insert(issueComments).values({ companyId: f.companyId, issueId: f.issueId, authorType: "user", authorUserId: "responsible-user", body: "Continue reviewing the accepted plan in Plan mode." }).returning();
try {
const heartbeat = heartbeatService(db);
const next = await heartbeat.wakeup(f.agentId, { source: "automation", triggerDetail: "system", reason: "issue_commented", requestedByActorType: "user", requestedByActorId: "responsible-user",
payload: { issueId: f.issueId, commentId: comment!.id }, contextSnapshot: { issueId: f.issueId, taskId: f.issueId, wakeCommentId: comment!.id, wakeCommentIds: [comment!.id] } });
expect(next).not.toBeNull(); expect(next!.id).not.toBe(f.runId);
const [admitted] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, next!.id));
expect(admitted).toMatchObject({ status: "queued", retryOfRunId: null });
expect(admitted!.contextSnapshot?.wakeCommentIds).toContain(comment!.id);
const [agent] = await db.select().from(agents).where(eq(agents.id, f.agentId));
expect(agent!.adapterConfig.acpxSessionMode).toBe("plan");
expect(await hasCommittedNativeCursorPlanWait(db, f)).toBe(false);
expect(mockExecutePaperclipNativeSession).not.toHaveBeenCalled();
} finally {
await db.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date() }).where(and(eq(heartbeatRuns.companyId, f.companyId), inArray(heartbeatRuns.status, ["running", "queued"])));
await db.update(agentWakeupRequests).set({ status: "cancelled", finishedAt: new Date() }).where(and(eq(agentWakeupRequests.companyId, f.companyId), inArray(agentWakeupRequests.status, ["queued", "claimed"])));
}
});
it.each(["delivery", "assignment", "admission", "user_request", "result_or_decision"] as const)("revokes an accepted Cursor plan passive wait after %s changes", async change => {
const f = await seedAcceptedCursorPlanWait();
await finalizeNativeRun({ db, runId: f.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
if (change === "delivery") await db.update(issueQuestionResponseDeliveries).set({ acknowledgedAt: null }).where(eq(issueQuestionResponseDeliveries.id, f.deliveryId));
if (change === "assignment") await db.update(issues).set({ assigneeAgentId: null }).where(eq(issues.id, f.issueId));
if (change === "admission") {
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId));
const profile = structuredClone(run!.runnerProfileJson!);
(profile.nativeExecutionInput as { provider: { cursorMode: string } }).provider.cursorMode = "agent";
await db.update(heartbeatRuns).set({ runnerProfileJson: profile }).where(eq(heartbeatRuns.id, f.runId));
}
if (change === "user_request") await db.insert(issueComments).values({ companyId: f.companyId, issueId: f.issueId, authorType: "user", authorUserId: "responsible-user", body: "Continue reviewing this plan in Plan mode." });
if (change === "result_or_decision") {
const [accepted] = await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, f.runId));
await db.update(nativeRunResults).set({ canonicalSha256: "changed" }).where(eq(nativeRunResults.id, accepted!.id));
expect(await hasCommittedNativeCursorPlanWait(db, f)).toBe(false);
await db.update(nativeRunResults).set({ canonicalSha256: accepted!.canonicalSha256 }).where(eq(nativeRunResults.id, accepted!.id));
expect(await hasCommittedNativeCursorPlanWait(db, f)).toBe(true);
await db.update(issues).set({ lastStatusDecisionId: null }).where(eq(issues.id, f.issueId));
}
expect(await hasCommittedNativeCursorPlanWait(db, f)).toBe(false);
});
it("rechecks accepted Cursor plan delivery under the status transaction before presentation or effects", async () => {
const f = await seedAcceptedCursorPlanWait();
await finalizeNativeRun({ db, runId: f.runId, workspaceFinalizeStatus: "succeeded", failpoint: "status_projection" });
const [coordinator] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId));
const [issue] = await db.select().from(issues).where(eq(issues.id, f.issueId));
await db.update(issueQuestionResponseDeliveries).set({ payloadSha256: "changed" }).where(eq(issueQuestionResponseDeliveries.id, f.deliveryId));
await expect(commitNativeStatusDecision({ db, companyId: f.companyId, issueId: f.issueId, runId: f.runId, assessmentId: coordinator!.assessmentId!,
priorStatus: issue!.status, priorStatusVersion: issue!.statusVersion, priorDecisionId: issue!.lastStatusDecisionId,
decision: { policyVersion: "phase6-v7", statusAction: "in_progress", toStatus: "in_progress", reasonCode: "native_plan_accepted_waiting_for_continuation", unblockDescriptor: null, effects: [] },
requireCursorPlanWaitSource: f.proof.source })).rejects.toBeInstanceOf(NativeStatusRaceError);
expect(await db.select().from(statusDecisions).where(eq(statusDecisions.runId, f.runId))).toHaveLength(0);
expect(await db.select().from(issueComments).where(eq(issueComments.createdByRunId, f.runId))).toHaveLength(0);
});
it("preserves explicit semantic finish priority over accepted Cursor plan evidence", async () => {
const f = await seedAcceptedCursorPlanWait(true);
await finalizeNativeRun({ db, runId: f.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
expect(await hasCommittedNativeCursorPlanWait(db, f)).toBe(false);
const decisions = await db.select().from(statusDecisions).where(eq(statusDecisions.runId, f.runId));
expect(decisions).toHaveLength(1); expect(decisions[0]!.reasonCode).not.toBe("native_plan_accepted_waiting_for_continuation");
});
});
async function seedNativePassiveBoardResponse(
continuationKind:
"response_wake" | "same_agent" | "retry" = "response_wake",
@@ -16627,4 +16406,237 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
.where(eq(heartbeatRuns.id, runId));
expect(runs).toHaveLength(1);
});
describe("accepted Cursor planning boundaries", () => {
const nativeImplementation = mockExecutePaperclipNativeSession.getMockImplementation();
beforeEach(() => {
mockExecutePaperclipNativeSession.mockImplementation(async () => { throw new Error("Accepted-plan tests must never execute a provider"); });
});
afterEach(() => { mockExecutePaperclipNativeSession.mockImplementation(nativeImplementation!); });
async function seedAcceptedCursorPlanWait(semanticFinish = false, sourceSequenceOffset = 0) {
const f = await seedStrandedIssueFixture({ status: "in_progress", runStatus: "succeeded", livenessState: "advanced" });
const instance = randomUUID(), interactionId = randomUUID();
const contractInput = { db, companyId: f.companyId,
issue: { id: f.issueId, title: "Review the plan before further work", description: "Explicit completion required" }, actorId: "test" };
const { row: persistedContract, contract } = await ensureNativeCompletionContract(contractInput);
const reused = await ensureNativeCompletionContract(contractInput);
expect(reused.row.id).toBe(persistedContract.id);
expect(reused.contract).toEqual(contract);
const contractId = persistedContract.id, contractSha = persistedContract.canonicalSha256;
// Production binds policy/schema as well as the body; a body-only hash is not a valid contract receipt.
expect(contractSha).not.toBe(nativeSha256(contract));
const model = "gpt-5.6-luna[context=272k,reasoning=medium,fast=false]";
await db.update(agents).set({ adapterType: "paperclip_runner", adapterConfig: { provider: "acpx", acpxAgent: "cursor", model, acpxSessionMode: "plan", acpxPermissionMode: "approve-all" } }).where(eq(agents.id, f.agentId));
await db.update(agentWakeupRequests).set({ status: "completed" }).where(eq(agentWakeupRequests.id, f.wakeupRequestId));
await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: f.issueId, nativeSessionId: f.runId,
completionContractId: contractId, completionContractSha256: contractSha, runnerInstanceId: instance,
runnerProfileJson: { nativeExecutionInput: { binding: { companyId: f.companyId, issueId: f.issueId, runId: f.runId, agentId: f.agentId }, provider: { kind: "acpx", agent: "cursor", model, mode: "plan", permissionMode: "approve-all",
profile: paperclipRunner.resolveQualifiedAcpxProfile("cursor", model) }, session: { normalizedSessionId: f.runId },
completionContract: { id: contractId, sha256: contractSha, contract } } } }).where(eq(heartbeatRuns.id, f.runId));
const planId = `plan-${"a".repeat(64)}`;
const questionSet = { schema: "paperclip.question_set.v1", title: "Native plan", description: "Exact accepted revision", questions: [{ id: planId, prompt: "Proceed?", required: true,
answerMode: "single_select", options: [{ id: "accept", label: "Accept" }, { id: "reject", label: "Reject" }, { id: "cancel", label: "Cancel" }] }] };
const [interaction] = await db.insert(issueThreadInteractions).values({ id: interactionId, companyId: f.companyId, issueId: f.issueId, sourceRunId: f.runId,
createdByAgentId: f.agentId, kind: "ask_user_questions", status: "answered", continuationPolicy: "none",
idempotencyKey: `paperclip-runner-question:${f.runId}:request`, resolvedByUserId: "responsible-user", resolvedAt: new Date(Date.now() - 1000),
payload: { version: 1, questions: [{ id: planId, prompt: "Proceed?", selectionMode: "single", required: true, allowOther: false, options: [{ id: "accept", label: "Accept" }, { id: "reject", label: "Reject" }, { id: "cancel", label: "Cancel" }] }], runtimeRequestId: "request", questionSet: questionSet as never }, result: { version: 1, answers: [{ questionId: planId, optionIds: ["accept"] }] } }).returning();
const answer = buildQuestionResponseDeliveryEnvelope(interaction as never);
const [delivery] = await db.insert(issueQuestionResponseDeliveries).values({ companyId: f.companyId, issueId: f.issueId, interactionId,
sourceRunId: f.runId, targetRunId: f.runId, correlationId: randomUUID(), status: "delivered", deliveryMode: "steered", acknowledgedAt: new Date(), payloadSha256: nativeSha256(answer) }).returning();
const port = new PaperclipControlPlanePort(db, { companyId: f.companyId, issueId: f.issueId, runId: f.runId, agentId: f.agentId,
sessionId: f.runId, completionContractId: contractId, completionContractSha256: contractSha, sourceInstanceId: instance, controlPlaneSourceInstanceId: `control-${f.runId}` });
const events = [
{ eventType: "runtime_request.created", payload: { request: { schema: "paperclip.runtime_request.v2", requestKind: "runtime", requestId: "request", type: "input", status: "pending", turnId: "turn", itemId: "native-plan-tool", prompt: "Review plan",
origin: { adapter: "acpx-runtime-sidecar", provider: "cursor", method: "cursor/create_plan" }, input: questionSet } } },
{ eventType: "tool.execution.started", payload: { schema: "paperclip.tool.execution.v1", executionId: "native-plan-tool", transport: "builtin", operation: "execute", status: "running", readOnly: false, namespace: null, name: "Create Plan", target: null, inputUpdated: true, output: null, outputBytes: 0, outputTruncated: false, outputDigest: null, progress: null, exitCode: null, durationMs: null } },
{ eventType: "runtime_request.resolved", payload: { requestId: "request", turnId: "turn", action: "submit", response: answer.response } },
{ eventType: "tool.execution.completed", payload: { schema: "paperclip.tool.execution.v1", executionId: "native-plan-tool", transport: "builtin", operation: "execute", status: "completed", readOnly: false, namespace: null, name: "Create Plan", target: null, inputUpdated: false, output: null, outputBytes: 0, outputTruncated: false, outputDigest: null, progress: null, exitCode: null, durationMs: null } },
{ eventType: "turn.completed", payload: { status: "completed", error: null } },
];
for (const [index, event] of events.entries()) await port.appendEvent({ schema: "paperclip.prp.event.v1", sourceEventId: `${instance}:${sourceSequenceOffset + index + 1}`, sourceSeq: sourceSequenceOffset + index + 1,
sourceInstanceId: instance, sourceKind: "runner", runId: f.runId, normalizedSessionId: f.runId, turnId: "turn", schemaVersion: 1, priority: 0,
emittedAt: new Date().toISOString(), ...event } as paperclipRunner.PrpEvent);
const proof = await readNativePlanWait(db, f);
expect(proof).not.toBeNull();
const result = semanticFinish ? { ...proof!.result, reportedWorkDisposition: "done" as const, summary: "Explicit semantic finish wins", continuation: undefined,
completionClaim: { contractRevision: "1", objectiveSatisfied: true, criteria: [{ criterionId: "objective", status: "satisfied" as const, evidenceRefs: [] }], remainingWork: [] } } : proof!.result;
await port.completeRun({ result, turnId: "turn", terminal: { schema: "paperclip.prp.terminal.v1", runTerminalState: "succeeded", turnTerminalState: "completed", reportedWorkDisposition: result.reportedWorkDisposition } });
return { ...f, interactionId, deliveryId: delivery!.id, proof: proof! };
}
it("keeps an accepted Cursor plan passive across restart, future configuration changes, and paused recovery", async () => {
const f = await seedAcceptedCursorPlanWait();
await finalizeNativeRun({ db, runId: f.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
await finalizeNativeRun({ db, runId: f.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
expect(await hasCommittedNativePlanWait(db, f)).toBe(true);
expect(await db.select().from(statusDecisions).where(eq(statusDecisions.runId, f.runId))).toEqual([expect.objectContaining({ reasonCode: "native_plan_accepted_waiting_for_continuation", toStatus: "in_progress" })]);
expect(await db.select().from(statusDecisionEffects).where(eq(statusDecisionEffects.issueId, f.issueId))).toEqual([expect.objectContaining({ effectKind: "issue_status_projection", targetType: "issue", deliveryState: "delivered" })]);
expect(await db.select().from(issueComments).where(eq(issueComments.createdByRunId, f.runId))).toEqual([expect.objectContaining({ body: expect.stringContaining("next message") })]);
const current = paperclipRunner.resolveQualifiedAcpxProfile("cursor", "gpt-5.6-luna[context=272k,reasoning=medium,fast=false]");
const resolver = vi.spyOn(paperclipRunner, "resolveQualifiedAcpxProfile").mockReturnValue({ ...current,
agentProfileVersion: current.agentProfileVersion + 1, commandDigest: `sha256:${"b".repeat(64)}` });
try {
expect(await readNativePlanWait(db, f)).toBeNull(); // Old profile cannot create a new wait.
for (const status of ["idle", "paused"] as const) {
await db.update(agents).set({ status, adapterConfig: { provider: "acpx", acpxAgent: "cursor",
model: "future-model", acpxSessionMode: "agent", acpxPermissionMode: "approve-reads" } }).where(eq(agents.id, f.agentId));
expect(await hasCommittedNativePlanWait(db, f)).toBe(true);
const recovered = await heartbeatService(db).reconcileStrandedAssignedIssues();
expect(recovered.continuationRequeued).toBe(0); expect(recovered.escalated).toBe(0);
}
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId));
const changed = structuredClone(run!.runnerProfileJson!);
(changed.nativeExecutionInput as { provider: { profile: { commandDigest: string } } }).provider.profile.commandDigest = "tampered-original-profile";
await db.update(heartbeatRuns).set({ runnerProfileJson: changed }).where(eq(heartbeatRuns.id, f.runId));
expect(await hasCommittedNativePlanWait(db, f)).toBe(false);
await db.update(heartbeatRuns).set({ runnerProfileJson: run!.runnerProfileJson }).where(eq(heartbeatRuns.id, f.runId));
expect(await hasCommittedNativePlanWait(db, f)).toBe(true);
const [tool] = await db.select().from(heartbeatRunEvents).where(and(eq(heartbeatRunEvents.runId, f.runId), eq(heartbeatRunEvents.eventType, "tool.execution.completed")));
const changedTool = structuredClone(tool!.payload!) as { prpEvent: { payload: { name: string } } };
changedTool.prpEvent.payload.name = "mutated committed tool evidence";
await db.update(heartbeatRunEvents).set({ payload: changedTool, sourcePayloadSha256: nativeSha256(changedTool.prpEvent) }).where(eq(heartbeatRunEvents.id, tool!.id));
expect(await hasCommittedNativePlanWait(db, f)).toBe(false);
await db.update(heartbeatRunEvents).set({ payload: tool!.payload, sourcePayloadSha256: tool!.sourcePayloadSha256 }).where(eq(heartbeatRunEvents.id, tool!.id));
expect(await hasCommittedNativePlanWait(db, f)).toBe(true);
} finally { resolver.mockRestore(); }
expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.companyId))).toHaveLength(1);
expect(mockAdapterExecute).not.toHaveBeenCalled(); expect(mockExecutePaperclipNativeSession).not.toHaveBeenCalled();
});
it("retains an exact historical Cursor6 committed wait across more than 1000 ignored progress rows without allowing new unbound admission", async () => {
const progressCount = 1002;
const f = await seedAcceptedCursorPlanWait(false, progressCount);
await finalizeNativeRun({ db, runId: f.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId));
const profile = structuredClone(run!.runnerProfileJson!) as any;
Object.assign(profile.nativeExecutionInput.provider.profile, { agentProfileVersion: 6, commandDigest: "sha256:377dcea64a727ce799cc112458d4b40ba4bc6574cd6c6f7233b6efd5917a6c4b" });
await db.update(heartbeatRuns).set({ runnerProfileJson: profile }).where(eq(heartbeatRuns.id, f.runId));
await db.delete(heartbeatRunEvents).where(and(eq(heartbeatRunEvents.runId, f.runId), inArray(heartbeatRunEvents.eventType, ["tool.execution.started", "tool.execution.completed"])));
// Reconstruct the persisted Cursor6 receipt format using its unchanged
// source facts. This historical format had no tool lifecycle binding.
const rows = await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, f.runId));
const event = (kind: string) => (rows.find(r => r.eventType === kind)!.payload as any).prpEvent;
const [contract] = await db.select().from(completionContracts).where(eq(completionContracts.id, run!.completionContractId!));
const [interaction] = await db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.id, f.interactionId));
const [delivery] = await db.select().from(issueQuestionResponseDeliveries).where(eq(issueQuestionResponseDeliveries.id, f.deliveryId));
profile.nativeExecutionInput.provider.cursorMode = profile.nativeExecutionInput.provider.mode;
delete profile.nativeExecutionInput.provider.mode;
await db.update(heartbeatRuns).set({ runnerProfileJson: profile }).where(eq(heartbeatRuns.id, f.runId));
const legacy = { ...f.proof.source, schema: "paperclip.native_cursor_plan_wait.v1" }; delete legacy.toolExecutionId; delete legacy.toolLifecycleSha256;
legacy.authoritySha256 = nativeSha256({ admission: profile.nativeExecutionInput, contract, created: event("runtime_request.created"), resolved: event("runtime_request.resolved"), terminal: event("turn.completed"), interaction, delivery, resolvedAt: interaction!.resolvedAt!.toISOString(), acknowledgedAt: delivery!.acknowledgedAt!.toISOString() });
const [decision] = await db.select().from(statusDecisions).where(eq(statusDecisions.runId, f.runId));
const [savedResult] = await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, f.runId));
const resultJson = structuredClone(savedResult!.resultJson!) as any;
resultJson.result.continuation.idempotencyKey = resultJson.result.continuation.idempotencyKey.replace("native-plan-wait:", "cursor-plan-wait:");
const canonicalSha256 = nativeSha256(resultJson);
await db.update(nativeRunResults).set({ resultJson, canonicalSha256 }).where(eq(nativeRunResults.id, savedResult!.id));
const decisionJson = { ...decision!.decisionJson } as any;
delete decisionJson.planWait; delete decisionJson.planWaitResult;
decisionJson.cursorPlanWait = legacy;
decisionJson.cursorPlanWaitResult = { resultId: savedResult!.id, resultSha256: canonicalSha256 };
await db.update(statusDecisions).set({ decisionJson }).where(eq(statusDecisions.id, decision!.id));
expect(await readNativePlanWait(db, f)).toBeNull();
expect(await hasCommittedNativePlanWait(db, f)).toBe(true);
// Fill the reserved source prefix with genuine durable progress rows. Shift
// only DB ordering keys; the original PRP facts and committed receipt stay
// byte-identical. Cursor6 never queried these rows, even above its budget.
await db.update(heartbeatRunEvents).set({ seq: sql`${heartbeatRunEvents.seq} + ${progressCount}` }).where(eq(heartbeatRunEvents.runId, f.runId));
const progress = Array.from({ length: progressCount }, (_, index) => {
const seq = index + 1;
const prpEvent = { ...event("runtime_request.created"), sourceEventId: `${run!.runnerInstanceId}:${seq}`, sourceSeq: seq, eventType: "tool.execution.progressed",
payload: { schema: "paperclip.tool.execution.v1", executionId: "earlier-tool", transport: "builtin", operation: "read", status: "running" } };
return { companyId: f.companyId, runId: f.runId, agentId: f.agentId, seq, eventType: prpEvent.eventType, payload: { prpEvent }, sourceInstanceId: run!.runnerInstanceId,
sourceEventId: prpEvent.sourceEventId, sourceSeq: seq, sourcePayloadSha256: nativeSha256(prpEvent), protocolSchemaVersion: 1 };
});
await db.insert(heartbeatRunEvents).values(progress);
expect(await hasCommittedNativePlanWait(db, f)).toBe(true);
const [unchangedDecision] = await db.select().from(statusDecisions).where(eq(statusDecisions.id, decision!.id));
expect(unchangedDecision!.decisionJson!.cursorPlanWait).toEqual(legacy);
// Completed rows belonged to the old query. A later completion must still
// invalidate the normal-terminal boundary rather than being filtered away.
const completion = { ...event("turn.completed"), sourceEventId: `${run!.runnerInstanceId}:99999`, sourceSeq: 99999, eventType: "tool.execution.completed", payload: { ...progress[0]!.payload.prpEvent.payload, status: "completed" } };
const [extra] = await db.insert(heartbeatRunEvents).values({ ...progress[0]!, seq: 99999, eventType: completion.eventType, payload: { prpEvent: completion }, sourceEventId: completion.sourceEventId, sourceSeq: 99999, sourcePayloadSha256: nativeSha256(completion) }).returning();
expect(await hasCommittedNativePlanWait(db, f)).toBe(false);
await db.delete(heartbeatRunEvents).where(eq(heartbeatRunEvents.id, extra!.id));
expect(await hasCommittedNativePlanWait(db, f)).toBe(true);
profile.nativeExecutionInput.provider.profile.commandDigest = "tampered-history";
await db.update(heartbeatRuns).set({ runnerProfileJson: profile }).where(eq(heartbeatRuns.id, f.runId));
expect(await hasCommittedNativePlanWait(db, f)).toBe(false);
});
it("admits a later ordinary user message after an accepted Cursor plan without changing Plan mode or replaying its run", async () => {
const f = await seedAcceptedCursorPlanWait();
await finalizeNativeRun({ db, runId: f.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
// Occupy the single dispatch slot: this checks actual user-wake admission
// without executing any provider or fabricating a second semantic result.
const occupied = randomUUID();
await db.update(agents).set({ runtimeConfig: { heartbeat: { wakeOnDemand: true, maxConcurrentRuns: 1 } } }).where(eq(agents.id, f.agentId));
await db.insert(heartbeatRuns).values({ id: occupied, companyId: f.companyId, agentId: f.agentId, status: "running", startedAt: new Date(), contextSnapshot: {} });
const [comment] = await db.insert(issueComments).values({ companyId: f.companyId, issueId: f.issueId, authorType: "user", authorUserId: "responsible-user", body: "Continue reviewing the accepted plan in Plan mode." }).returning();
try {
const heartbeat = heartbeatService(db);
const next = await heartbeat.wakeup(f.agentId, { source: "automation", triggerDetail: "system", reason: "issue_commented", requestedByActorType: "user", requestedByActorId: "responsible-user",
payload: { issueId: f.issueId, commentId: comment!.id }, contextSnapshot: { issueId: f.issueId, taskId: f.issueId, wakeCommentId: comment!.id, wakeCommentIds: [comment!.id] } });
expect(next).not.toBeNull(); expect(next!.id).not.toBe(f.runId);
const [admitted] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, next!.id));
expect(admitted).toMatchObject({ status: "queued", retryOfRunId: null });
expect(admitted!.contextSnapshot?.wakeCommentIds).toContain(comment!.id);
const [agent] = await db.select().from(agents).where(eq(agents.id, f.agentId));
expect(agent!.adapterConfig.acpxSessionMode).toBe("plan");
expect(await hasCommittedNativePlanWait(db, f)).toBe(false);
expect(mockExecutePaperclipNativeSession).not.toHaveBeenCalled();
} finally {
await db.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date() }).where(and(eq(heartbeatRuns.companyId, f.companyId), inArray(heartbeatRuns.status, ["running", "queued"])));
await db.update(agentWakeupRequests).set({ status: "cancelled", finishedAt: new Date() }).where(and(eq(agentWakeupRequests.companyId, f.companyId), inArray(agentWakeupRequests.status, ["queued", "claimed"])));
}
});
it.each(["delivery", "assignment", "admission", "user_request", "result_or_decision"] as const)("revokes an accepted Cursor plan passive wait after %s changes", async change => {
const f = await seedAcceptedCursorPlanWait();
await finalizeNativeRun({ db, runId: f.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
if (change === "delivery") await db.update(issueQuestionResponseDeliveries).set({ acknowledgedAt: null }).where(eq(issueQuestionResponseDeliveries.id, f.deliveryId));
if (change === "assignment") await db.update(issues).set({ assigneeAgentId: null }).where(eq(issues.id, f.issueId));
if (change === "admission") {
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId));
const profile = structuredClone(run!.runnerProfileJson!);
(profile.nativeExecutionInput as { provider: { cursorMode: string } }).provider.cursorMode = "agent";
await db.update(heartbeatRuns).set({ runnerProfileJson: profile }).where(eq(heartbeatRuns.id, f.runId));
}
if (change === "user_request") await db.insert(issueComments).values({ companyId: f.companyId, issueId: f.issueId, authorType: "user", authorUserId: "responsible-user", body: "Continue reviewing this plan in Plan mode." });
if (change === "result_or_decision") {
const [accepted] = await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, f.runId));
await db.update(nativeRunResults).set({ canonicalSha256: "changed" }).where(eq(nativeRunResults.id, accepted!.id));
expect(await hasCommittedNativePlanWait(db, f)).toBe(false);
await db.update(nativeRunResults).set({ canonicalSha256: accepted!.canonicalSha256 }).where(eq(nativeRunResults.id, accepted!.id));
expect(await hasCommittedNativePlanWait(db, f)).toBe(true);
await db.update(issues).set({ lastStatusDecisionId: null }).where(eq(issues.id, f.issueId));
}
expect(await hasCommittedNativePlanWait(db, f)).toBe(false);
});
it("rechecks accepted Cursor plan delivery under the status transaction before presentation or effects", async () => {
const f = await seedAcceptedCursorPlanWait();
await finalizeNativeRun({ db, runId: f.runId, workspaceFinalizeStatus: "succeeded", failpoint: "status_projection" });
const [coordinator] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId));
const [issue] = await db.select().from(issues).where(eq(issues.id, f.issueId));
await db.update(issueQuestionResponseDeliveries).set({ payloadSha256: "changed" }).where(eq(issueQuestionResponseDeliveries.id, f.deliveryId));
await expect(commitNativeStatusDecision({ db, companyId: f.companyId, issueId: f.issueId, runId: f.runId, assessmentId: coordinator!.assessmentId!,
priorStatus: issue!.status, priorStatusVersion: issue!.statusVersion, priorDecisionId: issue!.lastStatusDecisionId,
decision: { policyVersion: "phase6-v7", statusAction: "in_progress", toStatus: "in_progress", reasonCode: "native_plan_accepted_waiting_for_continuation", unblockDescriptor: null, effects: [] },
requirePlanWaitSource: f.proof.source })).rejects.toBeInstanceOf(NativeStatusRaceError);
expect(await db.select().from(statusDecisions).where(eq(statusDecisions.runId, f.runId))).toHaveLength(0);
expect(await db.select().from(issueComments).where(eq(issueComments.createdByRunId, f.runId))).toHaveLength(0);
});
it("preserves explicit semantic finish priority over accepted Cursor plan evidence", async () => {
const f = await seedAcceptedCursorPlanWait(true);
await finalizeNativeRun({ db, runId: f.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
expect(await hasCommittedNativePlanWait(db, f)).toBe(false);
const decisions = await db.select().from(statusDecisions).where(eq(statusDecisions.runId, f.runId));
expect(decisions).toHaveLength(1); expect(decisions[0]!.reasonCode).not.toBe("native_plan_accepted_waiting_for_continuation");
});
});
});
@@ -78,7 +78,12 @@ async function createApp(
return app;
}
describe("GET /invites/:token/test-resolution", () => {
describe("GET /invites/:token/test-resolution", { sequential: true }, () => {
beforeAll(async () => {
// Route transformation is fixture setup, not part of the network assertions.
await loadAppModules();
});
beforeEach(() => {
vi.clearAllMocks();
});
+42 -14
View File
@@ -41,20 +41,48 @@ function receiptNotice(version: 1 | 2 = 1): Record<string, any> {
const receiptSchemaValue = (notice: Record<string, any>) => notice.details.find((detail: any) => detail.name === "schema").value;
describe("redaction", () => {
it("preserves credential-related prose, metadata, and dotted filenames", () => {
const input = {
body: "Keep the private key in a secret manager. Document credential handling and token permissions. Use bearer tokens for authentication. Prefer bearer authentication.",
tokenPolicy: "least privilege",
secretStorage: "vault",
credentialHandling: "harness",
authorizationRequired: true,
path: "deployment.credentials.example.md",
};
expect(sanitizeRecord(input)).toEqual(input);
expect(redactSensitiveText(input.body)).toBe(input.body);
expect(sanitizeRecord({ credentials: { provider: "opaque-value" }, passwordValue: "opaque-value", authorization_code: "opaque-value" }))
.toEqual({ credentials: REDACTED_EVENT_VALUE, passwordValue: REDACTED_EVENT_VALUE, authorization_code: REDACTED_EVENT_VALUE });
expect(redactSensitiveText("--token-budget 4000 SECRET_STORAGE=vault")).toBe("--token-budget 4000 SECRET_STORAGE=vault");
it("preserves the actual Copilot receipt producer discriminator through nested durable redaction", () => {
const events: Array<Record<string, any>> = [];
const projector = createCopilotToolEvidence({ sessionId: "session", turnId: "turn", workingDirectory: "/workspace",
active: () => true, emit: event => events.push(event) });
const receipt = appendSemanticToolReceipt({ tool: "finish_task", callId: "call", arguments: {} },
{ content: [{ type: "text", text: "accepted" }] }).receipt;
expect(receipt.schema).toBe("paperclip.semantic_tool_receipt.v2");
projector.captureSemanticReceipt()!(receipt);
expect(events).toHaveLength(1);
expect(events[0]!.payload.category).toBe("paperclip_semantic_tool_receipt_v2");
expect(events[0]!.payload.details).toHaveLength(8);
const input = { prpEvent: { schema: "paperclip.prp.event.v1", schemaVersion: 1,
eventType: "provider.notice.recorded", payload: events[0]!.payload } };
expect(redactEventPayload(input)).toEqual(input);
expect(redactEventPayload(redactEventPayload(input))).toEqual(input);
});
it.each([1, 2] as const)("preserves public v%s receipt identifiers as text", version => {
const notice = receiptNotice(version);
expect(redactEventPayload(notice)).toEqual(notice);
// Master's JWT detector recognizes encoded headers, so a dotted public
// identifier needs no receipt-shaped exemption to survive redaction.
expect(redactEventPayload({ value: `paperclip.semantic_tool_receipt.v${version}` }))
.toEqual({ value: `paperclip.semantic_tool_receipt.v${version}` });
});
it("redacts credentials in receipt-shaped data and every adjacent field", () => {
const jwt = "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.signature12345678";
const notice = receiptNotice();
notice.details.find((d: any) => d.name === "operationId").value = jwt;
notice.provenance.sessionId = jwt;
const redacted = redactEventPayload(notice)! as Record<string, any>;
expect(receiptSchemaValue(redacted)).toBe("paperclip.semantic_tool_receipt.v1");
expect(redacted.details.find((d: any) => d.name === "operationId").value).toBe(REDACTED_EVENT_VALUE);
expect(redacted.provenance.sessionId).toBe(REDACTED_EVENT_VALUE);
expect(redactEventPayload({ notice, password: "canary", arbitrary: jwt }))
.toMatchObject({ password: REDACTED_EVENT_VALUE, arbitrary: REDACTED_EVENT_VALUE });
notice.details.find((d: any) => d.name === "schema").value = jwt;
notice.summary = "Authorization: Bearer canary-token";
const hostile = redactEventPayload(notice)!;
expect(receiptSchemaValue(hostile)).toBe(REDACTED_EVENT_VALUE);
expect(JSON.stringify(hostile)).not.toContain("canary-token");
});
it("keeps the discriminator allowlist in exact PRP v1 schema parity", () => {
@@ -0,0 +1,163 @@
import { createHash } from "node:crypto";
import { chmodSync, linkSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, expect, it, vi } from "vitest";
const hooks = vi.hoisted(() => ({ mkdir: undefined as undefined | ((path: string) => void), rename: undefined as undefined | ((source: string, destination: string) => void), open: undefined as undefined | ((path: string) => void), read: undefined as undefined | ((path: string, bytes: number) => void) }));
vi.mock("node:fs/promises", async (original) => {
const fs = await original<typeof import("node:fs/promises")>();
return { ...fs,
mkdir: async (...args: Parameters<typeof fs.mkdir>) => { hooks.mkdir?.(String(args[0])); return fs.mkdir(...args); },
rename: async (...args: Parameters<typeof fs.rename>) => { hooks.rename?.(String(args[0]), String(args[1])); return fs.rename(...args); },
open: async (...args: Parameters<typeof fs.open>) => { hooks.open?.(String(args[0])); const file = await fs.open(...args); const read = file.read.bind(file); file.read = (async (...readArgs: any[]) => { const result = await (read as any)(...readArgs); hooks.read?.(String(args[0]), result.bytesRead); return result; }) as typeof file.read; return file; },
};
});
vi.mock("../vendor/paperclip-runner/index.js", async () => await import("../../../packages/paperclip-runner/src/drivers/acpx/qualified-profiles.js"));
import { QUALIFIED_ACPX_PROFILES } from "../../../packages/paperclip-runner/src/drivers/acpx/qualified-profiles.js";
import { createRemotePiCompanionManifest, importRemotePiCompanion, inventoryRemoteCompanion, resolveRemotePiCompanion, selectRemotePiCompanion } from "../services/native-runtime/remote-pi-companion.js";
const roots: string[] = [];
afterEach(() => { hooks.mkdir = undefined; hooks.rename = undefined; hooks.open = undefined; hooks.read = undefined; for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); vi.restoreAllMocks(); });
const hash = (bytes: string | Buffer) => createHash("sha256").update(bytes).digest("hex");
const canonical = (v: any): string => Array.isArray(v) ? `[${v.map(canonical).join(",")}]` : v && typeof v === "object" ? `{${Object.keys(v).sort().map(k => `${JSON.stringify(k)}:${canonical(v[k])}`).join(",")}}` : JSON.stringify(v);
async function fixture() {
const root = realpathSync(mkdtempSync(join(tmpdir(), "paperclip-companion-"))); roots.push(root);
const source = "a".repeat(40); const serverRoot = join(root, "server"); const directory = join(root, "release");
mkdirSync(join(serverRoot, "dist"), { recursive: true }); writeFileSync(join(serverRoot, "package.json"), '{"name":"@paperclipai/server"}'); writeFileSync(join(serverRoot, "dist/build-info.json"), JSON.stringify({ commit: source }));
mkdirSync(join(directory, "bin"), { recursive: true }); mkdirSync(join(directory, "provider-pack"));
// Synthetic ELF header; these tests never launch it or claim native qualification.
const elf = Buffer.alloc(128); Buffer.from([127, 69, 76, 70, 2, 1]).copy(elf); elf.writeUInt16LE(62, 18); writeFileSync(join(directory, "bin/paperclip-runnerd"), elf, { mode: 0o755 });
const payload = { runnerSourceRevision: source, target: { platform: "linux", architecture: "x64" }, candidateProviders: { pi: { qualification: "qualified", profileDigest: QUALIFIED_ACPX_PROFILES.pi.commandDigest, path: "provider-assets/pi/linux-x64" } } };
writeFileSync(join(directory, "provider-pack/provider-pack.json"), JSON.stringify({ schema: "paperclip-runner/remote-provider-pack/v1", digest: `sha256:${hash(canonical(payload))}`, payload }));
const manifest = await createRemotePiCompanionManifest(directory, source); const bytes = JSON.stringify(manifest); writeFileSync(join(directory, "companion.json"), bytes); return { root, directory, serverRoot, sha256: hash(bytes), manifest, source };
}
it("imports the release-generated closure and resolves target bytes without environment overrides", async () => {
const f = await fixture(); const result = await importRemotePiCompanion(f); expect(result.status).toBe("imported_verified");
expect(result.runnerBinary).toBe(join(f.serverRoot, "remote-companions/linux-x64/bin/paperclip-runnerd"));
expect(readFileSync(result.runnerBinary!)).toEqual(readFileSync(join(f.directory, "bin/paperclip-runnerd")));
expect((await importRemotePiCompanion(f)).status).toBe("verified_existing"); expect((await resolveRemotePiCompanion({ serverRoot: f.serverRoot }))?.manifestSha256).toBe(f.sha256);
expect(readdirSync(join(f.serverRoot, "remote-companions"))).toEqual(["linux-x64"]);
});
it.each(["sha", "source", "profile", "target", "daemon", "extra", "mode", "outside-link", "outside-hardlink", "pack"])("rejects %s before publishing", async kind => {
const f = await fixture();
if (kind === "sha") f.sha256 = "b".repeat(64);
if (kind === "source") writeFileSync(join(f.serverRoot, "dist/build-info.json"), JSON.stringify({ commit: "b".repeat(40) }));
if (kind === "profile" || kind === "target") { Object.assign(f.manifest, kind === "profile" ? { profileDigest: "sha256:" + "b".repeat(64) } : { target: "darwin-arm64" }); const bytes = JSON.stringify(f.manifest); writeFileSync(join(f.directory, "companion.json"), bytes); f.sha256 = hash(bytes); }
if (kind === "daemon") writeFileSync(join(f.directory, "bin/paperclip-runnerd"), "changed");
if (kind === "extra") writeFileSync(join(f.directory, "extra"), "unlisted");
if (kind === "mode") chmodSync(join(f.directory, "bin/paperclip-runnerd"), 0o777);
if (kind === "outside-link") symlinkSync("../../server/package.json", join(f.directory, "provider-pack/escape"));
if (kind === "outside-hardlink") linkSync(join(f.directory, "bin/paperclip-runnerd"), join(f.root, "alias"));
if (kind === "pack") writeFileSync(join(f.directory, "provider-pack/provider-pack.json"), "{}");
await expect(importRemotePiCompanion(f)).rejects.toThrow(); expect(() => readdirSync(join(f.serverRoot, "remote-companions/linux-x64"))).toThrow();
});
it("copies contained symlinks and breaks only fully owned internal hardlinks", async () => {
const f = await fixture(); linkSync(join(f.directory, "bin/paperclip-runnerd"), join(f.directory, "bin/alias")); symlinkSync("paperclip-runnerd", join(f.directory, "bin/link"));
f.manifest = await createRemotePiCompanionManifest(f.directory, f.source); const bytes = JSON.stringify(f.manifest); writeFileSync(join(f.directory, "companion.json"), bytes); f.sha256 = hash(bytes);
expect((await importRemotePiCompanion(f)).status).toBe("imported_verified");
});
it("fails closed on cache corruption, foreign authority and workspace-contained cache", async () => {
const f = await fixture(); const result = await importRemotePiCompanion(f);
await expect(resolveRemotePiCompanion({ serverRoot: f.serverRoot, workspaceRoot: f.root })).rejects.toThrow("workspace");
await expect(resolveRemotePiCompanion({ serverRoot: f.serverRoot, workspaceRoot: join(result.root!, "provider-pack") })).rejects.toThrow("workspace");
writeFileSync(result.runnerBinary!, "corrupted"); await expect(resolveRemotePiCompanion({ serverRoot: f.serverRoot })).rejects.toThrow("inventory");
await expect(importRemotePiCompanion(f)).rejects.toThrow();
});
it("preserves an existing empty destination and releases only its import lock", async () => {
const f = await fixture(); mkdirSync(join(f.serverRoot, "remote-companions/linux-x64"), { recursive: true, mode: 0o700 }); chmodSync(join(f.serverRoot, "remote-companions"), 0o700);
await expect(importRemotePiCompanion(f)).rejects.toThrow(); expect(readdirSync(join(f.serverRoot, "remote-companions/linux-x64"))).toEqual([]); expect(readdirSync(join(f.serverRoot, "remote-companions"))).toEqual(["linux-x64"]);
});
it("rejects an unsafe cache parent and leaves it intact", async () => {
const f = await fixture(); const outside = join(f.root, "outside"); mkdirSync(outside); symlinkSync(outside, join(f.serverRoot, "remote-companions"));
await expect(importRemotePiCompanion(f)).rejects.toThrow(); expect(readdirSync(outside)).toEqual([]);
});
it("reports missing installation without manufacturing a companion", async () => { const f = await fixture(); expect(await resolveRemotePiCompanion({ serverRoot: f.serverRoot })).toBeNull(); });
it("selects only normal remote Pi, preserving explicit overrides and C/C admission", async () => {
const f = await fixture(); await importRemotePiCompanion(f); const workspaceRoot = join(f.root, "workspace"); mkdirSync(workspaceRoot);
const input = { serverRoot: f.serverRoot, workspaceRoot, remote: true, provider: { kind: "acpx", agent: "pi" } };
expect((await selectRemotePiCompanion(input))?.target).toBe("linux-x64");
for (const patch of [{ remote: false }, { provider: { kind: "acpx", agent: "cursor" } }, { provider: { kind: "acpx", agent: "copilot" } }, { provider: { kind: "codex" } }, { binaryOverride: "/operator/runnerd" }, { packOverride: "/operator/pack" }]) expect(await selectRemotePiCompanion({ ...input, ...patch })).toBeNull();
});
it("rejects an appeared empty destination without replacing it", async () => {
const f = await fixture(); const output = join(f.serverRoot, "remote-companions/linux-x64");
hooks.mkdir = path => { if (path !== output) return; hooks.mkdir = undefined; mkdirSync(output, { mode: 0o700 }); };
await expect(importRemotePiCompanion(f)).rejects.toThrow(); expect(readdirSync(output)).toEqual([]);
expect(readdirSync(join(f.serverRoot, "remote-companions"))).toEqual(["linux-x64"]);
});
it("drains owned output, staging and lock cleanup after publication failure", async () => {
const f = await fixture(); hooks.rename = () => { hooks.rename = undefined; throw new Error("injected publication failure"); };
await expect(importRemotePiCompanion(f)).rejects.toThrow("injected publication failure");
expect(readdirSync(join(f.serverRoot, "remote-companions"))).toEqual([]);
});
it("preserves a replaced staging root while draining the other owned cleanup", async () => {
const f = await fixture(); let replaced = "";
hooks.rename = source => {
hooks.rename = undefined; replaced = source.slice(0, source.lastIndexOf("/"));
renameSync(replaced, replaced + "-original"); mkdirSync(replaced, { mode: 0o700 }); writeFileSync(join(replaced, "foreign"), "preserve");
throw new Error("replaced staging");
};
await expect(importRemotePiCompanion(f)).rejects.toThrow("cleanup incomplete"); expect(readFileSync(join(replaced, "foreign"), "utf8")).toBe("preserve");
expect(readdirSync(join(f.serverRoot, "remote-companions"))).not.toContain(".import-linux-x64.lock");
expect(readdirSync(join(f.serverRoot, "remote-companions"))).not.toContain("linux-x64");
});
it.each(["new", "existing"])("defers actual SIGTERM on the %s import path and drains owned cleanup", async (mode) => {
const f = await fixture(); if (mode === "existing") await importRemotePiCompanion(f);
const { build } = await import("esbuild");
const { execFile } = await import("node:child_process");
const { promisify } = await import("node:util");
const bundle = join(f.root, "companion.mjs");
await build({ entryPoints: [new URL("../services/native-runtime/remote-pi-companion.ts", import.meta.url).pathname], outfile: bundle,
platform: "node", format: "esm", target: "node24", bundle: true, logLevel: "silent",
plugins: [{ name: "source-owned-profile-only", setup(builder) {
builder.onResolve({ filter: /vendor\/paperclip-runner\/index\.js$/ }, () => ({ path: new URL("../../../packages/paperclip-runner/src/drivers/acpx/qualified-profiles.ts", import.meta.url).pathname }));
} }],
});
const script = join(f.root, "cancel.mjs");
writeFileSync(script, `import {importRemotePiCompanion} from './companion.mjs';
import {readdirSync,existsSync} from 'node:fs';
let cancelled=false, checkpoints=0,sent=false; const cancel=()=>{cancelled=true}; process.on('SIGTERM',cancel);
try { await importRemotePiCompanion({...JSON.parse(process.argv[2]),checkCancelled(){ checkpoints++; if(!sent&&existsSync(process.argv[3]+'/.import-linux-x64.lock')){sent=true;process.kill(process.pid,'SIGTERM');} if(cancelled)throw Error('owned cancellation'); }}); throw Error('unexpected success'); }
catch(error){ if(error.message!=='owned cancellation')throw error; console.log(JSON.stringify({cancelled,checkpoints,remaining:readdirSync(process.argv[3])})); }
finally {process.off('SIGTERM',cancel);}
`);
const { stdout } = await promisify(execFile)(process.execPath, [script, JSON.stringify({ directory: f.directory, sha256: f.sha256, serverRoot: f.serverRoot }), join(f.serverRoot, "remote-companions")], { env: { PATH: "/usr/bin:/bin", LANG: "C.UTF-8" }, timeout: 10_000, maxBuffer: 65536 });
expect(JSON.parse(stdout)).toMatchObject({ cancelled: true, remaining: mode === "existing" ? ["linux-x64"] : [] });
});
it("honors deadline expiry after re-verifying an existing cache without deleting it", async () => {
const f = await fixture(); await importRemotePiCompanion(f); let expired = false;
hooks.open = path => { if(path === join(f.serverRoot, "remote-companions/linux-x64/companion.json")) expired = true; };
vi.spyOn(Date, "now").mockImplementation(() => expired ? 600_001 : 0);
await expect(importRemotePiCompanion(f)).rejects.toThrow("deadline exceeded");
expect(readdirSync(join(f.serverRoot, "remote-companions"))).toEqual(["linux-x64"]);
});
async function largeFixture() {
const f = await fixture(); writeFileSync(join(f.directory, "provider-pack/large"), Buffer.alloc(8 * 1024 ** 2, 0x61));
f.manifest = await createRemotePiCompanionManifest(f.directory, f.source);
const bytes = JSON.stringify(f.manifest); writeFileSync(join(f.directory, "companion.json"), bytes); f.sha256 = hash(bytes); return f;
}
it("yields to unrelated event-loop work between chunks of a real cache file", async () => {
const f = await largeFixture(); await importRemotePiCompanion(f);
const file = join(f.serverRoot, "remote-companions/linux-x64/provider-pack/large");
let reads = 0; let serviced = false; let servicedBeforeNextRead = false;
hooks.read = (path, bytes) => { if(path !== file || !bytes) return; reads++; if(reads === 1) setImmediate(() => { serviced = true; }); else servicedBeforeNextRead ||= serviced; };
expect((await resolveRemotePiCompanion({serverRoot:f.serverRoot}))?.manifestSha256).toBe(f.sha256);
expect(reads).toBeGreaterThanOrEqual(8); expect(servicedBeforeNextRead).toBe(true);
});
it("cancels during a large existing-cache file before reading the whole file and preserves it", async () => {
const f = await largeFixture(); await importRemotePiCompanion(f);
const file = join(f.serverRoot, "remote-companions/linux-x64/provider-pack/large"); let readBytes = 0;
hooks.read = (path, bytes) => { if(path === file) readBytes += bytes; };
await expect(importRemotePiCompanion({...f, checkCancelled(){ if(readBytes) throw Error("cancel during cache bytes"); }})).rejects.toThrow("cancel during cache bytes");
expect(readBytes).toBeGreaterThan(0); expect(readBytes).toBeLessThan(8 * 1024 ** 2);
expect(readdirSync(join(f.serverRoot, "remote-companions"))).toEqual(["linux-x64"]);
});
it("rejects read-only directory modes before claiming a staging or output path", async () => {
const f = await fixture(); const dir = join(f.directory, "provider-pack"); chmodSync(dir, 0o500);
try {
await expect(importRemotePiCompanion({...f, checkCancelled(){}})).rejects.toThrow("owner read/write/search");
expect(() => readdirSync(join(f.serverRoot, "remote-companions"))).toThrow();
} finally { chmodSync(dir, 0o755); }
});