mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-10 12:37:55 +01:00
Reconcile Pi prerequisites with the integrated runtime while holding admission
This commit is contained in:
1 parent
20ce062701
commit
f620e7157b
313 files changed
+13086
-2187
No files matched your search
@@ -6,6 +6,7 @@ import { createRequire } from "node:module";
|
||||
import childProcess, { execFileSync } from "node:child_process";
|
||||
import { prepareAdapterExecutionTargetRuntime } from "@paperclipai/adapter-utils/execution-target";
|
||||
import { captureDirectorySnapshot } from "@paperclipai/adapter-utils/workspace-restore-merge";
|
||||
import { prepareSandboxManagedRuntime } from "@paperclipai/adapter-utils/sandbox-managed-runtime";
|
||||
import { agentDirectoryBaselineDigest, agentDirectoryProbeProgram, observeLocalAgentDirectory, probeAgentDirectory, retireAgentDirectoryTransferScratch, agentDirectoryTransferCleanupProgram } from "../services/agent-directory-probe.js";
|
||||
|
||||
describe("stable live agent directory observation", () => {
|
||||
@@ -98,6 +99,51 @@ describe("stable live agent directory observation", () => {
|
||||
expect(probeAgentDirectory(remote).digest).toBe(agentDirectoryBaselineDigest(baseline));
|
||||
} finally { await runtime?.cleanupWorkspaceSnapshot?.(); fs.rmSync(remote, { recursive: true, force: true }); }
|
||||
});
|
||||
it("keeps restart fallback transfer scratch outside the native memory directory", async () => {
|
||||
const remoteCwd = `${root}-remote`;
|
||||
const agentHome = join(remoteCwd, ".paperclip-runtime", "agent-files", "agent", "original-run");
|
||||
const scratch = join(remoteCwd, ".paperclip-runtime", "paperclip-runner", "agent-file-transfers", "agent", "original-run");
|
||||
fs.mkdirSync(agentHome, { recursive: true });
|
||||
fs.cpSync(root, agentHome, { recursive: true });
|
||||
const target = { kind: "remote" as const, transport: "sandbox" as const, remoteCwd,
|
||||
// A newly bound controller target may have no native sync hooks. Exercise
|
||||
// the production command/base64 fallback rather than a transport mock.
|
||||
runner: { execute: async (input: Parameters<import("@paperclipai/adapter-utils/command-managed-runtime").CommandManagedRuntimeRunner["execute"]>[0]) => ({
|
||||
stdout: execFileSync(input.command, input.args ?? [], { cwd: input.cwd, input: input.stdin, encoding: "utf8",
|
||||
env: { PATH: process.env.PATH, ...input.env }, timeout: input.timeoutMs, maxBuffer: 32 * 1024 * 1024 }),
|
||||
stderr: "", exitCode: 0, signal: null, timedOut: false, pid: null, startedAt: new Date().toISOString(),
|
||||
}) } };
|
||||
let runtime: Awaited<ReturnType<typeof prepareAdapterExecutionTargetRuntime>> | undefined;
|
||||
try {
|
||||
const baseline = await captureDirectorySnapshot(root);
|
||||
runtime = await prepareAdapterExecutionTargetRuntime({ target, runId: "restart-transfer", adapterKey: "agent-files",
|
||||
workspaceLocalDir: root, workspaceRemoteDir: agentHome, runtimeRootDir: scratch,
|
||||
syncWorkspace: true, workspaceInboundMode: "adopt_remote", workspaceBaseline: baseline,
|
||||
workspaceGitSnapshot: null, workspaceFileMode: "all", workspaceExclude: [".paperclip-runtime", ".paperclip-runtime/**"] });
|
||||
const memory = "0123456789abcdef0123456789abcdef\n";
|
||||
fs.writeFileSync(join(agentHome, "notes", "memory.txt"), memory);
|
||||
await runtime.restoreWorkspace();
|
||||
expect(fs.readFileSync(join(root, "notes", "memory.txt"), "utf8")).toBe(memory);
|
||||
// A transfer must not introduce a reserved path that the product's own
|
||||
// full agent-directory probe rejects, including after controller recovery.
|
||||
expect(() => probeAgentDirectory(agentHome)).not.toThrow();
|
||||
expect(fs.existsSync(join(agentHome, ".paperclip-runtime"))).toBe(false);
|
||||
expect(runtime.runtimeRootDir).toBe(scratch);
|
||||
expect(fs.readdirSync(scratch)).toEqual([]);
|
||||
} finally { await runtime?.cleanupWorkspaceSnapshot?.(); fs.rmSync(remoteCwd, { recursive: true, force: true }); }
|
||||
});
|
||||
it.each(["outside", "reserved-root", "relative", "traversal", "trailing-slash", "nul"])("rejects %s transfer scratch before filesystem work", async kind => {
|
||||
const reserved = join(root, ".paperclip-runtime");
|
||||
const invalid = {
|
||||
outside: `${root}-foreign/scratch`, "reserved-root": reserved, relative: ".paperclip-runtime/scratch",
|
||||
traversal: `${reserved}/../../scratch`, "trailing-slash": `${reserved}/scratch/`, nul: `${reserved}/scratch\0`,
|
||||
}[kind]!;
|
||||
await expect(prepareSandboxManagedRuntime({
|
||||
spec: { transport: "sandbox", provider: "fixture", sandboxId: "owned", remoteCwd: root, apiKey: null },
|
||||
adapterKey: "agent-files", client: {} as never, workspaceLocalDir: root, runtimeRootDir: invalid,
|
||||
})).rejects.toThrow("Transfer scratch must remain within the lease runtime tree");
|
||||
expect(fs.existsSync(reserved)).toBe(false);
|
||||
});
|
||||
it.each(["file", "extra-directory", "symlink", "race"])("fails closed on %s in transfer-owned scratch without deleting contents", kind => {
|
||||
const parent = join(root, ".paperclip-runtime"), scratch = join(parent, "agent-files");
|
||||
fs.mkdirSync(scratch, { recursive: true });
|
||||
|
||||
@@ -225,6 +225,8 @@ describe("persistent agent directories", () => {
|
||||
await copies.recoverStopped();
|
||||
expect((await copies.get(companyId, secondId))?.processStoppedAt).toBeNull();
|
||||
await stop(secondId);
|
||||
// Make the deferred maintenance retry due after the new owner stop proof.
|
||||
await db.update(agentInstructionWorkingCopies).set({ nextAttemptAt: null }).where(eq(agentInstructionWorkingCopies.runId, secondId));
|
||||
await copies.recoverStopped();
|
||||
}
|
||||
expect(await copies.get(companyId, secondId)).toMatchObject({ state: "saved", receipt: { cleanupPending: false } });
|
||||
@@ -556,7 +558,8 @@ describe("persistent agent directories", () => {
|
||||
}
|
||||
expect(execute).toHaveBeenCalledTimes(2);
|
||||
expect(execute).toHaveBeenLastCalledWith(expect.objectContaining({ lease: expect.objectContaining({ id: leaseId, providerLeaseId: "original-sandbox" }),
|
||||
command: "rm", args: ["-rf", "--", executionRoot], bypassSession: true }));
|
||||
command: "rm", args: ["-rf", "--", executionRoot,
|
||||
path.posix.join(remoteCwd, ".paperclip-runtime", "paperclip-runner", "agent-file-transfers", agentId, copy.runId)], bypassSession: true }));
|
||||
await copies.recoverCaptured();
|
||||
expect(execute).toHaveBeenCalledTimes(2);
|
||||
await copies.release(companyId, copy.runId);
|
||||
@@ -856,6 +859,9 @@ describe("persistent agent directories", () => {
|
||||
await fs.mkdir(path.join(root, "build"));
|
||||
await fs.writeFile(path.join(root, "build", "personal.txt"), "cache-like names are still agent files");
|
||||
const first = await prepare();
|
||||
const firstScratch = path.join(remoteCwd, ".paperclip-runtime", "paperclip-runner", "agent-file-transfers", agentId, first.runId);
|
||||
expect(await fs.readdir(firstScratch)).toEqual([]);
|
||||
await expect(fs.stat(path.join(first.executionRoot, ".paperclip-runtime"))).rejects.toMatchObject({ code: "ENOENT" });
|
||||
expect(await fs.readFile(path.join(first.executionRoot, "build", "personal.txt"), "utf8")).toBe("cache-like names are still agent files");
|
||||
await fs.mkdir(path.join(first.executionRoot, "notes"));
|
||||
await fs.writeFile(path.join(first.executionRoot, "notes", "bytes.bin"), Buffer.from([0, 128, 255]));
|
||||
@@ -865,6 +871,7 @@ describe("persistent agent directories", () => {
|
||||
expect((await execFile("git", ["-C", remoteCwd, "status", "--porcelain", "--untracked-files=all"])).stdout).toBe("?? task-only.txt\n");
|
||||
expect((await copies.collectStopped({ companyId, runId: first.runId, target: executionTarget }))?.state).toBe("saved");
|
||||
await copies.release(companyId, first.runId);
|
||||
await expect(fs.stat(firstScratch)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
expect((await copies.get(companyId, first.runId))?.receipt?.baseline).toBeUndefined();
|
||||
await expect(fs.stat(first.localRoot)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
await fs.rm(remoteCwd, { recursive: true });
|
||||
@@ -1188,8 +1195,8 @@ describe("persistent agent directories", () => {
|
||||
const execute = vi.fn(), restoreWorkspace = vi.fn(), cleanupWorkspaceSnapshot = vi.fn().mockResolvedValue(undefined);
|
||||
const executionTarget = { kind: "remote" as const, transport: "sandbox" as const, environmentId, leaseId, remoteCwd, runner: { execute } };
|
||||
const shell = vi.spyOn(executionTargetTools, "runAdapterExecutionTargetShellCommand").mockResolvedValue({ exitCode: 0, signal: null, timedOut: false, stdout: "", stderr: "" });
|
||||
const stage = vi.spyOn(executionTargetTools, "prepareAdapterExecutionTargetRuntime").mockImplementation(async () => ({ target: executionTarget,
|
||||
workspaceRemoteDir: path.posix.join(remoteCwd, ".paperclip-runtime", "agent-files", agentId, runId), runtimeRootDir: null,
|
||||
const stage = vi.spyOn(executionTargetTools, "prepareAdapterExecutionTargetRuntime").mockImplementation(async options => ({ target: executionTarget,
|
||||
workspaceRemoteDir: path.posix.join(remoteCwd, ".paperclip-runtime", "agent-files", agentId, runId), runtimeRootDir: options.runtimeRootDir ?? null,
|
||||
assetDirs: {}, additionalSourceDirs: {}, additionalSourceFailures: [], workspaceSyncSnapshot: null, restoreWorkspace, cleanupWorkspaceSnapshot }));
|
||||
try {
|
||||
const copy = (await copies.prepare({ ...target(), runId, cwd: home, target: executionTarget }))!;
|
||||
|
||||
@@ -300,6 +300,10 @@ describe("registered run instruction copies", () => {
|
||||
expect((await revisions.readCurrent(target(), board()))?.content).toBe(initial);
|
||||
expect((await copies.list(companyId, agentId, board()))[0]).toMatchObject({ state: "pending_collection", content: null });
|
||||
await appendHeartbeatRunEvent(db, { ...target(), runId: copy.runId, eventType: "native.local_process_stopped", stream: "system" });
|
||||
// Recovery scheduled a retry while stop authority was missing. Make that
|
||||
// retry due before asking the restarted controller to collect the copy.
|
||||
await db.update(agentInstructionWorkingCopies).set({ nextAttemptAt: new Date(0) })
|
||||
.where(eq(agentInstructionWorkingCopies.runId, copy.runId));
|
||||
await copies.recoverStopped();
|
||||
expect((await revisions.readCurrent(target(), board()))?.content).toBe("edit before controller restart");
|
||||
});
|
||||
|
||||
@@ -1091,12 +1091,23 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
const companyId = randomUUID();
|
||||
const assignedAgentId = randomUUID();
|
||||
const replacementAgentId = randomUUID();
|
||||
await db.insert(companies).values({
|
||||
id: companyId,
|
||||
name: `Chat Test ${companyId.slice(0, 8)}`,
|
||||
issuePrefix: `C${companyId.replace(/-/g, "").toUpperCase()}`,
|
||||
requireBoardApprovalForNewAgents: false,
|
||||
});
|
||||
let inserted = false;
|
||||
// Retired fixtures retain company rows. Retry only the short-prefix unique
|
||||
// conflict; every other database error must still fail the test immediately.
|
||||
for (let attempt = 0; attempt < companyPrefixAttempts; attempt += 1) {
|
||||
const [company] = await db.insert(companies).values({
|
||||
id: companyId,
|
||||
name: `Chat Test ${companyId.slice(0, 8)}`,
|
||||
issuePrefix: nextIssuePrefix(),
|
||||
requireBoardApprovalForNewAgents: false,
|
||||
}).onConflictDoNothing({ target: companies.issuePrefix }).returning({ id: companies.id });
|
||||
if (company) {
|
||||
inserted = true;
|
||||
fixtureCompanies.add(companyId);
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!inserted) throw new Error(`Could not allocate a chat fixture company prefix after ${companyPrefixAttempts} attempts`);
|
||||
const now = new Date();
|
||||
await db
|
||||
.insert(authUsers)
|
||||
@@ -1614,6 +1625,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
await service.processPendingPublications();
|
||||
const providerRuntime = fakeRuntime.endpoints.get(endpointId);
|
||||
if (providerRuntime) providerRuntime.posts.length = 0;
|
||||
return setupFollowUpMessageId;
|
||||
}
|
||||
|
||||
async function configuredSlackEndpoint(
|
||||
@@ -28229,8 +28241,18 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
}),
|
||||
trigger: "direct_message",
|
||||
});
|
||||
await qualifySetupRoundTrip(service, endpoint.id, userId);
|
||||
const setupMessageId = await qualifySetupRoundTrip(service, endpoint.id, userId);
|
||||
await service.test(endpoint.id, "owner-user");
|
||||
if (provider === "telegram") {
|
||||
// Setup dispatches receipt cleanup asynchronously. Finish it before
|
||||
// measuring reaction removals owned by this fixture's working run.
|
||||
await service.processPendingReceiptReactions();
|
||||
await waitForProcessedReceiptRemoval(endpoint.id, {
|
||||
threadId: thread.thread.id,
|
||||
messageId: setupMessageId,
|
||||
emoji: "eyes",
|
||||
});
|
||||
}
|
||||
const [conversation] = await service.listConversations(endpoint.id);
|
||||
const runId = randomUUID();
|
||||
await db.insert(heartbeatRuns).values({
|
||||
@@ -62562,7 +62584,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
},
|
||||
);
|
||||
|
||||
it("orders reversed GitHub edit and delete callbacks behind their durable root", async () => {
|
||||
it.each([false, true])("orders reversed GitHub edit and delete callbacks behind their durable root (root processed before replay: %s)", async (rootProcessedFirst) => {
|
||||
const fixture = await seedCompany();
|
||||
const deferred: Array<() => void | Promise<void>> = [];
|
||||
const { callbacks, endpoint, service, wakeup, webhookSecret } =
|
||||
@@ -62665,8 +62687,28 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
// One root-conversation drain plus one durable-ingress callback per HTTP
|
||||
// request is queued. The duplicate delivery callback becomes a no-op.
|
||||
expect(deferred).toHaveLength(4);
|
||||
if (rootProcessedFirst) {
|
||||
await deferred.shift()?.();
|
||||
await vi.waitFor(async () => {
|
||||
const [root] = await db
|
||||
.select({ state: chatDeliveries.state })
|
||||
.from(chatDeliveries)
|
||||
.where(eq(chatDeliveries.endpointId, endpoint.id));
|
||||
expect(root.state).toBe("processed");
|
||||
});
|
||||
}
|
||||
await drainDeferred();
|
||||
// The scheduler callbacks start background promises. Wait for durable
|
||||
// replay admission before draining the conversation work it schedules.
|
||||
await vi.waitFor(async () => {
|
||||
const deliveries = await db
|
||||
.select({ id: chatDeliveries.id })
|
||||
.from(chatDeliveries)
|
||||
.where(eq(chatDeliveries.endpointId, endpoint.id));
|
||||
expect(deliveries).toHaveLength(3);
|
||||
});
|
||||
await vi.waitFor(async () => {
|
||||
await drainDeferred();
|
||||
const deliveries = await db
|
||||
.select({
|
||||
eventKind: chatDeliveries.eventKind,
|
||||
|
||||
@@ -12,6 +12,7 @@ const mockUpdateExecutionWorkspace = vi.hoisted(() => vi.fn());
|
||||
const mockLogActivity = vi.hoisted(() => vi.fn());
|
||||
const mockLoggerInfo = vi.hoisted(() => vi.fn());
|
||||
const mockGetEnvironment = vi.hoisted(() => vi.fn());
|
||||
const mockGetLease = vi.hoisted(() => vi.fn());
|
||||
|
||||
vi.mock("../services/environment-execution-target.js", () => ({
|
||||
resolveEnvironmentExecutionTarget: mockResolveEnvironmentExecutionTarget,
|
||||
@@ -30,6 +31,7 @@ vi.mock("../services/environments.js", () => ({
|
||||
environmentService: vi.fn(() => ({
|
||||
ensureLocalEnvironment: vi.fn(),
|
||||
getById: mockGetEnvironment,
|
||||
getLeaseById: mockGetLease,
|
||||
acquireLease: vi.fn(),
|
||||
releaseLease: vi.fn(),
|
||||
updateLeaseMetadata: mockUpdateLeaseMetadata,
|
||||
@@ -815,3 +817,56 @@ describe("admitted native lifecycle recovery", () => {
|
||||
expect(environment.config.reuseLease).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("live remote runner lease reattachment", () => {
|
||||
const input = {
|
||||
companyId: "company-1", selectedEnvironmentId: "env-1", localEnvironmentId: "local",
|
||||
adapterType: "paperclip_runner", admittedLifecycleMode: "per_turn" as const,
|
||||
issueId: "task-1", heartbeatRunId: "run-1", agentId: "agent-1",
|
||||
persistedExecutionWorkspace: { id: "ew-1", mode: "shared_workspace" as const },
|
||||
executionWorkspaceSettings: null,
|
||||
reattachRemoteLease: { leaseId: "lease-1", providerLeaseId: "original-sandbox", remoteCwd: "/remote/workspace" },
|
||||
};
|
||||
const originalLease = () => makeLease({
|
||||
provider: "daytona", providerLeaseId: "original-sandbox", issueId: "task-1", executionWorkspaceId: "ew-1",
|
||||
metadata: { agentId: "agent-1", remoteCwd: "/remote/workspace",
|
||||
sandboxLeaseAcquisition: { outcome: "created", providerLeaseId: "original-sandbox" } },
|
||||
});
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
mockGetEnvironment.mockResolvedValue({ ...makeEnvironment("sandbox"), config: { provider: "daytona", reuseLease: false } });
|
||||
mockGetLease.mockResolvedValue(originalLease());
|
||||
});
|
||||
it("reattaches the original active ephemeral lease without acquiring or resuming a sandbox", async () => {
|
||||
const savedLease = originalLease();
|
||||
mockGetLease.mockResolvedValue(savedLease);
|
||||
const runtime = makeMockRuntime();
|
||||
const result = await environmentRunOrchestrator({} as never, { environmentRuntime: runtime }).acquireForRun(input);
|
||||
expect(mockGetLease).toHaveBeenCalledWith("lease-1");
|
||||
expect(runtime.acquireRunLease).not.toHaveBeenCalled();
|
||||
expect(result.lease).toEqual(savedLease);
|
||||
expect(result.leaseContext.executionWorkspaceId).toBe("ew-1");
|
||||
expect(result.environment.config.reuseLease).toBe(false);
|
||||
});
|
||||
it.each([
|
||||
{ companyId: "foreign" }, { environmentId: "foreign" }, { heartbeatRunId: "another-run" },
|
||||
{ issueId: "another-task" }, { executionWorkspaceId: "another-workspace" },
|
||||
{ providerLeaseId: "replacement" }, { provider: "another-provider" },
|
||||
{ status: "released" }, { expiresAt: new Date(0) }, { releasedAt: new Date() }, { cleanupStatus: "pending" },
|
||||
{ metadata: { agentId: "another-agent", remoteCwd: "/remote/workspace" } },
|
||||
{ metadata: { agentId: "agent-1", remoteCwd: "/other/workspace" } },
|
||||
] as Partial<EnvironmentLease>[])("fails closed before provider acquisition for a mismatched lease %j", async (override) => {
|
||||
mockGetLease.mockResolvedValue({ ...originalLease(), ...override });
|
||||
const runtime = makeMockRuntime();
|
||||
await expect(environmentRunOrchestrator({} as never, { environmentRuntime: runtime }).acquireForRun(input))
|
||||
.rejects.toThrow("native_remote_recovery_lease_mismatch");
|
||||
expect(runtime.acquireRunLease).not.toHaveBeenCalled();
|
||||
});
|
||||
it("never replaces a missing original lease", async () => {
|
||||
mockGetLease.mockResolvedValue(null);
|
||||
const runtime = makeMockRuntime();
|
||||
await expect(environmentRunOrchestrator({} as never, { environmentRuntime: runtime }).acquireForRun(input))
|
||||
.rejects.toThrow("native_remote_recovery_lease_mismatch");
|
||||
expect(runtime.acquireRunLease).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -6679,13 +6679,24 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("destroys scoped reusable plugin-backed sandbox leases", async () => {
|
||||
it.each([
|
||||
{ status: "active", scope: "workspace" },
|
||||
{ status: "failed", scope: "workspace" },
|
||||
{ status: "failed", scope: "issue" },
|
||||
] as const)("destroys $status reusable plugin-backed sandbox leases scoped to an $scope", async ({ status, scope }) => {
|
||||
const { pluginId, companyId, runId, executionWorkspaceId, reusableLease } =
|
||||
await seedReusablePluginSandboxLease();
|
||||
await db
|
||||
.update(heartbeatRuns)
|
||||
.set({ status: "succeeded" })
|
||||
.set({ status: status === "failed" ? "failed" : "succeeded" })
|
||||
.where(eq(heartbeatRuns.id, runId));
|
||||
if (status === "failed") await environmentService(db).releaseLease(reusableLease.id, "failed");
|
||||
const issueId = randomUUID();
|
||||
if (scope === "issue") {
|
||||
await db.insert(issues).values({ id: issueId, companyId, title: "Failed reusable lease cleanup", status: "done", priority: "medium" });
|
||||
await db.update(environmentLeases).set({ issueId }).where(eq(environmentLeases.id, reusableLease.id));
|
||||
}
|
||||
const selection = scope === "issue" ? { issueId } : { executionWorkspaceId };
|
||||
|
||||
const workerManager = {
|
||||
isRunning: vi.fn((id: string) => id === pluginId),
|
||||
@@ -6699,9 +6710,12 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
|
||||
} as unknown as PluginWorkerManager;
|
||||
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
||||
|
||||
// A known issue/workspace id does not authorize another company's teardown.
|
||||
expect(await runtimeWithPlugin.destroyReusableSandboxLeases({ companyId: randomUUID(), ...selection })).toEqual([]);
|
||||
expect(workerManager.call).not.toHaveBeenCalled();
|
||||
const destroyed = await runtimeWithPlugin.destroyReusableSandboxLeases({
|
||||
companyId,
|
||||
executionWorkspaceId,
|
||||
...selection,
|
||||
failureReason: "execution_workspace_closed",
|
||||
});
|
||||
|
||||
@@ -6724,9 +6738,10 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("does not destroy a scoped reusable lease while its run is active", async () => {
|
||||
it.each(["active", "failed"] as const)("does not destroy a %s scoped reusable lease while its run is active", async (status) => {
|
||||
const { pluginId, companyId, executionWorkspaceId, reusableLease } =
|
||||
await seedReusablePluginSandboxLease();
|
||||
if (status === "failed") await environmentService(db).releaseLease(reusableLease.id, "failed");
|
||||
const workerManager = {
|
||||
isRunning: vi.fn((id: string) => id === pluginId),
|
||||
call: vi.fn(async () => undefined),
|
||||
@@ -6752,9 +6767,106 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
|
||||
expect(workerManager.call).not.toHaveBeenCalled();
|
||||
await expect(
|
||||
environmentService(db).getLeaseById(reusableLease.id),
|
||||
).resolves.toMatchObject({ status: "active" });
|
||||
).resolves.toMatchObject({ status });
|
||||
});
|
||||
|
||||
it.each(["issue", "workspace", "environment"] as const)(
|
||||
"durably claims failed reusable cleanup before %s teardown and recovers a provider failure",
|
||||
async (scope) => {
|
||||
const seeded = await seedReusablePluginSandboxLease();
|
||||
await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, seeded.runId));
|
||||
await environmentService(db).releaseLease(seeded.reusableLease.id, "failed");
|
||||
const issueId = randomUUID();
|
||||
if (scope === "issue") {
|
||||
await db.insert(issues).values({ id: issueId, companyId: seeded.companyId, title: "Cleanup crash window", status: "done" });
|
||||
await db.update(environmentLeases).set({ issueId }).where(eq(environmentLeases.id, seeded.reusableLease.id));
|
||||
}
|
||||
let enterProvider!: () => void;
|
||||
let finishProvider!: () => void;
|
||||
const entered = new Promise<void>((resolve) => { enterProvider = resolve; });
|
||||
const finish = new Promise<void>((resolve) => { finishProvider = resolve; });
|
||||
let unavailable = true;
|
||||
const call = vi.fn(async (_id: string, method: string) => {
|
||||
if (method !== "environmentDestroyLease") throw new Error(`Unexpected method ${method}`);
|
||||
if (unavailable) {
|
||||
enterProvider();
|
||||
await finish;
|
||||
throw new Error("Provider response lost");
|
||||
}
|
||||
return { providerLeaseId: seeded.reusableLease.providerLeaseId, state: "destroyed" };
|
||||
});
|
||||
const makeRuntime = () => environmentRuntimeService(db, { pluginWorkerManager: {
|
||||
isRunning: () => true, call,
|
||||
getWorker: () => ({ supportedMethods: ["environmentDestroyLease"] }),
|
||||
} as unknown as PluginWorkerManager });
|
||||
const runtime = makeRuntime();
|
||||
const close = (service: ReturnType<typeof makeRuntime>) => scope === "environment"
|
||||
? service.destroyReusableSandboxLeasesForEnvironment({ environmentId: seeded.environment.id })
|
||||
: service.destroyReusableSandboxLeases({ companyId: seeded.companyId,
|
||||
...(scope === "issue" ? { issueId } : { executionWorkspaceId: seeded.executionWorkspaceId }) });
|
||||
const closing = close(runtime);
|
||||
try {
|
||||
await entered;
|
||||
// This independently persisted row is what survives controller loss
|
||||
// during the provider call. No provider receipt has arrived yet.
|
||||
expect(await environmentService(db).getLeaseById(seeded.reusableLease.id)).toMatchObject({
|
||||
status: "pending_cleanup", cleanupStatus: "failed", companyId: seeded.companyId,
|
||||
providerLeaseId: seeded.reusableLease.providerLeaseId,
|
||||
metadata: { pendingCleanupAttemptId: expect.any(String), pendingCleanupInFlight: true,
|
||||
pendingCleanupLeaseExpiresAtMs: expect.any(Number) },
|
||||
});
|
||||
await close(makeRuntime());
|
||||
expect(await heartbeatService(db, { environmentRuntime: makeRuntime() }).sweepPendingCleanupLeases({ backoffMs: 0 }))
|
||||
.toEqual({ swept: 0, destroyed: 0, capped: 0 });
|
||||
expect(call).toHaveBeenCalledOnce();
|
||||
} finally {
|
||||
finishProvider();
|
||||
await closing;
|
||||
}
|
||||
expect(await environmentService(db).getLeaseById(seeded.reusableLease.id)).toMatchObject({
|
||||
status: "pending_cleanup", metadata: { pendingCleanupInFlight: false },
|
||||
});
|
||||
unavailable = false;
|
||||
expect(await heartbeatService(db, { environmentRuntime: makeRuntime() }).sweepPendingCleanupLeases({ backoffMs: 0 }))
|
||||
.toEqual({ swept: 1, destroyed: 1, capped: 0 });
|
||||
expect(call).toHaveBeenCalledTimes(2);
|
||||
expect(await environmentService(db).getLeaseById(seeded.reusableLease.id)).toMatchObject({
|
||||
status: "expired", cleanupStatus: "success", metadata: { remoteExecutionTermination: { state: "destroyed" } },
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it.each(["queued", "scheduled_retry", "running"] as const)(
|
||||
"fences scoped cleanup when its holding run becomes %s after selection", async (status) => {
|
||||
const seeded = await seedReusablePluginSandboxLease();
|
||||
await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, seeded.runId));
|
||||
await environmentService(db).releaseLease(seeded.reusableLease.id, "failed");
|
||||
const originalService = environmentsModule.environmentService;
|
||||
const serviceSpy = vi.spyOn(environmentsModule, "environmentService").mockImplementation((client) => {
|
||||
const service = originalService(client);
|
||||
return { ...service, getById: async (id: string) => {
|
||||
const environment = await service.getById(id);
|
||||
if (id === seeded.environment.id) {
|
||||
await db.update(heartbeatRuns).set({ status }).where(eq(heartbeatRuns.id, seeded.runId));
|
||||
}
|
||||
return environment;
|
||||
} };
|
||||
});
|
||||
const call = vi.fn(async () => undefined);
|
||||
const runtime = environmentRuntimeService(db, { pluginWorkerManager: {
|
||||
isRunning: () => true, call, getWorker: () => ({ supportedMethods: ["environmentDestroyLease"] }),
|
||||
} as unknown as PluginWorkerManager });
|
||||
try {
|
||||
expect(await runtime.destroyReusableSandboxLeases({ companyId: seeded.companyId,
|
||||
executionWorkspaceId: seeded.executionWorkspaceId })).toEqual([]);
|
||||
expect(call).not.toHaveBeenCalled();
|
||||
expect(await originalService(db).getLeaseById(seeded.reusableLease.id)).toMatchObject({ status: "failed" });
|
||||
} finally {
|
||||
serviceSpy.mockRestore();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("destroys reusable plugin-backed sandbox leases scoped to an environment", async () => {
|
||||
const { pluginId, runId, reusableLease } = await seedReusablePluginSandboxLease();
|
||||
// The holding run is finished, so the reservation is stale and destroyable.
|
||||
@@ -6796,9 +6908,51 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps a reusable lease held by an in-flight run out of the environment-scoped destroy", async () => {
|
||||
it("destroys a failed reusable sandbox after successful release before deleting its environment", async () => {
|
||||
const { pluginId, companyId, runId, environment, reusableLease } = await seedReusablePluginSandboxLease();
|
||||
const workerManager = {
|
||||
isRunning: vi.fn((id: string) => id === pluginId),
|
||||
call: vi.fn(async (_pluginId: string, method: string) => {
|
||||
if (method === "environmentReleaseLease") {
|
||||
return { providerLeaseId: reusableLease.providerLeaseId, state: "stopped" };
|
||||
}
|
||||
if (method === "environmentDestroyLease") {
|
||||
// The provider call must already have a durable cleanup reference.
|
||||
expect(await environmentService(db).getLeaseById(reusableLease.id)).toMatchObject({
|
||||
status: "pending_cleanup", environmentId: environment.id,
|
||||
});
|
||||
return { providerLeaseId: reusableLease.providerLeaseId, state: "destroyed" };
|
||||
}
|
||||
throw new Error(`Unexpected plugin method: ${method}`);
|
||||
}),
|
||||
getWorker: vi.fn(() => ({ supportedMethods: ["environmentReleaseLease", "environmentDestroyLease"] })),
|
||||
} as unknown as PluginWorkerManager;
|
||||
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
||||
await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, runId));
|
||||
|
||||
const released = await runtimeWithPlugin.releaseRunLeases(runId, "failed", undefined, undefined, true);
|
||||
expect(released[0]?.lease).toMatchObject({
|
||||
id: reusableLease.id, status: "failed", cleanupStatus: "success",
|
||||
metadata: { remoteExecutionTermination: { state: "stopped" } },
|
||||
});
|
||||
const result = await runtimeWithPlugin.destroyReusableSandboxLeasesForEnvironment({
|
||||
environmentId: environment.id, failureReason: "environment_deleted",
|
||||
});
|
||||
expect(result).toEqual({ destroyed: 1, failed: 0, skippedLiveRun: 0 });
|
||||
expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentDestroyLease",
|
||||
expect.objectContaining({ companyId, environmentId: environment.id, providerLeaseId: reusableLease.providerLeaseId }),
|
||||
31234);
|
||||
await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({
|
||||
status: "expired", cleanupStatus: "success",
|
||||
metadata: { remoteExecutionTermination: { state: "destroyed" } },
|
||||
});
|
||||
expect((await environmentService(db).removeIfDeletable(environment.id))?.id).toBe(environment.id);
|
||||
});
|
||||
|
||||
it.each(["active", "failed"] as const)("keeps a %s reusable lease held by an in-flight run out of the environment-scoped destroy", async (status) => {
|
||||
const { pluginId, reusableLease } = await seedReusablePluginSandboxLease();
|
||||
// seedEnvironment leaves the holding run in `running` status.
|
||||
if (status === "failed") await environmentService(db).releaseLease(reusableLease.id, "failed");
|
||||
|
||||
const workerManager = {
|
||||
isRunning: vi.fn((id: string) => id === pluginId),
|
||||
@@ -6816,7 +6970,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
|
||||
expect(workerManager.call).not.toHaveBeenCalled();
|
||||
// The lease keeps its reusable status, so the delete guard still blocks.
|
||||
await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({
|
||||
status: "active",
|
||||
status,
|
||||
leasePolicy: "reuse_by_environment",
|
||||
});
|
||||
});
|
||||
|
||||
@@ -636,6 +636,18 @@ describeEmbeddedPostgres("environmentService leases", () => {
|
||||
expect(releasedImpact?.reusableSandboxLeaseCount).toBe(1);
|
||||
expect(await svc.removeIfDeletable(environmentId)).toBeNull();
|
||||
|
||||
// A failed run can release its reusable sandbox successfully without
|
||||
// destroying it. The provider handle still needs environment-scoped cleanup.
|
||||
await svc.releaseLease(lease.id, "failed", {
|
||||
failureReason: "adapter_or_run_failure",
|
||||
cleanupStatus: "success",
|
||||
});
|
||||
const failedImpact = await svc.getDeleteBlastRadius(environmentId);
|
||||
expect.soft(failedImpact?.canDelete).toBe(false);
|
||||
expect.soft(failedImpact?.deleteBlockedReasons).toContain("reusable_sandbox_lease");
|
||||
expect.soft(failedImpact?.reusableSandboxLeaseCount).toBe(1);
|
||||
expect(await svc.removeIfDeletable(environmentId)).toBeNull();
|
||||
|
||||
const rows = await db.select().from(environments).where(eq(environments.id, environmentId));
|
||||
expect(rows).toHaveLength(1);
|
||||
const storedLease = await svc.getLeaseById(lease.id);
|
||||
|
||||
@@ -134,50 +134,65 @@ describe("workspace Git scan route load regression", () => {
|
||||
...unavailableMethods(),
|
||||
};
|
||||
const app = createLoadApp(db, companyId, service);
|
||||
const pendingResponses = Array.from({ length: 500 }, (_, index) => request(app)
|
||||
// Model one busy API server, not 500 independent ephemeral listeners whose
|
||||
// accept/close races can reset clients before they reach the scheduler.
|
||||
const server = app.listen(0, "127.0.0.1");
|
||||
await new Promise<void>(resolve => server.once("listening", resolve));
|
||||
const pendingResponses = Array.from({ length: 500 }, (_, index) => request(server)
|
||||
.get(`/api/issues/issue-${index}/file-resources/list`)
|
||||
.set("x-test-actor", `actor-${index % 73}`)
|
||||
.query({ mode: "changed" })
|
||||
.then((response) => response));
|
||||
// Observe every client rejection immediately, including on assertion failure.
|
||||
const settledResponses = Promise.allSettled(pendingResponses);
|
||||
try {
|
||||
await vi.waitFor(
|
||||
() => expect(scheduler.snapshot().totals.singleFlightJoins).toBe(498),
|
||||
{ timeout: 15_000, interval: 20 },
|
||||
);
|
||||
const loadedSnapshot = scheduler.snapshot();
|
||||
expect(loadedSnapshot).toMatchObject({ activeCount: 2, queuedCount: 0, inFlightCount: 2 });
|
||||
|
||||
await vi.waitFor(
|
||||
() => expect(scheduler.snapshot().totals.singleFlightJoins).toBe(498),
|
||||
{ timeout: 15_000, interval: 20 },
|
||||
);
|
||||
const loadedSnapshot = scheduler.snapshot();
|
||||
expect(loadedSnapshot).toMatchObject({ activeCount: 2, queuedCount: 0, inFlightCount: 2 });
|
||||
const healthLatencies: number[] = [];
|
||||
for (let index = 0; index < 25; index += 1) {
|
||||
const startedAt = performance.now();
|
||||
const response = await request(server).get("/api/health");
|
||||
healthLatencies.push(performance.now() - startedAt);
|
||||
expect(response.status).toBe(200);
|
||||
}
|
||||
healthLatencies.sort((left, right) => left - right);
|
||||
const healthP99Ms = healthLatencies[Math.ceil(healthLatencies.length * 0.99) - 1]!;
|
||||
expect(healthP99Ms).toBeLessThan(250);
|
||||
|
||||
const healthLatencies: number[] = [];
|
||||
for (let index = 0; index < 25; index += 1) {
|
||||
const startedAt = performance.now();
|
||||
const response = await request(app).get("/api/health");
|
||||
healthLatencies.push(performance.now() - startedAt);
|
||||
expect(response.status).toBe(200);
|
||||
releaseScans();
|
||||
const responses = (await settledResponses).map(result => {
|
||||
if (result.status === "rejected") throw result.reason;
|
||||
return result.value;
|
||||
});
|
||||
const outcomeCounts = responses.reduce<Record<number, number>>((counts, response) => {
|
||||
counts[response.status] = (counts[response.status] ?? 0) + 1;
|
||||
return counts;
|
||||
}, {});
|
||||
expect(outcomeCounts).toEqual({ 200: 500 });
|
||||
expect({ runnerCalls, peakActive }).toEqual({ runnerCalls: 2, peakActive: 2 });
|
||||
expect(scheduler.snapshot()).toMatchObject({ activeCount: 0, queuedCount: 0, inFlightCount: 0 });
|
||||
|
||||
console.info("workspace Git scan regression metrics", {
|
||||
requests: responses.length,
|
||||
repositories: roots.length,
|
||||
underlyingScans: runnerCalls,
|
||||
peakActive,
|
||||
peakQueued: loadedSnapshot.queuedCount,
|
||||
outcomes: outcomeCounts,
|
||||
healthP99Ms: Math.round(healthP99Ms * 100) / 100,
|
||||
unreapedChildCount: 0,
|
||||
});
|
||||
} finally {
|
||||
releaseScans();
|
||||
server.closeAllConnections();
|
||||
await settledResponses;
|
||||
await new Promise<void>((resolve, reject) => server.close(error => error ? reject(error) : resolve()));
|
||||
}
|
||||
healthLatencies.sort((left, right) => left - right);
|
||||
const healthP99Ms = healthLatencies[Math.ceil(healthLatencies.length * 0.99) - 1]!;
|
||||
expect(healthP99Ms).toBeLessThan(250);
|
||||
|
||||
releaseScans();
|
||||
const responses = await Promise.all(pendingResponses);
|
||||
const outcomeCounts = responses.reduce<Record<number, number>>((counts, response) => {
|
||||
counts[response.status] = (counts[response.status] ?? 0) + 1;
|
||||
return counts;
|
||||
}, {});
|
||||
expect(outcomeCounts).toEqual({ 200: 500 });
|
||||
expect({ runnerCalls, peakActive }).toEqual({ runnerCalls: 2, peakActive: 2 });
|
||||
expect(scheduler.snapshot()).toMatchObject({ activeCount: 0, queuedCount: 0, inFlightCount: 0 });
|
||||
|
||||
console.info("workspace Git scan regression metrics", {
|
||||
requests: responses.length,
|
||||
repositories: roots.length,
|
||||
underlyingScans: runnerCalls,
|
||||
peakActive,
|
||||
peakQueued: loadedSnapshot.queuedCount,
|
||||
outcomes: outcomeCounts,
|
||||
healthP99Ms: Math.round(healthP99Ms * 100) / 100,
|
||||
unreapedChildCount: 0,
|
||||
});
|
||||
}, 60_000);
|
||||
|
||||
it.each([
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { ensureNativeCompletionContract } from "../services/native-runtime/completion-contracts.js";
|
||||
import { readNativeCursorPlanWait, hasCommittedNativeCursorPlanWait } from "../services/native-runtime/native-cursor-plan-wait.js";
|
||||
import { readNativePlanWait, hasCommittedNativePlanWait } from "../services/native-runtime/native-plan-wait.js";
|
||||
import { nativeSha256 } from "../services/native-runtime/canonical.js";
|
||||
import { buildQuestionResponseDeliveryEnvelope } from "../services/question-response-delivery.js";
|
||||
import * as executionContinuation from "../services/execution-continuation.js";
|
||||
@@ -7930,8 +7930,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
||||
});
|
||||
expect(next?.contextSnapshot?.wakeCommentIds).toEqual([pending!.id, go!.id]);
|
||||
expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, deferred!.id)))[0]).toMatchObject({ status: "coalesced", runId: next!.id });
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
expect((await heartbeat.getRun(next!.id))?.status).not.toBe("running");
|
||||
await vi.waitFor(async () => expect((await heartbeat.getRun(next!.id))?.status).not.toBe("running"));
|
||||
});
|
||||
|
||||
it.each(["dedicated deferred donor", "non-coalescing recipient", "persistent agent conversation"] as const)(
|
||||
@@ -14537,226 +14536,6 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
||||
expect(recoveryAction.ownerType).toBe("board");
|
||||
});
|
||||
|
||||
describe("accepted Cursor planning boundaries", () => {
|
||||
const nativeImplementation = mockExecutePaperclipNativeSession.getMockImplementation();
|
||||
beforeEach(() => {
|
||||
mockExecutePaperclipNativeSession.mockImplementation(async () => { throw new Error("Accepted-plan tests must never execute a provider"); });
|
||||
});
|
||||
afterEach(() => { mockExecutePaperclipNativeSession.mockImplementation(nativeImplementation!); });
|
||||
|
||||
async function seedAcceptedCursorPlanWait(semanticFinish = false, sourceSequenceOffset = 0) {
|
||||
const f = await seedStrandedIssueFixture({ status: "in_progress", runStatus: "succeeded", livenessState: "advanced" });
|
||||
const instance = randomUUID(), interactionId = randomUUID();
|
||||
const contractInput = { db, companyId: f.companyId,
|
||||
issue: { id: f.issueId, title: "Review the plan before further work", description: "Explicit completion required" }, actorId: "test" };
|
||||
const { row: persistedContract, contract } = await ensureNativeCompletionContract(contractInput);
|
||||
const reused = await ensureNativeCompletionContract(contractInput);
|
||||
expect(reused.row.id).toBe(persistedContract.id);
|
||||
expect(reused.contract).toEqual(contract);
|
||||
const contractId = persistedContract.id, contractSha = persistedContract.canonicalSha256;
|
||||
// Production binds policy/schema as well as the body; a body-only hash is not a valid contract receipt.
|
||||
expect(contractSha).not.toBe(nativeSha256(contract));
|
||||
const model = "gpt-5.6-luna[context=272k,reasoning=medium,fast=false]";
|
||||
await db.update(agents).set({ adapterType: "paperclip_runner", adapterConfig: { provider: "acpx", acpxAgent: "cursor", model, acpxSessionMode: "plan", acpxPermissionMode: "approve-all" } }).where(eq(agents.id, f.agentId));
|
||||
await db.update(agentWakeupRequests).set({ status: "completed" }).where(eq(agentWakeupRequests.id, f.wakeupRequestId));
|
||||
await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: f.issueId, nativeSessionId: f.runId,
|
||||
completionContractId: contractId, completionContractSha256: contractSha, runnerInstanceId: instance,
|
||||
runnerProfileJson: { nativeExecutionInput: { binding: { companyId: f.companyId, issueId: f.issueId, runId: f.runId, agentId: f.agentId }, provider: { kind: "acpx", agent: "cursor", model, cursorMode: "plan", permissionMode: "approve-all",
|
||||
profile: paperclipRunner.resolveQualifiedAcpxProfile("cursor", model) }, session: { normalizedSessionId: f.runId },
|
||||
completionContract: { id: contractId, sha256: contractSha, contract } } } }).where(eq(heartbeatRuns.id, f.runId));
|
||||
const planId = `plan-${"a".repeat(64)}`;
|
||||
const questionSet = { schema: "paperclip.question_set.v1", title: "Native plan", description: "Exact accepted revision", questions: [{ id: planId, prompt: "Proceed?", required: true,
|
||||
answerMode: "single_select", options: [{ id: "accept", label: "Accept" }, { id: "reject", label: "Reject" }, { id: "cancel", label: "Cancel" }] }] };
|
||||
const [interaction] = await db.insert(issueThreadInteractions).values({ id: interactionId, companyId: f.companyId, issueId: f.issueId, sourceRunId: f.runId,
|
||||
createdByAgentId: f.agentId, kind: "ask_user_questions", status: "answered", continuationPolicy: "none",
|
||||
idempotencyKey: `paperclip-runner-question:${f.runId}:request`, resolvedByUserId: "responsible-user", resolvedAt: new Date(Date.now() - 1000),
|
||||
payload: { version: 1, questions: [{ id: planId, prompt: "Proceed?", selectionMode: "single", required: true, allowOther: false, options: [{ id: "accept", label: "Accept" }, { id: "reject", label: "Reject" }, { id: "cancel", label: "Cancel" }] }], runtimeRequestId: "request", questionSet: questionSet as never }, result: { version: 1, answers: [{ questionId: planId, optionIds: ["accept"] }] } }).returning();
|
||||
const answer = buildQuestionResponseDeliveryEnvelope(interaction as never);
|
||||
const [delivery] = await db.insert(issueQuestionResponseDeliveries).values({ companyId: f.companyId, issueId: f.issueId, interactionId,
|
||||
sourceRunId: f.runId, targetRunId: f.runId, correlationId: randomUUID(), status: "delivered", deliveryMode: "steered", acknowledgedAt: new Date(), payloadSha256: nativeSha256(answer) }).returning();
|
||||
const port = new PaperclipControlPlanePort(db, { companyId: f.companyId, issueId: f.issueId, runId: f.runId, agentId: f.agentId,
|
||||
sessionId: f.runId, completionContractId: contractId, completionContractSha256: contractSha, sourceInstanceId: instance, controlPlaneSourceInstanceId: `control-${f.runId}` });
|
||||
const events = [
|
||||
{ eventType: "runtime_request.created", payload: { request: { schema: "paperclip.runtime_request.v2", requestKind: "runtime", requestId: "request", type: "input", status: "pending", turnId: "turn", itemId: "native-plan-tool", prompt: "Review plan",
|
||||
origin: { adapter: "acpx-runtime-sidecar", provider: "cursor", method: "cursor/create_plan" }, input: questionSet } } },
|
||||
{ eventType: "tool.execution.started", payload: { schema: "paperclip.tool.execution.v1", executionId: "native-plan-tool", transport: "builtin", operation: "execute", status: "running", readOnly: false, namespace: null, name: "Create Plan", target: null, inputUpdated: true, output: null, outputBytes: 0, outputTruncated: false, outputDigest: null, progress: null, exitCode: null, durationMs: null } },
|
||||
{ eventType: "runtime_request.resolved", payload: { requestId: "request", turnId: "turn", action: "submit", response: answer.response } },
|
||||
{ eventType: "tool.execution.completed", payload: { schema: "paperclip.tool.execution.v1", executionId: "native-plan-tool", transport: "builtin", operation: "execute", status: "completed", readOnly: false, namespace: null, name: "Create Plan", target: null, inputUpdated: false, output: null, outputBytes: 0, outputTruncated: false, outputDigest: null, progress: null, exitCode: null, durationMs: null } },
|
||||
{ eventType: "turn.completed", payload: { status: "completed", error: null } },
|
||||
];
|
||||
for (const [index, event] of events.entries()) await port.appendEvent({ schema: "paperclip.prp.event.v1", sourceEventId: `${instance}:${sourceSequenceOffset + index + 1}`, sourceSeq: sourceSequenceOffset + index + 1,
|
||||
sourceInstanceId: instance, sourceKind: "runner", runId: f.runId, normalizedSessionId: f.runId, turnId: "turn", schemaVersion: 1, priority: 0,
|
||||
emittedAt: new Date().toISOString(), ...event } as paperclipRunner.PrpEvent);
|
||||
const proof = await readNativeCursorPlanWait(db, f);
|
||||
expect(proof).not.toBeNull();
|
||||
const result = semanticFinish ? { ...proof!.result, reportedWorkDisposition: "done" as const, summary: "Explicit semantic finish wins", continuation: undefined,
|
||||
completionClaim: { contractRevision: "1", objectiveSatisfied: true, criteria: [{ criterionId: "objective", status: "satisfied" as const, evidenceRefs: [] }], remainingWork: [] } } : proof!.result;
|
||||
await port.completeRun({ result, turnId: "turn", terminal: { schema: "paperclip.prp.terminal.v1", runTerminalState: "succeeded", turnTerminalState: "completed", reportedWorkDisposition: result.reportedWorkDisposition } });
|
||||
return { ...f, interactionId, deliveryId: delivery!.id, proof: proof! };
|
||||
}
|
||||
|
||||
it("keeps an accepted Cursor plan passive across restart, future configuration changes, and paused recovery", async () => {
|
||||
const f = await seedAcceptedCursorPlanWait();
|
||||
await finalizeNativeRun({ db, runId: f.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
|
||||
await finalizeNativeRun({ db, runId: f.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
|
||||
expect(await hasCommittedNativeCursorPlanWait(db, f)).toBe(true);
|
||||
expect(await db.select().from(statusDecisions).where(eq(statusDecisions.runId, f.runId))).toEqual([expect.objectContaining({ reasonCode: "native_plan_accepted_waiting_for_continuation", toStatus: "in_progress" })]);
|
||||
expect(await db.select().from(statusDecisionEffects).where(eq(statusDecisionEffects.issueId, f.issueId))).toEqual([expect.objectContaining({ effectKind: "issue_status_projection", targetType: "issue", deliveryState: "delivered" })]);
|
||||
expect(await db.select().from(issueComments).where(eq(issueComments.createdByRunId, f.runId))).toEqual([expect.objectContaining({ body: expect.stringContaining("next message") })]);
|
||||
const current = paperclipRunner.resolveQualifiedAcpxProfile("cursor", "gpt-5.6-luna[context=272k,reasoning=medium,fast=false]");
|
||||
const resolver = vi.spyOn(paperclipRunner, "resolveQualifiedAcpxProfile").mockReturnValue({ ...current,
|
||||
agentProfileVersion: current.agentProfileVersion + 1, commandDigest: `sha256:${"b".repeat(64)}` });
|
||||
try {
|
||||
expect(await readNativeCursorPlanWait(db, f)).toBeNull(); // Old profile cannot create a new wait.
|
||||
for (const status of ["idle", "paused"] as const) {
|
||||
await db.update(agents).set({ status, adapterConfig: { provider: "acpx", acpxAgent: "cursor",
|
||||
model: "future-model", acpxSessionMode: "agent", acpxPermissionMode: "approve-reads" } }).where(eq(agents.id, f.agentId));
|
||||
expect(await hasCommittedNativeCursorPlanWait(db, f)).toBe(true);
|
||||
const recovered = await heartbeatService(db).reconcileStrandedAssignedIssues();
|
||||
expect(recovered.continuationRequeued).toBe(0); expect(recovered.escalated).toBe(0);
|
||||
}
|
||||
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId));
|
||||
const changed = structuredClone(run!.runnerProfileJson!);
|
||||
(changed.nativeExecutionInput as { provider: { profile: { commandDigest: string } } }).provider.profile.commandDigest = "tampered-original-profile";
|
||||
await db.update(heartbeatRuns).set({ runnerProfileJson: changed }).where(eq(heartbeatRuns.id, f.runId));
|
||||
expect(await hasCommittedNativeCursorPlanWait(db, f)).toBe(false);
|
||||
await db.update(heartbeatRuns).set({ runnerProfileJson: run!.runnerProfileJson }).where(eq(heartbeatRuns.id, f.runId));
|
||||
expect(await hasCommittedNativeCursorPlanWait(db, f)).toBe(true);
|
||||
const [tool] = await db.select().from(heartbeatRunEvents).where(and(eq(heartbeatRunEvents.runId, f.runId), eq(heartbeatRunEvents.eventType, "tool.execution.completed")));
|
||||
const changedTool = structuredClone(tool!.payload!) as { prpEvent: { payload: { name: string } } };
|
||||
changedTool.prpEvent.payload.name = "mutated committed tool evidence";
|
||||
await db.update(heartbeatRunEvents).set({ payload: changedTool, sourcePayloadSha256: nativeSha256(changedTool.prpEvent) }).where(eq(heartbeatRunEvents.id, tool!.id));
|
||||
expect(await hasCommittedNativeCursorPlanWait(db, f)).toBe(false);
|
||||
await db.update(heartbeatRunEvents).set({ payload: tool!.payload, sourcePayloadSha256: tool!.sourcePayloadSha256 }).where(eq(heartbeatRunEvents.id, tool!.id));
|
||||
expect(await hasCommittedNativeCursorPlanWait(db, f)).toBe(true);
|
||||
} finally { resolver.mockRestore(); }
|
||||
expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.companyId))).toHaveLength(1);
|
||||
expect(mockAdapterExecute).not.toHaveBeenCalled(); expect(mockExecutePaperclipNativeSession).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("retains an exact historical Cursor6 committed wait across more than 1000 ignored progress rows without allowing new unbound admission", async () => {
|
||||
const progressCount = 1002;
|
||||
const f = await seedAcceptedCursorPlanWait(false, progressCount);
|
||||
await finalizeNativeRun({ db, runId: f.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
|
||||
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId));
|
||||
const profile = structuredClone(run!.runnerProfileJson!) as any;
|
||||
Object.assign(profile.nativeExecutionInput.provider.profile, { agentProfileVersion: 6, commandDigest: "sha256:377dcea64a727ce799cc112458d4b40ba4bc6574cd6c6f7233b6efd5917a6c4b" });
|
||||
await db.update(heartbeatRuns).set({ runnerProfileJson: profile }).where(eq(heartbeatRuns.id, f.runId));
|
||||
await db.delete(heartbeatRunEvents).where(and(eq(heartbeatRunEvents.runId, f.runId), inArray(heartbeatRunEvents.eventType, ["tool.execution.started", "tool.execution.completed"])));
|
||||
// Reconstruct the persisted Cursor6 receipt format using its unchanged
|
||||
// source facts. This historical format had no tool lifecycle binding.
|
||||
const rows = await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, f.runId));
|
||||
const event = (kind: string) => (rows.find(r => r.eventType === kind)!.payload as any).prpEvent;
|
||||
const [contract] = await db.select().from(completionContracts).where(eq(completionContracts.id, run!.completionContractId!));
|
||||
const [interaction] = await db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.id, f.interactionId));
|
||||
const [delivery] = await db.select().from(issueQuestionResponseDeliveries).where(eq(issueQuestionResponseDeliveries.id, f.deliveryId));
|
||||
const legacy = { ...f.proof.source }; delete legacy.toolExecutionId; delete legacy.toolLifecycleSha256;
|
||||
legacy.authoritySha256 = nativeSha256({ admission: profile.nativeExecutionInput, contract, created: event("runtime_request.created"), resolved: event("runtime_request.resolved"), terminal: event("turn.completed"), interaction, delivery, resolvedAt: interaction!.resolvedAt!.toISOString(), acknowledgedAt: delivery!.acknowledgedAt!.toISOString() });
|
||||
const [decision] = await db.select().from(statusDecisions).where(eq(statusDecisions.runId, f.runId));
|
||||
await db.update(statusDecisions).set({ decisionJson: { ...decision!.decisionJson, cursorPlanWait: legacy } }).where(eq(statusDecisions.id, decision!.id));
|
||||
expect(await readNativeCursorPlanWait(db, f)).toBeNull();
|
||||
expect(await hasCommittedNativeCursorPlanWait(db, f)).toBe(true);
|
||||
// Fill the reserved source prefix with genuine durable progress rows. Shift
|
||||
// only DB ordering keys; the original PRP facts and committed receipt stay
|
||||
// byte-identical. Cursor6 never queried these rows, even above its budget.
|
||||
await db.update(heartbeatRunEvents).set({ seq: sql`${heartbeatRunEvents.seq} + ${progressCount}` }).where(eq(heartbeatRunEvents.runId, f.runId));
|
||||
const progress = Array.from({ length: progressCount }, (_, index) => {
|
||||
const seq = index + 1;
|
||||
const prpEvent = { ...event("runtime_request.created"), sourceEventId: `${run!.runnerInstanceId}:${seq}`, sourceSeq: seq, eventType: "tool.execution.progressed",
|
||||
payload: { schema: "paperclip.tool.execution.v1", executionId: "earlier-tool", transport: "builtin", operation: "read", status: "running" } };
|
||||
return { companyId: f.companyId, runId: f.runId, agentId: f.agentId, seq, eventType: prpEvent.eventType, payload: { prpEvent }, sourceInstanceId: run!.runnerInstanceId,
|
||||
sourceEventId: prpEvent.sourceEventId, sourceSeq: seq, sourcePayloadSha256: nativeSha256(prpEvent), protocolSchemaVersion: 1 };
|
||||
});
|
||||
await db.insert(heartbeatRunEvents).values(progress);
|
||||
expect(await hasCommittedNativeCursorPlanWait(db, f)).toBe(true);
|
||||
const [unchangedDecision] = await db.select().from(statusDecisions).where(eq(statusDecisions.id, decision!.id));
|
||||
expect(unchangedDecision!.decisionJson!.cursorPlanWait).toEqual(legacy);
|
||||
// Completed rows belonged to the old query. A later completion must still
|
||||
// invalidate the normal-terminal boundary rather than being filtered away.
|
||||
const completion = { ...event("turn.completed"), sourceEventId: `${run!.runnerInstanceId}:99999`, sourceSeq: 99999, eventType: "tool.execution.completed", payload: { ...progress[0]!.payload.prpEvent.payload, status: "completed" } };
|
||||
const [extra] = await db.insert(heartbeatRunEvents).values({ ...progress[0]!, seq: 99999, eventType: completion.eventType, payload: { prpEvent: completion }, sourceEventId: completion.sourceEventId, sourceSeq: 99999, sourcePayloadSha256: nativeSha256(completion) }).returning();
|
||||
expect(await hasCommittedNativeCursorPlanWait(db, f)).toBe(false);
|
||||
await db.delete(heartbeatRunEvents).where(eq(heartbeatRunEvents.id, extra!.id));
|
||||
expect(await hasCommittedNativeCursorPlanWait(db, f)).toBe(true);
|
||||
profile.nativeExecutionInput.provider.profile.commandDigest = "tampered-history";
|
||||
await db.update(heartbeatRuns).set({ runnerProfileJson: profile }).where(eq(heartbeatRuns.id, f.runId));
|
||||
expect(await hasCommittedNativeCursorPlanWait(db, f)).toBe(false);
|
||||
});
|
||||
|
||||
it("admits a later ordinary user message after an accepted Cursor plan without changing Plan mode or replaying its run", async () => {
|
||||
const f = await seedAcceptedCursorPlanWait();
|
||||
await finalizeNativeRun({ db, runId: f.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
|
||||
// Occupy the single dispatch slot: this checks actual user-wake admission
|
||||
// without executing any provider or fabricating a second semantic result.
|
||||
const occupied = randomUUID();
|
||||
await db.update(agents).set({ runtimeConfig: { heartbeat: { wakeOnDemand: true, maxConcurrentRuns: 1 } } }).where(eq(agents.id, f.agentId));
|
||||
await db.insert(heartbeatRuns).values({ id: occupied, companyId: f.companyId, agentId: f.agentId, status: "running", startedAt: new Date(), contextSnapshot: {} });
|
||||
const [comment] = await db.insert(issueComments).values({ companyId: f.companyId, issueId: f.issueId, authorType: "user", authorUserId: "responsible-user", body: "Continue reviewing the accepted plan in Plan mode." }).returning();
|
||||
try {
|
||||
const heartbeat = heartbeatService(db);
|
||||
const next = await heartbeat.wakeup(f.agentId, { source: "automation", triggerDetail: "system", reason: "issue_commented", requestedByActorType: "user", requestedByActorId: "responsible-user",
|
||||
payload: { issueId: f.issueId, commentId: comment!.id }, contextSnapshot: { issueId: f.issueId, taskId: f.issueId, wakeCommentId: comment!.id, wakeCommentIds: [comment!.id] } });
|
||||
expect(next).not.toBeNull(); expect(next!.id).not.toBe(f.runId);
|
||||
const [admitted] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, next!.id));
|
||||
expect(admitted).toMatchObject({ status: "queued", retryOfRunId: null });
|
||||
expect(admitted!.contextSnapshot?.wakeCommentIds).toContain(comment!.id);
|
||||
const [agent] = await db.select().from(agents).where(eq(agents.id, f.agentId));
|
||||
expect(agent!.adapterConfig.acpxSessionMode).toBe("plan");
|
||||
expect(await hasCommittedNativeCursorPlanWait(db, f)).toBe(false);
|
||||
expect(mockExecutePaperclipNativeSession).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
await db.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date() }).where(and(eq(heartbeatRuns.companyId, f.companyId), inArray(heartbeatRuns.status, ["running", "queued"])));
|
||||
await db.update(agentWakeupRequests).set({ status: "cancelled", finishedAt: new Date() }).where(and(eq(agentWakeupRequests.companyId, f.companyId), inArray(agentWakeupRequests.status, ["queued", "claimed"])));
|
||||
}
|
||||
});
|
||||
|
||||
it.each(["delivery", "assignment", "admission", "user_request", "result_or_decision"] as const)("revokes an accepted Cursor plan passive wait after %s changes", async change => {
|
||||
const f = await seedAcceptedCursorPlanWait();
|
||||
await finalizeNativeRun({ db, runId: f.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
|
||||
if (change === "delivery") await db.update(issueQuestionResponseDeliveries).set({ acknowledgedAt: null }).where(eq(issueQuestionResponseDeliveries.id, f.deliveryId));
|
||||
if (change === "assignment") await db.update(issues).set({ assigneeAgentId: null }).where(eq(issues.id, f.issueId));
|
||||
if (change === "admission") {
|
||||
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId));
|
||||
const profile = structuredClone(run!.runnerProfileJson!);
|
||||
(profile.nativeExecutionInput as { provider: { cursorMode: string } }).provider.cursorMode = "agent";
|
||||
await db.update(heartbeatRuns).set({ runnerProfileJson: profile }).where(eq(heartbeatRuns.id, f.runId));
|
||||
}
|
||||
if (change === "user_request") await db.insert(issueComments).values({ companyId: f.companyId, issueId: f.issueId, authorType: "user", authorUserId: "responsible-user", body: "Continue reviewing this plan in Plan mode." });
|
||||
if (change === "result_or_decision") {
|
||||
const [accepted] = await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, f.runId));
|
||||
await db.update(nativeRunResults).set({ canonicalSha256: "changed" }).where(eq(nativeRunResults.id, accepted!.id));
|
||||
expect(await hasCommittedNativeCursorPlanWait(db, f)).toBe(false);
|
||||
await db.update(nativeRunResults).set({ canonicalSha256: accepted!.canonicalSha256 }).where(eq(nativeRunResults.id, accepted!.id));
|
||||
expect(await hasCommittedNativeCursorPlanWait(db, f)).toBe(true);
|
||||
await db.update(issues).set({ lastStatusDecisionId: null }).where(eq(issues.id, f.issueId));
|
||||
}
|
||||
expect(await hasCommittedNativeCursorPlanWait(db, f)).toBe(false);
|
||||
});
|
||||
|
||||
it("rechecks accepted Cursor plan delivery under the status transaction before presentation or effects", async () => {
|
||||
const f = await seedAcceptedCursorPlanWait();
|
||||
await finalizeNativeRun({ db, runId: f.runId, workspaceFinalizeStatus: "succeeded", failpoint: "status_projection" });
|
||||
const [coordinator] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId));
|
||||
const [issue] = await db.select().from(issues).where(eq(issues.id, f.issueId));
|
||||
await db.update(issueQuestionResponseDeliveries).set({ payloadSha256: "changed" }).where(eq(issueQuestionResponseDeliveries.id, f.deliveryId));
|
||||
await expect(commitNativeStatusDecision({ db, companyId: f.companyId, issueId: f.issueId, runId: f.runId, assessmentId: coordinator!.assessmentId!,
|
||||
priorStatus: issue!.status, priorStatusVersion: issue!.statusVersion, priorDecisionId: issue!.lastStatusDecisionId,
|
||||
decision: { policyVersion: "phase6-v7", statusAction: "in_progress", toStatus: "in_progress", reasonCode: "native_plan_accepted_waiting_for_continuation", unblockDescriptor: null, effects: [] },
|
||||
requireCursorPlanWaitSource: f.proof.source })).rejects.toBeInstanceOf(NativeStatusRaceError);
|
||||
expect(await db.select().from(statusDecisions).where(eq(statusDecisions.runId, f.runId))).toHaveLength(0);
|
||||
expect(await db.select().from(issueComments).where(eq(issueComments.createdByRunId, f.runId))).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("preserves explicit semantic finish priority over accepted Cursor plan evidence", async () => {
|
||||
const f = await seedAcceptedCursorPlanWait(true);
|
||||
await finalizeNativeRun({ db, runId: f.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
|
||||
expect(await hasCommittedNativeCursorPlanWait(db, f)).toBe(false);
|
||||
const decisions = await db.select().from(statusDecisions).where(eq(statusDecisions.runId, f.runId));
|
||||
expect(decisions).toHaveLength(1); expect(decisions[0]!.reasonCode).not.toBe("native_plan_accepted_waiting_for_continuation");
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
async function seedNativePassiveBoardResponse(
|
||||
continuationKind:
|
||||
"response_wake" | "same_agent" | "retry" = "response_wake",
|
||||
@@ -16627,4 +16406,237 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
||||
.where(eq(heartbeatRuns.id, runId));
|
||||
expect(runs).toHaveLength(1);
|
||||
});
|
||||
describe("accepted Cursor planning boundaries", () => {
|
||||
const nativeImplementation = mockExecutePaperclipNativeSession.getMockImplementation();
|
||||
beforeEach(() => {
|
||||
mockExecutePaperclipNativeSession.mockImplementation(async () => { throw new Error("Accepted-plan tests must never execute a provider"); });
|
||||
});
|
||||
afterEach(() => { mockExecutePaperclipNativeSession.mockImplementation(nativeImplementation!); });
|
||||
|
||||
async function seedAcceptedCursorPlanWait(semanticFinish = false, sourceSequenceOffset = 0) {
|
||||
const f = await seedStrandedIssueFixture({ status: "in_progress", runStatus: "succeeded", livenessState: "advanced" });
|
||||
const instance = randomUUID(), interactionId = randomUUID();
|
||||
const contractInput = { db, companyId: f.companyId,
|
||||
issue: { id: f.issueId, title: "Review the plan before further work", description: "Explicit completion required" }, actorId: "test" };
|
||||
const { row: persistedContract, contract } = await ensureNativeCompletionContract(contractInput);
|
||||
const reused = await ensureNativeCompletionContract(contractInput);
|
||||
expect(reused.row.id).toBe(persistedContract.id);
|
||||
expect(reused.contract).toEqual(contract);
|
||||
const contractId = persistedContract.id, contractSha = persistedContract.canonicalSha256;
|
||||
// Production binds policy/schema as well as the body; a body-only hash is not a valid contract receipt.
|
||||
expect(contractSha).not.toBe(nativeSha256(contract));
|
||||
const model = "gpt-5.6-luna[context=272k,reasoning=medium,fast=false]";
|
||||
await db.update(agents).set({ adapterType: "paperclip_runner", adapterConfig: { provider: "acpx", acpxAgent: "cursor", model, acpxSessionMode: "plan", acpxPermissionMode: "approve-all" } }).where(eq(agents.id, f.agentId));
|
||||
await db.update(agentWakeupRequests).set({ status: "completed" }).where(eq(agentWakeupRequests.id, f.wakeupRequestId));
|
||||
await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: f.issueId, nativeSessionId: f.runId,
|
||||
completionContractId: contractId, completionContractSha256: contractSha, runnerInstanceId: instance,
|
||||
runnerProfileJson: { nativeExecutionInput: { binding: { companyId: f.companyId, issueId: f.issueId, runId: f.runId, agentId: f.agentId }, provider: { kind: "acpx", agent: "cursor", model, mode: "plan", permissionMode: "approve-all",
|
||||
profile: paperclipRunner.resolveQualifiedAcpxProfile("cursor", model) }, session: { normalizedSessionId: f.runId },
|
||||
completionContract: { id: contractId, sha256: contractSha, contract } } } }).where(eq(heartbeatRuns.id, f.runId));
|
||||
const planId = `plan-${"a".repeat(64)}`;
|
||||
const questionSet = { schema: "paperclip.question_set.v1", title: "Native plan", description: "Exact accepted revision", questions: [{ id: planId, prompt: "Proceed?", required: true,
|
||||
answerMode: "single_select", options: [{ id: "accept", label: "Accept" }, { id: "reject", label: "Reject" }, { id: "cancel", label: "Cancel" }] }] };
|
||||
const [interaction] = await db.insert(issueThreadInteractions).values({ id: interactionId, companyId: f.companyId, issueId: f.issueId, sourceRunId: f.runId,
|
||||
createdByAgentId: f.agentId, kind: "ask_user_questions", status: "answered", continuationPolicy: "none",
|
||||
idempotencyKey: `paperclip-runner-question:${f.runId}:request`, resolvedByUserId: "responsible-user", resolvedAt: new Date(Date.now() - 1000),
|
||||
payload: { version: 1, questions: [{ id: planId, prompt: "Proceed?", selectionMode: "single", required: true, allowOther: false, options: [{ id: "accept", label: "Accept" }, { id: "reject", label: "Reject" }, { id: "cancel", label: "Cancel" }] }], runtimeRequestId: "request", questionSet: questionSet as never }, result: { version: 1, answers: [{ questionId: planId, optionIds: ["accept"] }] } }).returning();
|
||||
const answer = buildQuestionResponseDeliveryEnvelope(interaction as never);
|
||||
const [delivery] = await db.insert(issueQuestionResponseDeliveries).values({ companyId: f.companyId, issueId: f.issueId, interactionId,
|
||||
sourceRunId: f.runId, targetRunId: f.runId, correlationId: randomUUID(), status: "delivered", deliveryMode: "steered", acknowledgedAt: new Date(), payloadSha256: nativeSha256(answer) }).returning();
|
||||
const port = new PaperclipControlPlanePort(db, { companyId: f.companyId, issueId: f.issueId, runId: f.runId, agentId: f.agentId,
|
||||
sessionId: f.runId, completionContractId: contractId, completionContractSha256: contractSha, sourceInstanceId: instance, controlPlaneSourceInstanceId: `control-${f.runId}` });
|
||||
const events = [
|
||||
{ eventType: "runtime_request.created", payload: { request: { schema: "paperclip.runtime_request.v2", requestKind: "runtime", requestId: "request", type: "input", status: "pending", turnId: "turn", itemId: "native-plan-tool", prompt: "Review plan",
|
||||
origin: { adapter: "acpx-runtime-sidecar", provider: "cursor", method: "cursor/create_plan" }, input: questionSet } } },
|
||||
{ eventType: "tool.execution.started", payload: { schema: "paperclip.tool.execution.v1", executionId: "native-plan-tool", transport: "builtin", operation: "execute", status: "running", readOnly: false, namespace: null, name: "Create Plan", target: null, inputUpdated: true, output: null, outputBytes: 0, outputTruncated: false, outputDigest: null, progress: null, exitCode: null, durationMs: null } },
|
||||
{ eventType: "runtime_request.resolved", payload: { requestId: "request", turnId: "turn", action: "submit", response: answer.response } },
|
||||
{ eventType: "tool.execution.completed", payload: { schema: "paperclip.tool.execution.v1", executionId: "native-plan-tool", transport: "builtin", operation: "execute", status: "completed", readOnly: false, namespace: null, name: "Create Plan", target: null, inputUpdated: false, output: null, outputBytes: 0, outputTruncated: false, outputDigest: null, progress: null, exitCode: null, durationMs: null } },
|
||||
{ eventType: "turn.completed", payload: { status: "completed", error: null } },
|
||||
];
|
||||
for (const [index, event] of events.entries()) await port.appendEvent({ schema: "paperclip.prp.event.v1", sourceEventId: `${instance}:${sourceSequenceOffset + index + 1}`, sourceSeq: sourceSequenceOffset + index + 1,
|
||||
sourceInstanceId: instance, sourceKind: "runner", runId: f.runId, normalizedSessionId: f.runId, turnId: "turn", schemaVersion: 1, priority: 0,
|
||||
emittedAt: new Date().toISOString(), ...event } as paperclipRunner.PrpEvent);
|
||||
const proof = await readNativePlanWait(db, f);
|
||||
expect(proof).not.toBeNull();
|
||||
const result = semanticFinish ? { ...proof!.result, reportedWorkDisposition: "done" as const, summary: "Explicit semantic finish wins", continuation: undefined,
|
||||
completionClaim: { contractRevision: "1", objectiveSatisfied: true, criteria: [{ criterionId: "objective", status: "satisfied" as const, evidenceRefs: [] }], remainingWork: [] } } : proof!.result;
|
||||
await port.completeRun({ result, turnId: "turn", terminal: { schema: "paperclip.prp.terminal.v1", runTerminalState: "succeeded", turnTerminalState: "completed", reportedWorkDisposition: result.reportedWorkDisposition } });
|
||||
return { ...f, interactionId, deliveryId: delivery!.id, proof: proof! };
|
||||
}
|
||||
|
||||
it("keeps an accepted Cursor plan passive across restart, future configuration changes, and paused recovery", async () => {
|
||||
const f = await seedAcceptedCursorPlanWait();
|
||||
await finalizeNativeRun({ db, runId: f.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
|
||||
await finalizeNativeRun({ db, runId: f.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
|
||||
expect(await hasCommittedNativePlanWait(db, f)).toBe(true);
|
||||
expect(await db.select().from(statusDecisions).where(eq(statusDecisions.runId, f.runId))).toEqual([expect.objectContaining({ reasonCode: "native_plan_accepted_waiting_for_continuation", toStatus: "in_progress" })]);
|
||||
expect(await db.select().from(statusDecisionEffects).where(eq(statusDecisionEffects.issueId, f.issueId))).toEqual([expect.objectContaining({ effectKind: "issue_status_projection", targetType: "issue", deliveryState: "delivered" })]);
|
||||
expect(await db.select().from(issueComments).where(eq(issueComments.createdByRunId, f.runId))).toEqual([expect.objectContaining({ body: expect.stringContaining("next message") })]);
|
||||
const current = paperclipRunner.resolveQualifiedAcpxProfile("cursor", "gpt-5.6-luna[context=272k,reasoning=medium,fast=false]");
|
||||
const resolver = vi.spyOn(paperclipRunner, "resolveQualifiedAcpxProfile").mockReturnValue({ ...current,
|
||||
agentProfileVersion: current.agentProfileVersion + 1, commandDigest: `sha256:${"b".repeat(64)}` });
|
||||
try {
|
||||
expect(await readNativePlanWait(db, f)).toBeNull(); // Old profile cannot create a new wait.
|
||||
for (const status of ["idle", "paused"] as const) {
|
||||
await db.update(agents).set({ status, adapterConfig: { provider: "acpx", acpxAgent: "cursor",
|
||||
model: "future-model", acpxSessionMode: "agent", acpxPermissionMode: "approve-reads" } }).where(eq(agents.id, f.agentId));
|
||||
expect(await hasCommittedNativePlanWait(db, f)).toBe(true);
|
||||
const recovered = await heartbeatService(db).reconcileStrandedAssignedIssues();
|
||||
expect(recovered.continuationRequeued).toBe(0); expect(recovered.escalated).toBe(0);
|
||||
}
|
||||
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId));
|
||||
const changed = structuredClone(run!.runnerProfileJson!);
|
||||
(changed.nativeExecutionInput as { provider: { profile: { commandDigest: string } } }).provider.profile.commandDigest = "tampered-original-profile";
|
||||
await db.update(heartbeatRuns).set({ runnerProfileJson: changed }).where(eq(heartbeatRuns.id, f.runId));
|
||||
expect(await hasCommittedNativePlanWait(db, f)).toBe(false);
|
||||
await db.update(heartbeatRuns).set({ runnerProfileJson: run!.runnerProfileJson }).where(eq(heartbeatRuns.id, f.runId));
|
||||
expect(await hasCommittedNativePlanWait(db, f)).toBe(true);
|
||||
const [tool] = await db.select().from(heartbeatRunEvents).where(and(eq(heartbeatRunEvents.runId, f.runId), eq(heartbeatRunEvents.eventType, "tool.execution.completed")));
|
||||
const changedTool = structuredClone(tool!.payload!) as { prpEvent: { payload: { name: string } } };
|
||||
changedTool.prpEvent.payload.name = "mutated committed tool evidence";
|
||||
await db.update(heartbeatRunEvents).set({ payload: changedTool, sourcePayloadSha256: nativeSha256(changedTool.prpEvent) }).where(eq(heartbeatRunEvents.id, tool!.id));
|
||||
expect(await hasCommittedNativePlanWait(db, f)).toBe(false);
|
||||
await db.update(heartbeatRunEvents).set({ payload: tool!.payload, sourcePayloadSha256: tool!.sourcePayloadSha256 }).where(eq(heartbeatRunEvents.id, tool!.id));
|
||||
expect(await hasCommittedNativePlanWait(db, f)).toBe(true);
|
||||
} finally { resolver.mockRestore(); }
|
||||
expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.companyId))).toHaveLength(1);
|
||||
expect(mockAdapterExecute).not.toHaveBeenCalled(); expect(mockExecutePaperclipNativeSession).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("retains an exact historical Cursor6 committed wait across more than 1000 ignored progress rows without allowing new unbound admission", async () => {
|
||||
const progressCount = 1002;
|
||||
const f = await seedAcceptedCursorPlanWait(false, progressCount);
|
||||
await finalizeNativeRun({ db, runId: f.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
|
||||
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId));
|
||||
const profile = structuredClone(run!.runnerProfileJson!) as any;
|
||||
Object.assign(profile.nativeExecutionInput.provider.profile, { agentProfileVersion: 6, commandDigest: "sha256:377dcea64a727ce799cc112458d4b40ba4bc6574cd6c6f7233b6efd5917a6c4b" });
|
||||
await db.update(heartbeatRuns).set({ runnerProfileJson: profile }).where(eq(heartbeatRuns.id, f.runId));
|
||||
await db.delete(heartbeatRunEvents).where(and(eq(heartbeatRunEvents.runId, f.runId), inArray(heartbeatRunEvents.eventType, ["tool.execution.started", "tool.execution.completed"])));
|
||||
// Reconstruct the persisted Cursor6 receipt format using its unchanged
|
||||
// source facts. This historical format had no tool lifecycle binding.
|
||||
const rows = await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, f.runId));
|
||||
const event = (kind: string) => (rows.find(r => r.eventType === kind)!.payload as any).prpEvent;
|
||||
const [contract] = await db.select().from(completionContracts).where(eq(completionContracts.id, run!.completionContractId!));
|
||||
const [interaction] = await db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.id, f.interactionId));
|
||||
const [delivery] = await db.select().from(issueQuestionResponseDeliveries).where(eq(issueQuestionResponseDeliveries.id, f.deliveryId));
|
||||
profile.nativeExecutionInput.provider.cursorMode = profile.nativeExecutionInput.provider.mode;
|
||||
delete profile.nativeExecutionInput.provider.mode;
|
||||
await db.update(heartbeatRuns).set({ runnerProfileJson: profile }).where(eq(heartbeatRuns.id, f.runId));
|
||||
const legacy = { ...f.proof.source, schema: "paperclip.native_cursor_plan_wait.v1" }; delete legacy.toolExecutionId; delete legacy.toolLifecycleSha256;
|
||||
legacy.authoritySha256 = nativeSha256({ admission: profile.nativeExecutionInput, contract, created: event("runtime_request.created"), resolved: event("runtime_request.resolved"), terminal: event("turn.completed"), interaction, delivery, resolvedAt: interaction!.resolvedAt!.toISOString(), acknowledgedAt: delivery!.acknowledgedAt!.toISOString() });
|
||||
const [decision] = await db.select().from(statusDecisions).where(eq(statusDecisions.runId, f.runId));
|
||||
const [savedResult] = await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, f.runId));
|
||||
const resultJson = structuredClone(savedResult!.resultJson!) as any;
|
||||
resultJson.result.continuation.idempotencyKey = resultJson.result.continuation.idempotencyKey.replace("native-plan-wait:", "cursor-plan-wait:");
|
||||
const canonicalSha256 = nativeSha256(resultJson);
|
||||
await db.update(nativeRunResults).set({ resultJson, canonicalSha256 }).where(eq(nativeRunResults.id, savedResult!.id));
|
||||
const decisionJson = { ...decision!.decisionJson } as any;
|
||||
delete decisionJson.planWait; delete decisionJson.planWaitResult;
|
||||
decisionJson.cursorPlanWait = legacy;
|
||||
decisionJson.cursorPlanWaitResult = { resultId: savedResult!.id, resultSha256: canonicalSha256 };
|
||||
await db.update(statusDecisions).set({ decisionJson }).where(eq(statusDecisions.id, decision!.id));
|
||||
expect(await readNativePlanWait(db, f)).toBeNull();
|
||||
expect(await hasCommittedNativePlanWait(db, f)).toBe(true);
|
||||
// Fill the reserved source prefix with genuine durable progress rows. Shift
|
||||
// only DB ordering keys; the original PRP facts and committed receipt stay
|
||||
// byte-identical. Cursor6 never queried these rows, even above its budget.
|
||||
await db.update(heartbeatRunEvents).set({ seq: sql`${heartbeatRunEvents.seq} + ${progressCount}` }).where(eq(heartbeatRunEvents.runId, f.runId));
|
||||
const progress = Array.from({ length: progressCount }, (_, index) => {
|
||||
const seq = index + 1;
|
||||
const prpEvent = { ...event("runtime_request.created"), sourceEventId: `${run!.runnerInstanceId}:${seq}`, sourceSeq: seq, eventType: "tool.execution.progressed",
|
||||
payload: { schema: "paperclip.tool.execution.v1", executionId: "earlier-tool", transport: "builtin", operation: "read", status: "running" } };
|
||||
return { companyId: f.companyId, runId: f.runId, agentId: f.agentId, seq, eventType: prpEvent.eventType, payload: { prpEvent }, sourceInstanceId: run!.runnerInstanceId,
|
||||
sourceEventId: prpEvent.sourceEventId, sourceSeq: seq, sourcePayloadSha256: nativeSha256(prpEvent), protocolSchemaVersion: 1 };
|
||||
});
|
||||
await db.insert(heartbeatRunEvents).values(progress);
|
||||
expect(await hasCommittedNativePlanWait(db, f)).toBe(true);
|
||||
const [unchangedDecision] = await db.select().from(statusDecisions).where(eq(statusDecisions.id, decision!.id));
|
||||
expect(unchangedDecision!.decisionJson!.cursorPlanWait).toEqual(legacy);
|
||||
// Completed rows belonged to the old query. A later completion must still
|
||||
// invalidate the normal-terminal boundary rather than being filtered away.
|
||||
const completion = { ...event("turn.completed"), sourceEventId: `${run!.runnerInstanceId}:99999`, sourceSeq: 99999, eventType: "tool.execution.completed", payload: { ...progress[0]!.payload.prpEvent.payload, status: "completed" } };
|
||||
const [extra] = await db.insert(heartbeatRunEvents).values({ ...progress[0]!, seq: 99999, eventType: completion.eventType, payload: { prpEvent: completion }, sourceEventId: completion.sourceEventId, sourceSeq: 99999, sourcePayloadSha256: nativeSha256(completion) }).returning();
|
||||
expect(await hasCommittedNativePlanWait(db, f)).toBe(false);
|
||||
await db.delete(heartbeatRunEvents).where(eq(heartbeatRunEvents.id, extra!.id));
|
||||
expect(await hasCommittedNativePlanWait(db, f)).toBe(true);
|
||||
profile.nativeExecutionInput.provider.profile.commandDigest = "tampered-history";
|
||||
await db.update(heartbeatRuns).set({ runnerProfileJson: profile }).where(eq(heartbeatRuns.id, f.runId));
|
||||
expect(await hasCommittedNativePlanWait(db, f)).toBe(false);
|
||||
});
|
||||
|
||||
it("admits a later ordinary user message after an accepted Cursor plan without changing Plan mode or replaying its run", async () => {
|
||||
const f = await seedAcceptedCursorPlanWait();
|
||||
await finalizeNativeRun({ db, runId: f.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
|
||||
// Occupy the single dispatch slot: this checks actual user-wake admission
|
||||
// without executing any provider or fabricating a second semantic result.
|
||||
const occupied = randomUUID();
|
||||
await db.update(agents).set({ runtimeConfig: { heartbeat: { wakeOnDemand: true, maxConcurrentRuns: 1 } } }).where(eq(agents.id, f.agentId));
|
||||
await db.insert(heartbeatRuns).values({ id: occupied, companyId: f.companyId, agentId: f.agentId, status: "running", startedAt: new Date(), contextSnapshot: {} });
|
||||
const [comment] = await db.insert(issueComments).values({ companyId: f.companyId, issueId: f.issueId, authorType: "user", authorUserId: "responsible-user", body: "Continue reviewing the accepted plan in Plan mode." }).returning();
|
||||
try {
|
||||
const heartbeat = heartbeatService(db);
|
||||
const next = await heartbeat.wakeup(f.agentId, { source: "automation", triggerDetail: "system", reason: "issue_commented", requestedByActorType: "user", requestedByActorId: "responsible-user",
|
||||
payload: { issueId: f.issueId, commentId: comment!.id }, contextSnapshot: { issueId: f.issueId, taskId: f.issueId, wakeCommentId: comment!.id, wakeCommentIds: [comment!.id] } });
|
||||
expect(next).not.toBeNull(); expect(next!.id).not.toBe(f.runId);
|
||||
const [admitted] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, next!.id));
|
||||
expect(admitted).toMatchObject({ status: "queued", retryOfRunId: null });
|
||||
expect(admitted!.contextSnapshot?.wakeCommentIds).toContain(comment!.id);
|
||||
const [agent] = await db.select().from(agents).where(eq(agents.id, f.agentId));
|
||||
expect(agent!.adapterConfig.acpxSessionMode).toBe("plan");
|
||||
expect(await hasCommittedNativePlanWait(db, f)).toBe(false);
|
||||
expect(mockExecutePaperclipNativeSession).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
await db.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date() }).where(and(eq(heartbeatRuns.companyId, f.companyId), inArray(heartbeatRuns.status, ["running", "queued"])));
|
||||
await db.update(agentWakeupRequests).set({ status: "cancelled", finishedAt: new Date() }).where(and(eq(agentWakeupRequests.companyId, f.companyId), inArray(agentWakeupRequests.status, ["queued", "claimed"])));
|
||||
}
|
||||
});
|
||||
|
||||
it.each(["delivery", "assignment", "admission", "user_request", "result_or_decision"] as const)("revokes an accepted Cursor plan passive wait after %s changes", async change => {
|
||||
const f = await seedAcceptedCursorPlanWait();
|
||||
await finalizeNativeRun({ db, runId: f.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
|
||||
if (change === "delivery") await db.update(issueQuestionResponseDeliveries).set({ acknowledgedAt: null }).where(eq(issueQuestionResponseDeliveries.id, f.deliveryId));
|
||||
if (change === "assignment") await db.update(issues).set({ assigneeAgentId: null }).where(eq(issues.id, f.issueId));
|
||||
if (change === "admission") {
|
||||
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId));
|
||||
const profile = structuredClone(run!.runnerProfileJson!);
|
||||
(profile.nativeExecutionInput as { provider: { cursorMode: string } }).provider.cursorMode = "agent";
|
||||
await db.update(heartbeatRuns).set({ runnerProfileJson: profile }).where(eq(heartbeatRuns.id, f.runId));
|
||||
}
|
||||
if (change === "user_request") await db.insert(issueComments).values({ companyId: f.companyId, issueId: f.issueId, authorType: "user", authorUserId: "responsible-user", body: "Continue reviewing this plan in Plan mode." });
|
||||
if (change === "result_or_decision") {
|
||||
const [accepted] = await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, f.runId));
|
||||
await db.update(nativeRunResults).set({ canonicalSha256: "changed" }).where(eq(nativeRunResults.id, accepted!.id));
|
||||
expect(await hasCommittedNativePlanWait(db, f)).toBe(false);
|
||||
await db.update(nativeRunResults).set({ canonicalSha256: accepted!.canonicalSha256 }).where(eq(nativeRunResults.id, accepted!.id));
|
||||
expect(await hasCommittedNativePlanWait(db, f)).toBe(true);
|
||||
await db.update(issues).set({ lastStatusDecisionId: null }).where(eq(issues.id, f.issueId));
|
||||
}
|
||||
expect(await hasCommittedNativePlanWait(db, f)).toBe(false);
|
||||
});
|
||||
|
||||
it("rechecks accepted Cursor plan delivery under the status transaction before presentation or effects", async () => {
|
||||
const f = await seedAcceptedCursorPlanWait();
|
||||
await finalizeNativeRun({ db, runId: f.runId, workspaceFinalizeStatus: "succeeded", failpoint: "status_projection" });
|
||||
const [coordinator] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId));
|
||||
const [issue] = await db.select().from(issues).where(eq(issues.id, f.issueId));
|
||||
await db.update(issueQuestionResponseDeliveries).set({ payloadSha256: "changed" }).where(eq(issueQuestionResponseDeliveries.id, f.deliveryId));
|
||||
await expect(commitNativeStatusDecision({ db, companyId: f.companyId, issueId: f.issueId, runId: f.runId, assessmentId: coordinator!.assessmentId!,
|
||||
priorStatus: issue!.status, priorStatusVersion: issue!.statusVersion, priorDecisionId: issue!.lastStatusDecisionId,
|
||||
decision: { policyVersion: "phase6-v7", statusAction: "in_progress", toStatus: "in_progress", reasonCode: "native_plan_accepted_waiting_for_continuation", unblockDescriptor: null, effects: [] },
|
||||
requirePlanWaitSource: f.proof.source })).rejects.toBeInstanceOf(NativeStatusRaceError);
|
||||
expect(await db.select().from(statusDecisions).where(eq(statusDecisions.runId, f.runId))).toHaveLength(0);
|
||||
expect(await db.select().from(issueComments).where(eq(issueComments.createdByRunId, f.runId))).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("preserves explicit semantic finish priority over accepted Cursor plan evidence", async () => {
|
||||
const f = await seedAcceptedCursorPlanWait(true);
|
||||
await finalizeNativeRun({ db, runId: f.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
|
||||
expect(await hasCommittedNativePlanWait(db, f)).toBe(false);
|
||||
const decisions = await db.select().from(statusDecisions).where(eq(statusDecisions.runId, f.runId));
|
||||
expect(decisions).toHaveLength(1); expect(decisions[0]!.reasonCode).not.toBe("native_plan_accepted_waiting_for_continuation");
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
|
||||
});
|
||||
@@ -78,7 +78,12 @@ async function createApp(
|
||||
return app;
|
||||
}
|
||||
|
||||
describe("GET /invites/:token/test-resolution", () => {
|
||||
describe("GET /invites/:token/test-resolution", { sequential: true }, () => {
|
||||
beforeAll(async () => {
|
||||
// Route transformation is fixture setup, not part of the network assertions.
|
||||
await loadAppModules();
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
@@ -41,20 +41,48 @@ function receiptNotice(version: 1 | 2 = 1): Record<string, any> {
|
||||
const receiptSchemaValue = (notice: Record<string, any>) => notice.details.find((detail: any) => detail.name === "schema").value;
|
||||
|
||||
describe("redaction", () => {
|
||||
it("preserves credential-related prose, metadata, and dotted filenames", () => {
|
||||
const input = {
|
||||
body: "Keep the private key in a secret manager. Document credential handling and token permissions. Use bearer tokens for authentication. Prefer bearer authentication.",
|
||||
tokenPolicy: "least privilege",
|
||||
secretStorage: "vault",
|
||||
credentialHandling: "harness",
|
||||
authorizationRequired: true,
|
||||
path: "deployment.credentials.example.md",
|
||||
};
|
||||
expect(sanitizeRecord(input)).toEqual(input);
|
||||
expect(redactSensitiveText(input.body)).toBe(input.body);
|
||||
expect(sanitizeRecord({ credentials: { provider: "opaque-value" }, passwordValue: "opaque-value", authorization_code: "opaque-value" }))
|
||||
.toEqual({ credentials: REDACTED_EVENT_VALUE, passwordValue: REDACTED_EVENT_VALUE, authorization_code: REDACTED_EVENT_VALUE });
|
||||
expect(redactSensitiveText("--token-budget 4000 SECRET_STORAGE=vault")).toBe("--token-budget 4000 SECRET_STORAGE=vault");
|
||||
it("preserves the actual Copilot receipt producer discriminator through nested durable redaction", () => {
|
||||
const events: Array<Record<string, any>> = [];
|
||||
const projector = createCopilotToolEvidence({ sessionId: "session", turnId: "turn", workingDirectory: "/workspace",
|
||||
active: () => true, emit: event => events.push(event) });
|
||||
const receipt = appendSemanticToolReceipt({ tool: "finish_task", callId: "call", arguments: {} },
|
||||
{ content: [{ type: "text", text: "accepted" }] }).receipt;
|
||||
expect(receipt.schema).toBe("paperclip.semantic_tool_receipt.v2");
|
||||
projector.captureSemanticReceipt()!(receipt);
|
||||
expect(events).toHaveLength(1);
|
||||
expect(events[0]!.payload.category).toBe("paperclip_semantic_tool_receipt_v2");
|
||||
expect(events[0]!.payload.details).toHaveLength(8);
|
||||
const input = { prpEvent: { schema: "paperclip.prp.event.v1", schemaVersion: 1,
|
||||
eventType: "provider.notice.recorded", payload: events[0]!.payload } };
|
||||
expect(redactEventPayload(input)).toEqual(input);
|
||||
expect(redactEventPayload(redactEventPayload(input))).toEqual(input);
|
||||
});
|
||||
|
||||
it.each([1, 2] as const)("preserves public v%s receipt identifiers as text", version => {
|
||||
const notice = receiptNotice(version);
|
||||
expect(redactEventPayload(notice)).toEqual(notice);
|
||||
// Master's JWT detector recognizes encoded headers, so a dotted public
|
||||
// identifier needs no receipt-shaped exemption to survive redaction.
|
||||
expect(redactEventPayload({ value: `paperclip.semantic_tool_receipt.v${version}` }))
|
||||
.toEqual({ value: `paperclip.semantic_tool_receipt.v${version}` });
|
||||
});
|
||||
|
||||
it("redacts credentials in receipt-shaped data and every adjacent field", () => {
|
||||
const jwt = "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.signature12345678";
|
||||
const notice = receiptNotice();
|
||||
notice.details.find((d: any) => d.name === "operationId").value = jwt;
|
||||
notice.provenance.sessionId = jwt;
|
||||
const redacted = redactEventPayload(notice)! as Record<string, any>;
|
||||
expect(receiptSchemaValue(redacted)).toBe("paperclip.semantic_tool_receipt.v1");
|
||||
expect(redacted.details.find((d: any) => d.name === "operationId").value).toBe(REDACTED_EVENT_VALUE);
|
||||
expect(redacted.provenance.sessionId).toBe(REDACTED_EVENT_VALUE);
|
||||
expect(redactEventPayload({ notice, password: "canary", arbitrary: jwt }))
|
||||
.toMatchObject({ password: REDACTED_EVENT_VALUE, arbitrary: REDACTED_EVENT_VALUE });
|
||||
notice.details.find((d: any) => d.name === "schema").value = jwt;
|
||||
notice.summary = "Authorization: Bearer canary-token";
|
||||
const hostile = redactEventPayload(notice)!;
|
||||
expect(receiptSchemaValue(hostile)).toBe(REDACTED_EVENT_VALUE);
|
||||
expect(JSON.stringify(hostile)).not.toContain("canary-token");
|
||||
});
|
||||
|
||||
it("keeps the discriminator allowlist in exact PRP v1 schema parity", () => {
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { chmodSync, linkSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, expect, it, vi } from "vitest";
|
||||
const hooks = vi.hoisted(() => ({ mkdir: undefined as undefined | ((path: string) => void), rename: undefined as undefined | ((source: string, destination: string) => void), open: undefined as undefined | ((path: string) => void), read: undefined as undefined | ((path: string, bytes: number) => void) }));
|
||||
vi.mock("node:fs/promises", async (original) => {
|
||||
const fs = await original<typeof import("node:fs/promises")>();
|
||||
return { ...fs,
|
||||
mkdir: async (...args: Parameters<typeof fs.mkdir>) => { hooks.mkdir?.(String(args[0])); return fs.mkdir(...args); },
|
||||
rename: async (...args: Parameters<typeof fs.rename>) => { hooks.rename?.(String(args[0]), String(args[1])); return fs.rename(...args); },
|
||||
open: async (...args: Parameters<typeof fs.open>) => { hooks.open?.(String(args[0])); const file = await fs.open(...args); const read = file.read.bind(file); file.read = (async (...readArgs: any[]) => { const result = await (read as any)(...readArgs); hooks.read?.(String(args[0]), result.bytesRead); return result; }) as typeof file.read; return file; },
|
||||
};
|
||||
});
|
||||
vi.mock("../vendor/paperclip-runner/index.js", async () => await import("../../../packages/paperclip-runner/src/drivers/acpx/qualified-profiles.js"));
|
||||
import { QUALIFIED_ACPX_PROFILES } from "../../../packages/paperclip-runner/src/drivers/acpx/qualified-profiles.js";
|
||||
import { createRemotePiCompanionManifest, importRemotePiCompanion, inventoryRemoteCompanion, resolveRemotePiCompanion, selectRemotePiCompanion } from "../services/native-runtime/remote-pi-companion.js";
|
||||
const roots: string[] = [];
|
||||
afterEach(() => { hooks.mkdir = undefined; hooks.rename = undefined; hooks.open = undefined; hooks.read = undefined; for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); vi.restoreAllMocks(); });
|
||||
const hash = (bytes: string | Buffer) => createHash("sha256").update(bytes).digest("hex");
|
||||
const canonical = (v: any): string => Array.isArray(v) ? `[${v.map(canonical).join(",")}]` : v && typeof v === "object" ? `{${Object.keys(v).sort().map(k => `${JSON.stringify(k)}:${canonical(v[k])}`).join(",")}}` : JSON.stringify(v);
|
||||
async function fixture() {
|
||||
const root = realpathSync(mkdtempSync(join(tmpdir(), "paperclip-companion-"))); roots.push(root);
|
||||
const source = "a".repeat(40); const serverRoot = join(root, "server"); const directory = join(root, "release");
|
||||
mkdirSync(join(serverRoot, "dist"), { recursive: true }); writeFileSync(join(serverRoot, "package.json"), '{"name":"@paperclipai/server"}'); writeFileSync(join(serverRoot, "dist/build-info.json"), JSON.stringify({ commit: source }));
|
||||
mkdirSync(join(directory, "bin"), { recursive: true }); mkdirSync(join(directory, "provider-pack"));
|
||||
// Synthetic ELF header; these tests never launch it or claim native qualification.
|
||||
const elf = Buffer.alloc(128); Buffer.from([127, 69, 76, 70, 2, 1]).copy(elf); elf.writeUInt16LE(62, 18); writeFileSync(join(directory, "bin/paperclip-runnerd"), elf, { mode: 0o755 });
|
||||
const payload = { runnerSourceRevision: source, target: { platform: "linux", architecture: "x64" }, candidateProviders: { pi: { qualification: "qualified", profileDigest: QUALIFIED_ACPX_PROFILES.pi.commandDigest, path: "provider-assets/pi/linux-x64" } } };
|
||||
writeFileSync(join(directory, "provider-pack/provider-pack.json"), JSON.stringify({ schema: "paperclip-runner/remote-provider-pack/v1", digest: `sha256:${hash(canonical(payload))}`, payload }));
|
||||
const manifest = await createRemotePiCompanionManifest(directory, source); const bytes = JSON.stringify(manifest); writeFileSync(join(directory, "companion.json"), bytes); return { root, directory, serverRoot, sha256: hash(bytes), manifest, source };
|
||||
}
|
||||
it("imports the release-generated closure and resolves target bytes without environment overrides", async () => {
|
||||
const f = await fixture(); const result = await importRemotePiCompanion(f); expect(result.status).toBe("imported_verified");
|
||||
expect(result.runnerBinary).toBe(join(f.serverRoot, "remote-companions/linux-x64/bin/paperclip-runnerd"));
|
||||
expect(readFileSync(result.runnerBinary!)).toEqual(readFileSync(join(f.directory, "bin/paperclip-runnerd")));
|
||||
expect((await importRemotePiCompanion(f)).status).toBe("verified_existing"); expect((await resolveRemotePiCompanion({ serverRoot: f.serverRoot }))?.manifestSha256).toBe(f.sha256);
|
||||
expect(readdirSync(join(f.serverRoot, "remote-companions"))).toEqual(["linux-x64"]);
|
||||
});
|
||||
it.each(["sha", "source", "profile", "target", "daemon", "extra", "mode", "outside-link", "outside-hardlink", "pack"])("rejects %s before publishing", async kind => {
|
||||
const f = await fixture();
|
||||
if (kind === "sha") f.sha256 = "b".repeat(64);
|
||||
if (kind === "source") writeFileSync(join(f.serverRoot, "dist/build-info.json"), JSON.stringify({ commit: "b".repeat(40) }));
|
||||
if (kind === "profile" || kind === "target") { Object.assign(f.manifest, kind === "profile" ? { profileDigest: "sha256:" + "b".repeat(64) } : { target: "darwin-arm64" }); const bytes = JSON.stringify(f.manifest); writeFileSync(join(f.directory, "companion.json"), bytes); f.sha256 = hash(bytes); }
|
||||
if (kind === "daemon") writeFileSync(join(f.directory, "bin/paperclip-runnerd"), "changed");
|
||||
if (kind === "extra") writeFileSync(join(f.directory, "extra"), "unlisted");
|
||||
if (kind === "mode") chmodSync(join(f.directory, "bin/paperclip-runnerd"), 0o777);
|
||||
if (kind === "outside-link") symlinkSync("../../server/package.json", join(f.directory, "provider-pack/escape"));
|
||||
if (kind === "outside-hardlink") linkSync(join(f.directory, "bin/paperclip-runnerd"), join(f.root, "alias"));
|
||||
if (kind === "pack") writeFileSync(join(f.directory, "provider-pack/provider-pack.json"), "{}");
|
||||
await expect(importRemotePiCompanion(f)).rejects.toThrow(); expect(() => readdirSync(join(f.serverRoot, "remote-companions/linux-x64"))).toThrow();
|
||||
});
|
||||
it("copies contained symlinks and breaks only fully owned internal hardlinks", async () => {
|
||||
const f = await fixture(); linkSync(join(f.directory, "bin/paperclip-runnerd"), join(f.directory, "bin/alias")); symlinkSync("paperclip-runnerd", join(f.directory, "bin/link"));
|
||||
f.manifest = await createRemotePiCompanionManifest(f.directory, f.source); const bytes = JSON.stringify(f.manifest); writeFileSync(join(f.directory, "companion.json"), bytes); f.sha256 = hash(bytes);
|
||||
expect((await importRemotePiCompanion(f)).status).toBe("imported_verified");
|
||||
});
|
||||
it("fails closed on cache corruption, foreign authority and workspace-contained cache", async () => {
|
||||
const f = await fixture(); const result = await importRemotePiCompanion(f);
|
||||
await expect(resolveRemotePiCompanion({ serverRoot: f.serverRoot, workspaceRoot: f.root })).rejects.toThrow("workspace");
|
||||
await expect(resolveRemotePiCompanion({ serverRoot: f.serverRoot, workspaceRoot: join(result.root!, "provider-pack") })).rejects.toThrow("workspace");
|
||||
writeFileSync(result.runnerBinary!, "corrupted"); await expect(resolveRemotePiCompanion({ serverRoot: f.serverRoot })).rejects.toThrow("inventory");
|
||||
await expect(importRemotePiCompanion(f)).rejects.toThrow();
|
||||
});
|
||||
it("preserves an existing empty destination and releases only its import lock", async () => {
|
||||
const f = await fixture(); mkdirSync(join(f.serverRoot, "remote-companions/linux-x64"), { recursive: true, mode: 0o700 }); chmodSync(join(f.serverRoot, "remote-companions"), 0o700);
|
||||
await expect(importRemotePiCompanion(f)).rejects.toThrow(); expect(readdirSync(join(f.serverRoot, "remote-companions/linux-x64"))).toEqual([]); expect(readdirSync(join(f.serverRoot, "remote-companions"))).toEqual(["linux-x64"]);
|
||||
});
|
||||
it("rejects an unsafe cache parent and leaves it intact", async () => {
|
||||
const f = await fixture(); const outside = join(f.root, "outside"); mkdirSync(outside); symlinkSync(outside, join(f.serverRoot, "remote-companions"));
|
||||
await expect(importRemotePiCompanion(f)).rejects.toThrow(); expect(readdirSync(outside)).toEqual([]);
|
||||
});
|
||||
it("reports missing installation without manufacturing a companion", async () => { const f = await fixture(); expect(await resolveRemotePiCompanion({ serverRoot: f.serverRoot })).toBeNull(); });
|
||||
|
||||
it("selects only normal remote Pi, preserving explicit overrides and C/C admission", async () => {
|
||||
const f = await fixture(); await importRemotePiCompanion(f); const workspaceRoot = join(f.root, "workspace"); mkdirSync(workspaceRoot);
|
||||
const input = { serverRoot: f.serverRoot, workspaceRoot, remote: true, provider: { kind: "acpx", agent: "pi" } };
|
||||
expect((await selectRemotePiCompanion(input))?.target).toBe("linux-x64");
|
||||
for (const patch of [{ remote: false }, { provider: { kind: "acpx", agent: "cursor" } }, { provider: { kind: "acpx", agent: "copilot" } }, { provider: { kind: "codex" } }, { binaryOverride: "/operator/runnerd" }, { packOverride: "/operator/pack" }]) expect(await selectRemotePiCompanion({ ...input, ...patch })).toBeNull();
|
||||
});
|
||||
|
||||
it("rejects an appeared empty destination without replacing it", async () => {
|
||||
const f = await fixture(); const output = join(f.serverRoot, "remote-companions/linux-x64");
|
||||
hooks.mkdir = path => { if (path !== output) return; hooks.mkdir = undefined; mkdirSync(output, { mode: 0o700 }); };
|
||||
await expect(importRemotePiCompanion(f)).rejects.toThrow(); expect(readdirSync(output)).toEqual([]);
|
||||
expect(readdirSync(join(f.serverRoot, "remote-companions"))).toEqual(["linux-x64"]);
|
||||
});
|
||||
it("drains owned output, staging and lock cleanup after publication failure", async () => {
|
||||
const f = await fixture(); hooks.rename = () => { hooks.rename = undefined; throw new Error("injected publication failure"); };
|
||||
await expect(importRemotePiCompanion(f)).rejects.toThrow("injected publication failure");
|
||||
expect(readdirSync(join(f.serverRoot, "remote-companions"))).toEqual([]);
|
||||
});
|
||||
it("preserves a replaced staging root while draining the other owned cleanup", async () => {
|
||||
const f = await fixture(); let replaced = "";
|
||||
hooks.rename = source => {
|
||||
hooks.rename = undefined; replaced = source.slice(0, source.lastIndexOf("/"));
|
||||
renameSync(replaced, replaced + "-original"); mkdirSync(replaced, { mode: 0o700 }); writeFileSync(join(replaced, "foreign"), "preserve");
|
||||
throw new Error("replaced staging");
|
||||
};
|
||||
await expect(importRemotePiCompanion(f)).rejects.toThrow("cleanup incomplete"); expect(readFileSync(join(replaced, "foreign"), "utf8")).toBe("preserve");
|
||||
expect(readdirSync(join(f.serverRoot, "remote-companions"))).not.toContain(".import-linux-x64.lock");
|
||||
expect(readdirSync(join(f.serverRoot, "remote-companions"))).not.toContain("linux-x64");
|
||||
});
|
||||
it.each(["new", "existing"])("defers actual SIGTERM on the %s import path and drains owned cleanup", async (mode) => {
|
||||
const f = await fixture(); if (mode === "existing") await importRemotePiCompanion(f);
|
||||
const { build } = await import("esbuild");
|
||||
const { execFile } = await import("node:child_process");
|
||||
const { promisify } = await import("node:util");
|
||||
const bundle = join(f.root, "companion.mjs");
|
||||
await build({ entryPoints: [new URL("../services/native-runtime/remote-pi-companion.ts", import.meta.url).pathname], outfile: bundle,
|
||||
platform: "node", format: "esm", target: "node24", bundle: true, logLevel: "silent",
|
||||
plugins: [{ name: "source-owned-profile-only", setup(builder) {
|
||||
builder.onResolve({ filter: /vendor\/paperclip-runner\/index\.js$/ }, () => ({ path: new URL("../../../packages/paperclip-runner/src/drivers/acpx/qualified-profiles.ts", import.meta.url).pathname }));
|
||||
} }],
|
||||
});
|
||||
const script = join(f.root, "cancel.mjs");
|
||||
writeFileSync(script, `import {importRemotePiCompanion} from './companion.mjs';
|
||||
import {readdirSync,existsSync} from 'node:fs';
|
||||
let cancelled=false, checkpoints=0,sent=false; const cancel=()=>{cancelled=true}; process.on('SIGTERM',cancel);
|
||||
try { await importRemotePiCompanion({...JSON.parse(process.argv[2]),checkCancelled(){ checkpoints++; if(!sent&&existsSync(process.argv[3]+'/.import-linux-x64.lock')){sent=true;process.kill(process.pid,'SIGTERM');} if(cancelled)throw Error('owned cancellation'); }}); throw Error('unexpected success'); }
|
||||
catch(error){ if(error.message!=='owned cancellation')throw error; console.log(JSON.stringify({cancelled,checkpoints,remaining:readdirSync(process.argv[3])})); }
|
||||
finally {process.off('SIGTERM',cancel);}
|
||||
`);
|
||||
const { stdout } = await promisify(execFile)(process.execPath, [script, JSON.stringify({ directory: f.directory, sha256: f.sha256, serverRoot: f.serverRoot }), join(f.serverRoot, "remote-companions")], { env: { PATH: "/usr/bin:/bin", LANG: "C.UTF-8" }, timeout: 10_000, maxBuffer: 65536 });
|
||||
expect(JSON.parse(stdout)).toMatchObject({ cancelled: true, remaining: mode === "existing" ? ["linux-x64"] : [] });
|
||||
});
|
||||
|
||||
it("honors deadline expiry after re-verifying an existing cache without deleting it", async () => {
|
||||
const f = await fixture(); await importRemotePiCompanion(f); let expired = false;
|
||||
hooks.open = path => { if(path === join(f.serverRoot, "remote-companions/linux-x64/companion.json")) expired = true; };
|
||||
vi.spyOn(Date, "now").mockImplementation(() => expired ? 600_001 : 0);
|
||||
await expect(importRemotePiCompanion(f)).rejects.toThrow("deadline exceeded");
|
||||
expect(readdirSync(join(f.serverRoot, "remote-companions"))).toEqual(["linux-x64"]);
|
||||
});
|
||||
|
||||
async function largeFixture() {
|
||||
const f = await fixture(); writeFileSync(join(f.directory, "provider-pack/large"), Buffer.alloc(8 * 1024 ** 2, 0x61));
|
||||
f.manifest = await createRemotePiCompanionManifest(f.directory, f.source);
|
||||
const bytes = JSON.stringify(f.manifest); writeFileSync(join(f.directory, "companion.json"), bytes); f.sha256 = hash(bytes); return f;
|
||||
}
|
||||
it("yields to unrelated event-loop work between chunks of a real cache file", async () => {
|
||||
const f = await largeFixture(); await importRemotePiCompanion(f);
|
||||
const file = join(f.serverRoot, "remote-companions/linux-x64/provider-pack/large");
|
||||
let reads = 0; let serviced = false; let servicedBeforeNextRead = false;
|
||||
hooks.read = (path, bytes) => { if(path !== file || !bytes) return; reads++; if(reads === 1) setImmediate(() => { serviced = true; }); else servicedBeforeNextRead ||= serviced; };
|
||||
expect((await resolveRemotePiCompanion({serverRoot:f.serverRoot}))?.manifestSha256).toBe(f.sha256);
|
||||
expect(reads).toBeGreaterThanOrEqual(8); expect(servicedBeforeNextRead).toBe(true);
|
||||
});
|
||||
it("cancels during a large existing-cache file before reading the whole file and preserves it", async () => {
|
||||
const f = await largeFixture(); await importRemotePiCompanion(f);
|
||||
const file = join(f.serverRoot, "remote-companions/linux-x64/provider-pack/large"); let readBytes = 0;
|
||||
hooks.read = (path, bytes) => { if(path === file) readBytes += bytes; };
|
||||
await expect(importRemotePiCompanion({...f, checkCancelled(){ if(readBytes) throw Error("cancel during cache bytes"); }})).rejects.toThrow("cancel during cache bytes");
|
||||
expect(readBytes).toBeGreaterThan(0); expect(readBytes).toBeLessThan(8 * 1024 ** 2);
|
||||
expect(readdirSync(join(f.serverRoot, "remote-companions"))).toEqual(["linux-x64"]);
|
||||
});
|
||||
it("rejects read-only directory modes before claiming a staging or output path", async () => {
|
||||
const f = await fixture(); const dir = join(f.directory, "provider-pack"); chmodSync(dir, 0o500);
|
||||
try {
|
||||
await expect(importRemotePiCompanion({...f, checkCancelled(){}})).rejects.toThrow("owner read/write/search");
|
||||
expect(() => readdirSync(join(f.serverRoot, "remote-companions"))).toThrow();
|
||||
} finally { chmodSync(dir, 0o755); }
|
||||
});
|
||||
Reference in new issue
Block a user