import { execFileSync } from "node:child_process"; import { mkdirSync, unlinkSync } from "node:fs"; import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import { packageEvidence } from "./evidence.js"; import { RunnerApi } from "./api.js"; import { FixtureRegistry } from "./fixture-registry.js"; import { classifyFailure, shouldRetryFailure } from "./failure-classifier.js"; import { assertIsolatedServerEnvironment, buildPaperclipServerEnvironment, buildRunnerE2EProcessEnvironment, resolvePaperclipRemoteRunnerBinaryForHarness, resolvePaperclipRunnerBinaryForHarness, runnerE2EServerControlPaths, } from "./harness-env.js"; import { runnerExecutionById, runnerMatrix } from "./catalog.js"; import { assertEmbeddedDatabaseIsolation } from "./instance-isolation.js"; import { evaluateMatchers, persistedFinalRunMessage } from "./matchers.js"; import { assertSecretFree, findSecretLeak, findSecretLeakInJsonValues, findSecretLeakInDirectory, isEphemeralCodexRuntimeAuthFile, isEphemeralPostgresPidFile, isEphemeralPostgresScanFile, redactText, sanitizeJson, } from "./redaction.js"; import { parseDarwinSharedMemory } from "./shared-memory.js"; import { reserveRunnerE2EServerPort, runnerE2EServerPortConflictsWithDatabase, type LoopbackPortReservation, } from "./ports.js"; import { acceptedPlanSessionResetFailures, collectRunEvents, hasTerminalMalformedPlanConfirmation, isControlPlaneGovernedResponseWait, isNonExecutingReviewFenceRun, isOpenRouterDeepSeekHelloTerminalVariance, numberedPlanStepCount, providerSessionContinuityFailures, reasoningProjectionFailures, } from "./run-observations.js"; import { runnerE2EWebServerCommand } from "./web-server-command.js"; const cleanupDirectories: string[] = []; it("rejects reasoning mislabeled as assistant text without echoing private content", () => { const reasoning = { eventType: "item.delta", payload: { prpEvent: { eventType: "item.delta", payload: { kind: "agentMessage", text: "PRIVATE_THOUGHT", update: { kind: "reasoning" } }, } }, }; expect(reasoningProjectionFailures([reasoning])).toEqual([ "provider reasoning was projected as assistant text in 1 durable events", ]); const correctlyTyped = structuredClone(reasoning); correctlyTyped.payload.prpEvent.payload.kind = "reasoning"; expect(reasoningProjectionFailures([correctlyTyped])).toEqual([]); const assistant = structuredClone(reasoning); assistant.payload.prpEvent.payload.update.kind = "agentMessage"; expect(reasoningProjectionFailures([assistant])).toEqual([]); }); describe("complete run event evidence", () => { const page = Array.from({ length: 1000 }, (_, i) => ({ seq: i + 1, eventType: "item.delta" })); it("reads completion events beyond the first 1000 rows", async () => { const terminal = ["run.result.proposed", "run.result.accepted", "run.terminal"] .map((eventType, i) => ({ seq: 1001 + i, eventType })); const load = vi.fn().mockResolvedValueOnce(page).mockResolvedValueOnce(terminal); const events = await collectRunEvents(load); expect(events).toEqual([...page, ...terminal]); expect(load.mock.calls).toEqual([[0, 1000], [1000, 1000]]); }); it("checks for another page even at an exact page boundary", async () => { const load = vi.fn().mockResolvedValueOnce(page).mockResolvedValueOnce([]); expect(await collectRunEvents(load)).toEqual(page); expect(load).toHaveBeenCalledTimes(2); }); it.each([ null, {}, [{ eventType: "run.terminal" }], [{ seq: 0 }], [{ seq: -1 }], [{ seq: 1.5 }], [{ seq: "1" }], [{ seq: NaN }], [{ seq: Infinity }], [{ seq: 2 }, { seq: 1 }], [{ seq: 1 }, { seq: 1 }], [...page, { seq: 1001 }], ])("rejects malformed evidence page %#", async (malformed) => { await expect(collectRunEvents(async () => malformed)).rejects.toThrow("Run event evidence"); }); it("rejects a repeated cursor instead of accepting duplicate events", async () => { const load = vi.fn().mockResolvedValue(page); await expect(collectRunEvents(load)).rejects.toThrow("non-increasing sequence"); expect(load).toHaveBeenCalledTimes(2); }); it("propagates a missing later page without returning partial evidence", async () => { const load = vi.fn().mockResolvedValueOnce(page).mockRejectedValueOnce(new Error("Unavailable")); await expect(collectRunEvents(load)).rejects.toThrow("Unavailable"); }); it("fails closed when a stream never ends within the bounded capture", async () => { const load = vi.fn(async (afterSeq: number) => page.map((event) => ({ ...event, seq: event.seq + afterSeq }))); await expect(collectRunEvents(load)).rejects.toThrow("refusing incomplete evidence"); expect(load).toHaveBeenCalledTimes(100); }); }); afterEach(async () => { vi.unstubAllEnvs(); vi.unstubAllGlobals(); await Promise.all( cleanupDirectories .splice(0) .map((directory) => rm(directory, { recursive: true, force: true })), ); }); describe("runner E2E local binary resolution", () => { const localNativeExecution = runnerExecutionById( "core-compatibility.runner-codex.local.message-marker", ); const remoteNativeExecution = runnerExecutionById( "core-compatibility.runner-acpx-claude.daytona.message-marker", ); it("uses the debug runner binary built by the E2E workflow", () => { expect( resolvePaperclipRunnerBinaryForHarness( [localNativeExecution], "/repository", undefined, "linux", ), ).toBe( path.join( "/repository", "packages/paperclip-runner/runner/target/debug/paperclip-runnerd", ), ); }); it("preserves an explicit runner binary override", () => { expect( resolvePaperclipRunnerBinaryForHarness( [localNativeExecution], "/repository", "/custom/paperclip-runnerd", "linux", ), ).toBe("/custom/paperclip-runnerd"); }); it("uses and stages the same build-once binary for remote native cells", () => { const runnerBinary = resolvePaperclipRunnerBinaryForHarness( [remoteNativeExecution], "/repository", undefined, "linux", ); expect(runnerBinary).toBe( path.join( "/repository", "packages/paperclip-runner/runner/target/debug/paperclip-runnerd", ), ); expect( resolvePaperclipRemoteRunnerBinaryForHarness( [remoteNativeExecution], runnerBinary, undefined, "linux", ), ).toBe(runnerBinary); expect( resolvePaperclipRemoteRunnerBinaryForHarness( [localNativeExecution], runnerBinary, undefined, "linux", ), ).toBeUndefined(); expect( resolvePaperclipRemoteRunnerBinaryForHarness( [remoteNativeExecution], runnerBinary, undefined, "darwin", ), ).toBeUndefined(); expect( resolvePaperclipRemoteRunnerBinaryForHarness( [remoteNativeExecution], runnerBinary, "/cross-compiled/paperclip-runnerd", "darwin", ), ).toBe("/cross-compiled/paperclip-runnerd"); }); }); describe("runner E2E provider environment", () => { const legacyLocal = runnerExecutionById( "core-compatibility.legacy-opencode.local.message-marker", ); const legacyDaytona = runnerExecutionById( "core-compatibility.legacy-opencode.daytona.message-marker", ); const nativeOpenCode = runnerExecutionById( "core-compatibility.runner-opencode.local.message-marker", ); const breadthOpenCode = runnerMatrix.find( (execution) => execution.suite.id === "openrouter-model-breadth", )!; it("allows the pinned model only for isolated legacy OpenCode harnesses", () => { for (const execution of [legacyLocal, legacyDaytona]) { expect( buildRunnerE2EProcessEnvironment( { KEEP_ME: "yes", OPENCODE_ALLOW_ALL_MODELS: "ambient" }, [execution], ), ).toEqual({ KEEP_ME: "yes", OPENCODE_ALLOW_ALL_MODELS: "true", PAPERCLIP_ANNOUNCEMENTS_ENABLED: "false" }); } for (const execution of [nativeOpenCode, breadthOpenCode]) { expect( buildRunnerE2EProcessEnvironment( { KEEP_ME: "yes", OPENCODE_ALLOW_ALL_MODELS: "ambient" }, [execution], ), ).toEqual({ KEEP_ME: "yes", PAPERCLIP_ANNOUNCEMENTS_ENABLED: "false" }); } }); it("disables announcements through the server boundary for every runner cell", () => { for (const execution of runnerMatrix) { const source = { PAPERCLIP_ANNOUNCEMENTS_ENABLED: "true" }; const env = buildRunnerE2EProcessEnvironment(source, [execution]); expect(buildPaperclipServerEnvironment(env).PAPERCLIP_ANNOUNCEMENTS_ENABLED).toBe("false"); expect(source.PAPERCLIP_ANNOUNCEMENTS_ENABLED).toBe("true"); } }); }); describe("hiring capability opt-in", () => { it("enables API tools only when the manual hiring story is selected", () => { const hire = runnerMatrix.find((e) => e.suite.id === "everyday-workflows" && e.task.id === "hire-reuse")!; const delegate = runnerMatrix.find((e) => e.suite.id === "everyday-workflows" && e.task.id === "delegate-feedback")!; expect(buildRunnerE2EProcessEnvironment({}, [hire]).PAPERCLIP_RUNNER_API_TOOLS_ENABLED).toBe("true"); expect(buildRunnerE2EProcessEnvironment({}, [delegate]).PAPERCLIP_RUNNER_API_TOOLS_ENABLED).toBeUndefined(); expect(buildRunnerE2EProcessEnvironment({}, []).PAPERCLIP_RUNNER_API_TOOLS_ENABLED).toBeUndefined(); }); }); describe("runner E2E server port allocation", () => { it("rejects direct and derived embedded-Postgres collisions", () => { expect(runnerE2EServerPortConflictsWithDatabase(44_329)).toBe(true); expect(runnerE2EServerPortConflictsWithDatabase(54_329)).toBe(true); expect(runnerE2EServerPortConflictsWithDatabase(64_329)).toBe(true); expect(runnerE2EServerPortConflictsWithDatabase(43_123)).toBe(false); }); it("retries a derived collision while closing every reservation", async () => { const closed: number[] = []; const ports = [44_329, 43_123, 53_123]; const openPort = vi.fn(async (requestedPort: number) => { const port = requestedPort === 0 ? ports.shift() : requestedPort; if (port === undefined) throw new Error("Missing fake port"); return { port, close: async () => { closed.push(port); }, } satisfies LoopbackPortReservation; }); await expect(reserveRunnerE2EServerPort({ openPort })).resolves.toBe( 43_123, ); expect(openPort.mock.calls.map(([port]) => port)).toEqual([0, 0, 53_123]); expect(closed).toEqual([44_329, 53_123, 43_123]); }); }); describe("runner E2E sensitive API boundary", () => { it("keeps secret request bodies out of Playwright API tracing", async () => { vi.stubEnv("PAPERCLIP_RUNNER_E2E_PORT", "43123"); const playwrightPost = vi.fn(); const fetchMock = vi.fn().mockResolvedValue( new Response(JSON.stringify({ id: "secret-id" }), { status: 201, headers: { "content-type": "application/json" }, }), ); vi.stubGlobal("fetch", fetchMock); const api = new RunnerApi({ post: playwrightPost } as never); await expect( api.postSensitive("/api/companies/company-id/secrets", { value: "fixture-secret-value", }), ).resolves.toEqual({ id: "secret-id" }); expect(playwrightPost).not.toHaveBeenCalled(); expect(fetchMock).toHaveBeenCalledWith( new URL("http://127.0.0.1:43123/api/companies/company-id/secrets"), expect.objectContaining({ method: "POST" }), ); }); }); describe("runner E2E structured evidence scanning", () => { it("does not invent a secret shape across JSON syntax boundaries", () => { const value = { OPENROUTER_API_KEY: "generated-secret-reference-id", }; expect(findSecretLeak(JSON.stringify(value), [])).toBe( "secret-shaped value", ); expect(findSecretLeakInJsonValues(value, [])).toBeNull(); }); it("still rejects exact and provider-shaped values in nested JSON", () => { expect( findSecretLeakInJsonValues( { nested: ["safe", "campaign-secret-value"] }, ["campaign-secret-value"], ), ).toBe("exact secret value"); expect( findSecretLeakInJsonValues({ nested: "sk-proj-abcdefghijklmnop" }, []), ).toBe("secret-shaped value"); }); it("keeps fake Kimi and Grok credentials out of persisted payloads while retaining references", () => { const fakeCredentials = ["kimi-fixture-secret", "xai-fixture-secret"]; const payload = { env: { KIMI_MODEL_API_KEY: { type: "secret_ref", secretId: "kimi-ref", version: "latest" }, XAI_API_KEY: { type: "secret_ref", secretId: "xai-ref", version: "latest" }, }, log: "provider response redacted", }; expect(findSecretLeakInJsonValues(payload, fakeCredentials)).toBeNull(); expect(findSecretLeak(JSON.stringify(payload), fakeCredentials)).toBeNull(); expect(() => assertSecretFree(JSON.stringify(payload), fakeCredentials, "pending-profile.json")).not.toThrow(); expect(JSON.stringify(payload)).not.toContain(fakeCredentials[0]!); expect(JSON.stringify(payload)).not.toContain(fakeCredentials[1]!); }); }); describe("runner E2E fixture registry", () => { it("sets up in dependency order and tears down in reverse", async () => { const events: string[] = []; const registry = new FixtureRegistry() .register({ id: "company", setup: async () => { events.push("setup-company"); return "c"; }, teardown: async () => { events.push("teardown-company"); }, }) .register({ id: "agent", dependencies: ["company"], setup: async () => { events.push("setup-agent"); return "a"; }, teardown: async () => { events.push("teardown-agent"); }, }); const active = await registry.setupAll(); await active.teardown(); expect(events).toEqual([ "setup-company", "setup-agent", "teardown-agent", "teardown-company", ]); }); it("tears down partial setup after a failure", async () => { const cleanup = vi.fn(); const registry = new FixtureRegistry() .register({ id: "company", setup: async () => "c", teardown: cleanup }) .register({ id: "agent", dependencies: ["company"], setup: async () => { throw new Error("boom"); }, }); await expect(registry.setupAll()).rejects.toThrow("boom"); expect(cleanup).toHaveBeenCalledOnce(); }); it("preserves setup and partial-cleanup failures", async () => { const registry = new FixtureRegistry() .register({ id: "company", setup: async () => "c", teardown: async () => { throw new Error("cleanup transport returned 503"); }, }) .register({ id: "agent", dependencies: ["company"], setup: async () => { throw new Error("agent setup failed"); }, }); await expect(registry.setupAll()).rejects.toMatchObject({ name: "AggregateError", message: expect.stringContaining("cleanup failed"), errors: expect.arrayContaining([ expect.objectContaining({ message: "agent setup failed" }), expect.objectContaining({ message: "Fixture teardown failed" }), ]), }); }); }); describe("runner E2E matchers", () => { it("normalizes message text and evaluates state invariants", async () => { const results = await evaluateMatchers( [ { kind: "message_contains", expected: "PAPERCLIP_E2E_OK_nonce" }, { kind: "message_occurrences", expected: "PAPERCLIP_E2E_OK_nonce", count: 1, }, { kind: "issue_status", expected: "done" }, { kind: "runtime_mode", expected: "native" }, ], { message: " complete PAPERCLIP\\_E2E\\_OK\\_nonce ", issueStatus: "done", runtimeMode: "native", }, ); expect(results.every((result) => result.passed)).toBe(true); }); it("requires an exact number of complete visible markers", async () => { const [result] = await evaluateMatchers( [{ kind: "message_occurrences", expected: "FINAL_marker", count: 1 }], { message: "FINAL\\_marker\nFINAL_marker" }, ); expect(result).toMatchObject({ passed: false }); expect(result?.detail).toContain("observed 2"); }); it("matches finalized workspace files byte-for-byte", async () => { const [matched, extraLine] = await evaluateMatchers( [ { kind: "file_exact", path: "continuity.txt", expected: "T1\nT2\n" }, { kind: "file_exact", path: "duplicate.txt", expected: "T1\nT2\n" }, ], { files: { "continuity.txt": "T1\nT2\n", "duplicate.txt": "T1\nT2\nT2\n", }, }, ); expect(matched?.passed).toBe(true); expect(extraLine?.passed).toBe(false); }); it("normalizes ordered fragments and evaluates nested JSON Schema", async () => { const results = await evaluateMatchers( [ { kind: "message_ordered", expected: ["first marker", "second marker"], }, { kind: "json_schema", schema: { type: "object", required: ["run"], additionalProperties: false, properties: { run: { type: "object", required: ["status"], properties: { status: { const: "succeeded" } }, }, }, }, }, ], { message: "first marker\nsecond marker", json: { run: { status: "succeeded" } }, }, ); expect(results.every((result) => result.passed)).toBe(true); }); }); describe("runner E2E run observations", () => { it("retries only terminal Plan confirmations missing a revision-bound target", () => { const observation = { runs: [{ status: "succeeded" }], interactions: [ { kind: "request_confirmation", status: "pending", payload: { version: 1, prompt: "Approve the Plan?" }, }, ], minimumRunCount: 1, }; expect(hasTerminalMalformedPlanConfirmation(observation)).toBe(true); expect( hasTerminalMalformedPlanConfirmation({ ...observation, runs: [{ status: "running" }], }), ).toBe(false); expect( hasTerminalMalformedPlanConfirmation({ ...observation, interactions: [ { ...observation.interactions[0], payload: { target: { type: "issue_document", key: "plan", revisionId: "revision-1", }, }, }, ], }), ).toBe(false); }); it("retries only the zero-marker DeepSeek hello terminal emission variance", () => { const expectedMarker = "PC_H_nonce-1"; const observation = { suiteId: "openrouter-model-breadth", profileId: "openrouter-deepseek-deepseek-v4-flash-0731", taskId: "hello-complete", expectedMarker, finalRunMessage: "I'll complete this deterministic hello task by calling paperclip_finish once.", allAgentMessages: "I'll complete this deterministic hello task by calling paperclip_finish once.", semanticSummary: expectedMarker, issueStatus: "done", runStatuses: ["succeeded"], matcherResults: [ { matcher: { kind: "message_exact", expected: expectedMarker }, passed: false, }, { matcher: { kind: "message_occurrences", expected: expectedMarker, count: 1, }, passed: false, }, { matcher: { kind: "issue_status", expected: "done" }, passed: true }, ], invariantFailures: [], }; expect(isOpenRouterDeepSeekHelloTerminalVariance(observation)).toBe(true); expect( isOpenRouterDeepSeekHelloTerminalVariance({ ...observation, allAgentMessages: `${expectedMarker}\n${expectedMarker}`, }), ).toBe(false); expect( isOpenRouterDeepSeekHelloTerminalVariance({ ...observation, semanticSummary: "different-summary", }), ).toBe(false); expect( isOpenRouterDeepSeekHelloTerminalVariance({ ...observation, invariantFailures: ["missing native terminal event"], }), ).toBe(false); expect( isOpenRouterDeepSeekHelloTerminalVariance({ ...observation, matcherResults: [ ...observation.matcherResults, { matcher: { kind: "environment", expected: "local" }, passed: false, }, ], }), ).toBe(false); }); it("counts provider-equivalent numbered Plan step formats", () => { expect(numberedPlanStepCount("1. First\n2) Second")).toBe(2); expect( numberedPlanStepCount( "# Plan\n\nStep 1 — First\n\nStep 2 — Second\n\nStep 3: Verify", ), ).toBe(3); expect( numberedPlanStepCount("## **Step 1** — First\n- **2.** Second"), ).toBe(2); }); it("excludes only queued continuations fenced while awaiting review", () => { expect( isNonExecutingReviewFenceRun({ status: "cancelled", errorCode: "issue_continuation_waiting_on_review", }), ).toBe(true); expect( isNonExecutingReviewFenceRun({ status: "failed", errorCode: "issue_continuation_waiting_on_review", }), ).toBe(false); expect( isNonExecutingReviewFenceRun({ status: "cancelled", errorCode: "provider_failure", }), ).toBe(false); }); it("recognizes only authoritative control-plane governed response waits", () => { const event = { eventType: "run.result.accepted", payload: { prpEvent: { schema: "paperclip.prp.event.v1", eventType: "run.result.accepted", sourceKind: "control_plane", payload: { result: { schema: "paperclip.run_result.v1", reportedWorkDisposition: "yielded", evidence: [{ ref: "interaction:pending" }], artifacts: [ { kind: "issue_thread_interaction", ref: "interaction:pending", }, ], continuation: { kind: "response_wake", idempotencyKey: "interaction-response:pending", }, }, }, }, }, }; expect(isControlPlaneGovernedResponseWait([event])).toBe(true); expect( isControlPlaneGovernedResponseWait([ { ...event, payload: { prpEvent: { ...event.payload.prpEvent, sourceKind: "runner", }, }, }, ]), ).toBe(false); expect( isControlPlaneGovernedResponseWait([ { ...event, payload: { prpEvent: { ...event.payload.prpEvent, payload: { result: { ...event.payload.prpEvent.payload.result, artifacts: [], }, }, }, }, }, ]), ).toBe(false); expect( isControlPlaneGovernedResponseWait([ event, { ...event, payload: structuredClone(event.payload) }, ]), ).toBe(false); }); it("requires continuity except for an explicit accepted-Plan reset", () => { const initial = { id: "initial", sessionIdBefore: null, sessionIdAfter: "session-one", }; const resumed = { id: "resumed", sessionIdBefore: "session-one", sessionIdAfter: "session-one", }; const acceptedPlan = { id: "accepted-plan", sessionIdBefore: null, sessionIdAfter: "session-two", contextSnapshot: { forceFreshSession: true, workspaceRefreshReason: "accepted_plan_confirmation", source: "issue.interaction.accept", interactionStatus: "accepted", }, }; expect( providerSessionContinuityFailures("codex", [ initial, resumed, acceptedPlan, ]), ).toEqual([]); expect( providerSessionContinuityFailures("codex", [ initial, { ...acceptedPlan, sessionIdAfter: "session-one" }, ]), ).toEqual([ "expected accepted Plan run accepted-plan to rotate the codex provider session", ]); expect( providerSessionContinuityFailures("codex", [ initial, { ...resumed, sessionIdAfter: "session-two" }, ]), ).toEqual([ "expected codex to preserve its provider session for run resumed", ]); expect( acceptedPlanSessionResetFailures( "acpx", initial.sessionIdAfter, acceptedPlan, ), ).toEqual([]); expect( acceptedPlanSessionResetFailures("acpx", initial.sessionIdAfter, { ...acceptedPlan, sessionIdBefore: initial.sessionIdAfter, }), ).toEqual([ "expected accepted Plan run accepted-plan to start without a prior provider session", ]); expect( acceptedPlanSessionResetFailures("acpx", initial.sessionIdAfter, resumed), ).toBeNull(); }); }); describe("runner E2E failure policy", () => { it("classifies sandbox file-transfer RPC deadlines without hiding other RPC defects", () => { for (const method of ["environmentSyncIn", "environmentSyncOut"]) { expect(classifyFailure(new Error( `Stopped waiting for everyday recover-controller settled: native execution failed native_session_interrupted: RPC call "${method}" timed out after 330000ms`, ))).toBe("transient_infrastructure"); } expect(classifyFailure(new Error('RPC call "run.attach" timed out after 330000ms'))) .toBe("candidate_failure"); }); it.each([ "native_session_close_unrecoverable: provider transport failed", "Provider connection closed: runner did not durably suspend before checkpoint", "native_session_close_unrecoverable: provider transport timed out; runner did not durably suspend before checkpoint", ])("does not retry controller session-close defects: %s", (message) => { const failureClass = classifyFailure(new Error(message)); expect(failureClass).toBe("candidate_failure"); expect(shouldRetryFailure(failureClass)).toBe(false); }); it.each([ 'native_session_recovery_failed: Error: PRP command run.attach failed: {"result":{"code":"command_execution_failed","message":"failed to start ACPX provider: ACPX sidecar command session.open was rejected (retryable=false, classification=unclassified)"},"status":"failed"}', "native_session_recovery_failed: provider transport failed\nACPX sidecar command session.open was rejected (retryable = false, classification=session_not_found)", ])("does not retry explicit non-retryable ACPX recovery rejection: %s", (message) => { const failureClass = classifyFailure(new Error(message)); expect(failureClass).toBe("candidate_failure"); expect(shouldRetryFailure(failureClass)).toBe(false); }); it.each([ 'native_session_recovery_failed: PRP run.attach failed: {"message":"failed to start ACPX provider: ACPX sidecar command session.open was rejected (retryable=true, classification=network)","status":"failed"}', 'native_session_recovery_failed: PRP run.attach failed: {"message":"failed to start ACPX provider: ACPX sidecar command session.open was rejected","status":"failed"}', "native_session_recovery_failed: failed to start ACPX provider: ECONNRESET", ])("retains transient ACPX recovery retries: %s", (message) => { const failureClass = classifyFailure(new Error(message)); expect(failureClass).toBe("transient_infrastructure"); expect(shouldRetryFailure(failureClass)).toBe(true); }); it("disables both automatic retry classes when the policy is zero", () => { expect(shouldRetryFailure("transient_infrastructure", 0)).toBe(false); expect(shouldRetryFailure("provider_variance", 0)).toBe(false); expect(shouldRetryFailure("transient_infrastructure", 1)).toBe(true); expect(shouldRetryFailure("provider_variance", 1)).toBe(true); }); it("retries only transient infrastructure failures", () => { expect( classifyFailure(new Error("Daytona preview connection timed out")), ).toBe("transient_infrastructure"); expect( classifyFailure( new Error( "Browser bootstrap failed before task creation: New Task button timed out after a Vite 504", ), ), ).toBe("transient_infrastructure"); expect( shouldRetryFailure( classifyFailure(new Error("Daytona preview connection timed out")), ), ).toBe(true); expect( shouldRetryFailure(classifyFailure(new Error("marker matcher failed"))), ).toBe(false); expect(shouldRetryFailure("provider_variance")).toBe(true); expect( classifyFailure( new Error( "Timed out waiting for issue abc and heartbeat run terminal state", ), ), ).toBe("candidate_failure"); expect( classifyFailure( new Error("Provider request timed out during generation"), ), ).toBe("transient_infrastructure"); expect( classifyFailure( new Error( "runner_ingress_unavailable: paperclip-runnerd: cumulative ACK cannot move beyond the produced source cursor", ), ), ).toBe("transient_infrastructure"); expect( shouldRetryFailure(classifyFailure(new Error("invalid API key"))), ).toBe(false); expect( classifyFailure( new Error("Daytona lease cleanup failed: provider returned 503"), ), ).toBe("transient_infrastructure"); expect(classifyFailure(new Error("cleanup invariant failure"))).toBe( "cleanup_failure", ); }); }); describe("runner E2E server isolation", () => { it("shares restart control files beneath the isolated temporary root", () => { expect(runnerE2EServerControlPaths("/tmp/cell")).toEqual({ controlDirectory: path.join("/tmp/cell", "control"), restartRequestPath: path.join( "/tmp/cell", "control", "server-restart.request.json", ), restartAcknowledgementPath: path.join( "/tmp/cell", "control", "server-restart.ack.json", ), }); }); it("strips database and paid-provider credentials from the Paperclip process", () => { const env = buildPaperclipServerEnvironment( { PATH: "/bin", DATABASE_URL: "postgres://existing", DATABASE_MIGRATION_URL: "postgres://migration", OPENAI_API_KEY: "openai", ANTHROPIC_API_KEY: "anthropic", OPENROUTER_API_KEY: "openrouter", KIMI_MODEL_API_KEY: "kimi", XAI_API_KEY: "xai", GROK_AUTH_JSON: "grok-auth-json", DAYTONA_API_KEY: "daytona", XAI_ORG_ID: "xai-sensitive", GROK_HOME: "/outside/grok", OPENAI_ORG_ID: "also-provider-sensitive", PAPERCLIP_API_KEY: "ambient-board-key", PAPERCLIP_AGENT_API_KEY: "ambient-agent-key", PAPERCLIP_TASK_BRIDGE_TOKEN: "ambient-task-token", PAPERCLIP_SETUP_TOKEN: "ambient-setup-token", PAPERCLIP_SECRETS_MASTER_KEY: "ambient-master-key", PAPERCLIP_SECRETS_MASTER_KEY_FILE: "/outside/master.key", PAPERCLIP_STORAGE_S3_BUCKET: "production-bucket", }, { PAPERCLIP_HOME: "/tmp/cell/paperclip-home", PAPERCLIP_CONFIG: "/tmp/cell/paperclip-home/instances/e2e/config.json", XDG_CACHE_HOME: "/tmp/cell/xdg-cache", PAPERCLIP_AGENT_JWT_SECRET: "generated-agent-jwt", PAPERCLIP_DECISION_SIGNING_SECRET: "generated-decision-key", PAPERCLIP_TOOL_ACTION_SIGNING_SECRET: "generated-tool-key", BETTER_AUTH_SECRET: "generated-auth-key", }, ); expect(env.PATH).toBe("/bin"); expect(env.DATABASE_URL).toBeUndefined(); expect(env.OPENAI_API_KEY).toBeUndefined(); expect(env.KIMI_MODEL_API_KEY).toBeUndefined(); expect(env.XAI_API_KEY).toBeUndefined(); expect(env.GROK_AUTH_JSON).toBeUndefined(); expect(env.OPENAI_ORG_ID).toBeUndefined(); expect(env.XAI_ORG_ID).toBeUndefined(); expect(env.GROK_HOME).toBeUndefined(); expect(env.PAPERCLIP_API_KEY).toBeUndefined(); expect(env.PAPERCLIP_AGENT_API_KEY).toBeUndefined(); expect(env.XDG_CACHE_HOME).toBe("/tmp/cell/xdg-cache"); expect(env.PAPERCLIP_AGENT_JWT_SECRET).toBe("generated-agent-jwt"); expect(env.PAPERCLIP_TASK_BRIDGE_TOKEN).toBeUndefined(); expect(env.PAPERCLIP_SETUP_TOKEN).toBeUndefined(); expect(env.PAPERCLIP_SECRETS_MASTER_KEY).toBeUndefined(); expect(env.PAPERCLIP_SECRETS_MASTER_KEY_FILE).toBeUndefined(); expect(env.PAPERCLIP_STORAGE_S3_BUCKET).toBeUndefined(); expect(() => assertIsolatedServerEnvironment(env, { temporaryRoot: "/tmp/cell", paperclipHome: "/tmp/cell/paperclip-home", configPath: "/tmp/cell/paperclip-home/instances/e2e/config.json", }), ).not.toThrow(); }); it("uses absolute repository paths for the Playwright web server", () => { const command = runnerE2EWebServerCommand("/workspace/paperclip"); expect(command).toContain( "'/workspace/paperclip/cli/node_modules/tsx/dist/cli.mjs'", ); expect(command).toContain( "'/workspace/paperclip/tests/runner-e2e/server.ts'", ); }); it("accepts only embedded state paths beneath the temporary root", async () => { const root = await mkdtemp( path.join(os.tmpdir(), "runner-e2e-instance-isolation-test-"), ); cleanupDirectories.push(root); const database = path.join(root, "paperclip-home", "db"); const secretsKey = path.join( root, "paperclip-home", "secrets", "master.key", ); const configPath = path.join(root, "paperclip-home", "config.json"); await mkdir(database, { recursive: true }); await mkdir(path.dirname(secretsKey), { recursive: true }); await writeFile(secretsKey, "generated-master-key"); const config = { database: { mode: "embedded-postgres", embeddedPostgresDataDir: database, backup: { dir: path.join(root, "backups") }, }, logging: { logDir: path.join(root, "logs") }, storage: { provider: "local_disk", localDisk: { baseDir: path.join(root, "storage") }, }, secrets: { provider: "local_encrypted", strictMode: true, localEncrypted: { keyFilePath: secretsKey }, }, }; await writeFile(configPath, JSON.stringify(config)); await expect( assertEmbeddedDatabaseIsolation(configPath, root), ).resolves.toBeUndefined(); await writeFile( configPath, JSON.stringify({ ...config, storage: { ...config.storage, localDisk: { baseDir: "/outside/storage" }, }, }), ); await expect( assertEmbeddedDatabaseIsolation(configPath, root), ).rejects.toThrow("storage path escaped"); }); }); describe("runner E2E evidence redaction", () => { const secret = "sk-proj-supersecretvalue123456"; it("redacts exact and shaped credentials recursively", () => { expect(redactText(`token=${secret}`, [secret])).toBe("token=[REDACTED]"); expect(sanitizeJson({ nested: [secret] }, [secret])).toEqual({ nested: ["[REDACTED]"], }); expect( sanitizeJson( { metadata: { apiKey: "opaque-provider-issued-value", access_token: "opaque-access-token", apiKeyRef: "DAYTONA_API_KEY", }, }, [], ), ).toEqual({ metadata: { apiKey: "[REDACTED]", access_token: "[REDACTED]", apiKeyRef: "DAYTONA_API_KEY", }, }); expect(sanitizeJson("paperclip.runner-e2e.evidence/v1", [secret])).toBe( "paperclip.runner-e2e.evidence/v1", ); }); it("detects leaks and accepts sanitized evidence", () => { expect(findSecretLeak(Buffer.from(secret), [secret])).toBeTruthy(); expect(() => assertSecretFree("safe", [secret], "fixture")).not.toThrow(); expect(() => assertSecretFree( "sk-proj-documentationfixture123456", [secret], "API source", { includeShapes: false }, ), ).not.toThrow(); }); it("finds exact credentials across streamed persisted-state chunks", async () => { const root = await mkdtemp( path.join(os.tmpdir(), "runner-e2e-secret-scan-test-"), ); cleanupDirectories.push(root); await writeFile( path.join(root, "database.bin"), Buffer.concat([ Buffer.alloc(65_530, "x"), Buffer.from(secret), Buffer.alloc(32, "y"), ]), ); await expect( findSecretLeakInDirectory(root, [secret]), ).resolves.toMatchObject({ reason: "exact secret value" }); }); it("tolerates only the embedded PostgreSQL PID disappearing during shutdown and keeps scanning", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "runner-e2e-postgres-pid-race-")); cleanupDirectories.push(root); const pidFile = path.join(root, "instances", "test", "db", "postmaster.pid"); const persistedFile = path.join(path.dirname(pidFile), "z-persisted.bin"); await mkdir(path.dirname(pidFile), { recursive: true }); await writeFile(pidFile, "1234\n"); await writeFile(persistedFile, secret); await expect(findSecretLeakInDirectory(root, [secret], { ignoreFile: (file) => { if (file === pidFile) unlinkSync(file); // Removed after directory enumeration. return false; }, allowDisappearedFile: (file) => isEphemeralPostgresPidFile(root, file), })).resolves.toEqual({ file: persistedFile, reason: "exact secret value" }); expect(isEphemeralPostgresPidFile(root, path.join(root, "workspace", "postmaster.pid"))).toBe(false); expect(isEphemeralPostgresPidFile(root, path.join(root, "instances", "test", "db", "records.bin"))).toBe(false); }); it("handles a removed PostgreSQL relation but scans existing relation bytes", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "runner-e2e-postgres-relation-race-")); cleanupDirectories.push(root); const relation = path.join(root, "instances", "test", "db", "base", "16384", "16824"); const persisted = path.join(path.dirname(relation), "16825"); await mkdir(path.dirname(relation), { recursive: true }); await writeFile(relation, "old relation"); await writeFile(persisted, secret); await expect(findSecretLeakInDirectory(root, [secret], { ignoreFile: file => { if(file === relation)unlinkSync(file); return false; }, allowDisappearedFile: file => isEphemeralPostgresScanFile(root, file), })).resolves.toEqual({file:persisted,reason:"exact secret value"}); expect(isEphemeralPostgresScanFile(root,path.join(root,"workspace","base","16384","16824"))).toBe(false); expect(isEphemeralPostgresScanFile(root,path.join(root,"instances","test","db","base","records.json"))).toBe(false); }); it("still detects secrets in an existing PostgreSQL PID file", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "runner-e2e-postgres-pid-secret-")); cleanupDirectories.push(root); const pidFile = path.join(root, "instances", "test", "db", "postmaster.pid"); await mkdir(path.dirname(pidFile), { recursive: true }); await writeFile(pidFile, secret); await expect(findSecretLeakInDirectory(root, [secret], { allowDisappearedFile: (file) => isEphemeralPostgresPidFile(root, file), })).resolves.toEqual({ file: pidFile, reason: "exact secret value" }); }); it("fails when required persisted state disappears during scanning", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "runner-e2e-required-file-race-")); cleanupDirectories.push(root); const requiredFile = path.join(root, "records.json"); await writeFile(requiredFile, "{}"); await expect(findSecretLeakInDirectory(root, [secret], { ignoreFile: (file) => { unlinkSync(file); return false; }, allowDisappearedFile: (file) => isEphemeralPostgresPidFile(root, file), })).rejects.toMatchObject({ code: "ENOENT" }); }); it("does not suppress other I/O failures for the ephemeral PID path", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "runner-e2e-postgres-pid-io-")); cleanupDirectories.push(root); const pidFile = path.join(root, "instances", "test", "db", "postmaster.pid"); await mkdir(path.dirname(pidFile), { recursive: true }); await writeFile(pidFile, "1234\n"); await expect(findSecretLeakInDirectory(root, [secret], { ignoreFile: (file) => { unlinkSync(file); mkdirSync(file); return false; }, allowDisappearedFile: (file) => isEphemeralPostgresPidFile(root, file), })).rejects.toMatchObject({ code: "EISDIR" }); }); it("still reports missing mandatory pass evidence", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "runner-e2e-required-evidence-")); cleanupDirectories.push(root); const privateDir = path.join(root, "private"); await mkdir(privateDir); const packaged = await packageEvidence({ privateDir, uploadDir: path.join(root, "upload"), secrets: [secret], expectPassScreenshot: true, }); expect(packaged.missing).toContain("result.json"); expect(packaged.missing).toContain("final-state.png"); }); it("can ignore fake key shapes while scanning persisted package state", async () => { const root = await mkdtemp( path.join(os.tmpdir(), "runner-e2e-secret-shape-test-"), ); cleanupDirectories.push(root); await writeFile( path.join(root, "provider-fixture.test.ts"), 'const fake = "sk-proj-documentationfixture123456";\n', ); await expect( findSecretLeakInDirectory(root, [secret], { includeShapes: false }), ).resolves.toBeNull(); }); it("can narrowly exclude a verified ephemeral runtime credential file", async () => { const root = await mkdtemp( path.join(os.tmpdir(), "runner-e2e-ephemeral-auth-test-"), ); cleanupDirectories.push(root); const runtimeAuth = path.join(root, "codex-home", "auth.json"); const forbiddenConfig = path.join(root, "config.json"); await mkdir(path.dirname(runtimeAuth), { recursive: true }); await writeFile(runtimeAuth, secret, { mode: 0o600 }); await writeFile(forbiddenConfig, secret); await expect( findSecretLeakInDirectory(root, [secret], { includeShapes: false, ignoreFile: (file) => file === runtimeAuth, }), ).resolves.toMatchObject({ file: forbiddenConfig }); }); it("recognizes managed and durable Codex runtime auth files", () => { const root = path.join(os.tmpdir(), "paperclip-home"); expect( isEphemeralCodexRuntimeAuthFile( root, path.join( root, "instances/instance-1/companies/company-1/agents/agent-1/codex-home/auth.json", ), ), ).toBe(true); expect( isEphemeralCodexRuntimeAuthFile( root, path.join( root, "instances/instance-1/runtime/paperclip-runner/acpx/acpx/session-1/codex-home/auth.json", ), ), ).toBe(true); expect( isEphemeralCodexRuntimeAuthFile( root, path.join( root, "instances/instance-1/runtime/paperclip-runner/durable-sessions/session-1/codex-home/auth.json", ), ), ).toBe(true); expect( isEphemeralCodexRuntimeAuthFile( root, path.join(root, "instances/instance-1/runtime/auth.json"), ), ).toBe(false); expect( isEphemeralCodexRuntimeAuthFile( root, path.join( root, "instances/instance-1/runtime/paperclip-runner/durable-sessions/session-1/codex-home/config.toml", ), ), ).toBe(false); }); it("publishes only allowlisted sanitized files and reports source leaks", async () => { const root = await mkdtemp( path.join(os.tmpdir(), "runner-e2e-evidence-test-"), ); cleanupDirectories.push(root); const privateDir = path.join(root, "private"); const uploadDir = path.join(root, "upload"); await mkdir(privateDir, { recursive: true }); await writeFile( path.join(privateDir, "result.json"), JSON.stringify({ error: secret }), ); await writeFile(path.join(privateDir, "database.sqlite"), secret); const packaged = await packageEvidence({ privateDir, uploadDir, secrets: [secret], expectPassScreenshot: false, }); expect(packaged.leaks).toEqual([ { file: "result.json", reason: "exact secret value" }, ]); expect( await readFile(path.join(uploadDir, "result.json"), "utf8"), ).toContain("[REDACTED]"); await expect( readFile(path.join(uploadDir, "database.sqlite")), ).rejects.toThrow(); }); it("retains the two reviewed chat plan captures without admitting arbitrary chat PNGs", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "runner-e2e-chat-captures-")); cleanupDirectories.push(root); const privateDir = path.join(root, "private"); const uploadDir = path.join(root, "upload"); await mkdir(privateDir, { recursive: true }); for (const file of ["chat-plan-draft.png", "chat-plan-revised.png", "chat-secret.png", "chat-plan-extra.png"]) { await writeFile(path.join(privateDir, file), "fixture raster"); } const packaged = await packageEvidence({ privateDir, uploadDir, secrets: [secret], expectPassScreenshot: false }); expect(packaged.files.sort()).toEqual(["chat-plan-draft.png", "chat-plan-revised.png", "evidence-manifest.json"]); expect(packaged.leaks).toEqual([]); for (const file of packaged.files.filter((file) => file.endsWith(".png"))) { expect(await readFile(path.join(uploadDir, file), "utf8")).toBe("fixture raster"); } }); it("keeps raster evidence private to CI and rejects active SVG content", async () => { const root = await mkdtemp( path.join(os.tmpdir(), "runner-e2e-visual-evidence-test-"), ); cleanupDirectories.push(root); const privateDir = path.join(root, "private"); const uploadDir = path.join(root, "upload"); const playwrightOutput = path.join(privateDir, "playwright-output"); await mkdir(playwrightOutput, { recursive: true }); await writeFile(path.join(privateDir, "final-state.png"), "png"); await writeFile(path.join(playwrightOutput, "failure.webm"), "webm"); await writeFile( path.join(playwrightOutput, "active.svg"), "", ); const packaged = await packageEvidence({ privateDir, uploadDir, secrets: [secret], expectPassScreenshot: false, }); expect(packaged.files).toEqual( expect.arrayContaining([ "final-state.png", path.join("playwright-output", "failure.webm"), ]), ); expect(packaged.files).not.toContain( path.join("playwright-output", "active.svg"), ); await expect( readFile(path.join(playwrightOutput, "active.svg"), "utf8"), ).resolves.toContain("onload"); await expect( readFile(path.join(uploadDir, "playwright-output", "active.svg")), ).rejects.toThrow(); }); it("preserves valid JSON while redacting escaped command diagnostics", async () => { const root = await mkdtemp( path.join(os.tmpdir(), "runner-e2e-json-evidence-test-"), ); cleanupDirectories.push(root); const privateDir = path.join(root, "private"); const uploadDir = path.join(root, "upload"); await mkdir(path.join(privateDir, "snapshots"), { recursive: true }); await writeFile( path.join(privateDir, "snapshots", "api-state.json"), JSON.stringify({ log: String.raw`curl -H \"Authorization: Bearer temporary-run-token\" \\\n+ \"$PAPERCLIP_API_URL/api/issues\"`, }), ); await packageEvidence({ privateDir, uploadDir, secrets: [secret], expectPassScreenshot: false, }); const uploaded = await readFile( path.join(uploadDir, "snapshots", "api-state.json"), "utf8", ); expect(() => JSON.parse(uploaded)).not.toThrow(); expect(uploaded).toContain("[REDACTED]"); }); it("streams large ZIP evidence and detects exact secrets", async () => { const root = await mkdtemp( path.join(os.tmpdir(), "runner-e2e-zip-evidence-test-"), ); cleanupDirectories.push(root); const privateDir = path.join(root, "private"); const uploadDir = path.join(root, "upload"); const blobDir = path.join(privateDir, "blob-report"); await mkdir(blobDir, { recursive: true }); await writeFile( path.join(blobDir, "trace.txt"), Buffer.concat([Buffer.alloc(2 * 1024 * 1024, "x"), Buffer.from(secret)]), ); execFileSync("zip", ["-q", "report.zip", "trace.txt"], { cwd: blobDir, }); await rm(path.join(blobDir, "trace.txt")); const packaged = await packageEvidence({ privateDir, uploadDir, secrets: [secret], expectPassScreenshot: false, }); expect(packaged.leaks).toEqual([ { file: path.join("blob-report", "report.zip"), reason: "exact secret value", }, ]); await expect( readFile(path.join(uploadDir, "blob-report", "report.zip")), ).rejects.toThrow(); }); }); describe("runner E2E macOS shared-memory cleanup", () => { it("parses only shared-memory rows from ipcs output", () => { expect( parseDarwinSharedMemory( `IPC status from \nT ID KEY MODE OWNER GROUP CREATOR CGROUP NATTCH SEGSZ CPID LPID\nm 327709 0x028ed0ab --rw------- dotta staff dotta staff 0 56 52172 52172\ns 123 0x0 --ra------- dotta staff\n`, ), ).toEqual([ { id: "327709", owner: "dotta", attachments: 0, creatorPid: 52172, }, ]); }); }); describe("persisted final response selection", () => { const comments = [ { id: "attachment-comment", body: "Prepared file for this response.", createdByRunId: "run-1" }, { id: "reply", body: "FINAL", createdByRunId: "run-1" }, { id: "other-run", body: "unrelated", createdByRunId: "run-2" }, ]; const run = { id: "run-1", resultJson: { presentationDecision: { commentId: "reply" } } }; it("grades the real final comment independently from an attachment's preparation comment", () => { expect(persistedFinalRunMessage(comments, run)).toBe("FINAL"); }); it("fails closed when the selected final comment is missing or belongs to another run", () => { expect(persistedFinalRunMessage(comments.slice(0, 1), run)).toBe(""); expect(persistedFinalRunMessage(comments, { ...run, resultJson: { presentationDecision: { commentId: "other-run" } } })).toBe(""); }); it("keeps legacy fallback and does not replace absent visible text with a summary", () => { expect(persistedFinalRunMessage(comments, { id: "run-1" })).toBe("Prepared file for this response.\nFINAL"); expect(persistedFinalRunMessage([], { id: "run-1", resultJson: { summary: "FINAL" } })).toBe(""); }); }); describe("warm continuity grading scope", () => { it("checks workspace bytes, lifecycle, and ordered turn markers without exact response formatting", () => { const execution = runnerMatrix.find((cell) => cell.task.flow === "warm_three_turn")!; const matchers = execution.task.buildMatchers("test-nonce", execution); expect(matchers).toContainEqual({ kind: "message_occurrences", expected: "PAPERCLIP_E2E_WARM_T1_test-nonce", count: 1, }); expect(matchers).toContainEqual({ kind: "message_occurrences", expected: "PAPERCLIP_E2E_WARM_T2_test-nonce", count: 1, }); expect(matchers).toContainEqual({ kind: "message_occurrences", expected: "PAPERCLIP_E2E_WARM_T3_test-nonce", count: 1, }); expect(matchers).toContainEqual({ kind: "message_ordered", expected: [ "PAPERCLIP_E2E_WARM_T1_test-nonce", "PAPERCLIP_E2E_WARM_T2_test-nonce", "PAPERCLIP_E2E_WARM_T3_test-nonce", ], }); expect(matchers).toContainEqual({ kind: "file_exact", path: "daytona-warm-test-nonce.txt", expected: "T1-test-nonce\nT2-test-nonce\nT3-test-nonce\n", }); expect(matchers).toContainEqual({ kind: "issue_status", expected: "done" }); const hello = runnerMatrix.find((cell) => cell.task.id === "hello-complete")!; expect(hello.task.buildMatchers("test-nonce", hello).some((matcher) => matcher.kind === "message_exact")).toBe(true); }); it("accepts warm-turn prose while rejecting missing, duplicate, or out-of-order markers", async () => { const execution = runnerMatrix.find((cell) => cell.task.flow === "warm_three_turn")!; const matchers = execution.task.buildMatchers("test-nonce", execution) .filter((matcher) => matcher.kind.startsWith("message_")); const passing = await evaluateMatchers(matchers, { message: [ "Turn one is complete: PAPERCLIP_E2E_WARM_T1_test-nonce.", "Turn two is complete: PAPERCLIP_E2E_WARM_T2_test-nonce.", "Turn three is complete: PAPERCLIP_E2E_WARM_T3_test-nonce.", ].join("\n"), }); expect(passing.every((result) => result.passed)).toBe(true); const invalidMessages = [ "PAPERCLIP_E2E_WARM_T1_test-nonce PAPERCLIP_E2E_WARM_T1_test-nonce PAPERCLIP_E2E_WARM_T2_test-nonce PAPERCLIP_E2E_WARM_T3_test-nonce", "PAPERCLIP_E2E_WARM_T1_test-nonce PAPERCLIP_E2E_WARM_T3_test-nonce", "PAPERCLIP_E2E_WARM_T3_test-nonce PAPERCLIP_E2E_WARM_T2_test-nonce PAPERCLIP_E2E_WARM_T1_test-nonce", ]; for (const message of invalidMessages) { const results = await evaluateMatchers(matchers, { message }); expect(results.some((result) => !result.passed)).toBe(true); } }); });