diff --git a/doc/evals.md b/doc/evals.md index 8a57f31089e..f59035c2843 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -125,6 +125,8 @@ It requires a fresh owned agent with no prior work runs and uses three provider Cloud fleet operations and trusted CI remain private. Local eval gateway qualification does not qualify Cloud routes, and these selections stay outside `--all`. +The deployment-targeted [saved skill journey](../tests/runner-e2e/product-journeys/saved-skill-use.md) checks one company skill assigned by key, its exact admitted immutable version, a bound native source read/load, and a run-authored saved result from skill-only inputs. It uses the reusable Product journey target and remains unqualified until a real target run passes. + The explicit-only [case handoff workflow](../tests/runner-e2e/README.md#persistent-case-handoffs-explicit-only) contains arithmetic and receipt smoke cells for a same-task round trip among three native agents using real case/task APIs, saved documents, handoff receipts, and browser readback. Receipt smoke has diff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.ts index 21738faa5c0..9231fb7a089 100644 --- a/server/src/__tests__/environment-runtime.test.ts +++ b/server/src/__tests__/environment-runtime.test.ts @@ -3808,17 +3808,23 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); - await expect(runtimeWithPlugin.acquireRunLease({ - companyId, - environment, - issueId: null, - agentId, - heartbeatRunId: runId, - persistedExecutionWorkspace: { - id: executionWorkspaceId, - mode: "shared_workspace", - }, - })).rejects.toThrow("the lease was preserved and no replacement was created"); + // Keep the exact configured budget stable across deadline creation and RPC dispatch. + const now = vi.spyOn(Date, "now").mockReturnValue(Date.now()); + try { + await expect(runtimeWithPlugin.acquireRunLease({ + companyId, + environment, + issueId: null, + agentId, + heartbeatRunId: runId, + persistedExecutionWorkspace: { + id: executionWorkspaceId, + mode: "shared_workspace", + }, + })).rejects.toThrow("the lease was preserved and no replacement was created"); + } finally { + now.mockRestore(); + } expect(workerManager.call).toHaveBeenNthCalledWith(1, pluginId, "environmentResumeLease", expect.objectContaining({ driverKey: "fake-plugin", diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.md index 6e653494ad6..ce803748eb6 100644 --- a/tests/runner-e2e/FIXTURES.md +++ b/tests/runner-e2e/FIXTURES.md @@ -1,5 +1,13 @@ # Runner E2E fixture authoring +The deployment-targeted [saved skill journey](product-journeys/saved-skill-use.md) +creates and assigns one campaign-owned synthetic skill by key through public APIs, then +creates the measured task through the browser. The saved outcome must come from +one native run. Setup never supplies the output document or its native read +receipt. The ordinary Latest assignment, immutable created version and admitted +run version must agree. Skill ownership is journaled before creation and cleaned by the target +provider with the other owned resources. + [Chat feature evals](CHAT-FEATURES.md) load two authored YAML requests and seed only company, agents and project context through public APIs. Both agents use production role instructions; evaluator outcomes are never injected. Real browser diff --git a/tests/runner-e2e/PRODUCT-JOURNEYS.md b/tests/runner-e2e/PRODUCT-JOURNEYS.md index 1b73f1e566a..eec3849f077 100644 --- a/tests/runner-e2e/PRODUCT-JOURNEYS.md +++ b/tests/runner-e2e/PRODUCT-JOURNEYS.md @@ -26,6 +26,7 @@ never start a server or depend on a tenant filesystem. | `repository-continuation` | Native agent; two work runs; a pinned public read-only repository | Verify independent Git bundle history and workspace bytes across two turns, preserve the original random file, and reopen registered outputs after reload. See [repository-continuation.md](product-journeys/repository-continuation.md) | | `routine-scheduled-run` | Native agent; one work run; an owned project and routine | Browser-create a routine, enable its minute schedule, verify one scheduled invocation and run-attributed saved output, then pause and observe the next schedule boundary with no new work | | `company-access-isolation` | Two registered QA Cloud workspaces; disposable company-bound agent credentials; zero model runs | Switch through the real tenant Cloud menu and return. Require allowed task reads and task creation, foreign read/create denial in both directions, exact canonical creator evidence, and key revocation. See [COMPANY-ACCESS.md](product-journeys/COMPANY-ACCESS.md) | +| `saved-skill-use` | Native agent with skill-write capability; one work run; one synthetic company skill assigned by key using Latest | Load the assigned saved skill, use inputs available only in its source, and verify a same-run-authored task document after reload. Bind the immutable created version to the run's admitted skill-version map. Require a complete bound native source read/load and preserve the exact source and version. Works with Beta skills off. See [saved-skill-use.md](product-journeys/saved-skill-use.md) | | `file-input-answer` | Native agent; one work run; synthetic CSV uploaded through the composer | Verify original uploaded bytes, independently calculated accepted-row count and units, a run-attributed task document after reload, and preservation of the input attachment. The task prompt contains neither the random file reference nor the answers | | `inbox-triage` | Signed-in board user; a separately authenticated QA human author; a fresh self-assigned task; zero model runs | Mark an unread task read, archive it through Mine, reload both views, verify unchanged work status, then observe a real Blocked transition resurface it through the normal archive policy | | `task-create` | Company and signed-in board user | Create a task through the shared production composer, assign it to yourself, reopen and reload it, verify the saved prompt and no agent work | diff --git a/tests/runner-e2e/SECURITY.md b/tests/runner-e2e/SECURITY.md index 5abc39cb853..1c6969ebcf7 100644 --- a/tests/runner-e2e/SECURITY.md +++ b/tests/runner-e2e/SECURITY.md @@ -1,5 +1,12 @@ # Runner E2E security for a public repository +The deployment-targeted `saved-skill-use` journey creates one synthetic skill on +the campaign-owned agent. Its random receipt reference and numeric inputs occur +only in the saved skill source. Exact source, immutable version, ordinary assignment, native read +events and run-authored document revisions remain in the private bundle. The +existing public projection exports closed checks and the authenticated owned +task screenshot. The journey changes no shared agent, instance flag or budget. + Native `cursor-cloud` acceptance uses a separate callback-only loopback listener. Its external HTTPS tunnel must never target the local_trusted operator app. Credential-free ingress checks require `/` and `/api/health` to return 404 and diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index e5e598a5e78..3729583453d 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -127,6 +127,7 @@ describe("runner E2E catalog", () => { it("exports deployment journeys separately from the model matrix and default selection", () => { expect(productJourneys.map(journey => journey.id)).toEqual([ + "saved-skill-use", "task-edit-lifecycle", "company-access-isolation", "repository-continuation", diff --git a/tests/runner-e2e/journey-features.ts b/tests/runner-e2e/journey-features.ts index 1816d62bd96..157f2ffce49 100644 --- a/tests/runner-e2e/journey-features.ts +++ b/tests/runner-e2e/journey-features.ts @@ -19,6 +19,7 @@ function associate(suites: readonly string[], cases: Record; + | "repository-continuation" + | "saved-skill-use">; interface TaskState { currentIssue: Row & { id: string; status: string }; taskRuns: Array; comments: Row[]; interactions: Array; attachments: StoryDelivery[] } interface Options { deadlineAt: number; directory: string; capture(id: string, label: string, file: string): Promise } const active = (run: Row) => ["queued", "running"].includes(run.status); diff --git a/tests/runner-e2e/product-journeys/catalog.ts b/tests/runner-e2e/product-journeys/catalog.ts index 7467ae5d6bc..95e51b6f072 100644 --- a/tests/runner-e2e/product-journeys/catalog.ts +++ b/tests/runner-e2e/product-journeys/catalog.ts @@ -2,6 +2,7 @@ import { agentJourneyRuns } from "./success-contract.js"; export { agentJourneyRuns, agentJourneyRunCountValid } from "./success-contract.js"; import type { JourneyId, TargetConfig } from "./types.js"; export const productJourneys = [ + { id: "saved-skill-use", label: "Use a saved assigned skill and verify its run-authored result", requires: ["company", "board-user", "task-write", "agent", "skill-write"], capabilities: ["skills/policy", "skills/versions", "documents-artifacts/documents"], paid: true }, { id: "task-edit-lifecycle", label: "Edit task properties, complete and reopen with saved context", requires: ["company", "board-user", "task-write"], capabilities: ["tasks/properties", "tasks/completion"], paid: false }, { id: "company-access-isolation", label: "Switch QA workspaces and verify restricted agent isolation", requires: ["company", "board-user", "task-write", "company-access"], capabilities: ["companies/selection", "access/resources"], paid: false }, { id: "repository-continuation", label: "Continue repository work with the same saved workspace", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["projects/task-context", "documents-artifacts/attachments"], paid: true }, diff --git a/tests/runner-e2e/product-journeys/contracts.test.ts b/tests/runner-e2e/product-journeys/contracts.test.ts index b3d3bd7b441..a702406ee61 100644 --- a/tests/runner-e2e/product-journeys/contracts.test.ts +++ b/tests/runner-e2e/product-journeys/contracts.test.ts @@ -39,6 +39,32 @@ function connectReceipt(): OnboardingConnectEvidence { }; } describe("deployment journey contracts", () => { + it("discovers an ordinary assigned-skill journey without an experimental pin prerequisite", () => { + const discovered = productJourneys.find(journey => journey.id === "saved-skill-use")!; + expect(discovered).not.toHaveProperty("requiredSettings"); + expect(journeyRequiredChecks[discovered.id]?.[0]).toBe("saved-skill-assignment-bound"); + const config = parseTargetConfig({ ...local, journeys: ["saved-skill-use"], + limits: { durationMs: 600_000, modelBudgetCents: 100, maxRuns: 1 } }); + expect(preparationPreview(config).journeys).toContainEqual(discovered); + expect(config.baseline.features).toEqual({}); + }); + it("requires one native saved-skill run and every closed source/use/result check", () => { + const config = { ...local, id: "qa", mode: "existing-cloud", adapterModule: "/private/provider.mjs", + executionEnvironment: "daytona", journeys: ["saved-skill-use"], limits: { durationMs: 600_000, modelBudgetCents: 100, maxRuns: 1 } }; + expect(parseTargetConfig(config).journeys).toEqual(["saved-skill-use"]); + for (const maxRuns of [0, 2]) expect(() => parseTargetConfig({ ...config, limits: { ...config.limits, maxRuns } })).toThrow("agent_journeys_require_bounded_runs"); + const retained = result(); + Object.assign(retained, { caseId: "saved-skill-use", provider: "opencode", model: "configured-model", runtimeMode: "native", runIds: ["run"] }); + retained.productJourney!.checks = journeyRequiredChecks[retained.caseId]!.map(id => ({ id, passed: true })); + expect(() => validateRetainedRunnerResult(retained)).not.toThrow(); + for (const check of retained.productJourney!.checks) { + const missing = structuredClone(retained); + missing.productJourney!.checks = missing.productJourney!.checks.filter(row => row.id !== check.id); + expect(() => validateRetainedRunnerResult(missing)).toThrow("result.productJourney"); + } + retained.runIds = []; + expect(() => validateRetainedRunnerResult(retained)).toThrow("result.agentJourneyRun"); + }); it("accepts historical results without Connect diagnostics and optional closed receipts", () => { const retained = result(); retained.caseId = "cloud-onboarding"; diff --git a/tests/runner-e2e/product-journeys/launch.ts b/tests/runner-e2e/product-journeys/launch.ts index 35337dcc765..930112167c5 100644 --- a/tests/runner-e2e/product-journeys/launch.ts +++ b/tests/runner-e2e/product-journeys/launch.ts @@ -23,6 +23,7 @@ import { runRepositoryContinuation } from "./repository-continuation-flow.js"; import { runScheduledRoutine } from "./routine-schedule-flow.js"; import { taskDependencyResume } from "./task-dependency-resume.js"; import { runInputFile } from "./input-file-flow.js"; +import { runSavedSkillUse } from "./saved-skill-use-flow.js"; import { runAgentSettings } from "./agent-settings-flow.js"; import { companyAccessIsolation } from "./company-access-flow.js"; import { inboxTriage } from "./inbox-triage-flow.js"; @@ -118,6 +119,8 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri const source = resolveRunnerE2ESource({ sha, ref: null, workflowRunUrl: null }); const definitionHash = createHash("sha256").update(JSON.stringify(productJourneys)); for (const name of [ + "saved-skill-use-flow.ts", + "saved-skill-use-evidence.ts", "task-lifecycle.ts", "task-lifecycle-evidence.ts", "../../../packages/shared/src/validators/issue.ts", @@ -287,7 +290,9 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri result.provider = ready.agent.provider; result.model = ready.agent.model; result.runtimeMode = ready.agent.runtimeMode; - if (id === "repository-continuation") await runRepositoryContinuation(ready, session!, campaignId, evidence, + if (id === "saved-skill-use") await runSavedSkillUse(ready, session!, campaignId, evidence, + { deadlineAt, directory: path.join(outputDir, "private-stories", id) }); + else if (id === "repository-continuation") await runRepositoryContinuation(ready, session!, campaignId, evidence, { deadlineAt, directory: path.join(outputDir, "private-stories", id) }); else if (id === "agent-settings-persistence") await runAgentSettings(ready, session!, campaignId, evidence, { deadlineAt, directory: path.join(outputDir, "private-stories", id) }); diff --git a/tests/runner-e2e/product-journeys/saved-skill-use-codex.test.ts b/tests/runner-e2e/product-journeys/saved-skill-use-codex.test.ts new file mode 100644 index 00000000000..1bbf1b32a7f --- /dev/null +++ b/tests/runner-e2e/product-journeys/saved-skill-use-codex.test.ts @@ -0,0 +1,157 @@ +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { canonicalProviderEventsFromCodex } from "../../../packages/paperclip-runner/src/provider-events.js"; +import { createPrpSemanticToolInputEnvelope, createPrpSemanticToolResultEnvelope, digestPrpSemanticContent } from "../../../packages/paperclip-runner/src/protocol/semantic-tool-receipts.js"; +import { validatePrpEvent } from "../../../packages/paperclip-runner/src/protocol/replay-contract.js"; +import { redactEventPayload } from "../../../server/src/redaction.js"; +import { readAssignedSkill, RUNNER_BRIDGE_READ_OPERATIONS } from "../../../server/src/services/native-runtime/runner-bridge-tools.js"; +import { savedSkillFixture, savedSkillReadAssignment, savedSkillReadReceipts } from "./saved-skill-use-evidence.js"; + +type Row = Record; +const binding = { companyId: "company", agentId: "agent", runId: "run" }; +const fixture = savedSkillFixture("codex-campaign", "62ff548dfec748da8fef6d5fbf6cc23d"); +const key = `company/${binding.companyId}/${fixture.slug}`, runtimeName = `${fixture.slug}--0123456789`; +const correlation = { runId: "run", normalizedSessionId: "native-session", turnId: "turn", itemId: "item" }; +const callId = "assigned-read"; +const hash = (value: string | Buffer) => createHash("sha256").update(value).digest("hex"); + +function event(seq: number, eventType: string, payload: Row, itemId = correlation.itemId): Row { + return { ...binding, seq, eventType, protocolSchemaVersion: 1, sourceSeq: seq, + sourceInstanceId: "runner-instance", sourceEventId: `event-${seq}`, + payload: redactEventPayload({ prpEvent: { + schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", ...correlation, itemId, + priority: 0, emittedAt: "2026-11-03T12:00:00.000Z", + sourceSeq: seq, sourceInstanceId: "runner-instance", sourceEventId: `event-${seq}`, + eventType, payload, + } }) }; +} + +// Read the real immutable-file producer, not a hand-written success result. No model or DB is used. +async function observedRead(text = fixture.markdown, rawInput?: Row) { + const directory = await mkdtemp(path.join(tmpdir(), "saved-skill-native-read-")); + try { + const files = [{ path: "SKILL.md", sha256: hash(text), mode: 0o444, size: Buffer.byteLength(text) }]; + const digest = hash(JSON.stringify(files)), rootPath = path.join(directory, "bundles", digest); + const manifest = `${JSON.stringify({ schema: "paperclip.runtime-asset-manifest.v1", digest, + fileCount: 1, totalBytes: Buffer.byteLength(text), files })}\n`; + await mkdir(rootPath, { recursive: true }); + await mkdir(path.join(directory, "manifests")); + await writeFile(path.join(rootPath, "SKILL.md"), text); + await writeFile(path.join(directory, "manifests", `${digest}.json`), manifest); + const bundle = { rootPath, digest, manifestDigest: hash(manifest) }; + const input = rawInput ?? { offset: 0, skill: key, path: "SKILL.md" }; + const result = await readAssignedSkill({ skills: [{ key, runtimeName, versionId: "version", bundle }] } as any, input); + return { input, result }; + } finally { await rm(directory, { recursive: true, force: true }); } +} + +async function observations(text = fixture.markdown, rawInput?: Row) { + const { input, result } = await observedRead(text, rawInput); + const run: Row = { id: binding.runId, companyId: binding.companyId, agentId: binding.agentId, runtimeMode: "native", + runnerInstanceId: "runner-instance", nativeSessionId: correlation.normalizedSessionId, + contextSnapshot: { skillSelectionSnapshot: true, skillVersions: { [key]: "version" } }, + runnerProfileJson: { nativeExecutionInput: { provider: { kind: "codex" }, + runtimeContext: { skills: [{ key, versionId: "version", runtimeName }] } } } }; + const assignment = savedSkillReadAssignment({ key, currentVersionId: "version" }, + { entries: [{ key, versionId: "version", runtimeName }] }, run)!; + // Core091 provider_backend.rs uses these protocol envelopes, adds safe input, + // and names the successful fallback receipt operation_. + const semanticInput = { ...createPrpSemanticToolInputEnvelope({ operationId: "read_assigned_skill", callId, + correlation, content: input }), input }; + const semanticResult = createPrpSemanticToolResultEnvelope({ operationId: "read_assigned_skill", callId, + correlation, content: result, outcome: "succeeded", code: "semantic_tool_succeeded", retryable: false, + authorizationBoundary: "active_task", operationReceiptId: `operation_${callId}` }); + // Genuine converter output has dynamic transport and no invented first-party namespace/readOnly hint. + const dynamic = ["item/started", "item/completed"].map(method => canonicalProviderEventsFromCodex(method, + { item: { id: callId, type: "dynamicToolCall", tool: "read_assigned_skill", status: "completed" } })[0]!); + const events = [event(1, dynamic[0]!.eventType, dynamic[0]!.payload, dynamic[0]!.itemId), + event(2, "semantic_tool.input", { semantic_tool: semanticInput }), + event(3, "semantic_tool.result", { semantic_tool: semanticResult }), + event(4, dynamic[1]!.eventType, dynamic[1]!.payload, dynamic[1]!.itemId)]; + return { events, run, assignment, result, input }; +} + +const read = (value: Awaited>, f = fixture) => + savedSkillReadReceipts(value.events, binding, f, undefined, value.assignment, value.run); +const semantic = (value: Awaited>, index: number) => + value.events[index]!.payload.prpEvent.payload.semantic_tool as Row; + +describe("saved-skill Codex native authority", () => { + it("accepts the complete real read's authoritative input/result commitment after API redaction", async () => { + const value = await observations(); + expect(RUNNER_BRIDGE_READ_OPERATIONS.has("read_assigned_skill")).toBe(true); + expect(Object.keys(value.result).sort()).toEqual(["byteSize", "nextOffset", "path", "sha256", "skill", "text", "versionId"]); + for (const row of value.events) expect(validatePrpEvent(row.payload.prpEvent).ok).toBe(true); + expect(value.events[0]!.payload.prpEvent.payload).toMatchObject({ transport: "dynamic", readOnly: null, namespace: null }); + expect(read(value)).toEqual([{ runId: "run", executionId: callId, seq: 3 }]); + }); + + it("checks UTF-8 source bytes and canonical key order using the real Unicode producer", async () => { + const text = `${fixture.markdown}\nUnicode: café 🌍\n`; + const value = await observations(text); + expect(value.result.byteSize).toBe(Buffer.byteLength(text)); + expect(value.result.byteSize).toBeGreaterThan(text.length); + expect(read(value, { ...fixture, markdown: text })).toHaveLength(1); + }); + + it("accepts Core's omitted defaults and only its exact admitted runtime alias", async () => { + const value = await observations(fixture.markdown, { skill: runtimeName }); + expect(read(value)).toHaveLength(1); + }); + + it("rejects raw dynamic calls even when they copy the complete source and invent readOnly/namespace", async () => { + const value = await observations(); + value.events = [value.events[0]!, value.events[3]!]; + for (const row of value.events) Object.assign(row.payload.prpEvent.payload, + { name: "paperclip_read_assigned_skill", readOnly: true, namespace: "paperclip", + output: JSON.stringify({ type: "inputText", text: JSON.stringify(value.result) }), outputTruncated: false }); + const p = value.events[1]!.payload.prpEvent.payload; p.outputBytes = Buffer.byteLength(p.output); + expect(read(value)).toEqual([]); + }); + + it.each([ + ["missing input", (v: any) => { v.events.splice(1, 1); }], + ["missing result", (v: any) => { v.events.splice(2, 1); }], + ["duplicate input", (v: any) => { v.events.push(structuredClone(v.events[1])); }], + ["duplicate result", (v: any) => { v.events.push(structuredClone(v.events[2])); }], + ["result before input", (v: any) => { v.events[2].seq = 0; }], + ["foreign provider", (v: any) => { v.run.runnerProfileJson.nativeExecutionInput.provider.kind = "unknown"; }], + ["foreign runner", (v: any) => { v.run.runnerInstanceId = "foreign"; }], + ["foreign session", (v: any) => { v.run.nativeSessionId = "foreign"; }], + ["missing runner identity", (v: any) => { delete v.run.runnerInstanceId; }], + ["missing session identity", (v: any) => { delete v.run.nativeSessionId; }], + ["foreign run owner", (v: any) => { v.run.companyId = "foreign"; }], + ["missing native assignment", (v: any) => { v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills = []; }], + ["duplicate native assignment", (v: any) => { v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills.push(v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0]); }], + ["foreign native version", (v: any) => { v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0].versionId = "foreign"; }], + ["missing selection snapshot", (v: any) => { v.run.contextSnapshot.skillSelectionSnapshot = false; }], + ["foreign admitted version", (v: any) => { v.run.contextSnapshot.skillVersions[key] = "foreign"; }], + ["foreign call", (v: any) => { semantic(v, 2).callId = "foreign"; }], + ["foreign turn", (v: any) => { semantic(v, 2).correlation.turnId = "foreign"; }], + ["foreign item", (v: any) => { semantic(v, 2).correlation.itemId = "foreign"; }], + ["foreign source", (v: any) => { v.events[2].sourceInstanceId = v.events[2].payload.prpEvent.sourceInstanceId = "foreign"; }], + ["foreign operation", (v: any) => { for (const i of [1, 2]) semantic(v, i).operationId = "paperclip_read_assigned_skill"; }], + ["MCP carriers", (v: any) => { for (const i of [1, 2]) v.events[i].eventType = v.events[i].payload.prpEvent.eventType = i === 1 ? "mcp_app.tool_input" : "mcp_app.tool_result"; }], + ["failed result", (v: any) => { semantic(v, 2).outcome = "failed"; }], + ["wrong boundary", (v: any) => { semantic(v, 2).authorizationBoundary = "company"; }], + ["wrong receipt", (v: any) => { semantic(v, 2).operationReceiptId = "foreign"; }], + ["wrong input digest", (v: any) => { semantic(v, 1).content.digest = `sha256:${"0".repeat(64)}`; }], + ["foreign input key", (v: any) => { semantic(v, 1).input.skill = "foreign"; semantic(v, 1).content.digest = digestPrpSemanticContent(semantic(v, 1).input); }], + ["wrong source path", (v: any) => { semantic(v, 1).input.path = "other.md"; semantic(v, 1).content.digest = digestPrpSemanticContent(semantic(v, 1).input); }], + ["nonzero offset", (v: any) => { semantic(v, 1).input.offset = 1; semantic(v, 1).content.digest = digestPrpSemanticContent(semantic(v, 1).input); }], + ["wrong result digest", (v: any) => { semantic(v, 2).content.digest = `sha256:${"0".repeat(64)}`; }], + ["truncated disclosure", (v: any) => { semantic(v, 2).content.redactionDisposition = "redacted"; }], + ["truncated marker", (v: any) => { semantic(v, 2).outputTruncated = true; }], + ["partial page", (v: any) => { semantic(v, 2).content.digest = digestPrpSemanticContent({ ...v.result, nextOffset: 1 }); }], + ["missing page completion", (v: any) => { const result = { ...v.result }; delete result.nextOffset; semantic(v, 2).content.digest = digestPrpSemanticContent(result); }], + ["wrong source bytes", (v: any) => { semantic(v, 2).content.digest = digestPrpSemanticContent({ ...v.result, byteSize: v.result.byteSize + 1 }); }], + ["wrong source checksum", (v: any) => { semantic(v, 2).content.digest = digestPrpSemanticContent({ ...v.result, sha256: "0".repeat(64) }); }], + ["wrong version", (v: any) => { semantic(v, 2).content.digest = digestPrpSemanticContent({ ...v.result, versionId: "foreign" }); }], + ["copied nonce only", (v: any) => { semantic(v, 2).content.digest = digestPrpSemanticContent({ ...v.result, text: fixture.expected.reference }); }], + ] as const)("rejects authoritative-looking evidence with %s", async (_label, mutate) => { + const value = await observations(); mutate(value); expect(read(value)).toEqual([]); + }); +}); diff --git a/tests/runner-e2e/product-journeys/saved-skill-use-evidence.test.ts b/tests/runner-e2e/product-journeys/saved-skill-use-evidence.test.ts new file mode 100644 index 00000000000..3841766ef97 --- /dev/null +++ b/tests/runner-e2e/product-journeys/saved-skill-use-evidence.test.ts @@ -0,0 +1,311 @@ +import { createHash } from "node:crypto"; +import { describe, expect, it } from "vitest"; +import { canonicalProviderEventsFromOpenCodePart } from "../../../packages/paperclip-runner/src/provider-events.js"; +import { savedSkillDocumentAttributed, savedSkillFixture, savedSkillReadAssignment, savedSkillReadReceipts, savedSkillReadRuntimeName, savedSkillResultMatches, savedSkillSourceMatches } from "./saved-skill-use-evidence.js"; + +type Row = Record; +const fixture = savedSkillFixture("campaign-id", "62ff548dfec748da8fef6d5fbf6cc23d"); +const binding = { companyId: "company", agentId: "agent", issueId: "issue", runId: "run" }; +const assignment = { key: `company/${binding.companyId}/${fixture.slug}`, versionId: "version" }; +function native(seq: number, eventType: string, payload: Row): Row { + return { ...binding, seq, eventType, protocolSchemaVersion: 1, sourceSeq: seq, sourceInstanceId: "runner-instance", sourceEventId: `event-${seq}`, + payload: { prpEvent: { schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", runId: binding.runId, + sourceSeq: seq, sourceInstanceId: "runner-instance", sourceEventId: `event-${seq}`, turnId: "turn", eventType, payload } } }; +} +function readEvents(): Row[] { + const payload = { schema: "paperclip.tool.execution.v1", executionId: "read-skill", name: "read", operation: "read", + transport: "builtin", readOnly: true, target: `/workspace/.agents/skills/${fixture.slug}/SKILL.md` }; + return [native(1, "tool.execution.started", { ...payload, status: "running" }), + native(2, "tool.execution.completed", { ...payload, status: "completed", output: fixture.markdown, outputBytes: Buffer.byteLength(fixture.markdown), outputTruncated: false })]; +} +const runtimeName = `${fixture.slug}--0123456789`; +function wrappedReadEvents(): Row[] { + const output = `/workspace/.claude/skills/${runtimeName}/SKILL.md\nfile\n\n` + + fixture.markdown.trimEnd().split("\n").map((line, index) => `${index + 1}: ${line}`).join("\n") + + "\n(End of file - total 12 lines)\n"; + const events = readEvents(); + for (const row of events) Object.assign(row.payload.prpEvent.payload, { operation: "unknown", target: null }); + Object.assign(events[1]!.payload.prpEvent.payload, { output, outputBytes: Buffer.byteLength(output) }); + return events; +} +function assignedReadEvents(): Row[] { + const result = { skill: `company/${binding.companyId}/${fixture.slug}`, versionId: "version", path: "SKILL.md", + text: fixture.markdown, sha256: createHash("sha256").update(fixture.markdown).digest("hex"), + byteSize: Buffer.byteLength(fixture.markdown), nextOffset: null }; + const output = JSON.stringify({ type: "inputText", text: JSON.stringify(result) }); + const events = readEvents(); + for (const row of events) Object.assign(row.payload.prpEvent.payload, + { name: "paperclip_read_assigned_skill", operation: "unknown", target: null }); + Object.assign(events[1]!.payload.prpEvent.payload, { output, outputBytes: Buffer.byteLength(output) }); + return events; +} +function changeAssignedResult(events: Row[], change: (value: Row) => void) { + const payload = events[1]!.payload.prpEvent.payload, wrapper = JSON.parse(payload.output), result = JSON.parse(wrapper.text); + change(result); wrapper.text = JSON.stringify(result); payload.output = JSON.stringify(wrapper); +} +function source() { + const skill: Row = { id: "skill", companyId: binding.companyId, key: `company/${binding.companyId}/${fixture.slug}`, sourceType: "managed", name: fixture.name, slug: fixture.slug, + markdown: fixture.markdown, currentVersionId: "version" }; + const version: Row = { id: "version", companyId: binding.companyId, companySkillId: skill.id, + fileInventory: [{ path: "SKILL.md", kind: "skill", content: fixture.markdown }] }; + const assignments: Row = { desiredSkillEntries: [{ key: skill.key, versionId: null }] }; + return { skill, version, assignments }; +} +function output() { + const document: Row = { id: "document", companyId: binding.companyId, issueId: binding.issueId, key: fixture.key, format: "markdown", + latestRevisionId: "revision", latestRevisionNumber: 1, createdByAgentId: binding.agentId, createdByUserId: null, + updatedByAgentId: binding.agentId, updatedByUserId: null, body: JSON.stringify(fixture.expected) }; + const revision: Row = { id: "revision", documentId: document.id, companyId: binding.companyId, issueId: binding.issueId, + key: fixture.key, format: "markdown", revisionNumber: 1, createdByAgentId: binding.agentId, createdByUserId: null, createdByRunId: binding.runId, + body: document.body }; + return { document, revision }; +} + +describe("saved skill use evidence", () => { + it("keeps the random output reference in the skill, independently of its discoverable name", () => { + expect(fixture.markdown).toContain(fixture.expected.reference); + expect(fixture.name + fixture.slug).not.toContain(fixture.expected.reference); + const owned = savedSkillFixture("journey-1234-5678", fixture.expected.reference); + expect(owned.name).toBe(owned.slug); + expect(owned.name).toContain("journey-1234-5678"); + expect(owned.markdown).toContain(`\nname: ${owned.name}\n`); + expect(fixture.markdown).not.toContain("\\n"); + expect(() => savedSkillFixture("", fixture.expected.reference)).toThrow(); + expect(() => savedSkillFixture(fixture.expected.reference, fixture.expected.reference)).toThrow(); + expect(() => savedSkillFixture("campaign", "short")).toThrow(); + }); + it("requires an ordinary Latest assignment, its exact managed current version and original source bytes", () => { + const good = source(); + expect(savedSkillSourceMatches(good.skill, good.version, good.assignments, fixture, binding.companyId)).toBe(true); + const changes: Array<(value: ReturnType) => void> = [ + v => { v.skill.companyId = "foreign"; }, v => { v.version.companyId = "foreign"; }, + v => { v.version.companySkillId = "foreign"; }, v => { v.skill.slug = "foreign"; }, + v => { v.skill.name = "different saved name"; }, + v => { v.skill.sourceType = "local_path"; }, + v => { v.skill.key = "unknown"; v.assignments.desiredSkillEntries[0].key = "unknown"; }, + v => { v.skill.currentVersionId = "different"; }, + v => { v.skill.markdown += "changed"; }, v => { v.version.id = "stale"; }, + v => { v.version.fileInventory = []; }, v => { v.version.fileInventory[0].content += "changed"; }, + v => { v.version.fileInventory[0].encoding = "base64"; }, v => { v.assignments.desiredSkillEntries = []; }, + v => { v.assignments.desiredSkillEntries[0].versionId = "version"; }, + v => { v.assignments.desiredSkillEntries[0].versionId = "stale"; }, + v => { delete v.assignments.desiredSkillEntries[0].versionId; }, + v => { v.assignments.desiredSkillEntries[0].key = "foreign"; }, + v => { v.assignments.desiredSkillEntries.push(v.assignments.desiredSkillEntries[0]); }, + ]; + for (const change of changes) { + const wrong = source(); change(wrong); + expect(savedSkillSourceMatches(wrong.skill, wrong.version, wrong.assignments, fixture, binding.companyId)).toBe(false); + } + expect(savedSkillSourceMatches(undefined, good.version, good.assignments, fixture, binding.companyId)).toBe(false); + }); + it("accepts the bound successful source read and rejects missing, foreign, claimed and failed reads", () => { + expect(savedSkillReadReceipts(readEvents(), binding, fixture)).toEqual([{ runId: "run", executionId: "read-skill", seq: 2 }]); + const changes: Array<(events: Row[]) => void> = [ + e => { e.splice(0, 1); }, e => { e.pop(); }, e => { e[1].runId = "foreign"; }, + e => { e[1].companyId = "foreign"; }, e => { e[1].agentId = "foreign"; }, + e => { e[1].payload.prpEvent.runId = "foreign"; }, e => { e[1].payload.prpEvent.sourceKind = "provider"; }, + e => { e[1].sourceInstanceId = "foreign"; }, e => { e[1].sourceSeq += 1; }, + e => { e[1].payload.prpEvent.turnId = "foreign"; }, e => { e[1].payload.prpEvent.payload.status = "failed"; }, + e => { for (const row of e) delete row.payload.prpEvent.turnId; }, + e => { e[1].payload.prpEvent.payload.outputBytes = 0; }, e => { e[1].payload.prpEvent.payload.outputTruncated = true; }, + e => { delete e[1].payload.prpEvent.payload.outputTruncated; }, + e => { e[1].payload.prpEvent.payload.output = "I read the skill and followed it."; }, + e => { for (const row of e) row.payload.prpEvent.payload.target = `/workspace/other/${fixture.slug}/../SKILL.md`; }, + e => { for (const row of e) row.payload.prpEvent.payload.target = "/workspace/other-skill/SKILL.md"; }, + e => { for (const row of e) row.payload.prpEvent.payload.target = null; }, + e => { for (const row of e) Object.assign(row.payload.prpEvent.payload, { name: "bash", operation: "execute", transport: "process" }); }, + e => { e[1].payload.prpEvent.payload.executionId = "other-read"; }, + e => { e[1].payload.prpEvent.eventType = e[1].eventType = "item.completed"; }, + ]; + for (const change of changes) { + const wrong = readEvents(); change(wrong); + expect(savedSkillReadReceipts(wrong, binding, fixture)).toEqual([]); + } + expect(savedSkillReadReceipts([], binding, fixture)).toEqual([]); + }); + it("accepts the deployed OpenCode read wrapper for its exact version-bound runtime skill directory", () => { + expect(savedSkillReadReceipts(wrappedReadEvents(), binding, fixture, runtimeName)).toHaveLength(1); + expect(savedSkillReadReceipts(wrappedReadEvents(), binding, fixture)).toEqual([]); + }); + it("accepts the complete first-party assigned-skill receipt with exact immutable source identity and bytes", () => { + expect(savedSkillReadReceipts(assignedReadEvents(), binding, fixture, runtimeName, assignment)) + .toEqual([{ runId: "run", executionId: "read-skill", seq: 2 }]); + expect(savedSkillReadReceipts(assignedReadEvents(), binding, fixture, undefined, assignment)).toHaveLength(1); + expect(savedSkillReadReceipts(assignedReadEvents(), binding, fixture, runtimeName)).toEqual([]); + expect(savedSkillReadReceipts(assignedReadEvents(), binding, fixture, runtimeName, { ...assignment, key: "foreign" })).toEqual([]); + expect(savedSkillReadReceipts(assignedReadEvents(), binding, fixture, runtimeName, { ...assignment, versionId: "foreign" })).toEqual([]); + }); + it.each([{ kind: "codex" }, { kind: "acpx", agent: "cursor" }, { kind: "cursor_cloud" }])( + "binds the semantic source to common native skill entries for %j while preserving the OpenCode file guard", provider => { + const { skill } = source(); + const good = () => ({ assignments: { entries: [{ key: skill.key, versionId: "version", runtimeName }] }, + run: { contextSnapshot: { skillSelectionSnapshot: true, skillVersions: { [skill.key]: "version" } }, + runnerProfileJson: { nativeExecutionInput: { provider, + runtimeContext: { skills: [{ key: skill.key, versionId: "version", runtimeName }] } } } } }); + const valid = good(), selected = savedSkillReadAssignment(skill, valid.assignments, valid.run); + expect(selected).toEqual(assignment); + expect(savedSkillReadRuntimeName(skill, valid.assignments, valid.run)).toBeUndefined(); + expect(savedSkillReadReceipts(assignedReadEvents(), binding, fixture, undefined, selected)).toHaveLength(1); + expect(savedSkillReadReceipts(wrappedReadEvents(), binding, fixture, undefined, selected)).toEqual([]); + const changes: Array<(value: ReturnType) => void> = [ + v => { v.assignments.entries = []; }, + v => { v.assignments.entries[0]!.key = "foreign"; }, + v => { v.assignments.entries[0]!.versionId = "foreign"; }, + v => { v.assignments.entries.push(v.assignments.entries[0]!); }, + v => { v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills = []; }, + v => { v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0]!.key = "foreign"; }, + v => { v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0]!.versionId = "foreign"; }, + v => { v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills.push(v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0]!); }, + v => { v.run.contextSnapshot.skillSelectionSnapshot = false; }, + v => { v.run.contextSnapshot.skillVersions = {}; }, + v => { v.run.contextSnapshot.skillVersions[skill.key] = "foreign"; }, + ]; + for (const mutate of changes) { + const wrong = good(); mutate(wrong); + const selected = savedSkillReadAssignment(skill, wrong.assignments, wrong.run); + expect(selected).toBeUndefined(); + expect(savedSkillReadReceipts(assignedReadEvents(), binding, fixture, undefined, selected)).toEqual([]); + } + }); + it.each([ + ["foreign canonical key", (e: Row[]) => changeAssignedResult(e, r => { r.skill = "company/foreign/skill"; })], + ["foreign immutable version", (e: Row[]) => changeAssignedResult(e, r => { r.versionId = "foreign"; })], + ["wrong file", (e: Row[]) => changeAssignedResult(e, r => { r.path = "other/SKILL.md"; })], + ["incomplete page", (e: Row[]) => changeAssignedResult(e, r => { r.nextOffset = r.text.length; })], + ["missing pagination receipt", (e: Row[]) => changeAssignedResult(e, r => { delete r.nextOffset; })], + ["wrong checksum", (e: Row[]) => changeAssignedResult(e, r => { r.sha256 = "0".repeat(64); })], + ["wrong source byte count", (e: Row[]) => changeAssignedResult(e, r => { r.byteSize++; })], + ["partial procedure", (e: Row[]) => changeAssignedResult(e, r => { r.text = fixture.expected.reference; })], + ["changed full source", (e: Row[]) => changeAssignedResult(e, r => { r.text += "extra source"; })], + ["semantic failure with copied source", (e: Row[]) => changeAssignedResult(e, r => { r.outcome = "failed"; r.code = "runner_skill_not_assigned"; })], + ["shell copied receipt", (e: Row[]) => { for (const row of e) row.payload.prpEvent.payload.name = "bash"; }], + ["another first-party tool", (e: Row[]) => { for (const row of e) row.payload.prpEvent.payload.name = "paperclip_read_document"; }], + ["non-builtin transport", (e: Row[]) => { for (const row of e) row.payload.prpEvent.payload.transport = "process"; }], + ["mutating operation", (e: Row[]) => { for (const row of e) row.payload.prpEvent.payload.operation = "edit"; }], + ["not read only", (e: Row[]) => { e[1]!.payload.prpEvent.payload.readOnly = false; }], + ["start not read only", (e: Row[]) => { e[0]!.payload.prpEvent.payload.readOnly = false; }], + ["failed completion", (e: Row[]) => { e[1]!.payload.prpEvent.payload.status = "failed"; }], + ["truncated completion", (e: Row[]) => { e[1]!.payload.prpEvent.payload.outputTruncated = true; }], + ["missing start", (e: Row[]) => { e.splice(0, 1); }], + ["call mismatch", (e: Row[]) => { e[0]!.payload.prpEvent.payload.executionId = "another-call"; }], + ["foreign runner", (e: Row[]) => { e[0]!.sourceInstanceId = e[0]!.payload.prpEvent.sourceInstanceId = "another-runner"; }], + ["foreign run", (e: Row[]) => { e[1]!.runId = e[1]!.payload.prpEvent.runId = "another-run"; }], + ["foreign company", (e: Row[]) => { e[1]!.companyId = "another-company"; }], + ["foreign turn", (e: Row[]) => { e[1]!.payload.prpEvent.turnId = "another-turn"; }], + ["invalid wrapper", (e: Row[]) => { e[1]!.payload.prpEvent.payload.output = fixture.markdown; }], + ["wrong wrapper type", (e: Row[]) => { const p = e[1]!.payload.prpEvent.payload; p.output = p.output.replace("inputText", "outputText"); }], + ] as const)("rejects a first-party assigned-skill receipt with %s", (_label, mutate) => { + const events = assignedReadEvents(); mutate(events); + const completed = events.find(row => row.eventType === "tool.execution.completed")!; + completed.payload.prpEvent.payload.outputBytes = Buffer.byteLength(completed.payload.prpEvent.payload.output); + expect(savedSkillReadReceipts(events, binding, fixture, runtimeName, assignment)).toEqual([]); + }); + it.each([ + ["foreign path", (e: Row[]) => { e[1]!.payload.prpEvent.payload.output = e[1]!.payload.prpEvent.payload.output.replace(runtimeName, "foreign"); }], + ["relative path", (e: Row[]) => { e[1]!.payload.prpEvent.payload.output = e[1]!.payload.prpEvent.payload.output.replace("/workspace/", "workspace/"); }], + ["path traversal", (e: Row[]) => { e[1]!.payload.prpEvent.payload.output = e[1]!.payload.prpEvent.payload.output.replace("/workspace/", "/workspace/../"); }], + ["wrong file type", (e: Row[]) => { e[1]!.payload.prpEvent.payload.output = e[1]!.payload.prpEvent.payload.output.replace("file", "directory"); }], + ["missing wrapper", (e: Row[]) => { e[1]!.payload.prpEvent.payload.output = fixture.procedure; }], + ["partial procedure", (e: Row[]) => { e[1]!.payload.prpEvent.payload.output = e[1]!.payload.prpEvent.payload.output.replace("Do not create other tasks or deliverables.", ""); }], + ["copied shell output", (e: Row[]) => { for (const row of e) row.payload.prpEvent.payload.name = "bash"; }], + ["execute operation", (e: Row[]) => { for (const row of e) row.payload.prpEvent.payload.operation = "execute"; }], + ["not read only", (e: Row[]) => { e[1]!.payload.prpEvent.payload.readOnly = false; }], + ["start not read only", (e: Row[]) => { e[0]!.payload.prpEvent.payload.readOnly = false; }], + ["failed read", (e: Row[]) => { e[1]!.payload.prpEvent.payload.status = "failed"; }], + ["truncated read", (e: Row[]) => { e[1]!.payload.prpEvent.payload.outputTruncated = true; }], + ["missing start", (e: Row[]) => { e.shift(); }], + ["call mismatch", (e: Row[]) => { e[0]!.payload.prpEvent.payload.executionId = "another-call"; }], + ["foreign runner", (e: Row[]) => { e[0]!.sourceInstanceId = e[0]!.payload.prpEvent.sourceInstanceId = "another-runner"; }], + ["foreign run", (e: Row[]) => { e[1]!.runId = e[1]!.payload.prpEvent.runId = "another-run"; }], + ["foreign turn", (e: Row[]) => { e[1]!.payload.prpEvent.turnId = "another-turn"; }], + ] as const)("rejects a wrapped native read with %s", (_label, mutate) => { + const events = wrappedReadEvents(); mutate(events); + const completed = events.find(row => row.eventType === "tool.execution.completed")!; + completed.payload.prpEvent.payload.outputBytes = Buffer.byteLength(completed.payload.prpEvent.payload.output); + expect(savedSkillReadReceipts(events, binding, fixture, runtimeName)).toEqual([]); + }); + it("requires both API and admitted native input to bind the same immutable skill runtime name", () => { + const { skill } = source(); + const good = () => ({ assignments: { entries: [{ key: skill.key, runtimeName, versionId: "version" }] }, + run: { contextSnapshot: { skillSelectionSnapshot: true, skillVersions: { [skill.key]: "version" } }, + runnerProfileJson: { nativeExecutionInput: { provider: { kind: "opencode" }, + runtimeContext: { skills: [{ key: skill.key, runtimeName, versionId: "version" }] } } } } }); + const valid = good(); + expect(savedSkillReadRuntimeName(skill, valid.assignments, valid.run)).toBe(runtimeName); + const changes: Array<(value: ReturnType) => void> = [ + v => { v.assignments.entries = []; }, + v => { v.assignments.entries[0]!.key = "foreign"; }, + v => { v.assignments.entries[0]!.versionId = "stale"; }, + v => { v.assignments.entries[0]!.runtimeName = "foreign"; }, + v => { v.assignments.entries.push(v.assignments.entries[0]!); }, + v => { v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills = []; }, + v => { v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0]!.key = "foreign"; }, + v => { v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0]!.versionId = "stale"; }, + v => { v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0]!.runtimeName = "foreign"; }, + v => { v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills.push(v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0]!); }, + v => { v.run.contextSnapshot.skillSelectionSnapshot = false; }, + v => { v.run.contextSnapshot.skillVersions[skill.key] = "stale"; }, + v => { v.run.runnerProfileJson.nativeExecutionInput.provider.kind = "other"; }, + v => { v.assignments.entries[0]!.runtimeName = v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0]!.runtimeName = "../foreign"; }, + ]; + for (const mutate of changes) { + const wrong = good(); mutate(wrong); + const name = savedSkillReadRuntimeName(skill, wrong.assignments, wrong.run); + expect(name).toBeUndefined(); + expect(savedSkillReadReceipts(wrappedReadEvents(), binding, fixture, name)).toEqual([]); + } + }); + it("accepts the real OpenCode native skill-loader mapping, including its completed provider output", () => { + const part = (status: string) => ({ id: "load-skill", type: "tool", tool: "skill", state: { + status, output: `\n${fixture.procedure}\n`, + } }); + const events = ["pending", "completed"].map((status, index) => { + const event = canonicalProviderEventsFromOpenCodePart(part(status))[0]!; + return native(index + 1, event.eventType, event.payload); + }); + expect(savedSkillReadReceipts(events, binding, fixture)).toHaveLength(1); + for (const output of [fixture.procedure, `\n${fixture.procedure}\n`]) { + const wrong = structuredClone(events); wrong[1]!.payload.prpEvent.payload.output = output; + wrong[1]!.payload.prpEvent.payload.outputBytes = Buffer.byteLength(output); + expect(savedSkillReadReceipts(wrong, binding, fixture)).toEqual([]); + } + }); + it("allows known file-reader line prefixes without treating partial output as a complete read", () => { + const events = readEvents(); + events[1]!.payload.prpEvent.payload.output = fixture.markdown.split("\n").map((line, i) => `${i + 1}: ${line}`).join("\n"); + events[1]!.payload.prpEvent.payload.outputBytes = Buffer.byteLength(events[1]!.payload.prpEvent.payload.output); + expect(savedSkillReadReceipts(events, binding, fixture)).toHaveLength(1); + events[1]!.payload.prpEvent.payload.output = fixture.expected.reference; + events[1]!.payload.prpEvent.payload.outputBytes = Buffer.byteLength(fixture.expected.reference); + expect(savedSkillReadReceipts(events, binding, fixture)).toEqual([]); + }); + it("requires the exact derived result and a saved revision authored by this run", () => { + const good = output(); + expect(savedSkillResultMatches(`\u0060\u0060\u0060json\n${good.document.body}\n\u0060\u0060\u0060`, fixture.expected)).toBe(true); + expect(savedSkillDocumentAttributed(good.document, good.revision, fixture, binding)).toBe(true); + for (const body of ["I finished", "{}", JSON.stringify({ ...fixture.expected, extra: true }), + JSON.stringify({ ...fixture.expected, total: fixture.expected.total + 1 }), JSON.stringify({ ...fixture.expected, reference: "foreign" })]) { + expect(savedSkillResultMatches(body, fixture.expected)).toBe(false); + } + const changes: Array<(value: ReturnType) => void> = [ + v => { v.document.companyId = "foreign"; }, v => { v.document.issueId = "foreign"; }, + v => { v.document.createdByAgentId = "foreign"; }, v => { v.document.updatedByAgentId = "foreign"; }, + v => { v.document.createdByUserId = "owner"; }, v => { v.document.updatedByUserId = "owner"; }, + v => { v.document.key = "other"; }, v => { v.document.body = "{}"; }, + v => { v.document.latestRevisionId = null; }, v => { v.revision.id = "old"; }, + v => { delete v.document.latestRevisionNumber; delete v.revision.revisionNumber; }, + v => { v.revision.documentId = "foreign"; }, v => { v.revision.companyId = "foreign"; }, + v => { v.revision.issueId = "foreign"; }, v => { v.revision.key = "other"; }, + v => { v.revision.format = "unknown"; }, + v => { v.revision.createdByRunId = "foreign"; }, v => { v.revision.createdByAgentId = "foreign"; }, + v => { v.revision.createdByUserId = "owner"; }, v => { v.revision.revisionNumber = 2; }, + v => { v.revision.body = "{}"; }, + ]; + for (const change of changes) { + const wrong = output(); change(wrong); + expect(savedSkillDocumentAttributed(wrong.document, wrong.revision, fixture, binding)).toBe(false); + } + expect(savedSkillDocumentAttributed(undefined, good.revision, fixture, binding)).toBe(false); + }); +}); diff --git a/tests/runner-e2e/product-journeys/saved-skill-use-evidence.ts b/tests/runner-e2e/product-journeys/saved-skill-use-evidence.ts new file mode 100644 index 00000000000..0f4ede10092 --- /dev/null +++ b/tests/runner-e2e/product-journeys/saved-skill-use-evidence.ts @@ -0,0 +1,238 @@ +import { createHash } from "node:crypto"; +import { isBoundNativePrpEventRow, type NativePrpRunBinding } from "../native-event-envelope.js"; + +type Row = Record; +export interface SavedSkillFixture { + name: string; slug: string; key: "saved-skill-result"; markdown: string; procedure: string; + expected: { reference: string; total: number }; +} + +/** The public request names the skill; only its saved version contains these inputs. */ +export function savedSkillFixture(identity: string, reference: string): SavedSkillFixture { + if (!/^[a-f0-9]{32}$/.test(reference)) throw new Error("invalid_saved_skill_reference"); + if (!/^[a-z][a-z0-9]*(?:-[a-z0-9]+)*$/.test(identity) || identity.length > 48 || identity.includes(reference)) + throw new Error("invalid_saved_skill_identity"); + const slug = `qa-saved-skill-${identity}`, key = "saved-skill-result"; + const seed = createHash("sha256").update(reference).digest(); + const first = seed[0]! + 1, second = seed[1]! + 1; + const procedure = `# Saved receipt procedure\n\nThe receipt reference is ${reference}.\n` + + `Add the two amounts ${first} and ${second}.\n` + + `Save exactly one Paperclip task document with key ${key}. Its body must be JSON with exactly ` + + `two fields: reference (the receipt reference) and total (the sum of the two amounts).\n` + + "Check the saved document and finish the task. Do not create other tasks or deliverables.\n"; + return { name: slug, slug, key, procedure, + markdown: `---\nname: ${slug}\ndescription: Make a receipt using this saved procedure.\n---\n\n${procedure}`, + expected: { reference, total: first + second } }; +} + +export function savedSkillSourceMatches(skill: Row | undefined, version: Row | undefined, assignments: Row | undefined, + fixture: SavedSkillFixture, companyId: string): boolean { + if (!skill || !version || !assignments) return false; + const files = version.fileInventory; + const entries = assignments.desiredSkillEntries; + return Boolean(skill.id && skill.companyId === companyId && skill.sourceType === "managed" + && skill.name === fixture.name && skill.slug === fixture.slug && skill.key === `company/${companyId}/${fixture.slug}` + && skill.markdown === fixture.markdown && skill.currentVersionId + && version.id === skill.currentVersionId && version.companyId === companyId && version.companySkillId === skill.id + && Array.isArray(files) && files.length === 1 && files[0].path === "SKILL.md" && files[0].content === fixture.markdown + && (files[0].encoding === undefined || files[0].encoding === "utf8") + && Array.isArray(entries) && entries.filter(row => row.key === skill.key).length === 1 + && entries.find(row => row.key === skill.key)?.versionId === null); +} + +export function savedSkillResultMatches(body: unknown, expected: SavedSkillFixture["expected"]): boolean { + if (typeof body !== "string") return false; + const text = body.trim(), json = /^```(?:json)?\s*\n([\s\S]*?)\n```$/.exec(text)?.[1] ?? text; + try { + const value = JSON.parse(json); + return Boolean(value && typeof value === "object" && !Array.isArray(value) + && Object.keys(value).sort().join(",") === "reference,total" + && value.reference === expected.reference && value.total === expected.total); + } catch { return false; } +} + +export interface SavedSkillReadAssignment { key: string; versionId: string } + +/** All native providers share the canonical skill/version selection contract. */ +export function savedSkillReadAssignment(skill: Row, assignments: Row, run: Row): SavedSkillReadAssignment | undefined { + const input = run.runnerProfileJson?.nativeExecutionInput; + const assigned = Array.isArray(assignments.entries) ? assignments.entries.filter((row: Row) => row.key === skill.key) : []; + const native = Array.isArray(input?.runtimeContext?.skills) + ? input.runtimeContext.skills.filter((row: Row) => row.key === skill.key) : []; + const versions = run.contextSnapshot?.skillVersions; + if (typeof skill.key !== "string" || !skill.key.trim() || assigned.length !== 1 || native.length !== 1 + || typeof skill.currentVersionId !== "string" || !skill.currentVersionId + || run.contextSnapshot?.skillSelectionSnapshot !== true + || !versions || typeof versions !== "object" || Array.isArray(versions) || !Object.hasOwn(versions, skill.key) + || versions[skill.key] !== skill.currentVersionId + || assigned[0].versionId !== skill.currentVersionId || native[0].versionId !== skill.currentVersionId) return undefined; + return { key: skill.key, versionId: skill.currentVersionId }; +} + +/** OpenCode's file wrapper also needs the exact materialized directory identity. */ +export function savedSkillReadRuntimeName(skill: Row, assignments: Row, run: Row): string | undefined { + if (run.runnerProfileJson?.nativeExecutionInput?.provider?.kind !== "opencode" + || !savedSkillReadAssignment(skill, assignments, run)) return undefined; + const assigned = assignments.entries.find((row: Row) => row.key === skill.key); + const native = run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills.find((row: Row) => row.key === skill.key); + if (assigned.runtimeName !== native.runtimeName + || typeof native.runtimeName !== "string" || !/^[a-z0-9][a-z0-9_-]*$/i.test(native.runtimeName)) return undefined; + return native.runtimeName; +} + +/** Core's assigned-skill read verifies the pinned manifest and returns one UTF-8 page. */ +function assignedSkillReadText(output: string, binding: NativePrpRunBinding, fixture: SavedSkillFixture, versionId: string): string | undefined { + try { + const wrapper = JSON.parse(output); + if (wrapper?.type !== "inputText" || typeof wrapper.text !== "string" + || Object.keys(wrapper).sort().join(",") !== "text,type") return undefined; + const result = JSON.parse(wrapper.text); + if (Object.keys(result).sort().join(",") !== "byteSize,nextOffset,path,sha256,skill,text,versionId" + || result.skill !== `company/${binding.companyId}/${fixture.slug}` || result.versionId !== versionId + || result.path !== "SKILL.md" || result.nextOffset !== null || result.text !== fixture.markdown + || result.byteSize !== Buffer.byteLength(fixture.markdown) + || result.sha256 !== createHash("sha256").update(fixture.markdown).digest("hex")) return undefined; + return result.text; + } catch { return undefined; } +} + +// Core091 provider_bridge.rs hashes UTF-8 JSON with object keys ordered by UTF-16. +// This oracle reconstructs the complete pinned read result, not provider display text. +function semanticDigest(value: unknown): string { + const canonical = (value: unknown): string => Array.isArray(value) ? `[${value.map(canonical).join(",")}]` + : value && typeof value === "object" ? `{${Object.keys(value).sort().map(key => + `${JSON.stringify(key)}:${canonical((value as Row)[key])}`).join(",")}}` : JSON.stringify(value) ?? "null"; + return `sha256:${createHash("sha256").update(canonical(value), "utf8").digest("hex")}`; +} + +/** Codex's raw dynamic name, namespace and optional readOnly hint grant no authority. + * Its admitted first-party read instead has a controller-bound semantic input/result. + * Core's read_assigned_skill operation is read-only and verifies the immutable file. + */ +function codexAssignedSkillReceipts(bound: Row[], binding: NativePrpRunBinding, fixture: SavedSkillFixture, + assignment: SavedSkillReadAssignment | undefined, run: Row | undefined) { + const input = run?.runnerProfileJson?.nativeExecutionInput; + const skills = Array.isArray(input?.runtimeContext?.skills) + ? input.runtimeContext.skills.filter((skill: Row) => skill.key === assignment?.key) : []; + if (!run || run.runtimeMode !== "native" || input?.provider?.kind !== "codex" + || run.id !== binding.runId || run.companyId !== binding.companyId || run.agentId !== binding.agentId + || typeof run.runnerInstanceId !== "string" || !run.runnerInstanceId.trim() + || typeof run.nativeSessionId !== "string" || !run.nativeSessionId.trim() + || !assignment || assignment.key !== `company/${binding.companyId}/${fixture.slug}` || !assignment.versionId + || skills.length !== 1 || skills[0].versionId !== assignment.versionId + || run.contextSnapshot?.skillSelectionSnapshot !== true + || run.contextSnapshot?.skillVersions?.[assignment.key] !== assignment.versionId + || !fixture.markdown.includes(fixture.procedure.trim()) || !fixture.markdown.includes(fixture.expected.reference)) return []; + const expected = { skill: assignment.key, versionId: assignment.versionId, path: "SKILL.md", text: fixture.markdown, + sha256: createHash("sha256").update(fixture.markdown, "utf8").digest("hex"), + byteSize: Buffer.byteLength(fixture.markdown), nextOffset: null }; + const semantic = (row: Row): Row | undefined => row.payload.prpEvent.payload?.semantic_tool; + const content = (value: Row, digest: string) => value.content?.digest === digest + && value.content.redactionDisposition === "digest_only" && Array.isArray(value.content.references) + && value.content.references.length === 0 + && Object.keys(value.content).sort().join(",") === "digest,redactionDisposition,references"; + const correlated = (row: Row, phase: "input" | "result") => { + const event = row.payload.prpEvent, value = semantic(row), correlation = value?.correlation; + const keys = ["schema", "schemaVersion", "phase", "operationId", "callId", "correlation", "idempotencyKey", "content", + ...(phase === "input" ? ["input"] : ["outcome", "code", "retryable", "authorizationBoundary", "operationReceiptId"])]; + return Boolean(value && event.eventType === `semantic_tool.${phase}` && value.phase === phase + && value.schema === "paperclip.prp.semantic_tool.v1" && value.schemaVersion === 1 + && Object.keys(value).sort().join(",") === keys.sort().join(",") + && value.operationId === "read_assigned_skill" && typeof value.callId === "string" && value.callId.trim() + && value.idempotencyKey === null && Number.isSafeInteger(row.seq) + && event.sourceInstanceId === run.runnerInstanceId && event.normalizedSessionId === run.nativeSessionId + && typeof event.turnId === "string" && event.turnId.trim() && typeof event.itemId === "string" && event.itemId.trim() + && correlation && Object.keys(correlation).sort().join(",") === "itemId,normalizedSessionId,runId,turnId" + && correlation.runId === binding.runId && correlation.normalizedSessionId === run.nativeSessionId + && correlation.turnId === event.turnId && correlation.itemId === event.itemId); + }; + return bound.flatMap(start => { + if (!correlated(start, "input")) return []; + const initial = semantic(start)!, args = initial.input; + if (!args || typeof args !== "object" || Array.isArray(args) + || Object.keys(args).some(key => !["skill", "path", "offset"].includes(key)) + || !(args.skill === assignment.key || typeof skills[0].runtimeName === "string" + && skills[0].runtimeName.trim() && args.skill === skills[0].runtimeName) + || Object.hasOwn(args, "path") && args.path !== "SKILL.md" + || Object.hasOwn(args, "offset") && args.offset !== 0 || !content(initial, semanticDigest(args))) return []; + // Duplicates, reconciled alternatives and foreign same-call carriers cannot supply a second interpretation. + const pair = bound.filter(row => semantic(row)?.callId === initial.callId); + const end = pair.find(row => correlated(row, "result")); + if (pair.length !== 2 || !end || start.seq >= end.seq + || start.payload.prpEvent.sourceSeq >= end.payload.prpEvent.sourceSeq) return []; + const result = semantic(end)!; + if (semanticDigest(initial.correlation) !== semanticDigest(result.correlation) + || result.outcome !== "succeeded" || result.code !== "semantic_tool_succeeded" || result.retryable !== false + || result.authorizationBoundary !== "active_task" || result.operationReceiptId !== `operation_${initial.callId}` + || !content(result, semanticDigest(expected))) return []; + return [{ runId: binding.runId, executionId: String(initial.callId), seq: Number(end.seq) }]; + }); +} + +/** Successful built-in reads and native skill loads expose the actual skill text. + * A model claim, discovered name, shell echo, failed read or foreign run cannot qualify. */ +export function savedSkillReadReceipts(events: Row[], binding: NativePrpRunBinding, fixture: SavedSkillFixture, + runtimeName?: string, assignment?: SavedSkillReadAssignment, run?: Row) { + const bound = events.filter(row => isBoundNativePrpEventRow(row, binding)); + const builtin = bound.filter(row => { + const event = row.payload.prpEvent, payload = event.payload; + if (event.eventType !== "tool.execution.completed" || payload?.schema !== "paperclip.tool.execution.v1" + || payload.transport !== "builtin" || payload.status !== "completed" + || typeof event.turnId !== "string" || !event.turnId.trim() + || typeof payload.executionId !== "string" || !payload.executionId.trim() + || payload.outputTruncated !== false || typeof payload.output !== "string" || payload.outputBytes <= 0 + || payload.outputBytes !== Buffer.byteLength(payload.output)) return false; + const name = String(payload.name ?? "").toLowerCase(); + const target = typeof payload.target === "string" ? payload.target.replaceAll("\\", "/") : ""; + const reading = payload.operation === "read" && payload.readOnly === true && /^(read|read_file|file_read)$/.test(name) + && !target.split("/").includes("..") && target.endsWith(`/${fixture.slug}/SKILL.md`); + // OpenCode's skill tool names the loaded skill in its provider-visible result. + const loading = payload.operation === "execute" && name === "skill" + && payload.output.includes(``); + // Deployed OpenCode reports this read with operation unknown and target null. + // Its successful file-result wrapper supplies the exact materialized source path. + const wrapped = /^([^<>\r\n]+)<\/path>\r?\nfile<\/type>\r?\n\r?\n([\s\S]*)\r?\n<\/content>\s*$/.exec(payload.output); + const wrappedPath = wrapped?.[1]; + const wrappedRead = name === "read" && payload.readOnly === true && payload.target === null + && (payload.operation === "unknown" || payload.operation === "read") + && typeof runtimeName === "string" && /^[a-z0-9][a-z0-9_-]*$/i.test(runtimeName) + && wrappedPath?.startsWith("/") && !wrappedPath.includes("\\") && !wrappedPath.includes("\0") + && !wrappedPath.split("/").some(part => part === "." || part === "..") + && wrappedPath.endsWith(`/.claude/skills/${runtimeName}/SKILL.md`); + // The ordinary semantic tool is another native read path, not a filesystem wrapper. + // Its canonical key/version binding does not depend on a provider's directory name. + const assignedText = name === "paperclip_read_assigned_skill" && payload.readOnly === true && payload.target === null + && (payload.operation === "unknown" || payload.operation === "read") + && assignment?.key === `company/${binding.companyId}/${fixture.slug}` + && typeof assignment.versionId === "string" && assignment.versionId.trim() + ? assignedSkillReadText(payload.output, binding, fixture, assignment.versionId) : undefined; + if (!reading && !loading && !wrappedRead && assignedText === undefined) return false; + // File readers may prefix each source line with its line number. + const output = (assignedText ?? (wrappedRead ? wrapped![2]! : payload.output)).split("\n").map((line: string) => line.replace(/^\s*\d+(?:[:|]|→)\s?/, "")).join("\n"); + if (!output.includes(fixture.procedure.trim())) return false; + return bound.some(start => { + const initial = start.payload.prpEvent, p = initial.payload; + return initial.eventType === "tool.execution.started" && start.seq < row.seq + && initial.sourceInstanceId === event.sourceInstanceId && initial.turnId === event.turnId + && p?.schema === payload.schema && p.executionId === payload.executionId && p.name === payload.name + && p.transport === payload.transport && p.operation === payload.operation && p.status === "running" + && (!(wrappedRead || assignedText !== undefined) || p.readOnly === true); + }); + }).map(row => ({ runId: binding.runId, executionId: String(row.payload.prpEvent.payload.executionId), seq: Number(row.seq) })); + return [...builtin, ...codexAssignedSkillReceipts(bound, binding, fixture, assignment, run)]; +} + +export function savedSkillDocumentAttributed(document: Row | undefined, revision: Row | undefined, fixture: SavedSkillFixture, + binding: NativePrpRunBinding & { issueId: string }): boolean { + if (!document || !revision) return false; + return Boolean(document.id && document.companyId === binding.companyId && document.issueId === binding.issueId + && document.key === fixture.key && document.format === "markdown" && document.latestRevisionId + && Number.isSafeInteger(document.latestRevisionNumber) && document.latestRevisionNumber > 0 + && document.createdByAgentId === binding.agentId && document.createdByUserId === null + && document.updatedByAgentId === binding.agentId && document.updatedByUserId === null + && savedSkillResultMatches(document.body, fixture.expected) + && revision.id === document.latestRevisionId && revision.documentId === document.id + && revision.companyId === binding.companyId && revision.issueId === binding.issueId && revision.key === fixture.key && revision.format === document.format + && revision.createdByRunId === binding.runId && revision.createdByAgentId === binding.agentId && revision.createdByUserId === null + && revision.revisionNumber === document.latestRevisionNumber && savedSkillResultMatches(revision.body, fixture.expected)); +} diff --git a/tests/runner-e2e/product-journeys/saved-skill-use-flow.test.ts b/tests/runner-e2e/product-journeys/saved-skill-use-flow.test.ts new file mode 100644 index 00000000000..266d07514f5 --- /dev/null +++ b/tests/runner-e2e/product-journeys/saved-skill-use-flow.test.ts @@ -0,0 +1,370 @@ +import { createHash } from "node:crypto"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { canonicalProviderEventsFromOpenCodePart } from "../../../packages/paperclip-runner/src/provider-events.js"; +import { createPrpSemanticToolInputEnvelope, createPrpSemanticToolResultEnvelope } from "../../../packages/paperclip-runner/src/protocol/semantic-tool-receipts.js"; +import { parseFrontmatterMarkdown } from "../../../packages/shared/src/frontmatter.js"; +import { agentSkillSyncSchema } from "../../../packages/shared/src/validators/adapter-skills.js"; +import { createTaskThroughUi } from "../user-actions.js"; +import { runSavedSkillUse } from "./saved-skill-use-flow.js"; +import { savedSkillFixture, type SavedSkillFixture } from "./saved-skill-use-evidence.js"; +import type { JourneyContext, JourneyEvidence, TargetSession } from "./types.js"; + +type Row = Record; +vi.mock("../user-actions.js", async original => ({ ...await original(), createTaskThroughUi: vi.fn() })); +vi.mock("./task-document-flow.js", async original => ({ ...await original(), openDocumentAuthoring: vi.fn() })); +vi.mock("@playwright/test", () => ({ expect: (locator: { textContent(): Promise }) => ({ + async toContainText(text: string) { expect(await locator.textContent()).toContain(text); }, +}) })); +const directories: string[] = []; +afterEach(async () => { vi.clearAllMocks(); await Promise.all(directories.splice(0).map(directory => rm(directory, { recursive: true, force: true }))); }); + +async function setup(change: (kind: string, row: any) => any = (_kind, row) => row, campaignId = "campaign", betaSkillsEnabled = false, + wrappedRead: boolean | "assigned" | "codex" = false) { + const directory = await mkdtemp(path.join(os.tmpdir(), "saved-skill-flow-")); directories.push(directory); + let fixture: SavedSkillFixture; + let skill: Row, assigned = false, issueCreated = false, reloaded = false; + const savedSkills: Row[] = []; + const own = vi.fn(async (_resource: Row) => {}); + const evidence = { checks: [] } as unknown as JourneyEvidence; + const api = { + origin: "https://qa.example.test", + post: vi.fn(async (route: string, body: Row) => { + if (route === "/api/companies/company/skills") { + const reference = /receipt reference is ([a-f0-9]{32})/.exec(body.markdown)![1]!; + fixture = savedSkillFixture(campaignId, reference); + // createLocalSkill gives the parsed frontmatter name precedence over the API input name. + const savedName = parseFrontmatterMarkdown(body.markdown).frontmatter.name; + skill = { id: "skill", companyId: "company", key: `company/company/${fixture.slug}`, sourceType: "managed", slug: fixture.slug, + name: typeof savedName === "string" && savedName.trim() ? savedName.trim() : body.name, + markdown: body.markdown, currentVersionId: "version" }; + savedSkills.push(structuredClone(skill)); + return change("create-skill", structuredClone(skill)); + } + if (route === "/api/agents/agent/skills/sync?companyId=company") { + const parsed = agentSkillSyncSchema.parse(body); + // Core normalizes a string selection to Latest; only a non-null pin requires Beta skills. + const selections = parsed.desiredSkills.map(value => typeof value === "string" ? { key: value, versionId: null } : value); + if (!betaSkillsEnabled && selections.some(entry => entry.versionId !== null)) throw new Error("Beta skill version pins require Beta skills"); + expect(parsed).toEqual({ desiredSkills: [skill.key], mode: "add" }); + assigned = true; return {}; + } + throw new Error(`Unexpected mutation ${route}`); + }), + get: vi.fn(async (route: string) => { + if (route === "/api/instance/settings/experimental") return { enableBetaSkills: betaSkillsEnabled }; + if (route === "/api/companies/company/skills/skill") return change(reloaded ? "retained-skill" : "skill", structuredClone(skill)); + if (route === "/api/companies/company/skills/skill/versions/version") return change(reloaded ? "retained-version" : "version", { + id: "version", companyId: "company", companySkillId: "skill", fileInventory: [{ path: "SKILL.md", content: fixture.markdown }], + }); + if (route === "/api/agents/agent/skills?companyId=company") return change("assignment", { + desiredSkillEntries: assigned ? [{ key: skill.key, versionId: null }] : [], + entries: assigned ? [{ key: skill.key, versionId: "version", runtimeName: `${fixture.slug}--0123456789` }] : [], + }); + if (!issueCreated) throw new Error("Measured result was read before browser dispatch"); + if (route === "/api/issues/issue") return { id: "issue", identifier: "QA-1", companyId: "company", assigneeAgentId: "agent", status: "done", + description: change("description", vi.mocked(createTaskThroughUi).mock.calls.at(-1)![0].prompt) }; + if (route === "/api/issues/issue/runs") return change("runs", [{ runId: "run" }]); + if (route === "/api/issues/issue/comments?order=asc" || route === "/api/issues/issue/interactions") return []; + if (route === "/api/heartbeat-runs/run") return change("run", { id: "run", companyId: "company", agentId: "agent", issueId: "issue", + status: "succeeded", runtimeMode: "native", nativePhase: "committed", resultJson: null, + runnerInstanceId: "instance", nativeSessionId: "session", + contextSnapshot: { paperclipEnvironment: { id: "environment" }, skillSelectionSnapshot: true, skillVersions: { [skill.key]: "version" } }, + runnerProfileJson: { nativeExecutionInput: { provider: { kind: "opencode", model: "model" }, + runtimeContext: { skills: [{ key: skill.key, versionId: "version", runtimeName: `${fixture.slug}--0123456789` }] } } } }); + if (route.startsWith("/api/heartbeat-runs/run/events?")) { + if (wrappedRead === "codex") { + const correlation = { runId: "run", normalizedSessionId: "session", turnId: "turn", itemId: "item" }; + const input = { skill: skill.key }; + const result = change("assigned-read", { skill: skill.key, versionId: skill.currentVersionId, + path: "SKILL.md", text: fixture.markdown, sha256: createHash("sha256").update(fixture.markdown).digest("hex"), + byteSize: Buffer.byteLength(fixture.markdown), nextOffset: null }); + const semantic = [{ ...createPrpSemanticToolInputEnvelope({ operationId: "read_assigned_skill", callId: "load", + correlation, content: input }), input }, createPrpSemanticToolResultEnvelope({ operationId: "read_assigned_skill", + callId: "load", correlation, content: result, outcome: "succeeded", code: "semantic_tool_succeeded", + retryable: false, authorizationBoundary: "active_task", operationReceiptId: "operation_load" })]; + return semantic.map((value, i) => change("event", { runId: "run", companyId: "company", agentId: "agent", + seq: i + 1, sourceSeq: i + 1, sourceEventId: `event-${i}`, sourceInstanceId: "instance", protocolSchemaVersion: 1, + eventType: `semantic_tool.${value.phase}`, payload: { prpEvent: { ...correlation, + schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", sourceSeq: i + 1, + sourceEventId: `event-${i}`, sourceInstanceId: "instance", eventType: `semantic_tool.${value.phase}`, + payload: { semantic_tool: value } } } })); + } + return ["pending", "completed"].map((status, i) => { + const p = canonicalProviderEventsFromOpenCodePart({ id: "load", type: "tool", tool: "skill", state: { + status, output: `\n${fixture.procedure}\n`, + } })[0]!; + if (wrappedRead === true) { + const output = `/workspace/.claude/skills/${fixture.slug}--0123456789/SKILL.md\nfile\n\n` + + fixture.markdown.trimEnd().split("\n").map((line, index) => `${index + 1}: ${line}`).join("\n") + + "\n(End of file - total 12 lines)\n"; + Object.assign(p.payload, { name: "read", operation: "unknown", readOnly: true, target: null, + output: status === "completed" ? output : "", outputBytes: status === "completed" ? Buffer.byteLength(output) : 0 }); + } + if (wrappedRead === "assigned") { + const result = { skill: skill.key, versionId: skill.currentVersionId, path: "SKILL.md", text: fixture.markdown, + sha256: createHash("sha256").update(fixture.markdown).digest("hex"), byteSize: Buffer.byteLength(fixture.markdown), nextOffset: null }; + const output = JSON.stringify({ type: "inputText", text: JSON.stringify(change("assigned-read", result)) }); + Object.assign(p.payload, { name: "paperclip_read_assigned_skill", operation: "unknown", readOnly: true, target: null, + output: status === "completed" ? output : "", outputBytes: status === "completed" ? Buffer.byteLength(output) : 0 }); + } + return change("event", { runId: "run", companyId: "company", agentId: "agent", seq: i + 1, eventType: p.eventType, + sourceEventId: `event-${i}`, sourceInstanceId: "instance", sourceSeq: i + 1, protocolSchemaVersion: 1, + payload: { prpEvent: { schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", runId: "run", turnId: "turn", + eventType: p.eventType, sourceEventId: `event-${i}`, sourceInstanceId: "instance", sourceSeq: i + 1, payload: p.payload } } }); + }); + } + if (route === "/api/issues/issue/documents") return change("documents", [{ key: fixture.key }]); + if (route === `/api/issues/issue/documents/${fixture.key}`) return { id: "document", companyId: "company", issueId: "issue", + key: fixture.key, format: "markdown", body: JSON.stringify(fixture.expected), latestRevisionId: "revision", latestRevisionNumber: 1, + createdByAgentId: "agent", createdByUserId: null, updatedByAgentId: "agent", updatedByUserId: null }; + if (route === `/api/issues/issue/documents/${fixture.key}/revisions/revision`) return change("revision", { + id: "revision", companyId: "company", issueId: "issue", documentId: "document", key: fixture.key, format: "markdown", revisionNumber: 1, + createdByRunId: "run", createdByAgentId: "agent", createdByUserId: null, body: JSON.stringify(fixture.expected), + }); + throw new Error(`Unexpected observation ${route}`); + }), + }; + const page = { goto: vi.fn(async () => {}), reload: vi.fn(async () => { reloaded = true; }), + locator: vi.fn(() => ({ async textContent() { return JSON.stringify(fixture.expected); } })) }; + vi.mocked(createTaskThroughUi).mockImplementation(async input => { + expect(assigned).toBe(true); expect(input.prompt).not.toContain(fixture.expected.reference); + await input.beforeDispatch?.(); issueCreated = true; return { issueId: "issue", submittedAtMs: Date.now() }; + }); + const context = { api, page, company: { id: "company", issuePrefix: "QA" }, user: { id: "owner", name: "Owner" }, + environmentId: "environment", capabilities: ["company", "board-user", "task-write", "agent", "skill-write"], + agent: { id: "agent", name: "Owned agent", provider: "opencode", model: "model", runtimeMode: "native" } } as unknown as JourneyContext; + return { context, api, page, own, evidence, directory, savedSkills, + run: (timeoutMs = 5000) => runSavedSkillUse(context, { own } as unknown as TargetSession, campaignId, evidence, + { deadlineAt: Date.now() + timeoutMs, directory }) }; +} + +describe("saved skill journey orchestration", () => { + it.each([{ kind: "codex" }, { kind: "acpx", agent: "cursor" }, { kind: "cursor_cloud" }])( + "binds a common semantic read for provider %j without an OpenCode directory", async provider => { + const f = await setup((kind, row) => { + if (kind === "run") Object.assign(row.runnerProfileJson.nativeExecutionInput.provider, provider); + return row; + }, "campaign", false, provider.kind === "codex" ? "codex" : "assigned"); + f.context.agent!.provider = provider.kind; + await f.run(); + expect(f.evidence.checks).toHaveLength(7); + expect(f.evidence.checks.every(check => check.passed)).toBe(true); + expect(f.page.reload).toHaveBeenCalledOnce(); + }); + it.each(["codex", "acpx", "cursor_cloud"])("rejects missing or foreign semantic assignment evidence for %s", async provider => { + const changes: Array<{ target: string; mutate(row: Row): void }> = [ + { target: "assignment", mutate: row => { row.entries = []; } }, + { target: "run", mutate: row => { row.runnerProfileJson.nativeExecutionInput.runtimeContext.skills = []; } }, + { target: "run", mutate: row => { row.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0].versionId = "foreign"; } }, + { target: "assigned-read", mutate: row => { row.versionId = "foreign"; } }, + ]; + for (const { target, mutate } of changes) { + const f = await setup((kind, row) => { + if (kind === "run") Object.assign(row.runnerProfileJson.nativeExecutionInput.provider, { kind: provider, + ...(provider === "acpx" ? { agent: "cursor" } : {}) }); + if (kind === target) mutate(row); + return row; + }, "campaign", false, provider === "codex" ? "codex" : "assigned"); + f.context.agent!.provider = provider; + await expect(f.run()).rejects.toMatchObject({ outcome: "failed", code: "saved_skill_native_read_attributed" }); + expect(f.api.get.mock.calls.some(([route]) => route.includes("/documents"))).toBe(false); + expect(f.page.reload).not.toHaveBeenCalled(); + } + }); + it("verifies the ordinary semantic assigned-skill read through all seven checks with Beta skills off", async () => { + const f = await setup(undefined, "campaign", false, "assigned"); await f.run(); + expect(f.evidence.checks).toHaveLength(7); + expect(f.evidence.checks.every(check => check.passed)).toBe(true); + expect(f.page.reload).toHaveBeenCalledOnce(); + expect(f.api.post).toHaveBeenCalledTimes(2); + expect(createTaskThroughUi).toHaveBeenCalledOnce(); + expect(f.api.get.mock.calls.some(([route]) => route.includes("/settings/"))).toBe(false); + const receipt = JSON.parse(await readFile(path.join(f.directory, "saved-skill-use-checks.json"), "utf8")); + expect(receipt.receipts).toEqual([{ runId: "run", executionId: "load", seq: 2 }]); + expect(receipt.reopened).toEqual(receipt.document); + expect(receipt.retainedSkill).toEqual(receipt.skill); + }); + it.each([ + ["wrong receipt key", "assigned-read", (r: Row) => { r.skill = "foreign"; }], + ["wrong receipt version", "assigned-read", (r: Row) => { r.versionId = "stale"; }], + ["partial source page", "assigned-read", (r: Row) => { r.nextOffset = r.text.length; }], + ["semantic failure", "assigned-read", (r: Row) => { r.outcome = "failed"; }], + ["wrong assigned version", "assignment", (r: Row) => { r.entries[0].versionId = "stale"; }], + ["missing admitted skill", "run", (r: Row) => { r.runnerProfileJson.nativeExecutionInput.runtimeContext.skills = []; }], + ["wrong admitted version", "run", (r: Row) => { r.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0].versionId = "stale"; }], + ] as const)("rejects a semantic assigned-skill read with %s before reading its document", async (_label, target, mutate) => { + const f = await setup((kind, row) => { if (kind === target) mutate(row); return row; }, "campaign", false, "assigned"); + await expect(f.run()).rejects.toMatchObject({ outcome: "failed", code: "saved_skill_native_read_attributed" }); + expect(f.api.get.mock.calls.some(([route]) => route.includes("/documents"))).toBe(false); + expect(f.page.reload).not.toHaveBeenCalled(); + }); + it("assigns the input before browser dispatch and verifies native use, saved result and reload without setup supplying output", async () => { + const f = await setup(); await f.run(); + expect(f.own.mock.calls.map(call => call[0])).toEqual([ + expect.objectContaining({ kind: "skill", journeyId: "saved-skill-use" }), + expect.objectContaining({ kind: "skill", id: "skill" }), + expect.objectContaining({ kind: "issue", journeyId: "saved-skill-use" }), + expect.objectContaining({ kind: "issue", id: "issue" }), + ]); + expect(f.api.post).toHaveBeenCalledTimes(2); + expect(createTaskThroughUi).toHaveBeenCalledTimes(1); + expect(f.page.reload).toHaveBeenCalledTimes(1); + expect(f.evidence.checks).toHaveLength(7); + expect(f.evidence.checks.every(check => check.passed)).toBe(true); + const receipt = JSON.parse(await readFile(path.join(f.directory, "saved-skill-use-checks.json"), "utf8")); + expect(receipt.receipts).toEqual([{ runId: "run", executionId: "load", seq: 2 }]); + expect(receipt.reopened).toEqual(receipt.document); + expect(receipt.retainedSkill).toEqual(receipt.skill); + expect(receipt.retainedVersion).toEqual(receipt.version); + expect(receipt.retainedAssignments).toEqual(receipt.assignments); + }); + it("verifies the deployed wrapped native read and then independently checks the saved result and reload", async () => { + const f = await setup(undefined, "campaign", false, true); await f.run(); + expect(f.evidence.checks).toHaveLength(7); + expect(f.evidence.checks.every(check => check.passed)).toBe(true); + expect(f.page.reload).toHaveBeenCalledOnce(); + expect(f.api.get.mock.calls.some(([route]) => route.includes("/documents/saved-skill-result/revisions/"))).toBe(true); + }); + it.each([ + ["missing native skill", "run", (row: Row) => { row.runnerProfileJson.nativeExecutionInput.runtimeContext.skills = []; }], + ["wrong native version", "run", (row: Row) => { row.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0].versionId = "stale"; }], + ["wrong native name", "run", (row: Row) => { row.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0].runtimeName = "foreign"; }], + ["wrong assigned version", "assignment", (row: Row) => { row.entries[0].versionId = "stale"; }], + ["wrong assigned name", "assignment", (row: Row) => { row.entries[0].runtimeName = "foreign"; }], + ] as const)("rejects a complete wrapped read with %s before inspecting its document", async (_label, target, mutate) => { + const f = await setup((kind, row) => { if (kind === target) mutate(row); return row; }, "campaign", false, true); + await expect(f.run()).rejects.toMatchObject({ outcome: "failed", code: "saved_skill_native_read_attributed" }); + expect(f.api.get.mock.calls.some(([route]) => route.includes("/documents"))).toBe(false); + expect(f.page.reload).not.toHaveBeenCalled(); + }); + it.each([false, true])("uses ordinary Latest assignment with Beta skills %s and no setting read or write", async betaSkillsEnabled => { + const f = await setup(undefined, "campaign", betaSkillsEnabled); await f.run(); + expect(f.api.post.mock.calls[1]).toEqual(["/api/agents/agent/skills/sync?companyId=company", + { desiredSkills: [f.savedSkills[0]!.key], mode: "add" }]); + expect(f.api.get.mock.calls.some(([route]) => route.includes("/settings/"))).toBe(false); + expect(f.api.post.mock.calls.some(([route]) => route.includes("/settings/"))).toBe(false); + expect(f.evidence.checks).toHaveLength(7); + expect(f.evidence.checks.every(check => check.passed)).toBe(true); + }); + it.each([ + ["missing snapshot", (row: Row) => { delete row.contextSnapshot.skillSelectionSnapshot; }], + ["false snapshot", (row: Row) => { row.contextSnapshot.skillSelectionSnapshot = false; }], + ["missing versions", (row: Row) => { delete row.contextSnapshot.skillVersions; }], + ["null versions", (row: Row) => { row.contextSnapshot.skillVersions = null; }], + ["array versions", (row: Row) => { row.contextSnapshot.skillVersions = []; }], + ["foreign key", (row: Row) => { row.contextSnapshot.skillVersions = { foreign: "version" }; }], + ["wrong version", (row: Row) => { Object.keys(row.contextSnapshot.skillVersions).forEach(key => { row.contextSnapshot.skillVersions[key] = "stale"; }); }], + ] as const)("rejects an admitted run with %s before trusting its read or result", async (_label, mutate) => { + const f = await setup((kind, row) => { if (kind === "run") mutate(row); return row; }); + await expect(f.run()).rejects.toMatchObject({ outcome: "failed", code: "saved_skill_run_version_mismatch" }); + expect(f.api.get.mock.calls.some(([route]) => route.includes("/events?") || route.includes("/documents"))).toBe(false); + expect(f.page.reload).not.toHaveBeenCalled(); + }); + it.each([undefined, "stale"])("rejects a created-version receipt %s that cannot bind the immutable input", async currentVersionId => { + const f = await setup((kind, row) => kind === "create-skill" ? { ...row, currentVersionId } : row); + await expect(f.run()).rejects.toMatchObject({ outcome: "failed", code: "saved_skill_created_version_mismatch" }); + expect(f.api.post).toHaveBeenCalledTimes(1); + expect(createTaskThroughUi).not.toHaveBeenCalled(); + expect(f.own).toHaveBeenCalledTimes(2); + }); + it("rejects Latest source drift between creation and dispatch", async () => { + let reads = 0; + const f = await setup((kind, row) => kind === "skill" && ++reads === 2 ? { ...row, currentVersionId: "changed" } : row); + await expect(f.run()).rejects.toMatchObject({ outcome: "failed", code: "saved_skill_assignment_bound" }); + expect(reads).toBe(2); + expect(createTaskThroughUi).not.toHaveBeenCalled(); + }); + it("rejects another skill's complete native output even when the receipt document would be correct", async () => { + const f = await setup((kind, row) => { + if (kind === "event" && row.eventType === "tool.execution.completed") { + const payload = row.payload.prpEvent.payload; + payload.output = payload.output.replace(//, ''); + payload.outputBytes = Buffer.byteLength(payload.output); + } + return row; + }); + await expect(f.run()).rejects.toMatchObject({ outcome: "failed", code: "saved_skill_native_read_attributed" }); + expect(f.api.get.mock.calls.some(([route]) => route.includes("/documents"))).toBe(false); + }); + it.each([null, { presentationDecision: { commentAction: "none" } }])( + "qualifies authoritative committed finalization with optional resultJson %j", async resultJson => { + const f = await setup((kind, row) => kind === "run" ? { ...row, resultJson } : row); + await f.run(); + expect(f.evidence.checks).toHaveLength(7); + expect(f.evidence.checks.every(check => check.passed)).toBe(true); + }); + it.each([ + [null, null], + [undefined, { presentationDecision: { commentAction: "none" } }], + ["workspace_finalizing", null], + ["retryable_failure", { finalizationPhase: "committed" }], + ])("does not qualify Done and succeeded before native phase %s is committed", async (nativePhase, resultJson) => { + const f = await setup((kind, row) => kind === "run" ? { ...row, nativePhase, resultJson } : row); + await expect(f.run(250)).rejects.toMatchObject({ outcome: "failed", code: "saved_skill_task_not_completed" }); + expect(f.api.get.mock.calls.some(([route]) => route === "/api/heartbeat-runs/run")).toBe(true); + expect(f.api.get.mock.calls.some(([route]) => route.includes("/events?") || route.includes("/documents"))).toBe(false); + expect(f.page.reload).not.toHaveBeenCalled(); + const receipt = JSON.parse(await readFile(path.join(f.directory, "saved-skill-use-checks.json"), "utf8")); + expect(receipt.state.taskRuns[0].nativePhase).toBe(nativePhase); + }); + it("waits for native commit before qualifying the already saved current-run result", async () => { + let runReads = 0; + const f = await setup((kind, row) => { + if (kind === "run") return { ...row, nativePhase: ++runReads === 1 ? "workspace_finalizing" : "committed", + resultJson: { finalizationPhase: "committed" } }; + if (kind === "documents") expect(runReads).toBe(2); + return row; + }); + await f.run(); + expect(runReads).toBe(2); + expect(f.evidence.checks.every(check => check.passed)).toBe(true); + }); + it.each([ + ["assignment", "saved_skill_assignment_bound", (row: Row) => ({ ...row, desiredSkillEntries: [{ key: "unknown", versionId: null }] })], + ["assignment", "saved_skill_assignment_bound", (row: Row) => ({ ...row, desiredSkillEntries: row.desiredSkillEntries.map((entry: Row) => ({ ...entry, versionId: "stale" })) })], + ["assignment", "saved_skill_assignment_bound", (row: Row) => ({ ...row, desiredSkillEntries: row.desiredSkillEntries.map(({ key }: Row) => ({ key })) })], + ["assignment", "saved_skill_assignment_bound", (row: Row) => ({ ...row, desiredSkillEntries: [] })], + ["skill", "saved_skill_assignment_bound", (row: Row) => ({ ...row, sourceType: "local_path" })], + ["version", "saved_skill_assignment_bound", (row: Row) => ({ ...row, fileInventory: [{ path: "SKILL.md", content: "changed source" }] })], + ["runs", "saved_skill_run_limit_exceeded", (row: Row[]) => [...row, { runId: "extra" }]], + ["event", "saved_skill_native_read_attributed", (row: Row) => ({ ...row, companyId: "foreign" })], + ["documents", "saved_skill_unexpected_documents", (row: Row[]) => [...row, { key: "unrelated" }]], + ["revision", "saved_skill_result_authored_by_run", (row: Row) => ({ ...row, createdByRunId: "foreign" })], + ["retained-skill", "saved_skill_source_preserved", (row: Row) => ({ ...row, currentVersionId: "changed" })], + ["retained-version", "saved_skill_source_preserved", (row: Row) => ({ ...row, fileInventory: [] })], + ] as const)("fails closed on changed %s evidence and retains its failure checkpoint", async (kind, code, change) => { + const f = await setup((which, row) => which === kind ? change(row as never) : row); + await expect(f.run()).rejects.toMatchObject({ outcome: "failed", code }); + expect(await readFile(path.join(f.directory, "saved-skill-use-checks.json"), "utf8")).toContain("fixture"); + if (kind === "assignment" || kind === "version" || kind === "skill") expect(createTaskThroughUi).not.toHaveBeenCalled(); + else expect(createTaskThroughUi).toHaveBeenCalledTimes(1); + }); + it("journals uncertain creation once without replaying it or dispatching a task", async () => { + const f = await setup(); f.api.post.mockRejectedValueOnce(new Error("Response lost after POST")); + await expect(f.run()).rejects.toThrow("Response lost after POST"); + expect(f.api.post).toHaveBeenCalledTimes(1); + expect(f.own).toHaveBeenCalledTimes(1); + expect(createTaskThroughUi).not.toHaveBeenCalled(); + }); + it("recovers the exact persisted frontmatter name after a successful creation response is lost", async () => { + const campaignId = "journey-1793701020304-a1b2c3d4"; + const f = await setup((kind, row) => { + if (kind === "create-skill") throw new Error("Response lost after successful creation"); + return row; + }, campaignId); + await expect(f.run()).rejects.toThrow("Response lost after successful creation"); + expect(f.savedSkills).toHaveLength(1); + expect(f.api.post).toHaveBeenCalledTimes(1); + expect(f.own).toHaveBeenCalledTimes(1); + expect(createTaskThroughUi).not.toHaveBeenCalled(); + const intent = f.own.mock.calls[0]![0]; + const recovered = f.savedSkills.filter(row => row.companyId === f.context.company.id && row.name === intent.name); + expect(recovered).toEqual([f.savedSkills[0]]); + expect(intent.name).toContain(campaignId); + expect(f.savedSkills[0]!.name).toContain(campaignId); + expect(f.api.post.mock.calls[0]![1].name).toBe(f.savedSkills[0]!.name); + }); +}); diff --git a/tests/runner-e2e/product-journeys/saved-skill-use-flow.ts b/tests/runner-e2e/product-journeys/saved-skill-use-flow.ts new file mode 100644 index 00000000000..750de80f293 --- /dev/null +++ b/tests/runner-e2e/product-journeys/saved-skill-use-flow.ts @@ -0,0 +1,129 @@ +import { randomBytes } from "node:crypto"; +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { expect } from "@playwright/test"; +import { collectRunEvents } from "../run-observations.js"; +import { createTaskThroughUi } from "../user-actions.js"; +import { validateAgentTaskState, waitFor } from "./agent-flows.js"; +import { createObservedReader } from "./observation-read.js"; +import { openDocumentAuthoring } from "./task-document-flow.js"; +import { savedSkillDocumentAttributed, savedSkillFixture, savedSkillReadAssignment, savedSkillReadReceipts, savedSkillReadRuntimeName, savedSkillSourceMatches } from "./saved-skill-use-evidence.js"; +import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSession } from "./types.js"; + +type Row = Record; +const id = "saved-skill-use"; + +export async function runSavedSkillUse(context: JourneyContext, session: TargetSession, campaignId: string, + evidence: JourneyEvidence, options: { deadlineAt: number; directory: string }) { + const agent = context.agent; + if (!agent || agent.runtimeMode !== "native" || !context.environmentId) + throw new JourneyError("unsupported", "native_agent_and_environment_required"); + if (!context.capabilities.includes("skill-write")) throw new JourneyError("unsupported", "missing_skill_write"); + await mkdir(options.directory, { recursive: true, mode: 0o700 }); + const read = createObservedReader(context.api, options.deadlineAt); + const fixture = savedSkillFixture(campaignId, randomBytes(16).toString("hex")); + const title = `Journey QA saved skill ${campaignId}`; + const prompt = `Use /${fixture.slug} for this task. Follow that saved skill's procedure, verify its saved result, and finish. ` + + "Do not create child tasks or unrelated deliverables."; + const check = (name: string, passed: boolean) => { + evidence.checks.push({ id: name, passed }); + if (!passed) throw new JourneyError("failed", name.replaceAll("-", "_")); + }; + let checkpoint: Row = { fixture }; + let primaryFailure: unknown; + try { + const companyRoute = `/api/companies/${context.company.id}`; + await session.own({ kind: "skill", name: fixture.name, journeyId: id }); + const createdSkill = await context.api.post(`${companyRoute}/skills`, { + name: fixture.name, slug: fixture.slug, description: "Make a receipt using this saved procedure.", + markdown: fixture.markdown, idempotencyKey: `saved-skill-${campaignId}`, + }); + if (!createdSkill?.id) throw new JourneyError("failed", "saved_skill_creation_receipt_missing"); + await session.own({ kind: "skill", name: fixture.name, id: createdSkill.id, journeyId: id }); + checkpoint = { ...checkpoint, createdSkill }; + const skillRoute = `${companyRoute}/skills/${createdSkill.id}`; + const skill = await read.get(skillRoute); + if (!skill?.key || !skill.currentVersionId) throw new JourneyError("failed", "saved_skill_version_missing"); + if (skill.id !== createdSkill.id || !createdSkill.currentVersionId || skill.currentVersionId !== createdSkill.currentVersionId) + throw new JourneyError("failed", "saved_skill_created_version_mismatch"); + await context.api.post(`/api/agents/${agent.id}/skills/sync?companyId=${context.company.id}`, { + desiredSkills: [skill.key], mode: "add", + }); + const assignmentRoute = `/api/agents/${agent.id}/skills?companyId=${context.company.id}`; + const versionRoute = `${skillRoute}/versions/${skill.currentVersionId}`; + const [version, assignments, beforeDispatchSkill] = await Promise.all([ + read.get(versionRoute), read.get(assignmentRoute), read.get(skillRoute), + ]); + checkpoint = { ...checkpoint, skill, version, assignments, beforeDispatchSkill }; + check("saved-skill-assignment-bound", beforeDispatchSkill.id === createdSkill.id + && beforeDispatchSkill.currentVersionId === createdSkill.currentVersionId + && savedSkillSourceMatches(beforeDispatchSkill, version, assignments, fixture, context.company.id)); + + const created = await createTaskThroughUi({ page: context.page, origin: context.api.origin, + issuePrefix: context.company.issuePrefix, agentName: agent.name, title, prompt, workMode: "standard", requireExplicitTitle: true, + beforeDispatch: () => session.own({ kind: "issue", name: title, journeyId: id }) }); + await session.own({ kind: "issue", name: title, id: created.issueId, journeyId: id }); + const route = `/api/issues/${created.issueId}`; + const state = await waitFor(options.deadlineAt, async () => { + const [currentIssue, summaries, comments, interactions] = await Promise.all([ + read.get(route), read.get(`${route}/runs`), + read.get(`${route}/comments?order=asc`), read.get>(`${route}/interactions`), + ]); + if (!Array.isArray(summaries) || summaries.length > 1) throw new JourneyError("failed", "saved_skill_run_limit_exceeded"); + const taskRuns = await Promise.all(summaries.map(row => read.get(`/api/heartbeat-runs/${row.runId}`))); + const value = { currentIssue, taskRuns, comments, interactions, attachments: [] }; + validateAgentTaskState(context, created.issueId, value, 1); + checkpoint = { ...checkpoint, state: value }; + return value; + }, value => value.currentIssue.status === "done" && value.taskRuns.length === 1 && value.taskRuns[0]!.status === "succeeded" + && value.taskRuns[0]!.nativePhase === "committed" + && value.interactions.every(row => row.status !== "pending"), "saved_skill_task_not_completed"); + const admittedVersions = state.taskRuns[0]!.contextSnapshot?.skillVersions; + if (state.taskRuns[0]!.contextSnapshot?.skillSelectionSnapshot !== true || !admittedVersions + || typeof admittedVersions !== "object" || Array.isArray(admittedVersions) + || !Object.hasOwn(admittedVersions, skill.key) || admittedVersions[skill.key] !== version.id) + throw new JourneyError("failed", "saved_skill_run_version_mismatch"); + const binding = { companyId: context.company.id, agentId: agent.id, runId: state.taskRuns[0]!.id, issueId: created.issueId }; + const request = String(state.currentIssue.description ?? ""); + check("saved-skill-input-only-in-source", request.includes(`/${fixture.slug}`) && !request.includes(fixture.expected.reference) + && !state.comments.some(row => row.authorUserId && String(row.body ?? "").includes(fixture.expected.reference))); + const events = await collectRunEvents( + (afterSeq, limit) => read.get(`/api/heartbeat-runs/${binding.runId}/events?afterSeq=${afterSeq}&limit=${limit}`)); + const runtimeName = savedSkillReadRuntimeName(skill, assignments, state.taskRuns[0]!); + const assignment = savedSkillReadAssignment(skill, assignments, state.taskRuns[0]!); + const receipts = savedSkillReadReceipts(events, binding, fixture, runtimeName, assignment, state.taskRuns[0]!); + checkpoint = { ...checkpoint, events, receipts }; + check("saved-skill-native-read-attributed", receipts.length > 0); + const documents = await read.get(`${route}/documents`); + if (documents.length !== 1 || documents[0]?.key !== fixture.key) throw new JourneyError("failed", "saved_skill_unexpected_documents"); + const documentRoute = `${route}/documents/${fixture.key}`; + const document = await read.get(documentRoute); + const revision = await read.get(`${documentRoute}/revisions/${document.latestRevisionId}`); + checkpoint = { ...checkpoint, document, revision }; + check("saved-skill-result-authored-by-run", savedSkillDocumentAttributed(document, revision, fixture, binding)); + const url = `${context.api.origin}/${context.company.issuePrefix}/issues/${state.currentIssue.identifier ?? created.issueId}`; + await context.page.goto(url, { waitUntil: "domcontentloaded" }); + await context.page.reload(); + await openDocumentAuthoring(context); + await expect(context.page.locator(`#document-${fixture.key}`)).toContainText(fixture.expected.reference); + const [reopened, retainedSkill, retainedVersion, retainedAssignments] = await Promise.all([ + read.get(documentRoute), read.get(skillRoute), read.get(versionRoute), read.get(assignmentRoute), + ]); + checkpoint = { ...checkpoint, reopened, retainedSkill, retainedVersion, retainedAssignments }; + check("saved-skill-result-reopened", reopened.latestRevisionId === document.latestRevisionId + && savedSkillDocumentAttributed(reopened, revision, fixture, binding)); + check("saved-skill-source-preserved", savedSkillSourceMatches(retainedSkill, retainedVersion, retainedAssignments, fixture, context.company.id) + && retainedSkill.id === skill.id && retainedSkill.currentVersionId === skill.currentVersionId); + check("configured-model-and-environment", true); // All observed runs were independently validated above. + } catch (error) { + primaryFailure = error; + throw error; + } finally { + try { + await writeFile(path.join(options.directory, "saved-skill-use-checks.json"), JSON.stringify(checkpoint, null, 2), { mode: 0o600 }); + await read.preserve(options.directory); + } catch (error) { + if (!primaryFailure) throw error; + } + } +} diff --git a/tests/runner-e2e/product-journeys/saved-skill-use.md b/tests/runner-e2e/product-journeys/saved-skill-use.md new file mode 100644 index 00000000000..1ea01e75855 --- /dev/null +++ b/tests/runner-e2e/product-journeys/saved-skill-use.md @@ -0,0 +1,97 @@ +# Use a saved assigned skill + +`saved-skill-use` is a reusable Product E2E journey for an existing target. It +uses one native work run on the target provider's campaign-owned agent. It does +not enroll itself in a daily plan. Local calibration and catalog discovery are +separate from a real staging pass. + +The target supplies ordinary company skill-write access. Assignment uses +`desiredSkills: [canonicalSkillKey]`, which selects Latest and returns a stored +`versionId: null`. This path works with Beta skills disabled. No experimental +setting is read or changed. Experimental version pins need separate qualification. + +The fixture creates one synthetic company skill through the public skill API. +The journal name, API name and Markdown frontmatter name equal its slug. Each +contains the exact campaign ID, including its hyphens. Recovery can therefore +find the persisted skill by name if a successful creation response is lost. +It assigns the skill on the owned agent through the normal skill sync API. +The created skill must be managed, with the exact canonical company key. Its +creation receipt and separate reads must agree on the immutable current version. +A version read must return the exact `SKILL.md` bytes. The skill's +random receipt reference and two numeric inputs occur only in those saved bytes. +The slug and task prompt do not contain the reference. The task is created +through the production browser composer and explicitly names that skill. + +A pass requires all seven closed checks: + +- `saved-skill-assignment-bound`: exact company, managed skill, creation version, + source bytes and one ordinary Latest assignment before dispatch. The successful + run must have `contextSnapshot.skillSelectionSnapshot: true` and an admitted + `skillVersions[canonicalSkillKey]` equal to that immutable created version. +- `saved-skill-input-only-in-source`: the persisted request names the skill but + does not expose its receipt reference; no human comment supplies that reference. +- `saved-skill-native-read-attributed`: a complete successful native skill load or + source-file read, with start/completion identity bound to this company, agent, + run, turn and runner. Its native receipt must contain or cryptographically + commit to the complete saved procedure. +- `saved-skill-result-authored-by-run`: exactly one saved task document contains + the reference and independently calculated sum. Its revision belongs to the + successful native run and agent, without human authorship. The run's + authoritative `nativePhase` must be `committed` before the result qualifies. +- `saved-skill-result-reopened`: the browser reload shows that document, and its + independently read content and revision are unchanged. +- `saved-skill-source-preserved`: the original skill version, source bytes and + assignment remain unchanged after execution. +- `configured-model-and-environment`: the one successful run matches the owned + company, task, agent, configured native provider/model and execution environment. + +The read oracle accepts a completed built-in file read with the exact skill +directory target, or OpenCode's native `skill` load with its exact named +`skill_content` wrapper. Deployed OpenCode can instead report a native `read` +with an unknown operation and null target. That path requires its complete file +result wrapper: the file path must name the exact materialized skill directory. +Both the saved assignment entry and the admitted native execution input must +bind that runtime name to the same canonical key and immutable created version. +The native read must remain read-only, successful, paired by call/turn/runner, +untruncated and byte-complete, and its content must contain the full procedure. +A targetless filesystem read without this bound file wrapper cannot qualify. Discovery, +shell output, model prose, failed or truncated reads, foreign run receipts and +marker-only output cannot pass. + +The first-party `paperclip_read_assigned_skill` tool can also supply the read. +Its complete `inputText` result must contain the exact canonical skill key, +immutable created version, `SKILL.md` path, original Markdown, byte size and +SHA-256 checksum. `nextOffset` must be null. The same dual assignment and +admitted canonical key/version binding is required for every native provider. +The OpenCode materialized directory name is required only for its filesystem +wrapper; it is not a condition for the common first-party semantic read. +A semantic read failure can have a +completed tool status; it does not qualify. A partial page, copied tool result, +foreign version, missing start or unbound native event also fails closed. + +Codex's dynamic tool display is not first-party authority. Its raw name, +optional namespace or read-only hint cannot qualify a read. For Codex, the +grader also accepts Core's retained `semantic_tool.input` and `semantic_tool.result` +for the closed read-only operation `read_assigned_skill`. Both must name the +actual run's runner, native session, call, turn and item. The input must match +its own digest and select the canonical skill or its exact admitted alias, +`SKILL.md` and offset zero (including Core's defaults). The successful +`active_task` result must commit to the exact complete immutable read object: +canonical key, version, file, all UTF-8 source text, SHA-256, byte size and a +null next-page offset. The digest uses Core's canonical JSON key ordering. +Missing, duplicate, reordered, failed, foreign or differently hashed carriers +fail. This path does not relax the existing OpenCode built-in or filesystem guards. + +The existing campaign deadline bounds observations. Only failed read transport +operations can be retried; task input, skill creation and execution are not +replayed. Done status and a successful run alone cannot qualify incomplete native +finalization. The optional `resultJson.finalizationPhase` presentation field is +not required or accepted in place of `nativePhase`; compact or null result data +can qualify when the authoritative phase is committed. +The provider journals skill and task ownership before creation, then +collects evidence and stops/cleans only campaign-owned resources. The fixed skill +source, native events and saved document receipts stay in the private bundle. +The public report uses the existing closed checks and authenticated task capture. + +No instance-wide experimental setting, shared source agent, global model, +company budget or production workspace is changed by this journey. diff --git a/tests/runner-e2e/product-journeys/success-contract.ts b/tests/runner-e2e/product-journeys/success-contract.ts index fd9b6886ce8..d730a85f21b 100644 --- a/tests/runner-e2e/product-journeys/success-contract.ts +++ b/tests/runner-e2e/product-journeys/success-contract.ts @@ -1,5 +1,6 @@ /** Closed, target-independent success requirements used by dispatch and retained evidence. */ export const agentJourneyRuns = { + "saved-skill-use": 1, "repository-continuation": 2, "routine-scheduled-run": 1, "file-input-answer": 1, @@ -24,6 +25,7 @@ export function agentJourneyRunCountValid(id: string, count: unknown): boolean { return (count as number) >= minimum && (count as number) <= maximum; } export const journeyRequiredChecks: Record = { + "saved-skill-use": ["saved-skill-assignment-bound", "saved-skill-input-only-in-source", "saved-skill-native-read-attributed", "saved-skill-result-authored-by-run", "saved-skill-result-reopened", "saved-skill-source-preserved", "configured-model-and-environment"], "task-edit-lifecycle": ["task-lifecycle-browser-created", "task-lifecycle-visible-edits-persisted", "task-lifecycle-edits-reloaded", "task-lifecycle-completed-and-reopened", "task-lifecycle-unedited-context-preserved", "no-agent-work"], "company-access-isolation": ["isolation-owner-resource-controls", "cloud-workspace-switch-and-return", "workspace-data-scoped", "restricted-agent-positive-controls", "cross-workspace-agent-read-write-denied", "isolation-credential-revoked", "no-agent-work"], "repository-continuation": ["repository-configured-source-bound", "repository-first-commit-verified", "repository-original-files-saved", "repository-followup-stored-once", "repository-same-workspace-retained", "repository-second-commit-verified", "repository-original-and-result-reopened", "configured-model-and-environment"], diff --git a/tests/runner-e2e/product-journeys/types.ts b/tests/runner-e2e/product-journeys/types.ts index 2a776353237..51cacf91541 100644 --- a/tests/runner-e2e/product-journeys/types.ts +++ b/tests/runner-e2e/product-journeys/types.ts @@ -26,7 +26,8 @@ export type JourneyId = | "inbox-triage" | "routine-scheduled-run" | "repository-continuation" - | "company-access-isolation"; + | "company-access-isolation" + | "saved-skill-use"; export type TargetMode = "managed-self-hosted" | "existing-cloud" | "new-cloud"; export interface DeploymentIdentity { kind: "self-hosted" | "cloud";