diff --git a/tests/runner-e2e/product-journeys/saved-skill-use-evidence.ts b/tests/runner-e2e/product-journeys/saved-skill-use-evidence.ts index 0f4ede10092..ed8df09cd5f 100644 --- a/tests/runner-e2e/product-journeys/saved-skill-use-evidence.ts +++ b/tests/runner-e2e/product-journeys/saved-skill-use-evidence.ts @@ -96,6 +96,49 @@ function assignedSkillReadText(output: string, binding: NativePrpRunBinding, fix } catch { return undefined; } } +/** OpenCode's proxy forwards the builtin skill loader without an operation. + * Core091 therefore records unknown/false even though the loader exposes its saved body. + * Only this complete, immutable-version-bound native loader may use that shape. + */ +function nativeOpenCodeSkillLoad(row: Row, bound: Row[], binding: NativePrpRunBinding, fixture: SavedSkillFixture, + runtimeName: string | undefined, assignment: SavedSkillReadAssignment | undefined, run: Row | undefined): boolean { + const event = row.payload.prpEvent, payload = event.payload, input = run?.runnerProfileJson?.nativeExecutionInput; + const skills = Array.isArray(input?.runtimeContext?.skills) + ? input.runtimeContext.skills.filter((skill: Row) => skill.key === assignment?.key) : []; + if (!run || run.runtimeMode !== "native" || input?.provider?.kind !== "opencode" + || run.id !== binding.runId || run.companyId !== binding.companyId || run.agentId !== binding.agentId + || typeof run.runnerInstanceId !== "string" || !run.runnerInstanceId.trim() + || typeof run.nativeSessionId !== "string" || !run.nativeSessionId.trim() + || event.sourceInstanceId !== run.runnerInstanceId || event.normalizedSessionId !== run.nativeSessionId + || typeof event.itemId !== "string" || !event.itemId.trim() || !Number.isSafeInteger(row.seq) + || !assignment || assignment.key !== `company/${binding.companyId}/${fixture.slug}` + || typeof assignment.versionId !== "string" || !assignment.versionId.trim() + || skills.length !== 1 || skills[0].versionId !== assignment.versionId || skills[0].runtimeName !== runtimeName + || run.contextSnapshot?.skillSelectionSnapshot !== true + || run.contextSnapshot?.skillVersions?.[assignment.key] !== assignment.versionId + || typeof runtimeName !== "string" || !/^[a-z0-9][a-z0-9_-]*$/i.test(runtimeName) + || payload.name !== "skill" || payload.operation !== "unknown" || payload.readOnly !== false || payload.target !== null) return false; + const wrapper = /^\r\n]+)">\n# Skill: ([^\r\n]+)\n\n([\s\S]*?)\n\nBase directory for this skill: ([^\r\n]+)\n([\s\S]*)\n<\/skill_content>\s*$/.exec(payload.output); + if (!wrapper || wrapper[1] !== fixture.slug || wrapper[2] !== fixture.slug || wrapper[3] !== fixture.procedure.trim() + || !/\s*<\/skill_files>/.test(wrapper[5]!) + || (payload.output.match(//g) ?? []).length !== 1 + || (payload.output.match(//g) ?? []).length !== 1 + || (payload.output.match(/<\/skill_files>/g) ?? []).length !== 1) return false; + // The native loader prints an absolute filesystem path, not a URI. + const path = wrapper[4]!; + if (!path.startsWith("/") || /[\\\u0000-\u001f\u007f?#]/.test(path) || /%[a-f0-9]{2}/i.test(path) + || path.split("/").some(part => part === "." || part === "..") + || !path.replace(/\/$/, "").endsWith(`/.claude/skills/${runtimeName}`)) return false; + const pair = bound.filter(candidate => ["tool.execution.started", "tool.execution.completed"].includes(candidate.payload.prpEvent.eventType) + && candidate.payload.prpEvent.payload?.executionId === payload.executionId); + const start = pair.find(candidate => candidate.payload.prpEvent.eventType === "tool.execution.started"); + const initial = start?.payload.prpEvent, initialPayload = initial?.payload; + return Boolean(pair.length === 2 && start && Number.isSafeInteger(start.seq) && start.seq < row.seq + && initial.sourceSeq < event.sourceSeq && initial.normalizedSessionId === event.normalizedSessionId && initial.itemId === event.itemId + && initialPayload?.readOnly === false && initialPayload.target === null); +} + // Core091 provider_bridge.rs hashes UTF-8 JSON with object keys ordered by UTF-16. // This oracle reconstructs the complete pinned read result, not provider display text. function semanticDigest(value: unknown): string { @@ -189,6 +232,8 @@ export function savedSkillReadReceipts(events: Row[], binding: NativePrpRunBindi // OpenCode's skill tool names the loaded skill in its provider-visible result. const loading = payload.operation === "execute" && name === "skill" && payload.output.includes(``); + const nativeLoading = name === "skill" && payload.operation === "unknown" + && nativeOpenCodeSkillLoad(row, bound, binding, fixture, runtimeName, assignment, run); // Deployed OpenCode reports this read with operation unknown and target null. // Its successful file-result wrapper supplies the exact materialized source path. const wrapped = /^([^<>\r\n]+)<\/path>\r?\nfile<\/type>\r?\n\r?\n([\s\S]*)\r?\n<\/content>\s*$/.exec(payload.output); @@ -206,7 +251,7 @@ export function savedSkillReadReceipts(events: Row[], binding: NativePrpRunBindi && assignment?.key === `company/${binding.companyId}/${fixture.slug}` && typeof assignment.versionId === "string" && assignment.versionId.trim() ? assignedSkillReadText(payload.output, binding, fixture, assignment.versionId) : undefined; - if (!reading && !loading && !wrappedRead && assignedText === undefined) return false; + if (!reading && !loading && !nativeLoading && !wrappedRead && assignedText === undefined) return false; // File readers may prefix each source line with its line number. const output = (assignedText ?? (wrappedRead ? wrapped![2]! : payload.output)).split("\n").map((line: string) => line.replace(/^\s*\d+(?:[:|]|→)\s?/, "")).join("\n"); if (!output.includes(fixture.procedure.trim())) return false; diff --git a/tests/runner-e2e/product-journeys/saved-skill-use-native-loader.test.ts b/tests/runner-e2e/product-journeys/saved-skill-use-native-loader.test.ts new file mode 100644 index 00000000000..925ad58b5d9 --- /dev/null +++ b/tests/runner-e2e/product-journeys/saved-skill-use-native-loader.test.ts @@ -0,0 +1,129 @@ +import { describe, expect, it } from "vitest"; +import { canonicalProviderEventsFromOpenCodePart } from "../../../packages/paperclip-runner/src/provider-events.js"; +import { openCodeProxyToolNotification } from "../../../packages/paperclip-runner/src/cli/opencode-proxy-events.js"; +import { validatePrpEvent } from "../../../packages/paperclip-runner/src/protocol/replay-contract.js"; +import { savedSkillFixture, savedSkillReadAssignment, savedSkillReadReceipts, savedSkillReadRuntimeName } from "./saved-skill-use-evidence.js"; + +type Row = Record; +const binding = { companyId: "company", agentId: "agent", runId: "run" }; +const fixture = savedSkillFixture("native-loader", "62ff548dfec748da8fef6d5fbf6cc23d"); +const key = `company/${binding.companyId}/${fixture.slug}`, runtimeName = `${fixture.slug}--0123456789`; +const body = `\n# Skill: ${fixture.slug}\n\n${fixture.procedure.trim()}\n\n` + + `Base directory for this skill: /workspace/.claude/skills/${runtimeName}\n` + + "Relative paths are relative to this base directory.\n\n\n\n"; + +function observations() { + const skill = { key, currentVersionId: "version" }; + const assignments = { entries: [{ key, versionId: "version", runtimeName }] }; + const run: Row = { id: "run", companyId: "company", agentId: "agent", runtimeMode: "native", + runnerInstanceId: "runner", nativeSessionId: "session", + contextSnapshot: { skillSelectionSnapshot: true, skillVersions: { [key]: "version" } }, + runnerProfileJson: { nativeExecutionInput: { provider: { kind: "opencode" }, + runtimeContext: { skills: [{ key, versionId: "version", runtimeName }] } } } }; + const notifications: Row[] = []; + const events = ["pending", "completed"].map((status, index) => { + const direct = canonicalProviderEventsFromOpenCodePart({ id: "load", type: "tool", tool: "skill", + state: { status, output: index ? body : "" } })[0]!; + const forwarded = openCodeProxyToolNotification({ eventType: direct.eventType, threadId: "session", + turnId: "turn", payload: direct.payload })!; + notifications.push(forwarded); + // Actual Core091 retained builtin fixture. The real proxy drops operation; + // Rust provider_events.rs maps builtinToolCall to unknown, not TS's execute. + // This JS fixture does not claim to run the Rust converter itself. + const payload: Row = { ...direct.payload, operation: "unknown" }; + const eventType = direct.eventType; + return { ...binding, seq: index + 1, eventType, protocolSchemaVersion: 1, + sourceSeq: index + 1, sourceEventId: `event-${index}`, sourceInstanceId: "runner", + payload: { prpEvent: { schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", + priority: 0, emittedAt: "2026-11-04T04:00:00.000Z", + runId: "run", normalizedSessionId: "session", turnId: "turn", itemId: "item", + sourceSeq: index + 1, sourceEventId: `event-${index}`, sourceInstanceId: "runner", eventType, payload } } }; + }); + return { skill, assignments, run, events, notifications }; +} +type Observations = ReturnType; +const read = (value: Observations) => savedSkillReadReceipts(value.events, binding, fixture, + savedSkillReadRuntimeName(value.skill, value.assignments, value.run), + savedSkillReadAssignment(value.skill, value.assignments, value.run), value.run); +const completed = (v: Observations) => v.events[1]!.payload.prpEvent.payload; +function output(v: Observations, change: (text: string) => string) { + const payload = completed(v); payload.output = change(payload.output); payload.outputBytes = Buffer.byteLength(payload.output); +} + +describe("saved-skill native OpenCode builtin loader", () => { + it("keeps the real proxy's builtin identity and qualifies Core091's version-bound native receipt", () => { + const value = observations(); + expect(value.notifications[0]!.params.item).toMatchObject({ type: "builtinToolCall", tool: "skill", readOnlyHint: false }); + expect(value.notifications[0]!.params.item).not.toHaveProperty("operation"); + expect(completed(value)).toMatchObject({ operation: "unknown", readOnly: false, transport: "builtin" }); + for (const row of value.events) expect(validatePrpEvent(row.payload.prpEvent).ok).toBe(true); + expect(read(value)).toEqual([{ runId: "run", executionId: "load", seq: 2 }]); + }); + it("accepts an absolute local directory with a space and its optional trailing slash", () => { + const value = observations(); output(value, text => text.replace("/workspace/", "/work space/") + .replace(`${runtimeName}\n`, `${runtimeName}/\n`)); expect(read(value)).toHaveLength(1); + }); + it.each([ + ["missing start", (v: Observations) => { v.events.shift(); }], + ["missing completion", (v: Observations) => { v.events.pop(); }], + ["duplicate start", (v: Observations) => { v.events.push(structuredClone(v.events[0]!)); }], + ["duplicate completion", (v: Observations) => { v.events.push(structuredClone(v.events[1]!)); }], + ["foreign call", (v: Observations) => { v.events[0]!.payload.prpEvent.payload.executionId = "foreign"; }], + ["foreign run", (v: Observations) => { v.events[1]!.runId = v.events[1]!.payload.prpEvent.runId = "foreign"; }], + ["foreign company", (v: Observations) => { v.events[1]!.companyId = "foreign"; }], + ["foreign agent", (v: Observations) => { v.events[1]!.agentId = "foreign"; }], + ["foreign runner", (v: Observations) => { v.events[1]!.sourceInstanceId = v.events[1]!.payload.prpEvent.sourceInstanceId = "foreign"; }], + ["foreign turn", (v: Observations) => { v.events[0]!.payload.prpEvent.turnId = "foreign"; }], + ["foreign session", (v: Observations) => { v.events[0]!.payload.prpEvent.normalizedSessionId = "foreign"; }], + ["foreign item", (v: Observations) => { v.events[0]!.payload.prpEvent.itemId = "foreign"; }], + ["missing session", (v: Observations) => { delete v.run.nativeSessionId; }], + ["missing runner", (v: Observations) => { delete v.run.runnerInstanceId; }], + ["wrong native provider", (v: Observations) => { v.run.runnerProfileJson.nativeExecutionInput.provider.kind = "codex"; }], + ["wrong run owner", (v: Observations) => { v.run.companyId = "foreign"; }], + ["unselected skill", (v: Observations) => { v.run.contextSnapshot.skillSelectionSnapshot = false; }], + ["foreign admitted version", (v: Observations) => { v.run.contextSnapshot.skillVersions[key] = "foreign"; }], + ["foreign API version", (v: Observations) => { v.assignments.entries[0]!.versionId = "foreign"; }], + ["foreign API runtime directory", (v: Observations) => { v.assignments.entries[0]!.runtimeName = "foreign"; }], + ["foreign native version", (v: Observations) => { v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0].versionId = "foreign"; }], + ["duplicate native assignment", (v: Observations) => { v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills.push(v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0]); }], + ["failed load", (v: Observations) => { completed(v).status = "failed"; }], + ["truncated load", (v: Observations) => { completed(v).outputTruncated = true; }], + ["wrong bytes", (v: Observations) => { completed(v).outputBytes++; }], + ["another operation", (v: Observations) => { for (const row of v.events) row.payload.prpEvent.payload.operation = "read"; }], + ["wrong completed hint", (v: Observations) => { completed(v).readOnly = true; }], + ["wrong started hint", (v: Observations) => { v.events[0]!.payload.prpEvent.payload.readOnly = true; }], + ["started target", (v: Observations) => { v.events[0]!.payload.prpEvent.payload.target = "/foreign"; }], + ["completed target", (v: Observations) => { completed(v).target = "/foreign"; }], + ["reversed row order", (v: Observations) => { v.events[0]!.seq = 3; }], + ["reversed source order", (v: Observations) => { v.events[0]!.sourceSeq = v.events[0]!.payload.prpEvent.sourceSeq = 3; }], + ["raw dynamic loader", (v: Observations) => { for (const row of v.events) row.payload.prpEvent.payload.transport = "dynamic"; }], + ["MCP loader", (v: Observations) => { for (const row of v.events) row.payload.prpEvent.payload.transport = "mcp"; }], + ["copied shell output", (v: Observations) => { for (const row of v.events) row.payload.prpEvent.payload.name = "bash"; }], + ["foreign skill name", (v: Observations) => { output(v, text => text.replaceAll(fixture.slug, "foreign")); }], + ["foreign heading", (v: Observations) => { output(v, text => text.replace(`# Skill: ${fixture.slug}`, "# Skill: foreign")); }], + ["nonce only", (v: Observations) => { output(v, text => text.replace(fixture.procedure.trim(), fixture.expected.reference)); }], + ["changed source body", (v: Observations) => { output(v, text => text.replace(fixture.procedure.trim(), fixture.procedure.trim() + "\nChanged.")); }], + ["foreign base directory", (v: Observations) => { output(v, text => text.replace(runtimeName, "foreign")); }], + ["nonempty skill inventory", (v: Observations) => { output(v, text => text.replace("\n", "\nother.txt\n")); }], + ["duplicate inventory", (v: Observations) => { output(v, text => text.replace("", "")); }], + ["extra inventory closing tag", (v: Observations) => { output(v, text => text.replace("", "")); }], + ["extra wrapper closing tag", (v: Observations) => { output(v, text => text.replace("", "")); }], + ] as const)("rejects %s", (_label, mutate) => { const value = observations(); mutate(value); expect(read(value)).toEqual([]); }); + it.each([ + "https://example.test/.claude/skills/", "file://foreign/.claude/skills/", "file:///workspace/.claude/skills/", + "relative/.claude/skills/", "/workspace/../.claude/skills/", "/workspace/%2e%2e/.claude/skills/", + "/workspace%2fother/.claude/skills/", "/workspace%5cother/.claude/skills/", + "/workspace%00/.claude/skills/", "/workspace%0a/.claude/skills/", "/workspace\\other/.claude/skills/", + ])("rejects an unbound or ambiguous base address %s", prefix => { + const value = observations(); output(value, text => text.replace("/workspace/.claude/skills/", prefix)); + expect(read(value)).toEqual([]); + }); + it.each(["?query", "#fragment", "/../foreign", "%00", "%2Fextra"])("rejects URI suffix %s", suffix => { + const value = observations(); output(value, text => text.replace(`${runtimeName}\n`, `${runtimeName}${suffix}\n`)); + expect(read(value)).toEqual([]); + }); + it.each([["TAB", "\u0009"], ["ESC", "\u001b"], ["DEL", "\u007f"]])("rejects ASCII control %s in the base path", (_name, control) => { + const value = observations(); output(value, text => text.replace("/workspace/", `/work${control}space/`)); + expect(read(value)).toEqual([]); + }); +}); diff --git a/tests/runner-e2e/product-journeys/saved-skill-use.md b/tests/runner-e2e/product-journeys/saved-skill-use.md index 1ea01e75855..c0b8df87244 100644 --- a/tests/runner-e2e/product-journeys/saved-skill-use.md +++ b/tests/runner-e2e/product-journeys/saved-skill-use.md @@ -52,12 +52,23 @@ with an unknown operation and null target. That path requires its complete file result wrapper: the file path must name the exact materialized skill directory. Both the saved assignment entry and the admitted native execution input must bind that runtime name to the same canonical key and immutable created version. -The native read must remain read-only, successful, paired by call/turn/runner, +The file read must remain read-only, successful, paired by call/turn/runner, untruncated and byte-complete, and its content must contain the full procedure. A targetless filesystem read without this bound file wrapper cannot qualify. Discovery, shell output, model prose, failed or truncated reads, foreign run receipts and marker-only output cannot pass. +Core's native OpenCode proxy forwards its built-in `skill` loader without an +operation. Core records that receipt as `unknown` with a false read-only hint. +This closed loader path requires the actual OpenCode run, runner, native session, +turn, item and paired call. Both assignment records and the admitted version map +must bind the exact immutable skill and materialized directory. Its complete +`skill_content` wrapper and heading must name that skill. The saved procedure +body must match exactly. The local base-directory path must be absolute and name +that bound runtime directory without traversal or URI encodings, and `skill_files` +must be empty. The receipt must be successful, untruncated and byte-complete. +Other unknown operations, raw dynamic calls and unbound loaders still fail. + The first-party `paperclip_read_assigned_skill` tool can also supply the read. Its complete `inputText` result must contain the exact canonical skill key, immutable created version, `SKILL.md` path, original Markdown, byte size and