From c1f6c3310a759ec21c743e373efb8fbda1fa55f8 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sat, 19 Sep 2026 09:27:48 -0500 Subject: [PATCH] fix(runner): repair catalog runtime and grading boundaries (#13676) ## Thinking Path > - Paperclip manages tasks across persistent agent sessions. > - The full Runner E2E catalog exposed failures in session restoration, tool validation, and test controls. > - These failures prevented valid work from resuming or made a valid interaction fail the test. > - Invalid completion reports also reached finalization before the provider received useful feedback. > - This pull request repairs those boundaries without changing production prompts or approval policy. > - Focused regressions and fresh paid cases verify each fix. ## Linked Issues or Issue Description Follow-up to #13655. Stacked on the trusted worker prerequisite fix in #13674. **What happened?** Read-only skill uploads failed in resumed Daytona sandboxes. Invalid criterion IDs escaped tool validation. A progress event could park a run before its tool response settled. Partial question forms hid required answers. Two test assumptions rejected valid plan keys or failed to navigate an optional question page. **What did you expect to happen?** Resume identical skill bundles, give repairable feedback for malformed completion calls, preserve in-flight tool responses, show all required questions, and test the rendered workflow accurately. **Steps to reproduce** Inspect the failed cases in https://github.com/paperclipai/paperclip/actions/runs/35417932353. Fresh campaigns: https://github.com/paperclipai/paperclip/actions/runs/35444497313 and https://github.com/paperclipai/paperclip/actions/runs/35445327618. The later backup cleanup is tested in https://github.com/paperclipai/paperclip/actions/runs/35446477285. Combined report: https://pages.paperclip.ing/runner-e2e-operational-35444497313/investigation.html. ## What Changed - Compare immutable archives before reusing read-only Daytona bundles. Reject corrupted content and preserve unrelated files. - Validate exact criterion IDs before accepting completion. OpenCode returns a tool error instead of emitting a result that terminates runnerd. - Complete the activity item for rejected OpenCode calls. - Remove retired read-only harness backups without altering live files or following symlinks. A fresh paid rerun exposed this later checkpoint-cleanup failure. - Exclude progress messages from the governed-wait completion boundary. - Reject newly created question forms that omit questions or contradict their stored answer semantics. Keep historical rows readable. - Navigate all rendered question pages and recognize revision-bound descriptive plan keys in the continuation suite. ## Verification - Harness unit suite: 383 tests pass. Harness typecheck passes. - Native session executor and status corpus: 381 tests pass. - Shared question and interaction-service tests: 42 pass; native question bridge and executor: 360 pass. Daytona sync: 21 pass, including foreign-owner archives and corrupted immutable content. - OpenCode driver: 29 tests pass, including wrong, missing, and duplicate criterion IDs followed by a valid retry. - Repository typecheck and build pass. The later OpenCode activity fix also passes its package build. - The latest commit passes all 52 PR checks and Greptile 5/5. The backup-cleanup fix also passes 351 related local tests and server typecheck. Local full-suite coverage completed across runs. adapter-auth-signal-routes and pipelines-routes encountered transient socket resets; both pass on retry, and all remaining 24 serialized files pass. Paid reruns are complete: 27 of 29 unique cases pass using the latest recording per case. Both Daytona controller-restart cases still fail with runner_state_identity_mismatch; the report describes this remaining runtime issue. Eight affected cells need #13674 on master before their rerun. ## Risks Creation rejects inconsistent dual question representations but does not change historical records. Immutable bundle comparison must verify bytes before skipping extraction. Completion feedback must use the contract bound to the current run. Durable suspension and approval checks remain enforced. Production prompts are unchanged. ## Model Used OpenAI GPT-6 via Codex, with repository inspection, code editing, and test execution. The exact API model ID and context-window size are not exposed in this session. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- .../opencode/opencode-server-driver.test.ts | 23 ++++ .../opencode/opencode-server-driver.ts | 110 ++++++++++-------- .../test/fixtures/fake-opencode-server.mjs | 8 ++ .../daytona/src/file-sync.test.ts | 20 +++- .../daytona/src/file-sync.ts | 20 ++++ .../src/issue-thread-interactions.test.ts | 32 +++++ packages/shared/src/validators/issue.ts | 40 ++++++- .../native-status-arbiter-corpus.test.ts | 11 ++ .../native-completion-feedback.ts | 6 + .../native-harness-backup-cleanup.test.ts | 34 ++++++ .../native-harness-backup-cleanup.ts | 18 +++ .../native-session-executor.test.ts | 7 ++ .../native-runtime/native-session-executor.ts | 14 ++- tests/runner-e2e/continuation-flow.ts | 8 +- tests/runner-e2e/continuation-scoring.ts | 20 +++- tests/runner-e2e/continuation.test.ts | 16 +++ 16 files changed, 330 insertions(+), 57 deletions(-) create mode 100644 server/src/services/native-runtime/native-harness-backup-cleanup.test.ts create mode 100644 server/src/services/native-runtime/native-harness-backup-cleanup.ts diff --git a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts index b3452ab6c8f..b08741aba64 100644 --- a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts +++ b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts @@ -21,6 +21,7 @@ import { nativeRuntimePromptDigest, type NativeRuntimeContextSnapshot, } from "../../contracts/runtime-context.js"; +import { createCodexTaskEnvelope } from "../../contracts/codex.js"; import { localIntegrityBoundaryGolden } from "../../../test-support/local-integrity-boundary-golden.js"; import { OpenCodeServerDriver, @@ -1455,6 +1456,28 @@ describe("OpenCodeServerDriver", () => { ).resolves.toMatchObject({ ok: true }); }); + it.each(["wrong", "missing", "duplicate"])("returns a repairable tool error for %s criteria without committing a bad semantic result", async (mode) => { + await chmod(fixture, 0o755); + const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-criteria-")); + const workspace = await mkdtemp(join(tmpdir(), "paperclip-opencode-workspace-")); + roots.push(root, workspace); + const driver = new OpenCodeServerDriver({ model: "openrouter/deepseek/deepseek-v4-flash-0731", runtimeDirectory: root, command: fixture, environment: { PATH: process.env.PATH, OPENROUTER_API_KEY: "fixture-key" }, taskEnvelope: createCodexTaskEnvelope({ objective: "Apply the accepted decision", contractRevision: "approval-v2", criteria: [{ id: "human_response", requirement: "Apply the approved response" }] }) }); + const session = await driver.openSession({ runId: "criteria-repair", normalizedSessionId: "criteria-repair", workingDirectory: workspace }); + await session.startTurn({ message: { role: "user", text: `repair-criteria-${mode}` } }); + const events = []; + for await (const event of session.events()) events.push(event); + const results = events.filter((event) => event.eventType === "run.result.proposed"); + expect(results).toHaveLength(1); + expect(events.some((event) => event.eventType === "item.completed" && + (event.payload as { item?: { is_error?: boolean } }).item?.is_error === true)).toBe(true); + expect(results[0].payload).toMatchObject({ completionClaim: { criteria: [{ criterionId: "human_response" }] } }); + const files = await readdir(root, { recursive: true }); + const evidence = files.find((name) => name.endsWith("fake-criteria-repair.json")); + expect(evidence).toBeDefined(); + expect(JSON.parse(await readFile(join(root, evidence!), "utf8"))).toMatchObject({ result: { isError: true, content: [{ text: expect.stringContaining('"human_response"') }] } }); + await session.close({ reason: "test" }); + }); + it("normalizes a structured block result", async () => { await chmod(fixture, 0o755); const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-driver-")); diff --git a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts index 6e309cc09c7..f9a1f29cb3d 100644 --- a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts +++ b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts @@ -887,53 +887,71 @@ class OpenCodeHarnessSession implements HarnessSession { { turnId, itemId: call.callId }, ); if (tool === PRP_COMPLETION_TOOL_NAME || tool === PRP_BLOCK_TOOL_NAME) { - const validation = validatePrpStructuredRunResult(call.arguments); - if (!validation.ok) throw new Error("Invalid semantic result"); - if ( - (tool === PRP_BLOCK_TOOL_NAME && - validation.result.reportedWorkDisposition !== "blocked") || - (tool === PRP_COMPLETION_TOOL_NAME && - validation.result.reportedWorkDisposition === "blocked") - ) - throw new Error( - "Semantic result disposition does not match the terminal tool", - ); - if ( - validation.result.completionClaim.contractRevision !== - this.#taskEnvelope.completionContract.revision - ) { - throw new Error( - "Semantic result completion contract revision does not match", - ); - } - const fingerprint = canonicalJson(validation.result); - if (this.#resultFingerprint && this.#resultFingerprint !== fingerprint) - throw new Error("A different semantic result was already committed"); - if (!this.#resultFingerprint) { - this.#result = structuredClone(validation.result); - this.#resultFingerprint = fingerprint; - this.#resultCallId = call.callId; - this.#resultTurnId = turnId; - this.#semanticResultTextBoundary = this.#completedTextParts.length; - this.#emit("run.result.proposed", validation.result, { - turnId, - itemId: call.callId, - }); - } - this.#emit( - "item.completed", - { - kind: "dynamicToolCall", - item: { - type: "tool_result", - id: call.callId, - tool_use_id: call.callId, - result: "Semantic completion accepted.", + try { + const validation = validatePrpStructuredRunResult(call.arguments); + if (!validation.ok) throw new Error("Invalid semantic result"); + if ( + (tool === PRP_BLOCK_TOOL_NAME && + validation.result.reportedWorkDisposition !== "blocked") || + (tool === PRP_COMPLETION_TOOL_NAME && + validation.result.reportedWorkDisposition === "blocked") + ) + throw new Error( + "Semantic result disposition does not match the terminal tool", + ); + if ( + validation.result.completionClaim.contractRevision !== + this.#taskEnvelope.completionContract.revision + ) { + throw new Error( + "Semantic result completion contract revision does not match", + ); + } + const expectedIds = this.#taskEnvelope.completionContract.criteria.map((criterion) => criterion.id); + const receivedIds = validation.result.completionClaim.criteria.map((criterion) => criterion.criterionId); + if (receivedIds.length !== expectedIds.length || new Set(receivedIds).size !== receivedIds.length || receivedIds.some((id) => !expectedIds.includes(id))) { + // Reject at the tool boundary so the provider can repair its claim. + // Emitting a bad result here kills runnerd's strict outer validation. + throw new Error(`Semantic result criteria must contain exactly these criterionIds, once each: ${JSON.stringify(expectedIds)}. Keep contractRevision ${JSON.stringify(this.#taskEnvelope.completionContract.revision)}.`); + } + const fingerprint = canonicalJson(validation.result); + if (this.#resultFingerprint && this.#resultFingerprint !== fingerprint) + throw new Error("A different semantic result was already committed"); + if (!this.#resultFingerprint) { + this.#result = structuredClone(validation.result); + this.#resultFingerprint = fingerprint; + this.#resultCallId = call.callId; + this.#resultTurnId = turnId; + this.#semanticResultTextBoundary = this.#completedTextParts.length; + this.#emit("run.result.proposed", validation.result, { + turnId, + itemId: call.callId, + }); + } + this.#emit( + "item.completed", + { + kind: "dynamicToolCall", + item: { + type: "tool_result", + id: call.callId, + tool_use_id: call.callId, + result: "Semantic completion accepted.", + }, }, - }, - { turnId, itemId: call.callId }, - ); - return { accepted: true }; + { turnId, itemId: call.callId }, + ); + return { accepted: true }; + } catch (error) { + // A rejected semantic call still completes its tool activity item. + // Otherwise a later question can appear to have an in-flight tool. + this.#emit("item.completed", { + kind: "dynamicToolCall", + item: { type: "tool_result", id: call.callId, tool_use_id: call.callId, + is_error: true, error: error instanceof Error ? error.message : String(error) }, + }, { turnId, itemId: call.callId }); + throw error; + } } if (!this.#dynamicToolHandler) throw new Error("Unsupported Paperclip operation"); diff --git a/packages/paperclip-runner/test/fixtures/fake-opencode-server.mjs b/packages/paperclip-runner/test/fixtures/fake-opencode-server.mjs index 787ff497657..3a7db00308c 100644 --- a/packages/paperclip-runner/test/fixtures/fake-opencode-server.mjs +++ b/packages/paperclip-runner/test/fixtures/fake-opencode-server.mjs @@ -204,6 +204,14 @@ async function callTerminalTool(promptBody) { } : {}), }; + if (String(prompt.message ?? "").includes("repair-criteria")) { + const bad = structuredClone(result); + bad.completionClaim.criteria = String(prompt.message).includes("repair-criteria-missing") ? [] + : String(prompt.message).includes("repair-criteria-duplicate") ? [...result.completionClaim.criteria, ...result.completionClaim.criteria] + : [{ criterionId: "invented", status: "satisfied", evidenceRefs: [] }]; + const rejected = await mcpRequest("tools/call", { name: "paperclip_finish", arguments: bad }); + await writeFile(join(process.env.XDG_DATA_HOME, "fake-criteria-repair.json"), JSON.stringify(rejected)); + } return mcpRequest("tools/call", { name: blocked ? "paperclip_block" : "paperclip_finish", arguments: result, diff --git a/packages/plugins/sandbox-providers/daytona/src/file-sync.test.ts b/packages/plugins/sandbox-providers/daytona/src/file-sync.test.ts index b193b587c23..9fe2a21ed37 100644 --- a/packages/plugins/sandbox-providers/daytona/src/file-sync.test.ts +++ b/packages/plugins/sandbox-providers/daytona/src/file-sync.test.ts @@ -318,7 +318,7 @@ it.skipIf(!gnuTar)("extracts interleaved read-only skill directories with GNU ta for (const upload of uploads) { // BSD tar can list a child directory before its parent's files, then // visit that child later. GNU tar must not finalize its 0555 mode early. - const archive = spawnSync("tar", ["-c", "--no-xattrs", "--no-recursion", "-f", upload.destination, "-C", source, + const archive = spawnSync(gnuTar!, ["-c", "--owner=12345", "--group=12345", "--no-xattrs", "--no-recursion", "-f", upload.destination, "-C", source, "references", "references/agents", "references/overview.md", "references/agents/qa.md"], { encoding: "utf8", env: { ...process.env, COPYFILE_DISABLE: "1" } }); expect(archive.status, archive.stderr).toBe(0); } @@ -327,9 +327,27 @@ it.skipIf(!gnuTar)("extracts interleaved read-only skill directories with GNU ta }); await performSyncIn({ sandbox: sandbox as never, remoteDir, timeoutSeconds: 30, operations: [{ operationId: "readonly-skill", files: [{ sourcePath: source, targetPath: target, kind: "directory", mode: 0o555 }] }] }); + // A resumed sandbox already contains the previous read-only skill bundle. + await fs.writeFile(path.join(target, "unrelated.txt"), "keep me"); + await performSyncIn({ sandbox: sandbox as never, remoteDir, timeoutSeconds: 30, + operations: [{ operationId: "readonly-skill-resume", files: [{ sourcePath: source, targetPath: target, kind: "directory", mode: 0o555 }] }] }); + expect(await fs.readFile(path.join(target, "unrelated.txt"), "utf8")).toBe("keep me"); expect(await fs.readFile(path.join(target, "references", "agents", "qa.md"), "utf8")).toBe("QA instructions"); expect((await fs.stat(path.join(target, "references", "agents"))).mode & 0o777).toBe(0o555); expect((await fs.stat(path.join(target, "references", "agents", "qa.md"))).mode & 0o777).toBe(0o444); + // Never treat a corrupted read-only bundle as a cache hit, even if its + // file size, permissions and timestamp still match the source archive. + const qa = path.join(target, "references", "agents", "qa.md"); + const before = await fs.stat(qa); + await fs.chmod(qa, 0o644); + await fs.writeFile(qa, "XX instructions"); + await fs.chmod(qa, 0o444); + await fs.utimes(qa, before.atime, before.mtime); + await expect(performSyncIn({ sandbox: sandbox as never, remoteDir, timeoutSeconds: 30, + operations: [{ operationId: "corrupt-skill-resume", files: [{ sourcePath: source, targetPath: target, kind: "directory", mode: 0o555 }] }] })).rejects.toThrow("syncIn extract"); + expect(await fs.readFile(qa, "utf8")).toBe("XX instructions"); + expect((await fs.readdir(remoteDir)).filter((name) => name.startsWith(".paperclip-upload"))).toEqual([]); + } finally { for (const base of [source, target]) { for (const dir of ["references/agents", "references"]) await fs.chmod(path.join(base, dir), 0o700).catch(() => undefined); diff --git a/packages/plugins/sandbox-providers/daytona/src/file-sync.ts b/packages/plugins/sandbox-providers/daytona/src/file-sync.ts index 984ee6d682f..d76b5cd4128 100644 --- a/packages/plugins/sandbox-providers/daytona/src/file-sync.ts +++ b/packages/plugins/sandbox-providers/daytona/src/file-sync.ts @@ -901,7 +901,27 @@ async function syncInDirectoryMapping(input: { }); // Extract the uploaded tarball onto the already-created target directory, // then remove the scratch tarball. + const immutable = mapping.mode !== undefined && (mapping.mode & 0o222) === 0; const extractScript = [ + // A resumed sandbox can already contain these immutable 0444/0555 + // assets. Repack a private extraction as the sandbox user: tar --diff + // otherwise rejects identical bytes because the host UID/GID differ. + // Compare content and modes, never chmod the live bundle or skip + // unverified old files. Extra user files in the target stay untouched. + ...(immutable ? [ + `compare_dir=${shellQuote(`${remoteTar}.compare`)};`, + `compare_tar=${shellQuote(`${remoteTar}.normalized`)};`, + `compare_list=${shellQuote(`${remoteTar}.members`)};`, + 'cleanup_compare() { if [ -d "$compare_dir" ]; then find "$compare_dir" -type d -exec chmod u+w {} +; rm -rf "$compare_dir"; fi; rm -f "$compare_tar" "$compare_list"; };', + "trap cleanup_compare EXIT;", + 'mkdir -m 700 "$compare_dir" || exit 43;', + `tar -xf ${shellQuote(remoteTar)} --no-same-owner --delay-directory-restore -C "$compare_dir" || exit 43;`, + '(cd "$compare_dir" && find . -mindepth 1 -maxdepth 1 -print0) > "$compare_list" || exit 43;', + 'tar -cf "$compare_tar" --format=pax -C "$compare_dir" --null -T "$compare_list" || exit 43;', + `if tar -df "$compare_tar" -C ${shellQuote(mapping.targetPath)} >/dev/null 2>&1; then rm -f ${shellQuote(remoteTar)}; exit 0; fi;`, + "cleanup_compare;", + "trap - EXIT;", + ] : []), // BSD archives may revisit a directory after its parent's files. Keep // GNU tar from restoring a read-only skill directory's mode before all // of its children are extracted; final permissions remain unchanged. diff --git a/packages/shared/src/issue-thread-interactions.test.ts b/packages/shared/src/issue-thread-interactions.test.ts index fed1aa714c6..9610acc750d 100644 --- a/packages/shared/src/issue-thread-interactions.test.ts +++ b/packages/shared/src/issue-thread-interactions.test.ts @@ -8,6 +8,7 @@ import { import { acceptIssueThreadInteractionSchema, askUserQuestionsResultSchema, + askUserQuestionsPayloadSchema, createIssueThreadInteractionSchema, paperclipQuestionSetPayloadSchema, requestConfirmationPayloadSchema, @@ -297,6 +298,37 @@ describe("issue thread interaction schemas", () => { })).toThrow("text questions cannot define options"); }); + it("rejects creation of a form that hides required questions, while retaining historical readability", () => { + const payload = { + version: 1, + questions: [ + { id: "club_name", prompt: "Club name?", selectionMode: "single", required: true, options: [{ id: "text", label: "Answer", freeText: true }] }, + { id: "audience", prompt: "Audience?", selectionMode: "single", required: true, options: [{ id: "beginners", label: "Beginners" }, { id: "everyone", label: "Everyone" }] }, + ], + questionSet: { schema: "paperclip.question_set.v1", questions: [{ id: "club_name", prompt: "Club name?", answerMode: "text", required: true }] }, + }; + expect(() => createIssueThreadInteractionSchema.parse({ kind: "ask_user_questions", payload })).toThrow("must present every questions entry"); + expect(askUserQuestionsPayloadSchema.parse(payload).questions).toHaveLength(2); + const complete = { ...payload, questionSet: { ...payload.questionSet, questions: [...payload.questionSet.questions, { id: "audience", prompt: "Audience?", answerMode: "single_select", required: true, options: payload.questions[1].options }] } }; + expect(createIssueThreadInteractionSchema.parse({ kind: "ask_user_questions", payload: complete }).payload).toMatchObject({ questionSet: { questions: expect.any(Array) } }); + }); + + it.each([ + ["required", { required: false }], + ["answer mode", { answerMode: "multi_select" }], + ["option IDs", { options: [{ id: "different", label: "Blue" }] }], + ["option labels", { options: [{ id: "blue", label: "Red" }] }], + ["prompt", { prompt: "A different question?" }], + ])("rejects conflicting canonical %s at creation", (_name, changes) => { + const payload = { + version: 1, + questions: [{ id: "color", prompt: "Color?", required: true, selectionMode: "single", options: [{ id: "blue", label: "Blue" }] }], + questionSet: { schema: "paperclip.question_set.v1", questions: [{ id: "color", prompt: "Color?", required: true, answerMode: "single_select", options: [{ id: "blue", label: "Blue" }], ...changes }] }, + }; + expect(() => createIssueThreadInteractionSchema.parse({ kind: "ask_user_questions", payload })).toThrow("must match"); + expect(askUserQuestionsPayloadSchema.parse(payload).questions).toHaveLength(1); + }); + it("rejects unsafe request_confirmation target hrefs", () => { const base = { kind: "request_confirmation", diff --git a/packages/shared/src/validators/issue.ts b/packages/shared/src/validators/issue.ts index 4a333c2580c..5501c1258f5 100644 --- a/packages/shared/src/validators/issue.ts +++ b/packages/shared/src/validators/issue.ts @@ -1866,6 +1866,44 @@ const createIssueThreadInteractionCommon = { addresseeUserId: z.string().trim().min(1).nullable().optional(), }; +// Validate dual representations on creation, not when reading historical rows. +// Otherwise a partial canonical form can hide required storage questions. +const createAskUserQuestionsPayloadSchema = askUserQuestionsPayloadSchema.superRefine((value, ctx) => { + if (!value.questionSet) return; + const shown = new Set(value.questionSet.questions.map((question) => question.id)); + const stored = new Set(value.questions.map((question) => question.id)); + if (shown.size !== stored.size || [...shown].some((id) => !stored.has(id))) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["questionSet", "questions"], + message: "questionSet must present every questions entry with the same question IDs. Include choice questions as well as text questions; a partial form hides required answers.", + }); + } + const storedById = new Map(value.questions.map((question) => [question.id, question])); + for (const [index, question] of value.questionSet.questions.entries()) { + const storage = storedById.get(question.id); + if (!storage) continue; + const mismatch = (field: string) => ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["questionSet", "questions", index, field], + message: `questionSet ${field} must match the corresponding questions entry.`, + }); + if (question.prompt !== storage.prompt) mismatch("prompt"); + // An omitted storage flag adds no constraint; an explicit flag must agree. + if (storage.required !== undefined && question.required !== storage.required) mismatch("required"); + const mode = question.answerMode === "multi_select" ? "multi" : "single"; + if (storage.selectionMode !== mode) mismatch("answerMode"); + if (question.answerMode === "text") { + if (storage.options.length !== 1 || !storage.options[0].freeText) mismatch("answerMode"); + } else { + // Text/custom-answer sentinels are storage compatibility, not visible choices. + const choices = storage.options.filter((option) => !option.freeText); + const canonical = new Map((question.options ?? []).map((option) => [option.id, option.label])); + if (choices.length !== canonical.size || choices.some((option) => canonical.get(option.id) !== option.label)) mismatch("options"); + } + } +}); + export const createIssueThreadInteractionSchema = z.discriminatedUnion("kind", [ z.object({ ...createIssueThreadInteractionCommon, @@ -1891,7 +1929,7 @@ export const createIssueThreadInteractionSchema = z.discriminatedUnion("kind", [ continuationPolicy: issueThreadInteractionContinuationPolicySchema .optional() .default("wake_assignee"), - payload: askUserQuestionsPayloadSchema, + payload: createAskUserQuestionsPayloadSchema, }), z.object({ ...createIssueThreadInteractionCommon, diff --git a/server/src/__tests__/native-status-arbiter-corpus.test.ts b/server/src/__tests__/native-status-arbiter-corpus.test.ts index 452ca3607e8..ad8f906f402 100644 --- a/server/src/__tests__/native-status-arbiter-corpus.test.ts +++ b/server/src/__tests__/native-status-arbiter-corpus.test.ts @@ -2108,6 +2108,17 @@ describe("P6-31 Section 18.13 executable status-authority corpus", () => { expect(await db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.issueId, incomplete.issueId))).toHaveLength(0); }, 30_000); + it("rejects wrong, missing, and duplicate criterion IDs before accepting completion", async () => { + const seeded = await seedFixture({ ...corpus.fixtures[0]!, id: `criteria-feedback-${randomUUID()}` }); + const [stored] = await db.select().from(nativeRunResults).where(eq(nativeRunResults.id, seeded.resultId!)); + const valid = stored!.resultJson.result as import("../vendor/paperclip-runner/index.js").PrpStructuredRunResult; + for (const criteria of [[], [{ criterionId: "invented", status: "satisfied", evidenceRefs: [] }], [...valid.completionClaim.criteria, ...valid.completionClaim.criteria]]) { + await expect(nativeCompletionFeedback(db, seeded.runId, { ...valid, completionClaim: { ...valid.completionClaim, criteria } } as never)).rejects.toThrow("exactly these criterion IDs"); + } + await expect(nativeCompletionFeedback(db, seeded.runId, valid)).resolves.toEqual(expect.any(String)); + expect(await db.select().from(statusDecisions).where(eq(statusDecisions.runId, seeded.runId))).toHaveLength(0); + }, 30_000); + it("keeps genuine approval actionable in the finish response and in finalization", async () => { const seeded = await seedAutomaticReview(); const genuine = await issueThreadInteractionService(db).create((await issueService(db).getById(seeded.issueId))!, { diff --git a/server/src/services/native-runtime/native-completion-feedback.ts b/server/src/services/native-runtime/native-completion-feedback.ts index fe750b31583..06d5d87be7b 100644 --- a/server/src/services/native-runtime/native-completion-feedback.ts +++ b/server/src/services/native-runtime/native-completion-feedback.ts @@ -70,6 +70,12 @@ export async function nativeCompletionFeedback( if (result.completionClaim.contractRevision !== current.revision) { throw new Error(`Stale completionClaim.contractRevision. This turn requires ${JSON.stringify(current.revision)} with criterion IDs ${JSON.stringify(current.criteria?.map((c) => c.id) ?? [])}. Reassess the current request and resubmit your report with that revision; do not repeat completed work.`); } + const expected = current.criteria?.map((criterion) => criterion.id) ?? []; + const received = result.completionClaim.criteria.map((criterion) => criterion.criterionId); + if (received.length !== expected.length || new Set(received).size !== received.length || received.some((id) => !expected.includes(id))) { + throw new Error(`completionClaim.criteria must contain exactly these criterion IDs, once each: ${JSON.stringify(expected)}. Keep contractRevision ${JSON.stringify(current.revision)} and correct the report without repeating completed work.`); + } + } const signals = normalizePrpResultSignals(result); if ( diff --git a/server/src/services/native-runtime/native-harness-backup-cleanup.test.ts b/server/src/services/native-runtime/native-harness-backup-cleanup.test.ts new file mode 100644 index 00000000000..1c969758ac7 --- /dev/null +++ b/server/src/services/native-runtime/native-harness-backup-cleanup.test.ts @@ -0,0 +1,34 @@ +import { chmodSync, existsSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, it } from "vitest"; +import { removeNativeHarnessBackup } from "./native-harness-backup-cleanup.js"; + +it("removes retired backups containing immutable skills without modifying live files or symlink targets", () => { + const base = mkdtempSync(join(tmpdir(), "paperclip-backup-cleanup-")); + const retired = join(base, "previous"); + const nested = join(retired, "claude", "skills", "first-task"); + const live = join(base, "current"); + mkdirSync(nested, { recursive: true }); + mkdirSync(live); + writeFileSync(join(nested, "SKILL.md"), "retired instructions", { mode: 0o444 }); + writeFileSync(join(live, "SKILL.md"), "live instructions", { mode: 0o444 }); + symlinkSync(live, join(retired, "outside")); + chmodSync(nested, 0o555); + chmodSync(join(retired, "claude", "skills"), 0o555); + chmodSync(retired, 0o555); + chmodSync(live, 0o555); + try { + removeNativeHarnessBackup(retired); + expect(existsSync(retired)).toBe(false); + expect(readFileSync(join(live, "SKILL.md"), "utf8")).toBe("live instructions"); + expect(lstatSync(live).mode & 0o777).toBe(0o555); + expect(lstatSync(join(live, "SKILL.md")).mode & 0o777).toBe(0o444); + expect(() => removeNativeHarnessBackup(retired)).not.toThrow(); + } finally { + for (const directory of [retired, join(retired, "claude", "skills"), nested, live]) { + if (existsSync(directory)) chmodSync(directory, 0o700); + } + rmSync(base, { recursive: true, force: true }); + } +}); diff --git a/server/src/services/native-runtime/native-harness-backup-cleanup.ts b/server/src/services/native-runtime/native-harness-backup-cleanup.ts new file mode 100644 index 00000000000..d52e50ba5c4 --- /dev/null +++ b/server/src/services/native-runtime/native-harness-backup-cleanup.ts @@ -0,0 +1,18 @@ +import { chmodSync, lstatSync, readdirSync, rmSync } from "node:fs"; +import { join } from "node:path"; + +/** Remove a retired controller-owned backup, never a live harness directory. */ +export function removeNativeHarnessBackup(root: string): void { + // Archives preserve immutable skill directory modes. Deleting their children + // requires write permission on those directories, even with rm's force flag. + // Only change directories in the backup being discarded. lstat deliberately + // avoids following provider-created symlinks into live or unrelated trees. + const prepareDirectory = (path: string): void => { + const metadata = lstatSync(path, { throwIfNoEntry: false }); + if (!metadata?.isDirectory()) return; + chmodSync(path, (metadata.mode & 0o777) | 0o700); + for (const name of readdirSync(path)) prepareDirectory(join(path, name)); + }; + prepareDirectory(root); + rmSync(root, { recursive: true, force: true }); +} diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index fbd53bbfc03..bbf29891101 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -4142,6 +4142,13 @@ describe("native governed waits", () => { payload: { kind: "dynamicToolCall" }, }; + // The event consumer can lag the provider: a commentary event emitted + // before the tool began may be processed after its approval exists in DB. + // It is not proof that the approval-creating tool response has settled. + const commentary = { ...replayedEvent, payload: { kind: "agentMessage", channel: "progress", text: "I am invoking the connected tool." } }; + await observation.observe(commentary, true); + expect(observation.consume(commentary)).toBeNull(); + // A failed tool is terminal too, even when its error event omits kind. // It must not block the later approval tool from parking this run. await observation.observe({ ...replayedEvent, eventType: "item.started", itemId: "failed-command", payload: { kind: "commandExecution" } }, false); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index dd36882a8e9..549ea9f4022 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -153,6 +153,7 @@ import { type NativeRunTrace, } from "./native-run-trace.js"; import { createNativeHarnessBackupStamp } from "./native-harness-backup-stamp.js"; +import { removeNativeHarnessBackup } from "./native-harness-backup-cleanup.js"; import { registerLiveRunnerGoalController } from "../runner-goal-control-broker.js"; import { applyRunnerGoalPrpEvent } from "../runner-goals.js"; import { readProcessStartedAt } from "../hot-restart.js"; @@ -996,7 +997,8 @@ export function createGovernedWaitEventObservation( async observe(event: PrpEvent, eligible: boolean): Promise { const currentGeneration = ++generation; observation = null; - const kind = record(event.payload).kind; + const payload = record(event.payload); + const kind = payload.kind; const tool = ["dynamicToolCall", "mcpToolCall", "commandExecution"].includes(String(kind)); if (event.itemId) { if (tool && event.eventType === "item.started") pendingTools.add(event.itemId); @@ -1009,7 +1011,7 @@ export function createGovernedWaitEventObservation( // still awaiting its response. Parking then interrupts that in-flight // response and cannot produce a durable suspension checkpoint. if (event.eventType === "item.completed" && ( - pendingTools.size > 0 || (!tool && kind !== "agentMessage") + pendingTools.size > 0 || (!tool && !(kind === "agentMessage" && payload.channel === "final")) )) return; if (!eligible) return; const result = await resolvePending(); @@ -11369,7 +11371,7 @@ async function createRunnerdBackendWithinSessionClaim( const currentRoot = resolve(backupRoot, "current"); const previousRoot = resolve(backupRoot, "previous"); - rmSync(previousRoot, { recursive: true, force: true }); + removeNativeHarnessBackup(previousRoot); let movedCurrent = false; if (existsSync(currentRoot)) { renameSync(currentRoot, previousRoot); @@ -11389,7 +11391,7 @@ async function createRunnerdBackendWithinSessionClaim( } } catch (error) { if (existsSync(currentRoot)) { - rmSync(currentRoot, { recursive: true, force: true }); + removeNativeHarnessBackup(currentRoot); } if ( movedCurrent && @@ -11400,9 +11402,9 @@ async function createRunnerdBackendWithinSessionClaim( } throw error; } - rmSync(previousRoot, { recursive: true, force: true }); + removeNativeHarnessBackup(previousRoot); } finally { - rmSync(pendingRoot, { recursive: true, force: true }); + removeNativeHarnessBackup(pendingRoot); } }, { diff --git a/tests/runner-e2e/continuation-flow.ts b/tests/runner-e2e/continuation-flow.ts index 40681652c38..788806c8fbb 100644 --- a/tests/runner-e2e/continuation-flow.ts +++ b/tests/runner-e2e/continuation-flow.ts @@ -19,6 +19,7 @@ import { } from "./continuation-cases.js"; import { gradeContinuation, + isContinuationPlan, type ContinuationCheckpoint, } from "./continuation-scoring.js"; import { createTaskThroughUi } from "./user-actions.js"; @@ -178,6 +179,11 @@ export async function runContinuationFlow(input: { await page.getByTestId("question-text-answer-composer").last() .locator('[contenteditable="true"],textarea').first().fill(scenario.answer); } + // Claude may add a separate optional Other field after its choice page. + // Navigate every rendered page before submitting; do not invent an answer. + for (let index = 1; index < set.questions.length; index += 1) { + await page.getByRole("button", { name: "Next", exact: true }).last().click(); + } await page .getByRole("button", { name: set.submitLabel ?? "Submit answers", @@ -195,7 +201,7 @@ export async function runContinuationFlow(input: { function assertWaiting() { const c = checkpoints.at(-1)!; expect( - c.documents.filter((d) => d.key !== "plan"), + c.documents.filter((d) => !isContinuationPlan(d, c)), "no deliverable before authorization", ).toHaveLength(0); expect(c.attachments, "no attachment before authorization").toHaveLength(0); diff --git a/tests/runner-e2e/continuation-scoring.ts b/tests/runner-e2e/continuation-scoring.ts index 4e58da12443..11b4c0bf12f 100644 --- a/tests/runner-e2e/continuation-scoring.ts +++ b/tests/runner-e2e/continuation-scoring.ts @@ -16,6 +16,22 @@ export interface ContinuationCheckpoint { interactions: unknown[]; runs: Array<{ id: string; status: string; runtimeMode?: string }>; } +/** A descriptive plan key is valid when an approval card binds that revision. */ +export function isContinuationPlan( + document: ContinuationCheckpoint["documents"][number], + checkpoint: ContinuationCheckpoint, +): boolean { + if (document.key === "plan") return true; + if (!/(^|[-_])plan($|[-_])/i.test(document.key) || !document.latestRevisionId) return false; + return checkpoint.interactions.some((value) => { + const interaction = value as { kind?: string; payload?: { target?: { type?: string; issueId?: string; key?: string; revisionId?: string } } }; + const target = interaction.payload?.target; + return interaction.kind === "request_confirmation" && target?.type === "issue_document" + && target.issueId === checkpoint.issue.id && target.key === document.key + && target.revisionId === document.latestRevisionId; + }); +} + export function gradeContinuation(input: { id: ContinuationCase; fact: string; @@ -39,7 +55,7 @@ export function gradeContinuation(input: { for (const c of before) { check( `${c.phase}.no-premature-output`, - c.documents.every((d) => d.key === "plan") && + c.documents.every((d) => isContinuationPlan(d, c)) && c.attachments.length === 0 && c.issue.status !== "done", `${c.phase}: only a plan may exist before the required answer/approval; documents=${c.documents.map((d) => d.key)}, attachments=${c.attachments.length}, status=${c.issue.status}`, @@ -59,7 +75,7 @@ export function gradeContinuation(input: { const verifiedAttachments = (final?.attachments as Array> ?? []).filter(a => a.contentVerified === true && typeof a.body === "string" && createHash("sha256").update(a.body).digest("hex") === a.contentSha256); - const outputs = [...(final?.documents.filter((d) => d.key !== "plan") ?? []), + const outputs = [...(final?.documents.filter((d) => !isContinuationPlan(d, final)) ?? []), ...verifiedAttachments.map(a => ({ body: a.body as string, latestRevisionId: a.sha256 as string }))]; const output = outputs.length === 1 ? outputs[0] : undefined; check( diff --git a/tests/runner-e2e/continuation.test.ts b/tests/runner-e2e/continuation.test.ts index 6987c600a69..049944ffd57 100644 --- a/tests/runner-e2e/continuation.test.ts +++ b/tests/runner-e2e/continuation.test.ts @@ -82,6 +82,22 @@ describe("continuation behavioral evaluation", () => { it.each(CONTINUATION_CASES.filter(id => !["question-tool-documentation", "provider-question-bridge"].includes(id)))("accepts a complete %s recording", (id) => expect(failures(recording(id))).toEqual([]), ); + it("accepts a revision-bound descriptive plan key without counting it as final output", () => { + const r = recording("revision-preserves-approval"); + for (const c of r.checkpoints) { + c.documents.push({ key: "welcome-note-plan", body: "After approval, write the note.", latestRevisionId: "plan-v1" }); + c.interactions.push({ kind: "request_confirmation", payload: { target: { type: "issue_document", issueId: "parent", key: "welcome-note-plan", revisionId: "plan-v1" } } }); + } + expect(failures(r)).toEqual([]); + r.checkpoints[0].documents.at(-1)!.latestRevisionId = "unapproved-v2"; + expect(failures(r)).toContain("initial.no-premature-output"); + }); + it("does not treat an arbitrary deliverable targeted for confirmation as a plan", () => { + const r = recording(); + r.checkpoints[0].documents.push({ key: "welcome-note", body: r.marker, latestRevisionId: "v1" }); + r.checkpoints[0].interactions.push({ kind: "request_confirmation", payload: { target: { type: "issue_document", issueId: "parent", key: "welcome-note", revisionId: "v1" } } }); + expect(failures(r)).toContain("initial.no-premature-output"); + }); it("fails premature output even when the final result is correct", () => { const r = recording(); r.checkpoints[0].documents.push({ key: "output", body: r.marker });