From 47f81e850b14cc209a7cd5681accab860caefe55 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sat, 31 Oct 2026 08:54:03 -0500 Subject: [PATCH] fix(testops): verify unexpanded result job runs (#18462) ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - Chat feature evaluations send their results to TestOps. > - The final reader checks the result against its current GitHub run. > - A real result omitted the nested job object and had no presentation links. > - The reader rejected that passed result after a successful upload. > - This change joins the embedded job-run ID to the separate job-run response. > - The reader keeps the source, case, status, and attachment checks. ## Linked Issues or Issue Description Refs #18394 and #18322. **What happened?** The native chat test and cleanup passed. TestOps stored a passed result and three attachments. The final read-only step failed with `chat_testops_upload_mismatch`. The result response had a job-run ID, launch ID, and current GitHub run URL. It did not include the nested job object. Its presentation links were empty. **Expected behavior** The reader must accept that response when the separate job-run response proves the exact job, launch, and current GitHub run. It must reject conflicting identity or source data. **Steps to reproduce** 1. Return a valid result with an unexpanded job run and empty links. 2. Return the matching full job run from the existing job-run endpoint. 3. Call `verifyChatUpload`. The old reader rejects the result. **Paperclip version or commit** Observed on c2971badde04b602fa9fe698015e1f3653b39e6b. The fix starts from ab51cabbbd. **Deployment mode** GitHub Actions and AWS. This defect is in the TestOps report reader. ## What Changed - Join the exact embedded job-run ID to the separately checked job-run response. - Accept empty presentation links. Reject conflicting known run and commit links when present. - Reject conflicting embedded job fields. - Add regressions for the observed response and wrong job, run, launch, and source data. ## Verification - Three new tests fail against the parent reader. - `node --test tests/runner-e2e/chat-feature-testops-api.test.mjs`: 47 passed. - `node --test tests/runner-e2e/testops-chat-reporter.test.mjs`: 14 passed. - Both changed files pass `node --check`. `git diff --check` passes. - Local checks used Node 26.4. Normal CI must check the workflow runtime. - Full local typecheck, build, app, database, SDK, and provider runs were not run. This two-file MJS fix uses pure tests and mocks. - Normal CI [41205375203](https://github.com/paperclipai/paperclip/actions/runs/41205375203), attempt 1, passed on c7b5bf3238. Both required checks passed. Greptile is 5/5. The optional automated coverage note has a recorded disposition and is resolved. - The original feature verdict, backend pass, and failed final reader step remain separate. This change does not regrade that run. ## Risks - Empty presentation links do not prove a server commit link. Source identity comes from the trusted current GitHub context, selection, and bound export. - The separate job-run response remains required. All case, status, attachment, and deadline checks remain active. - The new reader has not yet run against the live service. CI and a later authorized run must qualify delivery. ## Model Used OpenAI Codex, `gpt-6.1-sol`, with reasoning, source inspection, and code execution. The host does not expose the context-window limit. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge Co-authored-by: Paperclip --- tests/runner-e2e/chat-feature-testops-api.mjs | 10 +++-- .../chat-feature-testops-api.test.mjs | 40 +++++++++++++++++-- 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/tests/runner-e2e/chat-feature-testops-api.mjs b/tests/runner-e2e/chat-feature-testops-api.mjs index 704dd5842a7..9b2b1ddfc3f 100644 --- a/tests/runner-e2e/chat-feature-testops-api.mjs +++ b/tests/runner-e2e/chat-feature-testops-api.mjs @@ -379,9 +379,13 @@ export async function verifyChatUpload(api, env, selected, exported, { wait = ms const result = await get(`/api/testresult/${row.id}`); if (result.id !== row.id || result.projectId !== 1 || result.launchId !== selected.launchId || result.testCaseId !== selected.caseId || result.fullName !== selected.fullName || result.jobRun?.id !== Number(selected.allureJobRunId) - || result.jobRun?.job?.id !== selected.jobId || result.jobRun?.launchId !== selected.launchId || result.jobRun?.url !== runUrl - || !Array.isArray(result.links) || !result.links.some(link => link.url === runUrl) - || !result.links.some(link => link.url === sourceUrl)) fail('chat_testops_upload_mismatch'); + || result.jobRun?.launchId !== selected.launchId || result.jobRun?.url !== runUrl + || result.jobRun.job != null && (result.jobRun.job.id !== run.job.id + || result.jobRun.job.type != null && result.jobRun.job.type !== run.job.type + || result.jobRun.job.url != null && !publicJobUrl(result.jobRun.job.url, env)) + || !Array.isArray(result.links) || result.links.some(link => typeof link.url === 'string' + && (link.url.startsWith('https://github.com/paperclipai/paperclip/actions/runs/') && link.url !== runUrl + || link.url.startsWith('https://github.com/paperclipai/paperclip/commit/') && link.url !== sourceUrl))) fail('chat_testops_upload_mismatch'); if (!['passed', 'failed', 'broken', 'skipped', 'unknown'].includes(result.status)) fail('chat_testops_upload_mismatch'); if (result.status !== 'unknown' && result.status !== exported.status) fail('chat_testops_upload_mismatch'); const attached = await get(`/api/testresult/attachment?${new URLSearchParams({ testResultId: String(row.id), page: '0', size: '100' })}`); diff --git a/tests/runner-e2e/chat-feature-testops-api.test.mjs b/tests/runner-e2e/chat-feature-testops-api.test.mjs index 4422a6e0d0b..9b2bc7c74f7 100644 --- a/tests/runner-e2e/chat-feature-testops-api.test.mjs +++ b/tests/runner-e2e/chat-feature-testops-api.test.mjs @@ -322,11 +322,11 @@ const uploadReceipt = { status: 'passed', cleanup: 'passed', caseId: 91, campaig { name: 'Execution provenance and checks', contentType: 'application/json', contentLength: 300 }, { name: 'Sanitized Product E2E record and accounting', contentType: 'application/json', contentLength: 800 }] }; const uploadRun = { id: 9, job: { id: 7, type: 'github' }, launchId: 10, url: 'https://github.com/paperclipai/paperclip/actions/runs/123', stage: 'finished' }; -function uploadApi(change = {}, absent = 0) { +function uploadApi(change = {}, absent = 0, runChange = {}) { let polls = 0; return async (route, method) => { assert.equal(method, 'GET'); - if (route === '/api/jobrun/9') return uploadRun; + if (route === '/api/jobrun/9') return { ...uploadRun, ...runChange }; if (route === '/api/launch/10') return { id: 10, projectId: 1 }; if (route.startsWith('/api/testresult?')) { polls++; return { content: polls <= absent ? [] : [{ id: 45, testCaseId: 91 }], totalElements: polls <= absent ? 0 : 1, totalPages: polls <= absent ? 0 : 1 }; } if (route === '/api/testresult/45') return { id: 45, projectId: 1, launchId: 10, testCaseId: 91, fullName, jobRun: uploadRun, status: 'passed', @@ -348,11 +348,45 @@ test('readback verifies exact current launch/case/source/status and safe attachm test('readback rejects wrong launch, case, current jobrun, source and contradictory result status', async () => { const { verifyChatUpload } = await import('./chat-feature-testops-api.mjs'); - for (const change of [{ launchId: 11 }, { testCaseId: 92 }, { jobRun: { ...uploadRun, id: 11 } }, { links: [] }, { status: 'failed' }]) { + for (const change of [{ launchId: 11 }, { testCaseId: 92 }, { jobRun: { ...uploadRun, id: 11 } }, { links: [{ url: 'https://github.com/paperclipai/paperclip/commit/' + 'b'.repeat(40) }] }, { status: 'failed' }]) { await assert.rejects(verifyChatUpload(uploadApi(change), uploadContext, uploadSelection, uploadReceipt), /upload_mismatch/); } }); +test('readback joins an unexpanded result jobrun to the authoritative jobrun response', async () => { + const { verifyChatUpload } = await import('./chat-feature-testops-api.mjs'); + const { job, ...embedded } = uploadRun; + const result = await verifyChatUpload(uploadApi({ jobRun: embedded }), uploadContext, uploadSelection, uploadReceipt); + assert.equal(result.resultId, 45); assert.equal(result.jobId, 7); assert.equal(result.verification, 'VERIFIED'); +}); + +test('readback accepts protected empty presentation links while retaining the source-bound current run', async () => { + const { verifyChatUpload } = await import('./chat-feature-testops-api.mjs'); + const { job, ...embedded } = uploadRun; + const result = await verifyChatUpload(uploadApi({ jobRun: embedded, links: [] }), uploadContext, uploadSelection, uploadReceipt); + assert.equal(result.targetSha, uploadContext.GITHUB_SHA); assert.equal(result.runId, '123'); + assert.equal(result.resultId, 45); assert.equal(result.verification, 'VERIFIED'); +}); + +test('readback rejects authoritative or embedded identity contradictions and a foreign source selection', async () => { + const { verifyChatUpload } = await import('./chat-feature-testops-api.mjs'); + const { job, ...embedded } = uploadRun; + for (const runChange of [{ job: { id: 8, type: 'github' } }, { job: { id: 7, type: 'gitlab' } }, + { url: 'https://github.com/paperclipai/paperclip/actions/runs/124' }, { launchId: 11 }]) { + await assert.rejects(verifyChatUpload(uploadApi({ jobRun: embedded, links: [] }, 0, runChange), uploadContext, uploadSelection, uploadReceipt), /jobrun_mismatch/); + } + for (const change of [{ jobRun: { ...embedded, id: 11 } }, { jobRun: { ...embedded, job: { id: 8 } } }, + { jobRun: { ...embedded, job: { id: 7, type: 'gitlab' } } }, { jobRun: { ...embedded, job: { id: 7, url: 'https://github.com/other/repo' } } }, + { jobRun: { ...embedded, launchId: 11 } }, { jobRun: { ...embedded, url: 'https://github.com/paperclipai/paperclip/actions/runs/124' } }, + { links: [{ url: 'https://github.com/paperclipai/paperclip/actions/runs/124' }] }, + { links: [{ url: 'https://github.com/paperclipai/paperclip/commit/' + 'b'.repeat(40) }] }]) { + await assert.rejects(verifyChatUpload(uploadApi(change), uploadContext, uploadSelection, uploadReceipt), /upload_mismatch/); + } + for (const selection of [{ ...uploadSelection, targetSha: 'b'.repeat(40) }, { ...uploadSelection, workflowSha: 'b'.repeat(40) }]) { + await assert.rejects(verifyChatUpload(uploadApi({ jobRun: embedded, links: [] }), uploadContext, selection, uploadReceipt), /upload_mismatch/); + } +}); + test('readback missing result or screenshot metadata is bounded unverified, never a fabricated pass', async () => { const { verifyChatUpload } = await import('./chat-feature-testops-api.mjs'); let waits = 0;