From c7b5bf3238b63b23c04d0e9824c2f18303c88edd Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 31 Oct 2026 08:23:10 -0500 Subject: [PATCH] fix(testops): join results to authoritative job runs Accept the observed unexpanded result DTO and protected empty links while retaining source-bound run identity and contradiction checks. Co-Authored-By: Paperclip --- tests/runner-e2e/chat-feature-testops-api.mjs | 10 +++-- .../chat-feature-testops-api.test.mjs | 40 +++++++++++++++++-- 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/tests/runner-e2e/chat-feature-testops-api.mjs b/tests/runner-e2e/chat-feature-testops-api.mjs index 704dd5842a7..9b2b1ddfc3f 100644 --- a/tests/runner-e2e/chat-feature-testops-api.mjs +++ b/tests/runner-e2e/chat-feature-testops-api.mjs @@ -379,9 +379,13 @@ export async function verifyChatUpload(api, env, selected, exported, { wait = ms const result = await get(`/api/testresult/${row.id}`); if (result.id !== row.id || result.projectId !== 1 || result.launchId !== selected.launchId || result.testCaseId !== selected.caseId || result.fullName !== selected.fullName || result.jobRun?.id !== Number(selected.allureJobRunId) - || result.jobRun?.job?.id !== selected.jobId || result.jobRun?.launchId !== selected.launchId || result.jobRun?.url !== runUrl - || !Array.isArray(result.links) || !result.links.some(link => link.url === runUrl) - || !result.links.some(link => link.url === sourceUrl)) fail('chat_testops_upload_mismatch'); + || result.jobRun?.launchId !== selected.launchId || result.jobRun?.url !== runUrl + || result.jobRun.job != null && (result.jobRun.job.id !== run.job.id + || result.jobRun.job.type != null && result.jobRun.job.type !== run.job.type + || result.jobRun.job.url != null && !publicJobUrl(result.jobRun.job.url, env)) + || !Array.isArray(result.links) || result.links.some(link => typeof link.url === 'string' + && (link.url.startsWith('https://github.com/paperclipai/paperclip/actions/runs/') && link.url !== runUrl + || link.url.startsWith('https://github.com/paperclipai/paperclip/commit/') && link.url !== sourceUrl))) fail('chat_testops_upload_mismatch'); if (!['passed', 'failed', 'broken', 'skipped', 'unknown'].includes(result.status)) fail('chat_testops_upload_mismatch'); if (result.status !== 'unknown' && result.status !== exported.status) fail('chat_testops_upload_mismatch'); const attached = await get(`/api/testresult/attachment?${new URLSearchParams({ testResultId: String(row.id), page: '0', size: '100' })}`); diff --git a/tests/runner-e2e/chat-feature-testops-api.test.mjs b/tests/runner-e2e/chat-feature-testops-api.test.mjs index 4422a6e0d0b..9b2bc7c74f7 100644 --- a/tests/runner-e2e/chat-feature-testops-api.test.mjs +++ b/tests/runner-e2e/chat-feature-testops-api.test.mjs @@ -322,11 +322,11 @@ const uploadReceipt = { status: 'passed', cleanup: 'passed', caseId: 91, campaig { name: 'Execution provenance and checks', contentType: 'application/json', contentLength: 300 }, { name: 'Sanitized Product E2E record and accounting', contentType: 'application/json', contentLength: 800 }] }; const uploadRun = { id: 9, job: { id: 7, type: 'github' }, launchId: 10, url: 'https://github.com/paperclipai/paperclip/actions/runs/123', stage: 'finished' }; -function uploadApi(change = {}, absent = 0) { +function uploadApi(change = {}, absent = 0, runChange = {}) { let polls = 0; return async (route, method) => { assert.equal(method, 'GET'); - if (route === '/api/jobrun/9') return uploadRun; + if (route === '/api/jobrun/9') return { ...uploadRun, ...runChange }; if (route === '/api/launch/10') return { id: 10, projectId: 1 }; if (route.startsWith('/api/testresult?')) { polls++; return { content: polls <= absent ? [] : [{ id: 45, testCaseId: 91 }], totalElements: polls <= absent ? 0 : 1, totalPages: polls <= absent ? 0 : 1 }; } if (route === '/api/testresult/45') return { id: 45, projectId: 1, launchId: 10, testCaseId: 91, fullName, jobRun: uploadRun, status: 'passed', @@ -348,11 +348,45 @@ test('readback verifies exact current launch/case/source/status and safe attachm test('readback rejects wrong launch, case, current jobrun, source and contradictory result status', async () => { const { verifyChatUpload } = await import('./chat-feature-testops-api.mjs'); - for (const change of [{ launchId: 11 }, { testCaseId: 92 }, { jobRun: { ...uploadRun, id: 11 } }, { links: [] }, { status: 'failed' }]) { + for (const change of [{ launchId: 11 }, { testCaseId: 92 }, { jobRun: { ...uploadRun, id: 11 } }, { links: [{ url: 'https://github.com/paperclipai/paperclip/commit/' + 'b'.repeat(40) }] }, { status: 'failed' }]) { await assert.rejects(verifyChatUpload(uploadApi(change), uploadContext, uploadSelection, uploadReceipt), /upload_mismatch/); } }); +test('readback joins an unexpanded result jobrun to the authoritative jobrun response', async () => { + const { verifyChatUpload } = await import('./chat-feature-testops-api.mjs'); + const { job, ...embedded } = uploadRun; + const result = await verifyChatUpload(uploadApi({ jobRun: embedded }), uploadContext, uploadSelection, uploadReceipt); + assert.equal(result.resultId, 45); assert.equal(result.jobId, 7); assert.equal(result.verification, 'VERIFIED'); +}); + +test('readback accepts protected empty presentation links while retaining the source-bound current run', async () => { + const { verifyChatUpload } = await import('./chat-feature-testops-api.mjs'); + const { job, ...embedded } = uploadRun; + const result = await verifyChatUpload(uploadApi({ jobRun: embedded, links: [] }), uploadContext, uploadSelection, uploadReceipt); + assert.equal(result.targetSha, uploadContext.GITHUB_SHA); assert.equal(result.runId, '123'); + assert.equal(result.resultId, 45); assert.equal(result.verification, 'VERIFIED'); +}); + +test('readback rejects authoritative or embedded identity contradictions and a foreign source selection', async () => { + const { verifyChatUpload } = await import('./chat-feature-testops-api.mjs'); + const { job, ...embedded } = uploadRun; + for (const runChange of [{ job: { id: 8, type: 'github' } }, { job: { id: 7, type: 'gitlab' } }, + { url: 'https://github.com/paperclipai/paperclip/actions/runs/124' }, { launchId: 11 }]) { + await assert.rejects(verifyChatUpload(uploadApi({ jobRun: embedded, links: [] }, 0, runChange), uploadContext, uploadSelection, uploadReceipt), /jobrun_mismatch/); + } + for (const change of [{ jobRun: { ...embedded, id: 11 } }, { jobRun: { ...embedded, job: { id: 8 } } }, + { jobRun: { ...embedded, job: { id: 7, type: 'gitlab' } } }, { jobRun: { ...embedded, job: { id: 7, url: 'https://github.com/other/repo' } } }, + { jobRun: { ...embedded, launchId: 11 } }, { jobRun: { ...embedded, url: 'https://github.com/paperclipai/paperclip/actions/runs/124' } }, + { links: [{ url: 'https://github.com/paperclipai/paperclip/actions/runs/124' }] }, + { links: [{ url: 'https://github.com/paperclipai/paperclip/commit/' + 'b'.repeat(40) }] }]) { + await assert.rejects(verifyChatUpload(uploadApi(change), uploadContext, uploadSelection, uploadReceipt), /upload_mismatch/); + } + for (const selection of [{ ...uploadSelection, targetSha: 'b'.repeat(40) }, { ...uploadSelection, workflowSha: 'b'.repeat(40) }]) { + await assert.rejects(verifyChatUpload(uploadApi({ jobRun: embedded, links: [] }), uploadContext, selection, uploadReceipt), /upload_mismatch/); + } +}); + test('readback missing result or screenshot metadata is bounded unverified, never a fabricated pass', async () => { const { verifyChatUpload } = await import('./chat-feature-testops-api.mjs'); let waits = 0;