diff --git a/tests/runner-e2e/chat-catalog-testops-streaming.test.mjs b/tests/runner-e2e/chat-catalog-testops-streaming.test.mjs index 525b77f65ba..4d6a6a1f21c 100644 --- a/tests/runner-e2e/chat-catalog-testops-streaming.test.mjs +++ b/tests/runner-e2e/chat-catalog-testops-streaming.test.mjs @@ -9,7 +9,8 @@ import { run } from './chat-completion-testops-cli.mjs'; import { prepareCaseBundle, readCaseBundle, publishCaseBundle, publicationSet } from './chat-catalog-testops-streaming.mjs'; const context = { catalog: 'full', runId: '123', attempt: '1', workflowSha: 'a'.repeat(40), targetSha: catalogTargetSha }; const caseIds = { 'tasks.create-backlog': 388, 'documents-artifacts.write-plan': 26, 'projects.project-defaults': 100 }; -const cases = fullCatalogCases.filter(row => Object.hasOwn(caseIds, row.key)).map(row => ({ ...row, caseId: caseIds[row.key] })); +const cases = fullCatalogCases.filter(row => Object.hasOwn(caseIds, row.key)).map(row => ({ ...row, caseId: caseIds[row.key], + registeredFullName: row.aliases[0] ?? row.fullName })); const selection = { schema: 'paperclip.chat-completion-testops-selection/v1', ...context, projectId: 1, planId: 102, jobId: 100, launchId: 12, allureJobRunId: '77', cases }; const digest = hash(JSON.stringify(selection)), current = cases.find(row => row.caseId === 388), runUrl = 'https://github.com/paperclipai/paperclip/actions/runs/123'; @@ -33,8 +34,15 @@ async function fixture(t, failed = false, owned = current, runtimeContext = cont const prepared = await prepareCaseBundle(root, output, runtimeContext, selection, owned.executionId, digest); return { root, output, prepared }; } -function server(prepared, { existing = false, uncertain = false, apply = true } = {}) { - let present = existing, posts = 0; const calls = []; +function server(prepared, { existing = false, uncertain = false, apply = true, scheduled = false, processingReads = 0 } = {}) { + let present = existing, posts = 0, pendingReads = processingReads; const calls = []; + const resultId = scheduled ? 3347 : 501; + const detail = () => { + const pending = !present || pendingReads-- > 0; + return { id: resultId, testCaseId: 388, projectId: 1, launchId: 12, + fullName: current.registeredFullName, jobRun: { id: 77, launchId: 12, url: runUrl }, links: [], + ...(!pending ? { status: prepared.exported[0].status } : {}) }; + }; const api = async (route, method = 'GET', body) => { calls.push({ route, method }); if (method === 'POST') { @@ -49,10 +57,11 @@ function server(prepared, { existing = false, uncertain = false, apply = true } } if (route === '/api/jobrun/77') return { id: 77, job: { id: 100, type: 'github' }, stage: 'finished', launchId: 12, url: runUrl }; if (route === '/api/launch/12') return { id: 12, projectId: 1, closed: false }; - if (route.startsWith('/api/testresult?')) return { content: [...(present ? [{ id: 501, testCaseId: 388 }] : []), - { id: 502, testCaseId: 26 }], totalElements: present ? 2 : 1, totalPages: 1, number: 0 }; - if (route === '/api/testresult/501') return { id: 501, testCaseId: 388, projectId: 1, launchId: 12, - fullName: current.fullName, status: prepared.exported[0].status, jobRun: { id: 77, launchId: 12, url: runUrl }, links: [] }; + if (route.startsWith('/api/testresult?')) return { content: scheduled ? cases.map(row => ({ + id: row.caseId === 388 ? resultId : row.caseId === 100 ? 3346 : 3345, testCaseId: row.caseId })) + : [...(present ? [{ id: resultId, testCaseId: 388 }] : []), { id: 502, testCaseId: 26 }], + totalElements: scheduled ? 3 : present ? 2 : 1, totalPages: 1, number: 0 }; + if (route === `/api/testresult/${resultId}`) return detail(); if (route.startsWith('/api/testresult/attachment?')) return { content: prepared.exported[0].attachments.map((row, index) => ({ ...row, id: 700 + index, missed: false, fromTestCase: false })), totalElements: 2, totalPages: 1 }; throw new Error(`Unexpected test route ${route}`); @@ -60,8 +69,8 @@ function server(prepared, { existing = false, uncertain = false, apply = true } return { api, calls, get posts() { return posts; } }; } const immediate = { now: () => 0, wait: async () => {} }; -async function publish(f, s, mark = async () => {}) { - return publishCaseBundle(s.api, f.root, f.output, f.prepared, context, selection, current.executionId, digest, mark, immediate); +async function publish(f, s, mark = async () => {}, options = immediate) { + return publishCaseBundle(s.api, f.root, f.output, f.prepared, context, selection, current.executionId, digest, mark, options); } test('one frozen singleton bundle publishes into the bound shared launch beside siblings', async t => { const f = await fixture(t), s = server(f.prepared); let markers = 0; @@ -76,9 +85,38 @@ test('an existing result is verified without another mutation or a regenerated U const value = await publish(f, s, async () => assert.fail('must not mark another upload')); assert.equal(value.uploadSent, false); assert.equal(s.posts, 0); assert.equal(value.verification, 'VERIFIED'); }); +test('an existing unknown outcome remains readback-only and never requests another upload', async t => { + const f = await fixture(t), s = server(f.prepared, { existing: true }), original = s.api; + s.api = async (...args) => { const row = await original(...args); if (args[0] === '/api/testresult/501') row.status = 'unknown'; return row; }; + await assert.rejects(publish(f, s, async () => assert.fail('must not mark another upload')), /upload_unverified/); + assert.equal(s.posts, 0); +}); +test('three scheduled launch rows permit one initial upload and remain unready until an outcome is processed', async t => { + const f = await fixture(t), s = server(f.prepared, { scheduled: true, processingReads: 1 }); + let markers = 0, waits = 0; + const value = await publish(f, s, async () => { markers++; }, { now: () => 0, wait: async () => { waits++; } }); + assert.equal(value.verification, 'VERIFIED'); assert.equal(value.uploadSent, true); + assert.equal(value.exported.status, 'passed'); assert.equal(value.results[0].resultId, 3347); + assert.equal(s.posts, 1); assert.equal(markers, 1); assert.equal(waits, 1); +}); +test('permanently scheduled readback stops at the original poll bound without another upload', async t => { + const f = await fixture(t), s = server(f.prepared, { scheduled: true, processingReads: Infinity }); + let waits = 0; + await assert.rejects(publish(f, s, async () => {}, { now: () => 0, wait: async () => { waits++; } }), /upload_unverified/); + assert.equal(s.posts, 1); assert.equal(waits, 11); +}); +test('scheduled details with a foreign jobrun, contradictory source or malformed status refuse before upload', async t => { + for (const change of [row => { row.jobRun.id = 78; }, row => { row.jobRun.url = runUrl + '0'; }, + row => { row.status = null; }]) { + const f = await fixture(t), s = server(f.prepared, { scheduled: true }), original = s.api; + s.api = async (...args) => { const row = await original(...args); if (args[0] === '/api/testresult/3347') change(row); return row; }; + await assert.rejects(publish(f, s, async () => assert.fail('must not mark an invalid result')), /result_mismatch/); + assert.equal(s.posts, 0); + } +}); test('an uncertain applied upload reconciles by GET while an absent upload stops without resend', async t => { for (const apply of [true, false]) { - const f = await fixture(t), s = server(f.prepared, { uncertain: true, apply }); + const f = await fixture(t), s = server(f.prepared, { uncertain: true, apply, scheduled: true }); if (apply) { const value = await publish(f, s); assert.equal(value.verification, 'VERIFIED'); assert.equal(value.uncertainUpload.kind, 'transport_or_response_failure'); @@ -253,7 +291,7 @@ test('the final CLI verifies all authenticated mixed-grade publications without const match = u.pathname.match(/^\/api\/testresult\/(\d+)$/); if (match) { const i = Number(match[1]) - 501; details.push(cases[i].caseId); - return Response.json({ id: Number(match[1]), projectId: 1, launchId: 12, testCaseId: cases[i].caseId, fullName: cases[i].fullName, + return Response.json({ id: Number(match[1]), projectId: 1, launchId: 12, testCaseId: cases[i].caseId, fullName: cases[i].registeredFullName, jobRun: { id: 77, launchId: 12, url: runUrl }, links: [], status: exported[i].status }); } assert.equal(u.pathname, '/api/testresult/attachment'); diff --git a/tests/runner-e2e/chat-completion-testops-api.mjs b/tests/runner-e2e/chat-completion-testops-api.mjs index 9a469bd398c..78236c02df8 100644 --- a/tests/runner-e2e/chat-completion-testops-api.mjs +++ b/tests/runner-e2e/chat-completion-testops-api.mjs @@ -52,6 +52,21 @@ export function isWorkflowUrl(url, workflowId, context = {}) { || context.catalog === 'full' && url === `https://github.com/paperclipai/paperclip/blob/master/${workflow}` || url === (context.catalog === 'full' ? 'https://github.com/paperclipai/paperclip/actions?query=workflow%3A%22Chat+catalog+TestOps%22' : 'https://github.com/paperclipai/paperclip/actions?query=workflow%3A%22Chat+completion+TestOps%22'); } +function validatedResultStatus(result, id, expected, selection, context, runUrl) { + const { targetSha } = catalogDefinition(context); + if (result?.id !== id || result.projectId !== 1 || result.launchId !== selection.launchId || result.testCaseId !== expected.caseId + || result.fullName !== (expected.registeredFullName ?? expected.fullName) || result.jobRun?.id !== Number(selection.allureJobRunId) + || result.jobRun?.launchId !== selection.launchId || result.jobRun?.url !== runUrl + || result.jobRun.job != null && (result.jobRun.job.id !== selection.jobId + || result.jobRun.job.type != null && result.jobRun.job.type !== 'github' + || result.jobRun.job.url != null && !isWorkflowUrl(result.jobRun.job.url, context.workflowId, context)) + || result.links != null && (!Array.isArray(result.links) || result.links.some(link => !link || typeof link.url === 'string' + && (link.url.startsWith('https://github.com/paperclipai/paperclip/actions/runs/') && link.url !== runUrl + || link.url.startsWith('https://github.com/paperclipai/paperclip/commit/') && link.url !== `https://github.com/paperclipai/paperclip/commit/${targetSha}`)))) reject('result_mismatch'); + if (!Object.hasOwn(result, 'status')) return undefined; + if (!['failed', 'broken', 'passed', 'skipped', 'unknown'].includes(result.status)) reject('result_mismatch'); + return result.status; +} function assertCase(actual, expected) { if (actual.id !== expected.caseId || actual.projectId !== 1 || actual.fullName !== (expected.registeredFullName ?? expected.fullName) || actual.automated !== true || actual.deleted === true) reject('case_mismatch'); @@ -269,16 +284,10 @@ async function verifyUploadScope(api, selection, context, exported, executionId, if (!expected || !positive(listedRow.id)) reject('foreign_result'); const wanted = exported.find(value => value.caseId === expected.caseId); const result = await get(`/api/testresult/${listedRow.id}`); - if (result.id !== listedRow.id || result.projectId !== 1 || result.launchId !== selection.launchId || result.testCaseId !== expected.caseId - || result.fullName !== (expected.registeredFullName ?? expected.fullName) || result.jobRun?.id !== Number(selection.allureJobRunId) - || result.jobRun?.launchId !== selection.launchId || result.jobRun?.url !== runUrl - || result.jobRun.job != null && (result.jobRun.job.id !== selection.jobId - || result.jobRun.job.type != null && result.jobRun.job.type !== 'github' - || result.jobRun.job.url != null && !isWorkflowUrl(result.jobRun.job.url, context.workflowId, context)) - || !Array.isArray(result.links) || result.links.some(link => typeof link.url === 'string' - && (link.url.startsWith('https://github.com/paperclipai/paperclip/actions/runs/') && link.url !== runUrl - || link.url.startsWith('https://github.com/paperclipai/paperclip/commit/') && link.url !== `https://github.com/paperclipai/paperclip/commit/${targetSha}`)) - || !['unknown', wanted.status].includes(result.status)) reject('result_mismatch'); + const status = validatedResultStatus(result, listedRow.id, expected, selection, context, runUrl); + if (context.catalog === 'full' && status === undefined) return { executionId: expected.executionId, caseId: expected.caseId, + resultId: result.id, cleanup: wanted.cleanup, attachmentsVerified: false }; + if (!Array.isArray(result.links) || !['unknown', wanted.status].includes(status)) reject('result_mismatch'); const attached = await get(`/api/testresult/attachment?${new URLSearchParams({ testResultId: String(result.id), page: '0', size: '100' })}`); const attachments = pageRows(attached); if (attached.totalElements !== attachments.length || attachments.length > 10 || new Set(attachments.map(row => row.id)).size !== attachments.length @@ -325,14 +334,18 @@ export async function inspectPilotCaseUpload(api, selection, context, executionI return api(route, 'GET', undefined, Math.min(30_000, remaining)); }; for (let poll = 0; poll < 12 && now() < deadline; poll++) { - const { run, launch } = await uploadBinding(get, selection, context); + const { run, launch, runUrl } = await uploadBinding(get, selection, context); if (launch.closed === true) reject('launch_mismatch'); const rows = await resultRows(get, selection.launchId, authorized.length, true); if (rows !== null) { if (new Set(rows.map(value => value.id)).size !== rows.length || new Set(rows.map(value => value.testCaseId)).size !== rows.length) reject('result_page'); if (rows.some(value => !positive(value.id) || !authorized.some(expected => expected.caseId === value.testCaseId))) reject('foreign_result'); - return { present: rows.some(value => value.testCaseId === row.caseId), jobRunStage: run.stage }; + const existing = rows.find(value => value.testCaseId === row.caseId); + if (!existing) return { present: false, jobRunStage: run.stage }; + const result = await get(`/api/testresult/${existing.id}`); + return { present: validatedResultStatus(result, existing.id, row, selection, context, runUrl) !== undefined, + jobRunStage: run.stage }; } if (poll < 11 && now() + 5000 < deadline) await wait(5000); }