diff --git a/tests/runner-e2e/reliability-testops-registration.mjs b/tests/runner-e2e/reliability-testops-registration.mjs index c9da1096cec..42d1c9706a2 100644 --- a/tests/runner-e2e/reliability-testops-registration.mjs +++ b/tests/runner-e2e/reliability-testops-registration.mjs @@ -94,8 +94,7 @@ export async function inspectReliability(api) { if (candidates.length !== 1 || !positive(candidates[0].job.id)) fail('standing_job'); const job = await api(`/api/job/${candidates[0].job.id}`); if (job?.id !== candidates[0].job.id || job.type !== 'github' || !workflowUrls.has(job.url)) fail('standing_job'); - const assignments = plan ? await api(`/api/testplan/${plan.id}/job`) : []; - if (!Array.isArray(assignments)) fail('plan_job'); + const assignments = plan ? jobAssignments(await api(`/api/testplan/${plan.id}/job`)) : []; return { caseIds: Object.fromEntries(found), plan, jobId: job.id, standingParameters: Array.isArray(candidates[0].job.parameters) ? candidates[0].job.parameters : [], assignments }; } @@ -134,8 +133,8 @@ export async function registerReliability(api) { plan = await api(`/api/testplan/${plan.id}`); } assertPlan(plan, ids); - let assignments = await api(`/api/testplan/${plan.id}/job`); - if (!Array.isArray(assignments) || assignments.length > 1 || assignments.some(row => row?.job?.id !== before.jobId)) fail('plan_job_conflict'); + let assignments = jobAssignments(await api(`/api/testplan/${plan.id}/job`)); + if (assignments.length > 1 || assignments.some(row => row?.job?.id !== before.jobId)) fail('plan_job_conflict'); if (!assignments.length) { await api(`/api/testplan/${plan.id}/assign`, 'POST', { jobId: before.jobId, selection: { inverted: false, manual: false, leafsInclude: ids } }); created.assignment = true; @@ -145,13 +144,21 @@ export async function registerReliability(api) { await api(`/api/testplan/${plan.id}/jobparameter`, 'POST', { jobId: before.jobId, parameters: before.standingParameters }); created.parameters = true; } - assignments = await api(`/api/testplan/${plan.id}/job`); - if (!Array.isArray(assignments) || assignments.length !== 1 || assignments[0]?.job?.id !== before.jobId + assignments = jobAssignments(await api(`/api/testplan/${plan.id}/job`)); + if (assignments.length !== 1 || assignments[0]?.job?.id !== before.jobId || assignments[0].testCasesCount !== ids.length || (before.standingParameters.length && !same(assignments[0].job.parameters ?? [], before.standingParameters))) fail('plan_job'); return { caseIds, plan, jobId: before.jobId, created }; } +// TestOps lists a plan's cases that no job covers as a job-less row, e.g. +// `[{ testCasesCount: 7 }]` for a freshly created plan. That row is not a job +// assignment; only rows naming a job are. +function jobAssignments(rows) { + if (!Array.isArray(rows)) fail('plan_job'); + return rows.filter(row => row?.job != null); +} + export function registrationReceipt(operation, state, env) { const caseIds = Object.fromEntries(reliabilityDefinitions.filter(row => state.caseIds[row.name]).map(row => [row.name, state.caseIds[row.name]])); const complete = Object.keys(caseIds).length === reliabilityDefinitions.length && positive(state.plan?.id) diff --git a/tests/runner-e2e/reliability-testops.test.mjs b/tests/runner-e2e/reliability-testops.test.mjs index c10727bd54c..4cda542b8c3 100644 --- a/tests/runner-e2e/reliability-testops.test.mjs +++ b/tests/runner-e2e/reliability-testops.test.mjs @@ -196,6 +196,8 @@ function fakeTestOps({ existing = [], plan = null, assignments = [], standing = if (method === 'GET' && url.pathname === '/api/testplan') return { content: state.plan ? [{ id: state.plan.id, name: state.plan.name }] : [], totalPages: 1 }; if (method === 'POST' && url.pathname === '/api/testplan') { state.plan = { id: 77, projectId: 1, name: body.name, treeSelection: body.treeSelection, testCasesCount: body.treeSelection.leafsInclude.length }; + // Live TestOps lists a new plan's uncovered cases as one job-less row. + state.assignments = [{ testCasesCount: body.treeSelection.leafsInclude.length }]; return { id: 77 }; } if (method === 'GET' && url.pathname === '/api/testplan/77') return state.plan; @@ -293,6 +295,20 @@ test('registration refuses a standing plan without exactly one trusted job or a await assert.rejects(registerReliability(api), /plan_job_conflict/); }); +test('a plan TestOps reports only as a job-less row is unassigned, so registration assigns the standing job', async () => { + const { api, state } = fakeTestOps(); + await registerReliability(api); + // Reproduces staging plan 135: created, then the run stopped before assignment. + state.assignments = [{ testCasesCount: 7 }]; + const inspected = await inspectReliability(api); + assert.deepEqual(inspected.assignments, []); + assert.equal(registrationReceipt('inspect', inspected, env).complete, false); + const repaired = await registerReliability(api); + assert.equal(repaired.created.assignment, true); + assert.deepEqual(state.assignments.map(row => row.job?.id), [50]); + assert.equal(registrationReceipt('register', repaired, { GITHUB_RUN_ID: '1', GITHUB_SHA: workflowSha }).complete, true); +}); + test('the workflows expose the reliability plan without new provider or reporting authority', async () => { const full = parse(await readFile(new URL('../../.github/workflows/runner-full-stack-e2e.yml', import.meta.url), 'utf8')); assert.deepEqual(full.on.workflow_dispatch.inputs.testops_plan.options, ['product-regression', 'reliability']);