diff --git a/tests/runner-e2e/CHAT-FEATURES.md b/tests/runner-e2e/CHAT-FEATURES.md index 5cf2ac9f0cf..9a04d43973c 100644 --- a/tests/runner-e2e/CHAT-FEATURES.md +++ b/tests/runner-e2e/CHAT-FEATURES.md @@ -225,6 +225,9 @@ Initial setup uses the existing workflow operations: binding. Only the observed single `Branch/defaultValue` definition for `refs/heads/master` is accepted. Bootstrap enables the job and assigns cases while preserving that definition; the plan uses its default branch. + Assignment inspection records unassigned and existing-job counts before + binding. A single unassigned bucket must cover the exact selected cases; + foreign jobs and unknown or incomplete buckets stop before writes. 2. Read the closed registration receipt. Store its observed `binding` JSON as `CHAT_COMPLETION_TESTOPS_BINDING` in `runner-e2e-testops-reporting`. 3. Use TestOps **Run**. For the initial operator launch, set an exact next diff --git a/tests/runner-e2e/chat-completion-testops-api.mjs b/tests/runner-e2e/chat-completion-testops-api.mjs index feaae741e62..e6eba6bc284 100644 --- a/tests/runner-e2e/chat-completion-testops-api.mjs +++ b/tests/runner-e2e/chat-completion-testops-api.mjs @@ -72,6 +72,11 @@ function assertBranchDefinition(job) { || job.parameters[0].name !== 'Branch' || job.parameters[0].defaultValue !== 'refs/heads/master' || Object.hasOwn(job.parameters[0], 'value')) reject('job_parameters'); } +function isUnassignedBucket(row, expectedCount) { + return row !== null && typeof row === 'object' && !Array.isArray(row) + && Object.keys(row).every(key => ['job', 'testCasesCount'].includes(key)) + && (row.job === null || !Object.hasOwn(row, 'job')) && row.testCasesCount === expectedCount; +} export async function validatePilotSelection(api, env, context, workflowId) { const { cases, targetSha, planName } = catalogDefinition(context); @@ -168,6 +173,7 @@ export async function completePilotRegistration(api, prepared, official, context const binding = { ...prepared, jobId: run.job.id }; assertPlan(await api(`/api/testplan/${binding.planId}`), binding, context); const job = await api(`/api/job/${binding.jobId}`); assertJob(job, binding, workflowId, context); + const assignments = await api(`/api/testplan/${binding.planId}/job`); await observe({ schema: 'paperclip.chat-completion-testops-registration-observed/v1', ...context, projectId: 1, jobId: job.id, launchId: official.launchId, allureJobRunId: official.jobRunId, parameterCount: Array.isArray(job.parameters) ? job.parameters.length : null, @@ -175,17 +181,22 @@ export async function completePilotRegistration(api, prepared, official, context id: positive(row?.id) ? row.id : null, name: row?.name === 'Branch' ? 'Branch' : 'UNRESOLVED', defaultValue: row?.defaultValue === 'refs/heads/master' ? 'refs/heads/master' : 'NOT_PROJECTED', runtimeValuePresent: Object.hasOwn(row ?? {}, 'value') })) : null, + assignmentCount: Array.isArray(assignments) ? assignments.length : null, + assignments: Array.isArray(assignments) ? assignments.slice(0, 4).map(row => ({ + kind: isUnassignedBucket(row, cases.length) ? 'unassigned' : row?.job?.id === binding.jobId ? 'owned' : 'foreign_or_unrecognized', + jobId: positive(row?.job?.id) ? row.job.id : null, + testCasesCount: Number.isSafeInteger(row?.testCasesCount) && row.testCasesCount >= 0 ? row.testCasesCount : null })) : null, qualification: 'inspection_only', actualResult: null }); assertBranchDefinition(job); - const assignments = await api(`/api/testplan/${binding.planId}/job`); - if (!Array.isArray(assignments) || assignments.length > 1 || assignments.some(row => row.job?.id !== binding.jobId || row.testCasesCount !== cases.length)) reject('plan_job_mismatch'); + if (!Array.isArray(assignments) || assignments.length > 1 || assignments.some(row => !isUnassignedBucket(row, cases.length) + && (row?.job?.id !== binding.jobId || row?.testCasesCount !== cases.length))) reject('plan_job_mismatch'); await markOnce('bind_owned_job'); // PATCH only the documented enable/integration fields. Preserve the opaque // definition exactly; no guessed parameter-definition write codec is used. await api(`/api/job/${binding.jobId}`, 'PATCH', { integrationId: binding.integrationId, canRun: true }); const patched = await api(`/api/job/${binding.jobId}`); assertJob(patched, binding, workflowId, context); assertBranchDefinition(patched); if (!same(patched.parameters, job.parameters) || (Object.hasOwn(patched, 'canRun') && patched.canRun !== true)) reject('job_parameters'); - if (!assignments.length) { + if (!assignments.length || isUnassignedBucket(assignments[0], cases.length)) { await markOnce('assign_plan'); await api(`/api/testplan/${binding.planId}/assign`, 'POST', { jobId: binding.jobId, selection: { inverted: false, manual: false, leafsInclude: binding.cases.map(row => row.caseId) } }); diff --git a/tests/runner-e2e/chat-completion-testops-api.test.mjs b/tests/runner-e2e/chat-completion-testops-api.test.mjs index ac52c469549..40a1da85cc2 100644 --- a/tests/runner-e2e/chat-completion-testops-api.test.mjs +++ b/tests/runner-e2e/chat-completion-testops-api.test.mjs @@ -92,6 +92,24 @@ test('registration keeps an existing exact assignment without assigning it twice assert.equal(f.calls.filter(row => row.route.endsWith('/assign')).length, 0); }); +for (const bucket of [{ job: null, testCasesCount: 3 }, { testCasesCount: 3 }]) { + test(`registration binds an explicit unassigned bucket (${Object.hasOwn(bucket, 'job') ? 'null' : 'omitted'} job)`, async () => { + const f = mutationApi({ assignments: [bucket] }), markers = []; let observed; + await completePilotRegistration(f.api, prepared, official, context, 999, async stage => markers.push(stage), async value => { observed = value; }); + assert.deepEqual(markers, ['bind_owned_job', 'assign_plan']); + assert.equal(f.calls.filter(row => row.route.endsWith('/assign')).length, 1); + assert.deepEqual(observed.assignments, [{ kind: 'unassigned', jobId: null, testCasesCount: 3 }]); + }); +} + +for (const bucket of [{ job: null, testCasesCount: 2 }, { job: null, testCasesCount: 3, otherJobId: 99 }]) { + test('registration rejects incomplete or unknown unassigned bucket data without writes', async () => { + const f = mutationApi({ assignments: [bucket] }); + await assert.rejects(completePilotRegistration(f.api, prepared, official, context, 999, async () => assert.fail('no mutation marker'))); + assert.equal(f.calls.filter(row => row.method !== 'GET').length, 0); + }); +} + test('bootstrap retains a closed observation before rejecting an unknown parameter shape', async () => { const f = mutationApi({ parameters: [{ id: 34, name: 'PRIVATE_NAME', defaultValue: 'PRIVATE_DEFAULT', value: 'PRIVATE_VALUE' }] }); let observed;