diff --git a/tests/runner-e2e/CHAT-FEATURES.md b/tests/runner-e2e/CHAT-FEATURES.md index 9a04d43973c..a66cf38a374 100644 --- a/tests/runner-e2e/CHAT-FEATURES.md +++ b/tests/runner-e2e/CHAT-FEATURES.md @@ -239,6 +239,12 @@ Initial setup uses the existing workflow operations: independent checks, private-evidence receipts and cleanup. Inspect screenshots through authenticated private-history retrieval. Retain earlier failed attempts. +Launch preflight searches use TestOps' typed name filter, encoded as Base64 of +UTF-8 JSON in the `search` query parameter. Plain name text returns HTTP 400. +The backlog and completion launchers share this encoding; the full-catalog +launcher uses the completion adapter. Existing nonce and once-only guards still +apply before a plan-run request. + The runtime accepts only `operation`, `ALLURE_JOB_RUN_ID` and reserved username metadata. It rejects arbitrary source/model/budget selectors, branch workflows, replays and foreign cases. All secret-bearing jobs recheck actual GitHub run and diff --git a/tests/runner-e2e/chat-completion-testops-api.mjs b/tests/runner-e2e/chat-completion-testops-api.mjs index e6eba6bc284..5b9f883c935 100644 --- a/tests/runner-e2e/chat-completion-testops-api.mjs +++ b/tests/runner-e2e/chat-completion-testops-api.mjs @@ -1,4 +1,4 @@ -import { testopsClient, endpoint } from './chat-feature-testops-api.mjs'; +import { testopsClient, endpoint, launchNameSearch } from './chat-feature-testops-api.mjs'; import { cases, planName, positive, reject, validateSelection, catalogDefinition } from './chat-completion-testops-contract.mjs'; import { reconcileFullCatalog } from './chat-full-catalog-testops-reconcile.mjs'; @@ -225,7 +225,7 @@ export async function launchPilotPlan(api, env, context, workflowId, markOnce) { if (!Array.isArray(assignments) || assignments.length !== 1 || assignments[0].job?.id !== binding.jobId || assignments[0].testCasesCount !== cases.length) reject('plan_job_mismatch'); const name = `${planName} · ${env.CHAT_COMPLETION_TESTOPS_LAUNCH_NONCE}`; - const existing = pageRows(await api(`/api/launch?${new URLSearchParams({ projectId: '1', search: name, page: '0', size: '100' })}`)); + const existing = pageRows(await api(`/api/launch?${new URLSearchParams({ projectId: '1', search: launchNameSearch(name), page: '0', size: '100' })}`)); if (existing.length) reject('launch_already_requested'); await markOnce('launch_plan'); const launched = await api(`/api/testplan/${binding.planId}/run`, 'POST', { launchName: name }); diff --git a/tests/runner-e2e/chat-completion-testops-api.test.mjs b/tests/runner-e2e/chat-completion-testops-api.test.mjs index 40a1da85cc2..4ee58ebe003 100644 --- a/tests/runner-e2e/chat-completion-testops-api.test.mjs +++ b/tests/runner-e2e/chat-completion-testops-api.test.mjs @@ -128,6 +128,22 @@ test('the fenced launcher requests one exact three-case TestOps launch', async ( launchName: `${planName} · ${operatorEnv.CHAT_COMPLETION_TESTOPS_LAUNCH_NONCE}` } }]); }); +test('completion launch preflight sends the typed UTF-8 name filter required by TestOps', async () => { + const f = mutationApi({ assignments: [{ job, testCasesCount: 3 }] }); + const api = async (route, ...args) => { + if (route.startsWith('/api/launch?')) { + const wire = new URL(route, 'https://paperclip.testops.cloud').searchParams.get('search'); + let filters; + try { filters = JSON.parse(Buffer.from(wire, 'base64').toString('utf8')); } + catch { throw new Error('TestOps HTTP 400: typed launch search required'); } + assert.deepEqual(filters, [{ id: 'name', type: 'string', value: `${planName} · ${operatorEnv.CHAT_COMPLETION_TESTOPS_LAUNCH_NONCE}` }]); + } + return f.api(route, ...args); + }; + await launchPilotPlan(api, operatorEnv, context, 999, async () => {}); + assert.equal(f.calls.filter(row => row.method === 'POST').length, 1); +}); + for (const changed of [{ GITHUB_RUN_NUMBER: '6' }, { CHAT_COMPLETION_TESTOPS_LAUNCH_NONCE: 'not-a-nonce' }, { CHAT_COMPLETION_TESTOPS_LAUNCH_RUN_NUMBER: '' }]) { test(`launcher rejects its invalid fence before requests: ${Object.keys(changed)[0]}`, async () => { const f = mutationApi(); diff --git a/tests/runner-e2e/chat-feature-testops-api.mjs b/tests/runner-e2e/chat-feature-testops-api.mjs index 8be6bc63fc3..2b4a4f1d049 100644 --- a/tests/runner-e2e/chat-feature-testops-api.mjs +++ b/tests/runner-e2e/chat-feature-testops-api.mjs @@ -2,6 +2,10 @@ export const endpoint = 'https://paperclip.testops.cloud'; export const planName = 'Chat feature evals'; export const fullName = 'paperclip.Product E2E.chat-features.create-backlog'; export const aliases = [fullName, 'paperclip.Product E2E.chat-features.tasks.create-backlog']; +export function launchNameSearch(name) { + // TestOps search uses the UI's typed filter carrier, not plain search text. + return Buffer.from(JSON.stringify([{ id: 'name', type: 'string', value: name }]), 'utf8').toString('base64'); +} class SetupFailure extends Error { constructor(code, status = null) { super(code); this.code = code; this.status = status; } } @@ -348,7 +352,7 @@ export async function launchChatPlan(api, env, markOnce, admission) { const assignments = await api(`/api/testplan/${known.planId}/job`); if (!Array.isArray(assignments) || assignments.length > 1 || assignments.some(row => row.job?.id !== known.jobId || row.testCasesCount !== 1)) fail('chat_testops_plan_job_mismatch'); const launchName = `${planName} · ${nonce}`; - const existing = rows(await api(`/api/launch?${new URLSearchParams({ projectId: '1', search: launchName, page: '0', size: '100' })}`)); + const existing = rows(await api(`/api/launch?${new URLSearchParams({ projectId: '1', search: launchNameSearch(launchName), page: '0', size: '100' })}`)); if (existing.length) { if (existing.length !== 1 || existing[0].name !== launchName || !id(existing[0].id)) fail('chat_testops_nonce_conflict'); return { schema: 'paperclip.chat-feature-testops-launch/v1', ...source, nonce, planId: known.planId, diff --git a/tests/runner-e2e/chat-feature-testops-api.test.mjs b/tests/runner-e2e/chat-feature-testops-api.test.mjs index 292411c1d22..318c616208c 100644 --- a/tests/runner-e2e/chat-feature-testops-api.test.mjs +++ b/tests/runner-e2e/chat-feature-testops-api.test.mjs @@ -196,6 +196,22 @@ test('launch configures only the exact registered public job/plan then issues on assert.ok(!server.calls.some(row => /jobrun.*rerun|dispatches/.test(row.route))); }); +test('backlog launch preflight sends the typed UTF-8 name filter required by TestOps', async () => { + const server = launchServer(); + const api = async (route, ...args) => { + if (route.startsWith('/api/launch?')) { + const wire = new URL(route, 'https://paperclip.testops.cloud').searchParams.get('search'); + let filters; + try { filters = JSON.parse(Buffer.from(wire, 'base64').toString('utf8')); } + catch { throw new Error('TestOps HTTP 400: typed launch search required'); } + assert.deepEqual(filters, [{ id: 'name', type: 'string', value: `Chat feature evals · ${launchEnv.CHAT_TESTOPS_LAUNCH_NONCE}` }]); + } + return server.api(route, ...args); + }; + await launchChatPlan(api, launchEnv, async () => {}, await fence()); + assert.equal(server.requests(), 1); +}); + test('observed launch nonce prevents a second run request across separate local entry attempts', async () => { const server = launchServer(); await launchChatPlan(server.api, launchEnv, async () => {}, await fence()); const result = await launchChatPlan(server.api, launchEnv, () => assert.fail('existing launch must not mark or write'), await fence());