diff --git a/tests/runner-e2e/CHAT-FEATURES.md b/tests/runner-e2e/CHAT-FEATURES.md index 864e25564a2..17b681cf0af 100644 --- a/tests/runner-e2e/CHAT-FEATURES.md +++ b/tests/runner-e2e/CHAT-FEATURES.md @@ -224,11 +224,13 @@ approved number. Missing or uncertain metadata stops; an uncertain request needs reconciliation rather than a new send. The nonce and launch search provide additional evidence. Normal TestOps **Run** uses `evaluate` and is independent of this one-time administrator fence. The launch nonce is protected configuration, -not a dispatch input. `CHAT_TESTOPS_OPERATION_PARAMETER_ID` is required only when -the registered job defines `operation`; the launcher sets that parameter to -`evaluate`. An official job with only `Branch` keeps its exact -`refs/heads/master` value and uses the fixed workflow's `evaluate` default. -Unknown parameters and other branch values are rejected. +not a dispatch input. Job parameter IDs are opaque. The launcher preserves the +API's exact ID/value pair for `refs/heads/master` and uses the fixed workflow's +`evaluate` default. An operation parameter is used only when the protected +`CHAT_TESTOPS_OPERATION_PARAMETER_ID` matches a returned ID with a `bootstrap` +or `evaluate` value; the launcher then sets it to `evaluate`. Other values, +duplicate IDs and conflicting configuration are rejected. No relationship to +global environment-variable IDs or names is inferred. `CHAT_TESTOPS_INTEGRATION_ID` names the reviewed existing project GitHub integration. The launcher verifies its project, type and enabled state, then binds diff --git a/tests/runner-e2e/chat-feature-testops-api.mjs b/tests/runner-e2e/chat-feature-testops-api.mjs index 0a0a16cbcfc..704dd5842a7 100644 --- a/tests/runner-e2e/chat-feature-testops-api.mjs +++ b/tests/runner-e2e/chat-feature-testops-api.mjs @@ -9,6 +9,7 @@ const fail = code => { const match = /^(testops_(?:api|auth)|chat_testops_github)_http_([1-5][0-9]{2})$/.exec(code); throw new SetupFailure(match ? `${match[1]}_http` : code, match ? Number(match[2]) : null); }; +const bootstrapRequestClasses = new Set(['JOB_RUN_GET', 'LAUNCH_GET', 'JOB_GET']); const diagnosticCodes = new Set(['untrusted_chat_testops_context', 'testops_token_missing', 'testops_auth_http', 'testops_api_http', 'testops_auth_invalid', 'testops_route_invalid', 'testops_response_too_large', 'testops_response_invalid', 'testops_metadata_page_incomplete', 'testops_case_identity_invalid', @@ -20,10 +21,14 @@ const diagnosticCodes = new Set(['untrusted_chat_testops_context', 'testops_toke 'chat_testops_job_id_mismatch', 'chat_testops_job_type_mismatch', 'chat_testops_job_parameters_shape_mismatch', 'chat_testops_job_parameters_limit', 'chat_testops_job_url_mismatch', 'chat_testops_jobrun_mismatch', 'chat_testops_launch_mismatch', 'chat_testops_selection_mismatch', 'chat_testops_parameter_mismatch', 'chat_testops_nonce_invalid', 'chat_testops_nonce_conflict', 'chat_testops_catalog_incomplete', 'chat_testops_launch_admission', 'chat_testops_github_http', 'chat_testops_integration_mismatch', 'chat_testops_upload_mismatch', 'chat_testops_upload_unverified']); export function failureProjection(error, stage) { + const request = error instanceof SetupFailure ? error.request : undefined; const stages = ['context', 'output_admission', 'authentication', 'metadata_inspection', 'plan_bootstrap', 'selection_validation', 'launch_admission', 'launch_preflight', 'plan_launch', 'bootstrap_job_metadata', 'result_export', 'upload_verification', 'bootstrap_context', 'receipt']; return { stage: stages.includes(stage) ? stage : 'unknown', code: error instanceof SetupFailure && diagnosticCodes.has(error.code) ? error.code : 'unknown_failure', - httpStatus: error instanceof SetupFailure && Number.isInteger(error.status) && error.status >= 100 && error.status < 600 ? error.status : null }; + httpStatus: error instanceof SetupFailure && Number.isInteger(error.status) && error.status >= 100 && error.status < 600 ? error.status : null, + ...(request && bootstrapRequestClasses.has(request.class) && id(request.resourceId) && ['STARTED', 'SUCCEEDED', 'FAILED'].includes(request.state) + ? { request: { class: request.class, resourceId: request.resourceId, state: request.state, + httpStatus: Number.isInteger(request.httpStatus) && request.httpStatus >= 100 && request.httpStatus < 600 ? request.httpStatus : null } } : {}) }; } const id = value => Number.isSafeInteger(value) && value > 0; const same = (a, b) => JSON.stringify(a) === JSON.stringify(b); @@ -186,63 +191,81 @@ async function registeredPlan(api, env) { const integration = await api(`/api/integration/${ids.integrationId}/project/1`); if (integration.id !== ids.integrationId || integration.projectId !== 1 || integration.info?.type !== 'github' || integration.disabled !== false) fail('chat_testops_integration_mismatch'); if (!Array.isArray(job.parameters) || job.parameters.length < 1 || job.parameters.length > 2) fail('chat_testops_parameter_mismatch'); - const seenIds = new Set(), seenNames = new Set(), parameters = []; + const seenIds = new Set(), parameters = []; + let hasOperation = false, hasRef = false; for (const parameter of job.parameters) { if (!id(parameter.id) || seenIds.has(parameter.id)) fail('chat_testops_parameter_mismatch'); seenIds.add(parameter.id); - const definition = await api(`/api/environment/${parameter.id}`); - if (definition.id !== parameter.id || !['Branch', 'operation'].includes(definition.name) || seenNames.has(definition.name)) fail('chat_testops_parameter_mismatch'); - seenNames.add(definition.name); - if (definition.name === 'Branch') { - if (parameter.value !== 'refs/heads/master') fail('chat_testops_parameter_mismatch'); - parameters.push({ id: parameter.id, value: parameter.value }); - } else { - if (ids.parameterId !== parameter.id || !['bootstrap', 'evaluate'].includes(parameter.value)) fail('chat_testops_parameter_mismatch'); + if (parameter.id === ids.parameterId) { + if (!['bootstrap', 'evaluate'].includes(parameter.value)) fail('chat_testops_parameter_mismatch'); + hasOperation = true; parameters.push({ id: parameter.id, value: 'evaluate' }); + } else { + if (hasRef || parameter.value !== 'refs/heads/master') fail('chat_testops_parameter_mismatch'); + hasRef = true; + parameters.push({ id: parameter.id, value: parameter.value }); } } - if (ids.parameterId !== null && !seenNames.has('operation')) fail('chat_testops_parameter_mismatch'); + if (ids.parameterId === null ? parameters.length !== 1 || !hasRef : !hasOperation) fail('chat_testops_parameter_mismatch'); return { ...ids, selected, plan, job, parameters }; } export async function bootstrapJobMetadata(api, env, expectedLaunchId) { - const strict = expectedLaunchId !== undefined; - const context = strict ? publicContext(env) : undefined; - if (strict && !id(expectedLaunchId)) fail('chat_testops_launch_mismatch'); - if (!positive(env.ALLURE_JOB_RUN_ID)) { - if (strict) fail('chat_testops_jobrun_mismatch'); - return { status: 'official_job_run_id_missing', actualResult: null }; + const requests = []; + const read = async (kind, resourceId, route) => { + const request = { class: kind, resourceId, state: 'STARTED', httpStatus: null }; + requests.push(request); + try { + const value = await api(route); + request.state = 'SUCCEEDED'; + return value; + } catch (error) { + request.state = 'FAILED'; + request.httpStatus = error instanceof SetupFailure ? error.status : null; + throw error; + } + }; + try { + const strict = expectedLaunchId !== undefined; + const context = strict ? publicContext(env) : undefined; + if (strict && !id(expectedLaunchId)) fail('chat_testops_launch_mismatch'); + if (!positive(env.ALLURE_JOB_RUN_ID)) { + if (strict) fail('chat_testops_jobrun_mismatch'); + return { status: 'official_job_run_id_missing', actualResult: null }; + } + const run = await read('JOB_RUN_GET', Number(env.ALLURE_JOB_RUN_ID), `/api/jobrun/${env.ALLURE_JOB_RUN_ID}`); + if (!id(run?.job?.id)) fail('chat_testops_jobrun_mismatch'); + if (strict) { + if (run.id !== Number(env.ALLURE_JOB_RUN_ID) || run.job.type !== 'github' || run.launchId !== expectedLaunchId + || run.url !== `https://github.com/paperclipai/paperclip/actions/runs/${context.runId}`) fail('chat_testops_jobrun_mismatch'); + const launch = await read('LAUNCH_GET', expectedLaunchId, `/api/launch/${expectedLaunchId}`); + if (launch.id !== expectedLaunchId || launch.projectId !== 1) fail('chat_testops_launch_mismatch'); + } + const job = await read('JOB_GET', run.job.id, `/api/job/${run.job.id}`); + if (job.id !== run.job.id) fail('chat_testops_job_id_mismatch'); + if (job.type !== 'github') fail('chat_testops_job_type_mismatch'); + if (!Array.isArray(job.parameters)) fail('chat_testops_job_parameters_shape_mismatch'); + if (job.parameters.length > 4) fail('chat_testops_job_parameters_limit'); + if (strict && !publicJobUrl(job.url, env)) fail('chat_testops_job_url_mismatch'); + const parameters = []; + const seen = new Set(); + for (const parameter of job.parameters) { + if (!id(parameter.id) || strict && seen.has(parameter.id)) fail('chat_testops_parameter_mismatch'); + seen.add(parameter.id); + const value = Object.getOwnPropertyDescriptor(parameter, 'value'); + const fixedRef = value && Object.hasOwn(value, 'value') && value.value === 'refs/heads/master'; + parameters.push({ id: parameter.id, kind: fixedRef ? 'fixed_default_ref' : 'UNRESOLVED', + value: fixedRef ? 'refs/heads/master' : 'NOT_PROJECTED' }); + } + return { ...(strict ? { schema: 'paperclip.chat-feature-testops-bootstrap-registration/v1', ...context, capturedAfterWatch: true, runnableBindingQualified: false } : {}), + status: 'observed', projectId: 1, jobId: job.id, allureJobRunId: env.ALLURE_JOB_RUN_ID, + publicWorkflowMatches: publicJobUrl(job.url, env), currentRunUrlMatches: run.url === `https://github.com/paperclipai/paperclip/actions/runs/${env.GITHUB_RUN_ID}`, + launchId: id(run.launchId) ? run.launchId : null, parameters, requests, actualResult: null }; + } catch (error) { + const failure = error instanceof SetupFailure ? error : new SetupFailure('unknown_failure'); + if (requests.length) failure.request = { ...requests.at(-1) }; + throw failure; } - const run = await api(`/api/jobrun/${env.ALLURE_JOB_RUN_ID}`); - if (!id(run?.job?.id)) fail('chat_testops_jobrun_mismatch'); - if (strict) { - if (run.id !== Number(env.ALLURE_JOB_RUN_ID) || run.job.type !== 'github' || run.launchId !== expectedLaunchId - || run.url !== `https://github.com/paperclipai/paperclip/actions/runs/${context.runId}`) fail('chat_testops_jobrun_mismatch'); - const launch = await api(`/api/launch/${expectedLaunchId}`); - if (launch.id !== expectedLaunchId || launch.projectId !== 1) fail('chat_testops_launch_mismatch'); - } - const job = await api(`/api/job/${run.job.id}`); - if (job.id !== run.job.id) fail('chat_testops_job_id_mismatch'); - if (job.type !== 'github') fail('chat_testops_job_type_mismatch'); - if (!Array.isArray(job.parameters)) fail('chat_testops_job_parameters_shape_mismatch'); - if (job.parameters.length > 4) fail('chat_testops_job_parameters_limit'); - if (strict && !publicJobUrl(job.url, env)) fail('chat_testops_job_url_mismatch'); - const parameters = []; - const seen = new Set(); - for (const parameter of job.parameters) { - if (!id(parameter.id) || strict && seen.has(parameter.id)) fail('chat_testops_parameter_mismatch'); - seen.add(parameter.id); - const definition = await api(`/api/environment/${parameter.id}`); - if (strict && definition.id !== parameter.id) fail('chat_testops_parameter_mismatch'); - const known = ['operation', 'ALLURE_JOB_RUN_ID', 'ALLURE_USERNAME', 'Branch'].includes(definition.name); - parameters.push({ id: parameter.id, name: known ? definition.name : 'UNKNOWN', - value: definition.name === 'operation' && ['bootstrap', 'evaluate', 'launch-plan'].includes(parameter.value) - || definition.name === 'Branch' && parameter.value === 'refs/heads/master' ? parameter.value : 'NOT_PROJECTED' }); - } - return { ...(strict ? { schema: 'paperclip.chat-feature-testops-bootstrap-registration/v1', ...context, capturedAfterWatch: true, runnableBindingQualified: false } : {}), - status: 'observed', projectId: 1, jobId: job.id, allureJobRunId: env.ALLURE_JOB_RUN_ID, - publicWorkflowMatches: publicJobUrl(job.url, env), currentRunUrlMatches: run.url === `https://github.com/paperclipai/paperclip/actions/runs/${env.GITHUB_RUN_ID}`, - launchId: id(run.launchId) ? run.launchId : null, parameters, actualResult: null }; } const launchAdmissions = new WeakSet(); diff --git a/tests/runner-e2e/chat-feature-testops-api.test.mjs b/tests/runner-e2e/chat-feature-testops-api.test.mjs index 6a7c5ce549a..4422a6e0d0b 100644 --- a/tests/runner-e2e/chat-feature-testops-api.test.mjs +++ b/tests/runner-e2e/chat-feature-testops-api.test.mjs @@ -168,16 +168,12 @@ async function fence(env = launchEnv, changes = {}, workflowChanges = {}, repoCh : url.includes('/actions/runs/') ? { ...ownRun, ...changes } : { id: 99, path: '.github/workflows/chat-feature-testops.yml', state: 'active', ...workflowChanges }) })); } -function launchServer({ wrongParameter = false, wrongAssignment = false, uncertainRun = false, integrationChange = {}, initialParameters = [{ id: 17, value: 'bootstrap' }], expectedParameters = [{ id: 17, value: 'evaluate' }], definitions = {}, jobUrl = 'https://github.com/paperclipai/paperclip/actions/workflows/chat-feature-testops.yml' } = {}) { +function launchServer({ wrongAssignment = false, uncertainRun = false, integrationChange = {}, initialParameters = [{ id: 17, value: 'bootstrap' }], expectedParameters = [{ id: 17, value: 'evaluate' }], jobUrl = 'https://github.com/paperclipai/paperclip/actions/workflows/chat-feature-testops.yml' } = {}) { const calls = [], parameters = structuredClone(initialParameters); let assigned = false, launched = false, requests = 0; const api = async (route, method, body) => { calls.push({ route, method, body }); if (route.startsWith('/api/launch?')) return { totalPages: 1, content: launched ? [{ id: 10, name: `Chat feature evals ยท ${'1'.repeat(32)}` }] : [] }; if (route === '/api/integration/2/project/1') return { id: 2, projectId: 1, info: { type: 'github' }, disabled: false, ...integrationChange }; - if (route.startsWith('/api/environment/')) { - const id = Number(route.split('/').at(-1)); - return definitions[id] ?? { id, name: id === 17 ? wrongParameter ? 'OTHER_PRIVATE_PARAMETER' : 'operation' : 'Branch' }; - } if (route === '/api/job/7') { if (method === 'PATCH') { assert.deepEqual(body, { integrationId: 2, canRun: true, parameters: expectedParameters }); parameters.splice(0, parameters.length, ...structuredClone(body.parameters)); } return { id: 7, type: 'github', url: jobUrl, parameters }; @@ -207,7 +203,7 @@ test('observed launch nonce prevents a second run request across separate local }); test('foreign assignment and unobserved operation parameter stop before any write', async () => { - for (const options of [{ wrongAssignment: true }, { wrongParameter: true }]) { + for (const options of [{ wrongAssignment: true }, { initialParameters: [{ id: 18, value: 'bootstrap' }] }]) { const server = launchServer(options); await assert.rejects(launchChatPlan(server.api, launchEnv, () => assert.fail('no marker'), await fence())); assert.equal(server.calls.some(row => row.method), false); assert.equal(server.requests(), 0); @@ -221,15 +217,14 @@ test('uncertain plan-run transport is sent once and never followed by another mu assert.equal(server.calls.at(-1).route, '/api/testplan/8/run'); }); -test('official bootstrap job metadata projects only fixed parameter names and operation enums', async () => { +test('official bootstrap job metadata leaves opaque parameter labels unresolved and private values unprojected', async () => { const job = await bootstrapJobMetadata(async route => { if (route === '/api/jobrun/9') return { id: 9, job: { id: 7 }, launchId: 10, url: 'https://github.com/paperclipai/paperclip/actions/runs/123' }; if (route === '/api/job/7') return { id: 7, type: 'github', url: 'https://github.com/paperclipai/paperclip/actions/workflows/chat-feature-testops.yml', parameters: [{ id: 17, value: 'bootstrap' }, { id: 18, value: 'SYNTHETIC_PRIVATE_VALUE' }] }; - if (route === '/api/environment/17') return { id: 17, name: 'operation' }; - return { id: 18, name: 'UNRELATED_NAME' }; + assert.fail('no namespace lookup or unrelated request'); }, publicEnv); - assert.equal(job.publicWorkflowMatches, true); assert.equal(job.parameters[0].value, 'bootstrap'); - assert.deepEqual(job.parameters[1], { id: 18, name: 'UNKNOWN', value: 'NOT_PROJECTED' }); + assert.equal(job.publicWorkflowMatches, true); assert.deepEqual(job.parameters[0], { id: 17, kind: 'UNRESOLVED', value: 'NOT_PROJECTED' }); + assert.deepEqual(job.parameters[1], { id: 18, kind: 'UNRESOLVED', value: 'NOT_PROJECTED' }); assert.ok(!JSON.stringify(job).includes('SYNTHETIC_PRIVATE_VALUE')); }); @@ -404,8 +399,6 @@ function strictBootstrapApi(patch = {}) { if (route === '/api/jobrun/9') return run; if (route === '/api/launch/10') return launch; if (route === '/api/job/7') return job; - if (route === '/api/environment/17') return { id: 17, name: 'operation', ...patch.definition }; - if (route === '/api/environment/18') return { id: 18, name: 'SYNTHETIC_PRIVATE_NAME' }; assert.fail(route); }; } @@ -415,7 +408,7 @@ test('post-watch reader binds actual current run source project launch and publi assert.equal(result.jobId, 7); assert.equal(result.runId, publicEnv.GITHUB_RUN_ID); assert.equal(result.workflowSha, publicEnv.GITHUB_SHA); assert.equal(result.capturedAfterWatch, true); assert.equal(result.runnableBindingQualified, false); assert.equal(result.actualResult, null); - assert.deepEqual(result.parameters[1], { id: 18, name: 'UNKNOWN', value: 'NOT_PROJECTED' }); + assert.deepEqual(result.parameters[1], { id: 18, kind: 'UNRESOLVED', value: 'NOT_PROJECTED' }); assert.ok(!JSON.stringify(result).includes('SYNTHETIC_PRIVATE')); }); @@ -423,14 +416,14 @@ test('post-watch reader rejects stale foreign project launch source and job iden for (const patch of [{ run: { id: 99 } }, { run: { url: 'https://github.com/paperclipai/paperclip/actions/runs/999' } }, { run: { launchId: 20 } }, { run: { job: { id: 7, type: 'foreign' } } }, { launch: { projectId: 2 } }, { launch: { id: 20 } }, { job: { url: 'https://github.com/other/repo/actions/workflows/chat-feature-testops.yml' } }, - { job: { type: 'foreign' } }, { definition: { id: 99 } }]) { + { job: { type: 'foreign' } }, { job: { parameters: [{ id: 0 }] } }]) { await assert.rejects(bootstrapJobMetadata(strictBootstrapApi(patch), publicEnv, 10)); } await assert.rejects(bootstrapJobMetadata(() => assert.fail('no API'), { ...publicEnv, GITHUB_SHA: 'untrusted' }, 10)); await assert.rejects(bootstrapJobMetadata(() => assert.fail('no API'), { ...publicEnv, ALLURE_JOB_RUN_ID: '' }, 10)); }); -test('post-watch reader caps parameter definitions and refuses duplicate IDs', async () => { +test('post-watch reader caps opaque parameters and refuses duplicate IDs', async () => { for (const parameters of [Array.from({ length: 5 }, (_, i) => ({ id: i + 1 })), [{ id: 17 }, { id: 17 }]]) { await assert.rejects(bootstrapJobMetadata(strictBootstrapApi({ job: { parameters } }), publicEnv, 10)); } @@ -460,7 +453,8 @@ test('official legacy URL admission rejects foreign workflow repo query credenti observedLegacyJobUrl.replace('https://github.com', 'https://user:password@github.com'), `${observedLegacyJobUrl}#extra`]) { await assert.rejects(validateChatSelection(selectedApi({ '/api/job/7': { id: 7, type: 'github', url } }), publicEnv), /chat_testops_job_mismatch/); await assert.rejects(bootstrapJobMetadata(strictBootstrapApi({ job: { url } }), publicEnv, 10), error => { - assert.deepEqual(failureProjection(error, 'bootstrap_context'), { stage: 'bootstrap_context', code: 'chat_testops_job_url_mismatch', httpStatus: null }); + assert.deepEqual(failureProjection(error, 'bootstrap_context'), { stage: 'bootstrap_context', code: 'chat_testops_job_url_mismatch', httpStatus: null, + request: { class: 'JOB_GET', resourceId: 7, state: 'SUCCEEDED', httpStatus: null } }); return true; }); const server = launchServer({ jobUrl: url }); @@ -481,7 +475,8 @@ test('bootstrap job predicate failures retain only distinct fixed codes and no o ]) { await assert.rejects(bootstrapJobMetadata(strictBootstrapApi({ job }), publicEnv, 10), error => { const projected = failureProjection(error, 'bootstrap_context'); - assert.deepEqual(projected, { stage: 'bootstrap_context', code, httpStatus: null }); + assert.deepEqual(projected, { stage: 'bootstrap_context', code, httpStatus: null, + request: { class: 'JOB_GET', resourceId: 7, state: 'SUCCEEDED', httpStatus: null } }); assert.ok(!JSON.stringify(projected).includes('SYNTHETIC_PRIVATE')); return true; }); @@ -508,33 +503,78 @@ test('launch preserves exact Branch and defaults evaluate only when operation is test('launch refuses empty unknown duplicate wrong Branch or conflicting operation before any mutation', async () => { const branch = { id: 19, value: 'refs/heads/master' }; - for (const [parameters, opId, definitions] of [ + for (const [parameters, opId] of [ [[], undefined, {}], [[{ id: 19, value: 'refs/heads/other' }], undefined, {}], [[branch, branch], undefined, {}], [[branch, { id: 20, value: 'refs/heads/master' }], undefined, {}], - [[{ id: 19, value: 'refs/heads/master' }], undefined, { 19: { id: 99, name: 'Branch' } }], + [[{ id: 0, value: 'refs/heads/master' }], undefined, {}], [[{ id: 19, value: 'PRIVATE_PARAMETER_VALUE' }], undefined, { 19: { id: 19, name: 'PRIVATE_PARAMETER_NAME' } }], [[branch], '17', {}], [[branch], 'bad-id', {}], + [[branch], '0', {}], + [[branch], ' ', {}], [[{ id: 17, value: 'bootstrap' }], undefined, {}], [[{ id: 17, value: 'bootstrap' }], '18', {}], [[{ id: 17, value: 'launch-plan' }], '17', {}], [[branch, { id: 17, value: 'bootstrap' }, { id: 20, value: 'refs/heads/master' }], '17', {}], ]) { const env = { ...launchEnv, CHAT_TESTOPS_OPERATION_PARAMETER_ID: opId }; - const server = launchServer({ initialParameters: parameters, definitions }); + const server = launchServer({ initialParameters: parameters }); await assert.rejects(launchChatPlan(server.api, env, async () => assert.fail('no mutation marker'), await fence(env)), /chat_testops_(?:identity_missing|parameter_mismatch)/); assert.equal(server.calls.some(call => call.method), false); assert.equal(server.requests(), 0); } }); -test('bootstrap projects only fixed Branch name and exact master value alongside genuine numeric ID', async () => { - const base = strictBootstrapApi({ job: { parameters: [{ id: 19, value: 'refs/heads/master' }] } }); - const result = await bootstrapJobMetadata(route => route === '/api/environment/19' ? { id: 19, name: 'Branch' } : base(route), publicEnv, 10); - assert.deepEqual(result.parameters, [{ id: 19, name: 'Branch', value: 'refs/heads/master' }]); - const hidden = strictBootstrapApi({ job: { parameters: [{ id: 19, value: 'PRIVATE_BRANCH_VALUE' }] } }); - const unknown = await bootstrapJobMetadata(route => route === '/api/environment/19' ? { id: 19, name: 'Branch' } : hidden(route), publicEnv, 10); - assert.deepEqual(unknown.parameters, [{ id: 19, name: 'Branch', value: 'NOT_PROJECTED' }]); +test('bootstrap projects an exact default ref without inventing an opaque parameter label', async () => { + const result = await bootstrapJobMetadata(strictBootstrapApi({ job: { parameters: [{ id: 19, value: 'refs/heads/master' }] } }), publicEnv, 10); + assert.deepEqual(result.parameters, [{ id: 19, kind: 'fixed_default_ref', value: 'refs/heads/master' }]); + const unknown = await bootstrapJobMetadata(strictBootstrapApi({ job: { parameters: [{ id: 19, value: 'PRIVATE_BRANCH_VALUE' }] } }), publicEnv, 10); + assert.deepEqual(unknown.parameters, [{ id: 19, kind: 'UNRESOLVED', value: 'NOT_PROJECTED' }]); assert.ok(!JSON.stringify(unknown).includes('PRIVATE_BRANCH')); }); + +test('opaque launch vector uses only returned IDs and never queries any environment namespace', async () => { + const env = { ...launchEnv, CHAT_TESTOPS_OPERATION_PARAMETER_ID: '' }; + const server = launchServer({ initialParameters: [{ id: 819, value: 'refs/heads/master' }], expectedParameters: [{ id: 819, value: 'refs/heads/master' }] }); + const result = await launchChatPlan(async (...args) => { assert.ok(!args[0].includes('/environment')); return server.api(...args); }, env, async () => {}, await fence(env)); + assert.equal(result.status, 'REQUESTED'); assert.equal(server.requests(), 1); +}); + +test('bootstrap successful reads expose only the three fixed source request classes and numeric IDs', async () => { + const result = await bootstrapJobMetadata(strictBootstrapApi(), publicEnv, 10); + assert.deepEqual(result.requests, [ + { class: 'JOB_RUN_GET', resourceId: 9, state: 'SUCCEEDED', httpStatus: null }, + { class: 'LAUNCH_GET', resourceId: 10, state: 'SUCCEEDED', httpStatus: null }, + { class: 'JOB_GET', resourceId: 7, state: 'SUCCEEDED', httpStatus: null }, + ]); +}); + +test('bootstrap actual HTTP client failure identifies each fixed request without reading private bodies', async () => { + for (const [failedRoute, kind, resourceId, count] of [['/api/jobrun/9', 'JOB_RUN_GET', 9, 1], ['/api/launch/10', 'LAUNCH_GET', 10, 2], ['/api/job/7', 'JOB_GET', 7, 3]]) { + const calls = [], base = strictBootstrapApi(); + const api = await testopsClient('SYNTHETIC_TOKEN', async (url, options) => { + const route = url.slice(endpoint.length); + if (route === '/api/uaa/oauth/token') return { ok: true, json: async () => ({ access_token: 'SYNTHETIC_ACCESS' }) }; + calls.push(route); assert.equal(options.method, 'GET'); + if (route === failedRoute) return { ok: false, status: 404, text: () => assert.fail('private error body must stay unopened') }; + return { ok: true, status: 200, text: async () => JSON.stringify(await base(route)) }; + }); + await assert.rejects(bootstrapJobMetadata(api, publicEnv, 10), error => { + const projected = failureProjection(error, 'bootstrap_context'); + assert.deepEqual(projected, { stage: 'bootstrap_context', code: 'testops_api_http', httpStatus: 404, + request: { class: kind, resourceId, state: 'FAILED', httpStatus: 404 } }); + assert.ok(!JSON.stringify(projected).includes('SYNTHETIC')); return true; + }); + assert.equal(calls.length, count); + } +}); + +test('bootstrap unknown transport errors retain only source-owned request class without private error text', async () => { + await assert.rejects(bootstrapJobMetadata(() => { throw new Error('PRIVATE_URL_TOKEN_RESPONSE'); }, publicEnv, 10), error => { + const projected = failureProjection(error, 'bootstrap_context'); + assert.deepEqual(projected, { stage: 'bootstrap_context', code: 'unknown_failure', httpStatus: null, + request: { class: 'JOB_RUN_GET', resourceId: 9, state: 'FAILED', httpStatus: null } }); + assert.ok(!JSON.stringify(projected).includes('PRIVATE')); return true; + }); +}); diff --git a/tests/runner-e2e/testops-chat-reporter.test.mjs b/tests/runner-e2e/testops-chat-reporter.test.mjs index 9e982d66f62..917d6912f90 100644 --- a/tests/runner-e2e/testops-chat-reporter.test.mjs +++ b/tests/runner-e2e/testops-chat-reporter.test.mjs @@ -200,7 +200,7 @@ test('post-watch facade runs exact official command with bounded pipes and prese assert.equal(result.runId, '123'); assert.equal(result.workflowSha, context.GITHUB_SHA); assert.equal(result.targetSha, context.GITHUB_SHA); assert.equal(result.currentRunUrlMatches, true); assert.equal(result.publicWorkflowMatches, true); assert.equal(result.capturedAfterWatch, true); assert.equal(result.runnableBindingQualified, false); assert.equal(result.actualResult, null); - assert.deepEqual(result.parameters, [{ id: 17, name: 'operation', value: 'bootstrap' }]); + assert.deepEqual(result.parameters, [{ id: 17, kind: 'UNRESOLVED', value: 'NOT_PROJECTED' }]); assert.ok(!bytes.toString().includes('SYNTHETIC')); assert.equal(f.env.ALLURE_JOB_RUN_ID, ''); assert.deepEqual(await readFile(path.join(f.state, 'job.json')), f.originalJob); const second = await captureSafeErrors(() => reporter('bootstrap-context', undefined, f.env, () => assert.fail('no second auth'), () => assert.fail('no second CLI')));