diff --git a/.github/workflows/chat-catalog-testops.yml b/.github/workflows/chat-catalog-testops.yml index 30eff3032a5..ca251a1859a 100644 --- a/.github/workflows/chat-catalog-testops.yml +++ b/.github/workflows/chat-catalog-testops.yml @@ -270,17 +270,27 @@ jobs: with: artifact-ids: ${{ needs.select.outputs.artifact_id }} path: chat-completion-testops-selection - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + # A single pattern match is flattened by this pinned action. Bind its + # exact name and explicit child destination instead of inferring a name. + - if: fromJSON(needs.select.outputs.execution_ids)[1] == null + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: chat-catalog-case-bundle-${{ github.run_id }}-1-0 + path: catalog-bundles/chat-catalog-case-bundle-${{ github.run_id }}-1-0 + - if: fromJSON(needs.select.outputs.execution_ids)[1] != null + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: pattern: chat-catalog-case-bundle-${{ github.run_id }}-1-* + merge-multiple: false path: catalog-bundles - name: Gather only safe singleton projections from immutable bundles - run: | - mkdir -m 700 catalog-projections - for bundle in catalog-bundles/*; do - test -d "$bundle/projection" - cp -R "$bundle/projection" "catalog-projections/${bundle##*/}" - done + env: + GH_TOKEN: ${{ github.token }} + TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} + ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} + EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.selection_sha256 }} + CHAT_TESTOPS_BUNDLES: ${{ github.workspace }}/catalog-bundles + run: node tests/runner-e2e/chat-completion-testops-cli.mjs gather-case-projections catalog-projections - name: Authenticate selection and validate all singleton exports before aggregation env: GH_TOKEN: ${{ github.token }} @@ -318,13 +328,31 @@ jobs: with: artifact-ids: ${{ needs.select.outputs.artifact_id }} path: chat-completion-testops-selection - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + # A single pattern match is flattened by this pinned action. Bind its + # exact name and explicit child destination instead of inferring a name. + - if: fromJSON(needs.select.outputs.execution_ids)[1] == null + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: chat-catalog-case-bundle-${{ github.run_id }}-1-0 + path: catalog-bundles/chat-catalog-case-bundle-${{ github.run_id }}-1-0 + - if: fromJSON(needs.select.outputs.execution_ids)[1] != null + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: pattern: chat-catalog-case-bundle-${{ github.run_id }}-1-* + merge-multiple: false path: catalog-bundles - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + # A single pattern match is flattened by this pinned action. Bind its + # exact name and explicit child destination instead of inferring a name. + - if: fromJSON(needs.select.outputs.execution_ids)[1] == null + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: chat-catalog-case-publication-${{ github.run_id }}-1-0 + path: catalog-publications/chat-catalog-case-publication-${{ github.run_id }}-1-0 + - if: fromJSON(needs.select.outputs.execution_ids)[1] != null + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: pattern: chat-catalog-case-publication-${{ github.run_id }}-1-* + merge-multiple: false path: catalog-publications - name: Authenticate every frozen publication and verify the exact original selection id: verification diff --git a/tests/runner-e2e/CHAT-FEATURES.md b/tests/runner-e2e/CHAT-FEATURES.md index 7432163eeba..60c7642d086 100644 --- a/tests/runner-e2e/CHAT-FEATURES.md +++ b/tests/runner-e2e/CHAT-FEATURES.md @@ -217,7 +217,7 @@ overlap. Exact requirements and limitations appear in each case's preconditions. A declared credential does not establish whether its runtime reference exists. This lane runs the app and fixtures from candidate -`51ef450754370c8a8f2cbe4bd18e1536bbc93966`. It retains the evaluated app from +`ab2019e976076ec0eb60b026844f3d9211cb0208`. It retains the evaluated app from `3b66c3c144360ddf32714a0742fd774b8ceafb16`. The first repair source added: owned SDK staging, public membership for three incidental human fixtures, company settings observation inside its original case deadline, and exact @@ -315,10 +315,23 @@ The SDK public-input registry is generated from the committed candidate tree. Co The full-catalog private evidence policy binds the launcher digest from the same reviewed manifest as the immutable candidate target. Every final GO reads `launch.ts` from that exact Git object and validates its digest against every selected singleton policy before submission. The legacy pilot launcher digest remains separately pinned. -### Repair source 4 (2026-11-03; candidate only) +### Historical repair source 4 (2026-11-03; candidate only) -The current target is the canonical revision above. This source adds remaining-original-deadline observation for primary guidance; independently expected normalized MCP results; fixed owned timeline-file reads through fixture board authority; finite supported CLI profile guidance with no identifier substitutions and fresh board/API/company/human readback; real completion-turn observation for plan review; and the reviewed true-price hire counterpart. Original failed attempts remain unchanged. +Historical target `51ef450754370c8a8f2cbe4bd18e1536bbc93966` adds remaining-original-deadline observation for primary guidance; independently expected normalized MCP results; fixed owned timeline-file reads through fixture board authority; finite supported CLI profile guidance with no identifier substitutions and fresh board/API/company/human readback; real completion-turn observation for plan review; and the reviewed true-price hire counterpart. Original failed attempts remain unchanged. The app-review journey now requires inspection of the exact full redacted arguments and audit hash in the normal shared review card, then verifies that authorized provider dispatch equals that proposal. The 17-line product disclosure is an explicit candidate-only overlay from unmerged core PR #18734; no backend permission or approval rule changes. The independent core guidance PR #18717 also remains unmerged. SDK public-source eligibility explicitly changes its expected restored lock from stale c7f230 to the authenticated credential-free build-input SHA256 `54a81b2ed3f1f85eaa085c034560ac248e384db34fbc7aba8c5a3846c29b3ce8`. Compiler/parser/Zod identities and all privacy/AST/custody guards remain enforced. This correction is not a plugin pass: case 115 must qualify with its original 300-second/one-native limits, and the other five SDK cases, including plugin 558's canary certificates, remain held. All case clocks and the controller model remain unchanged; Studio144's earlier explicit 2..3 accounting revision remains the only native-bound change. + +### Historical repair source 5 (2026-11-03; candidate only) + +The reviewed prior cut `de6d29e025683f5c28125bcaa4f38b41836d8516` has not run as a campaign lane. It adds case53 observation within its original300s budget, case74 actual rich-editor ordered-block inspection, case550 normal Projects-surface announcement fixture, case578 server-canonical secret-key metadata checks, and case466 canonical native status audit joins to the exact accepted task/run/worker decision. The timeline keeps its real exit1, human handoff and exit0 checks; legacy audit support remains. Queries, models and original per-case clocks/run counts stay fixed. Its candidate-only UI487 overlay includes reviewed pre-trim size guards from unmerged corePR18734/50600786. Historical Source4 `51ef450754370c8a8f2cbe4bd18e1536bbc93966` remains the source of launch1888; later master metadata never retargets that running workflow. These repairs and focused controls are not real feature acceptance. Other five SDK cases remain held pending actual115 evidence; no core PR is merged. + + +### Repair source 6 (2026-11-03; candidate only) + +The canonical candidate above retains Source5 and adds initial personal-secret and team-install observation within their original deadlines, real hire-review link readiness, and skill-update observation at genuine provider checkpoints after local native readiness. Case queries, controller model, permissions and declared clocks/run counts remain unchanged from Source5; Studio144 retains its earlier explicit 2..3 accounting revision. + +The reviewed product overlay from unmerged core PR #18742 at `61996a96dd09e90a5788e8711a94fc5ae631f425` allows a new Ask-first review for a strictly authenticated later same-person human intent following a declined request, including the normal reopened-comment source. Repeating the same intent stays denied; this grants no approval or execution authority. Both PostgreSQL route regressions ran and passed on that core head. The candidate preserves the compatible gateway and actual database row contracts; this is not a core merge or a qualified chat pass. Core PRs #18717, #18734 and #18742 remain unmerged. SDK115 completion remains unknown; the other five SDK cases remain held. + +For one selected case, the trusted controller downloads each exact bundle and publication artifact name into an explicit matching child directory. For multiple cases, it keeps separate named artifact directories. The gather command authenticates the frozen selection, every prepared bundle and its hashes before it copies the safe projections. Final coverage still authenticates the original publication receipts and verifies TestOps results without another export or upload. Empty, mixed, foreign and symbolic-link layouts fail closed. The producer's optional once-upload marker has an exact closed schema; it does not replace a verified publication receipt. diff --git a/tests/runner-e2e/chat-catalog-testops-streaming.mjs b/tests/runner-e2e/chat-catalog-testops-streaming.mjs index 6d27e7d30ae..50a31804a32 100644 --- a/tests/runner-e2e/chat-catalog-testops-streaming.mjs +++ b/tests/runner-e2e/chat-catalog-testops-streaming.mjs @@ -1,4 +1,4 @@ -import { lstat, readFile, readdir, realpath } from 'node:fs/promises'; +import { cp, lstat, mkdir, readFile, readdir, realpath } from 'node:fs/promises'; import path from 'node:path'; import { hash, validateSelection, reject } from './chat-completion-testops-contract.mjs'; import { readPilotExport, exportPilotAllure } from './chat-completion-testops-results.mjs'; @@ -95,33 +95,68 @@ export async function publishCaseBundle(api, root, output, prepared, context, se verification: verification.verification, results: verification.results }; } +// Artifact names remain authoritative. Singleton downloads must use an exact +// action `name` and that named child destination: the pinned downloader flattens +// a one-artifact pattern result even with merge-multiple:false. +async function artifactDirectories(root, kind, context, selection) { + if (context.catalog !== 'full') reject('stream_scope'); + const rows = validateSelection(selection, context, selection.allureJobRunId); + const stat = await lstat(root); + if (!stat.isDirectory() || stat.isSymbolicLink() || await realpath(root) !== path.resolve(root)) reject('stream_publications'); + const names = rows.map((_, index) => `chat-catalog-case-${kind}-${context.runId}-1-${index}`); + const actual = await readdir(root); + if (actual.length !== names.length || actual.some(name => !names.includes(name))) reject('stream_publications'); + const directories = names.map(name => path.join(root, name)); + for (const directory of directories) { + const info = await lstat(directory); + if (!info.isDirectory() || info.isSymbolicLink()) reject('stream_publications'); + const wanted = kind === 'bundle' ? ['allure', 'prepared.json', 'projection'] : ['publication.json', 'once-upload_case.json']; + const entries = await readdir(directory); + if (entries.some(name => !wanted.includes(name)) + || (kind === 'bundle' ? entries.length !== wanted.length : !entries.includes('publication.json'))) reject('stream_publications'); + if (kind === 'publication' && entries.includes('once-upload_case.json')) { + const marker = await artifactJson(directory, 'once-upload_case.json'); + if (Object.keys(marker).sort().join() !== 'stage,status' || marker.stage !== 'upload_case' + || marker.status !== 'STARTED_NO_AUTOMATIC_RESEND') reject('stream_publications'); + } + } + return directories; +} +async function artifactJson(directory, name) { + const file = path.join(directory, name), stat = await lstat(file); + if (!stat.isFile() || stat.isSymbolicLink() || stat.nlink !== 1 || stat.size > 65_536) reject('stream_publications'); + return JSON.parse(await readFile(file, 'utf8')); +} + +// Copy only authenticated safe projections. UUID-bearing Allure bytes are read +// for integrity, never regenerated or uploaded by this metadata-only command. +export async function gatherCaseProjections(bundlesRoot, output, context, selection, selectionSha256) { + const bundles = await artifactDirectories(bundlesRoot, 'bundle', context, selection); + for (const [index, bundle] of bundles.entries()) { + const prepared = await artifactJson(bundle, 'prepared.json'); + await readCaseBundle(path.join(bundle, 'projection'), path.join(bundle, 'allure'), prepared, context, + selection, selection.cases[index].executionId, selectionSha256); + } + await mkdir(output, { mode: 0o700 }); + for (const [index, bundle] of bundles.entries()) { + const destination = path.join(output, path.basename(bundle)); + await cp(path.join(bundle, 'projection'), destination, { recursive: true, force: false, errorOnExist: true }); + await readPilotExport(destination, context, caseSelection(selection, context, selection.cases[index].executionId)); + } +} + // The finalizer authenticates the already-emitted bytes and their original // grades. It never calls the random-UUID exporter or the upload endpoint. export async function publicationSet(bundlesRoot, publicationsRoot, context, selection, selectionSha256) { const rows = validateSelection(selection, context, selection.allureJobRunId); - const roots = [bundlesRoot, publicationsRoot]; - for (const [index, root] of roots.entries()) { - if (await realpath(root) !== path.resolve(root)) reject('stream_publications'); - const prefix = `chat-catalog-case-${index === 0 ? 'bundle' : 'publication'}-${context.runId}-1-`; - const names = await readdir(root); - if (names.length !== rows.length || names.some(name => !rows.some((_, i) => name === `${prefix}${i}`))) reject('stream_publications'); - } + const bundles = await artifactDirectories(bundlesRoot, 'bundle', context, selection); + const publications = await artifactDirectories(publicationsRoot, 'publication', context, selection); const exported = []; for (const [index, row] of rows.entries()) { - const bundle = path.join(bundlesRoot, `chat-catalog-case-bundle-${context.runId}-1-${index}`); - const publication = path.join(publicationsRoot, `chat-catalog-case-publication-${context.runId}-1-${index}`); - for (const directory of [bundle, publication]) { - const stat = await lstat(directory); - if (!stat.isDirectory() || stat.isSymbolicLink()) reject('stream_publications'); - } - const read = async (directory, name) => { - const stat = await lstat(path.join(directory, name)); - if (!stat.isFile() || stat.isSymbolicLink() || stat.nlink !== 1 || stat.size > 65_536) reject('stream_publications'); - return JSON.parse(await readFile(path.join(directory, name), 'utf8')); - }; - const prepared = await read(bundle, 'prepared.json'); + const bundle = bundles[index], publication = publications[index]; + const prepared = await artifactJson(bundle, 'prepared.json'); await readCaseBundle(path.join(bundle, 'projection'), path.join(bundle, 'allure'), prepared, context, selection, row.executionId, selectionSha256); - const value = await read(publication, 'publication.json'); + const value = await artifactJson(publication, 'publication.json'); const expected = { ...identity(selection, context, row.executionId, selectionSha256), schema: 'paperclip.chat-catalog-testops-case-publication/v1' }; if (Object.entries(expected).some(([key, wanted]) => value[key] !== wanted) || value.payloadSha256 !== prepared.payloadSha256 || value.projectionBindingSha256 !== prepared.projectionBindingSha256 diff --git a/tests/runner-e2e/chat-catalog-testops-streaming.test.mjs b/tests/runner-e2e/chat-catalog-testops-streaming.test.mjs index 8e8a462c74c..11e55adef1c 100644 --- a/tests/runner-e2e/chat-catalog-testops-streaming.test.mjs +++ b/tests/runner-e2e/chat-catalog-testops-streaming.test.mjs @@ -1,8 +1,11 @@ import test from 'node:test'; import assert from 'node:assert/strict'; -import { mkdtemp, mkdir, readFile, readdir, writeFile, rm, realpath, cp } from 'node:fs/promises'; +import { mkdtemp, mkdir, readFile, readdir, writeFile, rm, realpath, cp, symlink, rename, access } from 'node:fs/promises'; import path from 'node:path'; import { tmpdir } from 'node:os'; +import { createRequire } from 'node:module'; +import * as streaming from './chat-catalog-testops-streaming.mjs'; +const { parseDocument } = createRequire(import.meta.url)('yaml'); import { fullCatalogCases, catalogTargetSha } from './chat-full-catalog-testops-manifest.mjs'; import { hash, model } from './chat-completion-testops-contract.mjs'; import { run } from './chat-completion-testops-cli.mjs'; @@ -14,7 +17,7 @@ const cases = fullCatalogCases.filter(row => Object.hasOwn(caseIds, row.key)).ma const selection = { schema: 'paperclip.chat-completion-testops-selection/v1', ...context, projectId: 1, planId: 102, jobId: 100, launchId: 12, allureJobRunId: '77', cases }; const digest = hash(JSON.stringify(selection)), current = cases.find(row => row.caseId === 388), runUrl = 'https://github.com/paperclipai/paperclip/actions/runs/123'; -async function fixture(t, failed = false, owned = current, runtimeContext = context) { +async function fixture(t, failed = false, owned = current, runtimeContext = context, runtimeSelection = selection) { const base = await realpath(await mkdtemp(path.join(tmpdir(), 'chat-stream-test-'))); t.after(() => rm(base, { recursive: true, force: true })); const root = path.join(base, 'projection'), output = path.join(base, 'allure'); await mkdir(root); @@ -31,7 +34,7 @@ async function fixture(t, failed = false, owned = current, runtimeContext = cont finalScreenshot: failed ? null : { sha256: 'e'.repeat(64), bytes: 24 } }] }; for (const [name, bytes] of files) await writeFile(path.join(root, name), bytes); await writeFile(path.join(root, 'binding.json'), JSON.stringify(binding)); - const prepared = await prepareCaseBundle(root, output, runtimeContext, selection, owned.executionId, digest); + const prepared = await prepareCaseBundle(root, output, runtimeContext, runtimeSelection, owned.executionId, hash(JSON.stringify(runtimeSelection))); return { root, output, prepared }; } function server(prepared, { existing = false, uncertain = false, apply = true, scheduled = false, processingReads = 0 } = {}) { @@ -316,3 +319,211 @@ test('the final CLI verifies all authenticated mixed-grade publications without assert.doesNotMatch(stderr, /SYNTHETIC_/); } }); + +// Exact download-path decision from pinned download-artifact 3e5f45b, lines185–194. +// Even merge-multiple:false flattens a one-artifact pattern result. +const pinnedDownloadPath = (resolvedPath, artifact, artifacts, isSingleArtifactDownload, mergeMultiple = false) => + isSingleArtifactDownload || mergeMultiple || artifacts.length === 1 ? resolvedPath : path.join(resolvedPath, artifact.name); +async function downloaded(t, selected, available = selected.cases.length, runtimeContext = context) { + const base = await realpath(await mkdtemp(path.join(tmpdir(), 'chat-stream-download-'))); + t.after(() => rm(base, { recursive: true, force: true })); + const workflow = parseDocument(await readFile(new URL('../../.github/workflows/chat-catalog-testops.yml', import.meta.url), 'utf8')).toJS(); + const roots = { bundle: path.join(base, 'catalog-bundles'), publication: path.join(base, 'catalog-publications') }; + const active = step => !step.if || (step.if.includes('!= null') ? selected.cases.length > 1 : selected.cases.length === 1); + for (const role of ['bundle', 'publication']) { + const steps = workflow.jobs.verify.steps.filter(step => step.uses?.startsWith('actions/download-artifact@') + && (step.with.pattern ?? step.with.name ?? '').startsWith(`chat-catalog-case-${role}-`) && active(step)); + assert.equal(steps.length, 1); + const step = steps[0], artifacts = Array.from({ length: available }, (_, i) => ({ name: `chat-catalog-case-${role}-123-1-${i}` })); + await mkdir(roots[role]); + for (const [index, artifact] of artifacts.entries()) { + const destination = pinnedDownloadPath(path.join(base, step.with.path.replaceAll('${{ github.run_id }}', '123')), + artifact, artifacts, Boolean(step.with.name)); + await mkdir(destination, { recursive: true }); + const row = selected.cases[index]; + if (role === 'bundle') { + const f = await fixture(t, index === 1, row, runtimeContext, selected); + await cp(f.root, path.join(destination, 'projection'), { recursive: true }); + await cp(f.output, path.join(destination, 'allure'), { recursive: true }); + await writeFile(path.join(destination, 'prepared.json'), JSON.stringify(f.prepared)); + } else { + // Use the matching bundle's frozen UUID bytes; never independently export the receipt. + const bundlePath = artifacts.length === 1 && selected.cases.length > 1 ? roots.bundle + : path.join(roots.bundle, `chat-catalog-case-bundle-123-1-${index}`); + let prepared; + try { prepared = JSON.parse(await readFile(path.join(bundlePath, 'prepared.json'))); } + catch { prepared = JSON.parse(await readFile(path.join(roots.bundle, 'prepared.json'))); } + await writeFile(path.join(destination, 'publication.json'), JSON.stringify({ ...prepared, + schema: 'paperclip.chat-catalog-testops-case-publication/v1', exported: prepared.exported[0], verification: 'VERIFIED', + results: [{ executionId: row.executionId, caseId: row.caseId, attachmentsVerified: true }] })); + // The unchanged real upload-case producer retains this marker on a fresh POST. + await writeFile(path.join(destination, 'once-upload_case.json'), JSON.stringify({ stage: 'upload_case', status: 'STARTED_NO_AUTOMATIC_RESEND' })); + } + } + } + return { base, workflow, bundles: roots.bundle, publications: roots.publication, digest: hash(JSON.stringify(selected)) }; +} +test('actual pinned singleton download reaches safe gather', async t => { + const selected = { ...selection, cases: [current] }, f = await downloaded(t, selected); + const output = path.join(f.base, 'catalog-projections'); + await streaming.gatherCaseProjections(f.bundles, output, context, selected, f.digest); + assert.deepEqual(await readdir(output), ['chat-catalog-case-bundle-123-1-0']); +}); +test('actual pinned singleton download reaches final publication validation', async t => { + const selected = { ...selection, cases: [current] }, f = await downloaded(t, selected); + const exported = await publicationSet(f.bundles, f.publications, context, selected, f.digest); + assert.deepEqual(exported.map(row => row.status), ['passed']); +}); +test('actual pinned multi download preserves all mixed grades and missing singleton among a multi selection still refuses', async t => { + const f = await downloaded(t, selection); + assert.deepEqual((await publicationSet(f.bundles, f.publications, context, selection, f.digest)).map(row => row.status), ['passed','failed','passed']); + const incomplete = await downloaded(t, selection, 1); + await assert.rejects(publicationSet(incomplete.bundles, incomplete.publications, context, selection, incomplete.digest)); +}); + +test('named layouts reject empty, mixed, foreign, malformed and symlink entries before gathering', async t => { + const selected = { ...selection, cases: [current] }; + const changes = [ + async f => rm(path.join(f.bundles, 'chat-catalog-case-bundle-123-1-0'), { recursive: true }), + async f => writeFile(path.join(f.bundles, 'prepared.json'), '{}'), + async f => rename(path.join(f.bundles, 'chat-catalog-case-bundle-123-1-0'), path.join(f.bundles, 'chat-catalog-case-bundle-999-1-0')), + async f => writeFile(path.join(f.bundles, 'chat-catalog-case-bundle-123-1-0', 'unknown.json'), '{}'), + async f => { + const directory = path.join(f.bundles, 'chat-catalog-case-bundle-123-1-0'); + await rename(directory, path.join(f.base, 'outside')); + await symlink(path.join(f.base, 'outside'), directory); + }, + async f => { + const file = path.join(f.bundles, 'chat-catalog-case-bundle-123-1-0', 'projection', 'normalized-results.json'); + await rename(file, path.join(f.base, 'outside.json')); await symlink(path.join(f.base, 'outside.json'), file); + }, + async f => { + const file = path.join(f.bundles, 'chat-catalog-case-bundle-123-1-0', 'prepared.json'); + const prepared = JSON.parse(await readFile(file)); prepared.targetSha = 'f'.repeat(40); + await writeFile(file, JSON.stringify(prepared)); + }, + async f => { + const file = path.join(f.bundles, 'chat-catalog-case-bundle-123-1-0', 'allure'); + await rename(file, path.join(f.base, 'outside')); await symlink(path.join(f.base, 'outside'), file); + }, + ]; + for (const change of changes) { + const f = await downloaded(t, selected); await change(f); + const output = path.join(f.base, 'must-not-gather'); + await assert.rejects(streaming.gatherCaseProjections(f.bundles, output, context, selected, f.digest)); + await assert.rejects(access(output)); + await assert.rejects(publicationSet(f.bundles, f.publications, context, selected, f.digest)); + } + for (const change of [ + async f => writeFile(path.join(f.publications, 'chat-catalog-case-publication-123-1-0', 'failure.json'), JSON.stringify({ command: 'upload-case', code: 'chat_completion_testops_upload_unverified' })), + async f => writeFile(path.join(f.publications, 'chat-catalog-case-publication-123-1-0', 'once-upload_case.json'), JSON.stringify({ stage: 'upload_case', status: 'COMPLETE' })), + async f => { + const file = path.join(f.publications, 'chat-catalog-case-publication-123-1-0', 'once-upload_case.json'); + await rename(file, path.join(f.base, 'marker')); await symlink(path.join(f.base, 'marker'), file); + }, + async f => writeFile(path.join(f.publications, 'extra'), '{}'), + async f => writeFile(path.join(f.publications, 'chat-catalog-case-publication-123-1-0', 'extra'), '{}'), + async f => { + const file = path.join(f.publications, 'chat-catalog-case-publication-123-1-0', 'publication.json'); + await rename(file, path.join(f.base, 'outside')); await symlink(path.join(f.base, 'outside'), file); + }, + ]) { + const f = await downloaded(t, selected); await change(f); + await assert.rejects(publicationSet(f.bundles, f.publications, context, selected, f.digest)); + } +}); +test('the owning gather CLI authenticates selection and copies only unchanged projections without TestOps access', async t => { + const selected = { ...selection, cases: [current] }; + const actual = { ...context, operation: 'evaluate', actorId: 42, triggeringActorId: 42, workflowId: 99 }; + const f = await downloaded(t, selected, 1, actual), selectionRoot = path.join(f.base, 'chat-completion-testops-selection'); + await mkdir(selectionRoot); await writeFile(path.join(selectionRoot, 'testops-pilot-selection.json'), JSON.stringify(selected)); + const event = { repository: { full_name: 'paperclipai/paperclip', private: false, default_branch: 'master' }, + ref: 'refs/heads/master', sender: { id: 42 }, inputs: { operation: 'evaluate', ALLURE_JOB_RUN_ID: '77' } }; + const eventPath = path.join(f.base, 'event.json'); await writeFile(eventPath, JSON.stringify(event)); + const env = { GITHUB_ACTIONS: 'true', GITHUB_EVENT_NAME: 'workflow_dispatch', GITHUB_REPOSITORY: event.repository.full_name, + GITHUB_REF: event.ref, GITHUB_WORKFLOW_REF: 'paperclipai/paperclip/.github/workflows/chat-catalog-testops.yml@refs/heads/master', + GITHUB_SHA: context.workflowSha, GITHUB_RUN_ATTEMPT: '1', GITHUB_RUN_ID: '123', GITHUB_ACTOR_ID: '42', + TESTOPS_ALLOWED_ACTOR_IDS: '[42]', GITHUB_RUN_NUMBER: '1', GITHUB_EVENT_PATH: eventPath, + GH_TOKEN: 'SYNTHETIC_GITHUB_SECRET', RUNNER_TEMP: f.base, GITHUB_WORKSPACE: f.base, ALLURE_JOB_RUN_ID: '77', + EXPECTED_SELECTION_SHA256: f.digest, CHAT_TESTOPS_BUNDLES: f.bundles }; + const exec = async () => ({ stdout: JSON.stringify({ id: 123, run_attempt: 1, run_number: 1, workflow_id: 99, + head_sha: context.workflowSha, head_branch: 'master', event: 'workflow_dispatch', path: '.github/workflows/chat-catalog-testops.yml', + repository: event.repository, actor: { id: 42 }, triggering_actor: { id: 42 } }) }); + const output = path.join(f.base, 'projections'); + assert.equal(await run('gather-case-projections', output, { ...env, GITHUB_RUN_ATTEMPT: '2' }, { exec, + fetchImpl: async () => assert.fail('unsupported retry must not contact TestOps') }), 1); + await assert.rejects(access(output)); + assert.equal(await run('gather-case-projections', output, env, { exec, fetchImpl: async () => assert.fail('metadata gather must not contact TestOps') }), 0); + assert.deepEqual(await readdir(output), ['chat-catalog-case-bundle-123-1-0']); + const copied = path.join(output, 'chat-catalog-case-bundle-123-1-0'); + assert.deepEqual((await readdir(copied)).sort(), ['binding.json', 'junit.xml', 'normalized-results.json']); + for (const name of await readdir(copied)) assert.deepEqual(await readFile(path.join(copied, name)), + await readFile(path.join(f.bundles, 'chat-catalog-case-bundle-123-1-0', 'projection', name))); + const gather = f.workflow.jobs.aggregate.steps.find(step => step.name?.startsWith('Gather only')); + assert.match(gather.run, /chat-completion-testops-cli.mjs gather-case-projections catalog-projections/); + for (const job of ['aggregate', 'verify']) { + const downloads = f.workflow.jobs[job].steps.filter(step => step.with?.name?.startsWith('chat-catalog-case-bundle-')); + assert.equal(downloads.length, 1); assert.equal(downloads[0].with.name, 'chat-catalog-case-bundle-${{ github.run_id }}-1-0'); + assert.equal(downloads[0].with.path, 'catalog-bundles/chat-catalog-case-bundle-${{ github.run_id }}-1-0'); + } +}); + +test('singleton destination uses trusted run naming; the admitted producer remains first-attempt-only', async () => { + const workflow = parseDocument(await readFile(new URL('../../.github/workflows/chat-catalog-testops.yml', import.meta.url), 'utf8')).toJS(); + const callee = parseDocument(await readFile(new URL('../../.github/workflows/chat-catalog-testops-case.yml', import.meta.url), 'utf8')).toJS(); + for (const role of ['bundle', 'publication']) { + const step = workflow.jobs.verify.steps.find(step => step.with?.name?.startsWith(`chat-catalog-case-${role}-`)); + const name = step.with.name.replaceAll('${{ github.run_id }}', '987654321'); + const destination = step.with.path.replaceAll('${{ github.run_id }}', '987654321'); + assert.equal(path.basename(destination), name); + assert.equal(pinnedDownloadPath(destination, { name }, [{ name }], true), destination); + const job = role === 'bundle' ? callee.jobs.report : callee.jobs.upload; + const producer = job.steps.find(step => step.with?.name?.startsWith(`chat-catalog-case-${role}-`)); + assert.equal(producer.with.name.replaceAll('${{ github.run_id }}', '987654321').replaceAll('${{ fromJSON(inputs.cell).index }}', '0'), name); + if (role === 'publication') { + assert.match(producer.with.path, /once-upload_case.json/); + assert.match(producer.with.path, /failure.json/); + } + } +}); + +test('the unchanged upload CLI real publication and once files validate after a named singleton download', async t => { + const selected = { ...selection, cases: [current] }; + const actual = { ...context, operation: 'evaluate', actorId: 42, triggeringActorId: 42, workflowId: 99 }; + const f = await downloaded(t, selected, 1, actual), bundle = path.join(f.bundles, 'chat-catalog-case-bundle-123-1-0'); + const preparedFile = path.join(bundle, 'prepared.json'), prepared = JSON.parse(await readFile(preparedFile)); + const selectionRoot = path.join(f.base, 'chat-completion-testops-selection'); + await mkdir(selectionRoot); await writeFile(path.join(selectionRoot, 'testops-pilot-selection.json'), JSON.stringify(selected)); + const event = { repository: { full_name: 'paperclipai/paperclip', private: false, default_branch: 'master' }, + ref: 'refs/heads/master', sender: { id: 42 }, inputs: { operation: 'evaluate', ALLURE_JOB_RUN_ID: '77' } }; + const eventPath = path.join(f.base, 'event.json'); await writeFile(eventPath, JSON.stringify(event)); + const env = { GITHUB_ACTIONS: 'true', GITHUB_EVENT_NAME: 'workflow_dispatch', GITHUB_REPOSITORY: event.repository.full_name, + GITHUB_REF: event.ref, GITHUB_WORKFLOW_REF: 'paperclipai/paperclip/.github/workflows/chat-catalog-testops.yml@refs/heads/master', + GITHUB_SHA: context.workflowSha, GITHUB_RUN_ATTEMPT: '1', GITHUB_RUN_ID: '123', GITHUB_ACTOR_ID: '42', + TESTOPS_ALLOWED_ACTOR_IDS: '[42]', GITHUB_RUN_NUMBER: '1', GITHUB_EVENT_PATH: eventPath, + GH_TOKEN: 'SYNTHETIC_GITHUB_SECRET', RUNNER_TEMP: f.base, GITHUB_WORKSPACE: f.base, ALLURE_JOB_RUN_ID: '77', + ALLURE_TOKEN: 'SYNTHETIC_REPORT_TOKEN', EXPECTED_SELECTION_SHA256: f.digest, + ALLURE_RESULTS: path.join(bundle, 'allure'), CHAT_TESTOPS_PREPARED_FILE: preparedFile, CHAT_TESTOPS_EXECUTION_ID: current.executionId }; + const exec = async () => ({ stdout: JSON.stringify({ id: 123, run_attempt: 1, run_number: 1, workflow_id: 99, + head_sha: context.workflowSha, head_branch: 'master', event: 'workflow_dispatch', path: '.github/workflows/chat-catalog-testops.yml', + repository: event.repository, actor: { id: 42 }, triggering_actor: { id: 42 } }) }); + const transport = server(prepared); + const fetchImpl = async (url, options) => { + const u = new URL(url); + if (u.pathname === '/api/uaa/oauth/token') return Response.json({ access_token: 'SYNTHETIC_ACCESS_TOKEN' }); + const value = await transport.api(u.pathname + u.search, options.method, options.body); + if (u.pathname === '/api/testresult') { + value.content = value.content.filter(row => row.testCaseId === current.caseId); + value.totalElements = value.content.length; + } + return Response.json(value); + }; + assert.equal(await run('upload-case', path.join(bundle, 'projection'), env, { exec, fetchImpl }), 0); + assert.equal(transport.posts, 1); + const emitted = path.join(f.base, 'chat-completion-testops-upload-case'); + assert.deepEqual((await readdir(emitted)).sort(), ['once-upload_case.json', 'publication.json']); + const target = path.join(f.publications, 'chat-catalog-case-publication-123-1-0'); + await rm(target, { recursive: true }); await cp(emitted, target, { recursive: true }); + const exported = await publicationSet(f.bundles, f.publications, actual, selected, f.digest); + assert.equal(exported[0].status, 'passed'); assert.equal(transport.posts, 1, 'finalizer must not resend'); +}); diff --git a/tests/runner-e2e/chat-completion-testops-cli.mjs b/tests/runner-e2e/chat-completion-testops-cli.mjs index 3504ae87acc..f70f02fd48d 100644 --- a/tests/runner-e2e/chat-completion-testops-cli.mjs +++ b/tests/runner-e2e/chat-completion-testops-cli.mjs @@ -7,7 +7,7 @@ import { admitDispatch, hash, reject, validateSelection, evidencePolicy, catalog import { testopsClient, bootstrapPilotPlan, completePilotRegistration, validatePilotSelection, launchPilotPlan, verifyPilotUpload } from './chat-completion-testops-api.mjs'; import { parseOfficialBootstrapContext } from './testops-chat-reporter.mjs'; import { exportPilotAllure, readPilotExport } from './chat-completion-testops-results.mjs'; -import { prepareCaseBundle, publishCaseBundle, publicationSet } from './chat-catalog-testops-streaming.mjs'; +import { prepareCaseBundle, publishCaseBundle, publicationSet, gatherCaseProjections } from './chat-catalog-testops-streaming.mjs'; const execute = promisify(execFile); const fullCatalogFailureCodes = new Set([ @@ -94,7 +94,7 @@ export async function run(command, argument, env = process.env, { fetchImpl = fe let state; try { if (!['guard', 'bootstrap', 'bootstrap-context', 'selection', 'launch-plan', 'verify-selection', 'upload', 'verify-upload', - 'prepare-case-upload', 'upload-case', 'verify-stream'].includes(command)) reject('operation'); + 'prepare-case-upload', 'upload-case', 'verify-stream', 'gather-case-projections'].includes(command)) reject('operation'); const context = await actualContext(env, exec); if (command === 'guard') { await output(env, 'operation', context.operation); await output(env, 'target_sha', context.targetSha); @@ -106,7 +106,7 @@ export async function run(command, argument, env = process.env, { fetchImpl = fe } if (['bootstrap', 'bootstrap-context'].includes(command) && context.operation !== 'bootstrap' || command === 'launch-plan' && context.operation !== 'launch-plan' - || ['selection', 'upload', 'verify-upload', 'prepare-case-upload', 'upload-case', 'verify-stream'].includes(command) && context.operation !== 'evaluate') reject('operation'); + || ['selection', 'upload', 'verify-upload', 'prepare-case-upload', 'upload-case', 'verify-stream', 'gather-case-projections'].includes(command) && context.operation !== 'evaluate') reject('operation'); state = await stateDirectory(env, command === 'bootstrap-context' ? 'bootstrap' : command, command === 'bootstrap-context'); if (command === 'prepare-case-upload') { if (Object.hasOwn(env, 'ALLURE_TOKEN')) reject('worker_credential'); @@ -115,6 +115,12 @@ export async function run(command, argument, env = process.env, { fetchImpl = fe env.CHAT_TESTOPS_EXECUTION_ID, env.EXPECTED_SELECTION_SHA256); await writeReceipt(state, 'prepared.json', prepared); return 0; } + if (command === 'gather-case-projections') { + if (Object.hasOwn(env, 'ALLURE_TOKEN')) reject('worker_credential'); + const selected = await readSelection(path.join(env.GITHUB_WORKSPACE, 'chat-completion-testops-selection', 'testops-pilot-selection.json'), env, context); + await gatherCaseProjections(env.CHAT_TESTOPS_BUNDLES, path.resolve(argument), context, selected, env.EXPECTED_SELECTION_SHA256); + return 0; + } const api = await testopsClient(env.ALLURE_TOKEN, fetchImpl); if (command === 'bootstrap') { const prepared = await bootstrapPilotPlan(api, Number(env.CHAT_TESTOPS_INTEGRATION_ID), mutationMarker(state), context); diff --git a/tests/runner-e2e/chat-full-catalog-testops-manifest.json b/tests/runner-e2e/chat-full-catalog-testops-manifest.json index 1c03270a466..ad1ceed82ef 100644 --- a/tests/runner-e2e/chat-full-catalog-testops-manifest.json +++ b/tests/runner-e2e/chat-full-catalog-testops-manifest.json @@ -1,6 +1,6 @@ { "schema": "paperclip.chat-full-catalog-testops-manifest/v1", - "targetSha": "51ef450754370c8a8f2cbe4bd18e1536bbc93966", + "targetSha": "ab2019e976076ec0eb60b026844f3d9211cb0208", "metadataSourceSha": "de0941835cc8f0d01c982574ee258e110babdbeb", "profile": "runner-opencode", "model": "openrouter/deepseek/deepseek-v4.1-flash-1017", @@ -170,9 +170,9 @@ ], "repairMetadata": { "baseTargetSha": "3b66c3c144360ddf32714a0742fd774b8ceafb16", - "previousTargetSha": "43c44ccc823a8c972b247fa7159281887e6f5255", + "previousTargetSha": "de6d29e025683f5c28125bcaa4f38b41836d8516", "definitionSourceSha": "de0941835cc8f0d01c982574ee258e110babdbeb", - "candidateDescriptorSha256": "54760e1dca998c2ac2b6546f6daf5a13f399d2e9b125a465810b8fadf66c35d8", + "candidateDescriptorSha256": "d214f8bf1d6c8c737682999ca9eb9d4ea2a5945bebbccb96e9ac326d538b0629", "changedDefinitionFiles": [ { "path": "tests/runner-e2e/chat-scenarios/companies.yaml", @@ -191,8 +191,37 @@ "sha256": "18fbb9419bbdacb46eacdaa62dfa99feb9024faf2d1eef829aa70d43952575a9" } ], - "conditions": "New candidate measurements only; original definitions and all failed attempts remain historical. Source4 repairs remaining original observation, normalized-result and owned fixture authorization/profile boundaries, exact full-proposal approval evidence and optional real completion observation. Product487 disclosure is a reviewed candidate-only overlay from UNMERGED18734. SDK privacy/AST guards are unchanged; expected restored lock now binds authenticated BUILD-input54a81 rather than stale c7f230, explicitly a new runtime eligibility condition. Only144 retains its earlier reviewed2..3 native-accounting revision with actual3 accepted; all original clocks/model/human counts and all other native bounds stay fixed. No core merge, workflow/credential/permission expansion or regrade", + "conditions": "New candidate measurements only; original definitions, failed attempts and previousSource5 de6 remain immutable. Source6 retains all Source5 repairs and adds remaining ORIGINAL deadline plus exact canonical source guards for personal-secret and team-install initial observations, real hire-review link readiness, and genuine skill-update provider checkpoints after cheap native readiness. Candidate-only Core18742/61996 overlay permits only a new Ask-first review for a strictly authenticated later same-person human intent after decline, including issue.comment.reopen; same-intent replay remains denied, with no approval or provider authority granted. All queries/models/human/native counts/clocks remain as de6; earlier Studio144 explicit2..3 accounting remains. CorePRs18717/18734/18742 remain UNMERGED; SDK115 completion remainsUNKNOWN and otherfiveSDK cases stay held.", "changedFields": [ + { + "key": "activity.work-timeline", + "field": "sourceFiles", + "previous": [ + "chat-catalog-domains/internal-definition.ts", + "chat-catalog-domains/internal-support.ts", + "chat-catalog-domains/internal-activity.ts", + "chat-scenarios/activity.yaml", + "chat-catalog-domains/internal-work-timeline.ts", + "chat-catalog-domains/internal-timeline-verifier.ts", + "chat-catalog-domains/internal-privacy.ts", + "chat-catalog-domains/identity-auth.ts", + "chat-catalog-domains/internal-routine-audit.ts", + "../e2e/fetch-allowed-port.ts" + ], + "current": [ + "chat-catalog-domains/internal-definition.ts", + "chat-catalog-domains/internal-support.ts", + "chat-catalog-domains/internal-activity.ts", + "chat-scenarios/activity.yaml", + "chat-catalog-domains/internal-work-timeline.ts", + "chat-catalog-domains/internal-timeline-verifier.ts", + "chat-catalog-domains/internal-privacy.ts", + "chat-catalog-domains/identity-auth.ts", + "chat-catalog-domains/internal-routine-audit.ts", + "../e2e/fetch-allowed-port.ts", + "../../server/src/services/native-runtime/status-decision-committer.ts" + ] + }, { "key": "agents.choose-primary", "field": "sourceFiles", @@ -243,6 +272,7 @@ "../../server/src/routes/issues.ts", "../../server/src/routes/tool-access.ts", "../../server/src/services/tool-gateway.ts", + "../../server/src/services/tool-action-replay-intent.ts", "../../server/src/services/tool-action-review.ts", "../../server/src/services/tool-action-delivery.ts", "../../server/src/services/tool-access-policy.ts", @@ -452,7 +482,9 @@ "harness-env.ts", "../../packages/shared/src/announcements.ts", "../../server/src/services/announcement-feed.ts", - "../../server/src/routes/announcements.ts" + "../../server/src/routes/announcements.ts", + "../../ui/src/pages/Projects.tsx", + "../../ui/src/components/AnnouncementWell.tsx" ] }, { @@ -467,6 +499,25 @@ "previous": "paperclip.chat-catalog.identity.v1", "current": "paperclip.chat-catalog.answer-delivery.v2" }, + { + "key": "questions-and-approvals.formal-hire-revision", + "field": "sourceFiles", + "previous": [ + "chat-catalog-domains/identity-hire-revision.ts", + "chat-catalog-domains/identity-hire-model-choice.ts", + "chat-catalog-domains/identity-support.ts", + "chat-catalog-domains/internal-support.ts", + "chat-scenarios/questions-and-approvals.yaml" + ], + "current": [ + "chat-catalog-domains/identity-hire-revision.ts", + "chat-catalog-domains/identity-hire-model-choice.ts", + "chat-catalog-domains/identity-support.ts", + "chat-catalog-domains/internal-support.ts", + "chat-rendered-links.ts", + "chat-scenarios/questions-and-approvals.yaml" + ] + }, { "key": "questions-and-approvals.formal-hire-revision", "field": "oracleVersion", @@ -525,6 +576,38 @@ "previous": "paperclip.chat-catalog.identity.v1", "current": "paperclip.chat-catalog.plan-review.v2" }, + { + "key": "secrets.configure-company-secret", + "field": "sourceFiles", + "previous": [ + "chat-catalog-domains/external-company-secret.ts", + "chat-catalog-domains/external-private-browser.ts", + "chat-catalog-domains/external-private-browser-worker.ts", + "chat-catalog-domains/external-credential-language.ts", + "chat-catalog-domains/external-secret-scan.ts", + "chat-catalog-domains/external-support.ts", + "chat-catalog-domains/internal-support.ts", + "chat-catalog-domains/identity-support.ts", + "run-observations.ts", + "chat-flow.ts", + "chat-scenarios/secrets.yaml" + ], + "current": [ + "chat-catalog-domains/external-company-secret.ts", + "chat-catalog-domains/external-private-browser.ts", + "chat-catalog-domains/external-private-browser-worker.ts", + "chat-catalog-domains/external-credential-language.ts", + "chat-catalog-domains/external-secret-scan.ts", + "chat-catalog-domains/external-support.ts", + "chat-catalog-domains/internal-support.ts", + "chat-catalog-domains/identity-support.ts", + "live-fixtures.ts", + "../../server/src/services/secrets.ts", + "run-observations.ts", + "chat-flow.ts", + "chat-scenarios/secrets.yaml" + ] + }, { "key": "secrets.configure-company-secret", "field": "oracleVersion", @@ -546,7 +629,9 @@ "chat-catalog-domains/integrator-skills.ts", "chat-scenarios/skills.yaml", "chat-catalog-domains/integrator-skill-versions.ts", - "chat-catalog-domains/identity-support.ts" + "chat-catalog-domains/identity-support.ts", + "../../ui/src/components/MarkdownEditor.tsx", + "../../ui/src/pages/SkillStudio.tsx" ] }, { @@ -736,6 +821,35 @@ ], "candidateDescriptorSnapshot": "fixtures/chat-repair-candidate-descriptors.json", "descriptorChanges": [ + { + "key": "activity.work-timeline", + "field": "sourceFiles", + "previous": [ + "chat-catalog-domains/internal-definition.ts", + "chat-catalog-domains/internal-support.ts", + "chat-catalog-domains/internal-activity.ts", + "chat-scenarios/activity.yaml", + "chat-catalog-domains/internal-work-timeline.ts", + "chat-catalog-domains/internal-timeline-verifier.ts", + "chat-catalog-domains/internal-privacy.ts", + "chat-catalog-domains/identity-auth.ts", + "chat-catalog-domains/internal-routine-audit.ts", + "../e2e/fetch-allowed-port.ts" + ], + "current": [ + "chat-catalog-domains/internal-definition.ts", + "chat-catalog-domains/internal-support.ts", + "chat-catalog-domains/internal-activity.ts", + "chat-scenarios/activity.yaml", + "chat-catalog-domains/internal-work-timeline.ts", + "chat-catalog-domains/internal-timeline-verifier.ts", + "chat-catalog-domains/internal-privacy.ts", + "chat-catalog-domains/identity-auth.ts", + "chat-catalog-domains/internal-routine-audit.ts", + "../e2e/fetch-allowed-port.ts", + "../../server/src/services/native-runtime/status-decision-committer.ts" + ] + }, { "key": "agents.choose-primary", "field": "sourceFiles", @@ -786,6 +900,7 @@ "../../server/src/routes/issues.ts", "../../server/src/routes/tool-access.ts", "../../server/src/services/tool-gateway.ts", + "../../server/src/services/tool-action-replay-intent.ts", "../../server/src/services/tool-action-review.ts", "../../server/src/services/tool-action-delivery.ts", "../../server/src/services/tool-access-policy.ts", @@ -971,7 +1086,9 @@ "harness-env.ts", "../../packages/shared/src/announcements.ts", "../../server/src/services/announcement-feed.ts", - "../../server/src/routes/announcements.ts" + "../../server/src/routes/announcements.ts", + "../../ui/src/pages/Projects.tsx", + "../../ui/src/components/AnnouncementWell.tsx" ] }, { @@ -986,6 +1103,25 @@ "previous": "paperclip.chat-catalog.identity.v1", "current": "paperclip.chat-catalog.answer-delivery.v2" }, + { + "key": "questions-and-approvals.formal-hire-revision", + "field": "sourceFiles", + "previous": [ + "chat-catalog-domains/identity-hire-revision.ts", + "chat-catalog-domains/identity-hire-model-choice.ts", + "chat-catalog-domains/identity-support.ts", + "chat-catalog-domains/internal-support.ts", + "chat-scenarios/questions-and-approvals.yaml" + ], + "current": [ + "chat-catalog-domains/identity-hire-revision.ts", + "chat-catalog-domains/identity-hire-model-choice.ts", + "chat-catalog-domains/identity-support.ts", + "chat-catalog-domains/internal-support.ts", + "chat-rendered-links.ts", + "chat-scenarios/questions-and-approvals.yaml" + ] + }, { "key": "questions-and-approvals.formal-hire-revision", "field": "oracleVersion", @@ -1044,6 +1180,38 @@ "previous": "paperclip.chat-catalog.identity.v1", "current": "paperclip.chat-catalog.plan-review.v2" }, + { + "key": "secrets.configure-company-secret", + "field": "sourceFiles", + "previous": [ + "chat-catalog-domains/external-company-secret.ts", + "chat-catalog-domains/external-private-browser.ts", + "chat-catalog-domains/external-private-browser-worker.ts", + "chat-catalog-domains/external-credential-language.ts", + "chat-catalog-domains/external-secret-scan.ts", + "chat-catalog-domains/external-support.ts", + "chat-catalog-domains/internal-support.ts", + "chat-catalog-domains/identity-support.ts", + "run-observations.ts", + "chat-flow.ts", + "chat-scenarios/secrets.yaml" + ], + "current": [ + "chat-catalog-domains/external-company-secret.ts", + "chat-catalog-domains/external-private-browser.ts", + "chat-catalog-domains/external-private-browser-worker.ts", + "chat-catalog-domains/external-credential-language.ts", + "chat-catalog-domains/external-secret-scan.ts", + "chat-catalog-domains/external-support.ts", + "chat-catalog-domains/internal-support.ts", + "chat-catalog-domains/identity-support.ts", + "live-fixtures.ts", + "../../server/src/services/secrets.ts", + "run-observations.ts", + "chat-flow.ts", + "chat-scenarios/secrets.yaml" + ] + }, { "key": "secrets.configure-company-secret", "field": "oracleVersion", @@ -1076,7 +1244,9 @@ "chat-catalog-domains/integrator-skills.ts", "chat-scenarios/skills.yaml", "chat-catalog-domains/integrator-skill-versions.ts", - "chat-catalog-domains/identity-support.ts" + "chat-catalog-domains/identity-support.ts", + "../../ui/src/components/MarkdownEditor.tsx", + "../../ui/src/pages/SkillStudio.tsx" ] }, { @@ -1348,14 +1518,25 @@ }, "toolProposal487": { "corePr": 18734, - "coreHead": "17236e1736bbecdf04eeeee87ccb794c72d03da1", + "coreHead": "50600786f755b6602989f3b0f88cd090ff948196", "corePRUnmerged": true, - "condition": "Candidate-only exact shared-card bounded redacted full-argument/audit disclosure; no backend authority change. Actual human review/provider dispatch joined by oracle v2." + "condition": "Candidate-only reviewed disclosure plus bounded pre-trim guard; no authority change. CorePR18734 remainsunmerged.", + "replay": { + "corePR": 18742, + "coreHead": "61996a96dd09e90a5788e8711a94fc5ae631f425", + "corePRUnmerged": true, + "condition": "A new Ask-first request only for an authenticated later same-person human intent after the prior decline; same-intent retry and derived or foreign sources remain denied. No approval/execution authority change.", + "acceptedDirectSources": [ + "issue.comment", + "issue.comment.reopen" + ], + "actualPostAssertionPGCasesPassed": 2 + } } }, "candidateLauncherSnapshot": { "path": "fixtures/chat-repair-candidate-launcher.txt", - "targetSha": "51ef450754370c8a8f2cbe4bd18e1536bbc93966", + "targetSha": "ab2019e976076ec0eb60b026844f3d9211cb0208", "sourcePath": "tests/runner-e2e/launch.ts", "gitBlobOid": "dba1467e211611f37937e9e1fc4c0c5be4329cbc" } @@ -1909,7 +2090,8 @@ "chat-catalog-domains/internal-privacy.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/internal-routine-audit.ts", - "../e2e/fetch-allowed-port.ts" + "../e2e/fetch-allowed-port.ts", + "../../server/src/services/native-runtime/status-decision-committer.ts" ], "screenshot": "final-state.png", "evidenceKind": "private-final-state-png", @@ -2987,6 +3169,7 @@ "../../server/src/routes/issues.ts", "../../server/src/routes/tool-access.ts", "../../server/src/services/tool-gateway.ts", + "../../server/src/services/tool-action-replay-intent.ts", "../../server/src/services/tool-action-review.ts", "../../server/src/services/tool-action-delivery.ts", "../../server/src/services/tool-access-policy.ts", @@ -8118,7 +8301,9 @@ "harness-env.ts", "../../packages/shared/src/announcements.ts", "../../server/src/services/announcement-feed.ts", - "../../server/src/routes/announcements.ts" + "../../server/src/routes/announcements.ts", + "../../ui/src/pages/Projects.tsx", + "../../ui/src/components/AnnouncementWell.tsx" ], "screenshot": "final-state.png", "evidenceKind": "private-final-state-png", @@ -9421,6 +9606,7 @@ "chat-catalog-domains/identity-hire-model-choice.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", + "chat-rendered-links.ts", "chat-scenarios/questions-and-approvals.yaml" ], "screenshot": "final-state.png", @@ -10689,6 +10875,8 @@ "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", + "live-fixtures.ts", + "../../server/src/services/secrets.ts", "run-observations.ts", "chat-flow.ts", "chat-scenarios/secrets.yaml" @@ -10980,7 +11168,9 @@ "chat-catalog-domains/integrator-skills.ts", "chat-scenarios/skills.yaml", "chat-catalog-domains/integrator-skill-versions.ts", - "chat-catalog-domains/identity-support.ts" + "chat-catalog-domains/identity-support.ts", + "../../ui/src/components/MarkdownEditor.tsx", + "../../ui/src/pages/SkillStudio.tsx" ], "screenshot": "final-state.png", "evidenceKind": "private-final-state-png", diff --git a/tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs b/tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs index f8601a1762e..3b44ba66cc9 100644 --- a/tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs +++ b/tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs @@ -102,8 +102,8 @@ test('foreign identity, duplicated case and changed run-bound metadata are rejec test('repair source metadata binds the corrected candidate and explicit changed definitions', () => { const repair = manifest.source.repairMetadata; assert.equal(repair.baseTargetSha, '3b66c3c144360ddf32714a0742fd774b8ceafb16'); - assert.equal(repair.previousTargetSha, '43c44ccc823a8c972b247fa7159281887e6f5255'); - assert.equal(manifest.targetSha, '51ef450754370c8a8f2cbe4bd18e1536bbc93966'); + assert.equal(repair.previousTargetSha, 'de6d29e025683f5c28125bcaa4f38b41836d8516'); + assert.equal(manifest.targetSha, 'ab2019e976076ec0eb60b026844f3d9211cb0208'); assert.equal(repair.definitionSourceSha, manifest.metadataSourceSha); assert.equal(repair.candidateDescriptorSnapshot, 'fixtures/chat-repair-candidate-descriptors.json'); const descriptorBytes = readFileSync(new URL(repair.candidateDescriptorSnapshot, import.meta.url)); @@ -112,6 +112,10 @@ test('repair source metadata binds the corrected candidate and explicit changed assert.equal(declarations.length, 242); assert.equal(new Set(declarations.map(value => value.scenarioId)).size, 242); assert.deepEqual(repair.changedFields.map(value => [value.key, value.field]), [ + [ + "activity.work-timeline", + "sourceFiles" + ], [ "agents.choose-primary", "sourceFiles" @@ -188,6 +192,10 @@ test('repair source metadata binds the corrected candidate and explicit changed "questions-and-approvals.duplicate-answer", "oracleVersion" ], + [ + "questions-and-approvals.formal-hire-revision", + "sourceFiles" + ], [ "questions-and-approvals.formal-hire-revision", "oracleVersion" @@ -212,6 +220,10 @@ test('repair source metadata binds the corrected candidate and explicit changed "questions-and-approvals.review-plan", "oracleVersion" ], + [ + "secrets.configure-company-secret", + "sourceFiles" + ], [ "secrets.configure-company-secret", "oracleVersion" @@ -273,8 +285,8 @@ test('repair source metadata binds the corrected candidate and explicit changed assert.deepEqual(row(change.key)[change.field], change.current); assert.notDeepEqual(change.current, change.previous); } - assert.equal(repair.descriptorChanges.length, 38); - assert.equal(new Set(repair.descriptorChanges.map(change => change.key)).size, 27); + assert.equal(repair.descriptorChanges.length, 41); + assert.equal(new Set(repair.descriptorChanges.map(change => change.key)).size, 28); assert.deepEqual([...new Set(repair.descriptorChanges.map(change => change.field))].sort(), ['expectedRunCount', 'minimumRunCount', 'oracleVersion', 'setupPolicy', 'sourceFiles']); for (const change of repair.descriptorChanges) { assert.notDeepEqual(change.current, change.previous); @@ -304,4 +316,15 @@ test('repair source metadata binds the corrected candidate and explicit changed assert.equal(repair.runtimeConditions.sdkPublicProof.expectedLockSha256, "54a81b2ed3f1f85eaa085c034560ac248e384db34fbc7aba8c5a3846c29b3ce8"); assert.equal(repair.runtimeConditions.sdkPublicProof.runtimeCompilerParserZodIdentitiesUnchanged, true); assert.equal(repair.runtimeConditions.toolProposal487.corePRUnmerged, true); + assert.deepEqual(repair.runtimeConditions.toolProposal487.replay, { + "corePR": 18742, + "coreHead": "61996a96dd09e90a5788e8711a94fc5ae631f425", + "corePRUnmerged": true, + "condition": "A new Ask-first request only for an authenticated later same-person human intent after the prior decline; same-intent retry and derived or foreign sources remain denied. No approval/execution authority change.", + "acceptedDirectSources": [ + "issue.comment", + "issue.comment.reopen" + ], + "actualPostAssertionPGCasesPassed": 2 +}); }); diff --git a/tests/runner-e2e/fixtures/chat-repair-candidate-descriptors.json b/tests/runner-e2e/fixtures/chat-repair-candidate-descriptors.json index ca33fc8ec86..d5edf51a771 100644 --- a/tests/runner-e2e/fixtures/chat-repair-candidate-descriptors.json +++ b/tests/runner-e2e/fixtures/chat-repair-candidate-descriptors.json @@ -253,7 +253,8 @@ "chat-catalog-domains/internal-privacy.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/internal-routine-audit.ts", - "../e2e/fetch-allowed-port.ts" + "../e2e/fetch-allowed-port.ts", + "../../server/src/services/native-runtime/status-decision-committer.ts" ], "oracleVersion": "paperclip.chat-catalog.internal.v1", "coverageGaps": [], @@ -814,6 +815,7 @@ "../../server/src/routes/issues.ts", "../../server/src/routes/tool-access.ts", "../../server/src/services/tool-gateway.ts", + "../../server/src/services/tool-action-replay-intent.ts", "../../server/src/services/tool-action-review.ts", "../../server/src/services/tool-action-delivery.ts", "../../server/src/services/tool-access-policy.ts", @@ -3575,7 +3577,9 @@ "harness-env.ts", "../../packages/shared/src/announcements.ts", "../../server/src/services/announcement-feed.ts", - "../../server/src/routes/announcements.ts" + "../../server/src/routes/announcements.ts", + "../../ui/src/pages/Projects.tsx", + "../../ui/src/components/AnnouncementWell.tsx" ], "oracleVersion": "paperclip.chat-catalog.announcement-dismissal.v2", "expectedRunCount": 1, @@ -4196,6 +4200,7 @@ "chat-catalog-domains/identity-hire-model-choice.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", + "chat-rendered-links.ts", "chat-scenarios/questions-and-approvals.yaml" ], "oracleVersion": "paperclip.chat-catalog.hire-revision.native1017.v3", @@ -4887,6 +4892,8 @@ "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", + "live-fixtures.ts", + "../../server/src/services/secrets.ts", "run-observations.ts", "chat-flow.ts", "chat-scenarios/secrets.yaml" @@ -5064,7 +5071,9 @@ "chat-catalog-domains/integrator-skills.ts", "chat-scenarios/skills.yaml", "chat-catalog-domains/integrator-skill-versions.ts", - "chat-catalog-domains/identity-support.ts" + "chat-catalog-domains/identity-support.ts", + "../../ui/src/components/MarkdownEditor.tsx", + "../../ui/src/pages/SkillStudio.tsx" ], "oracleVersion": "paperclip.chat-catalog.skill-authoring.v2", "expectedRunCount": 1,