diff --git a/tests/runner-e2e/chat-full-catalog-testops-manifest.json b/tests/runner-e2e/chat-full-catalog-testops-manifest.json index 980c4d1ed62..b62bc085e4c 100644 --- a/tests/runner-e2e/chat-full-catalog-testops-manifest.json +++ b/tests/runner-e2e/chat-full-catalog-testops-manifest.json @@ -1,6 +1,6 @@ { "schema": "paperclip.chat-full-catalog-testops-manifest/v1", - "targetSha": "5aef4bffa2b106f2066bf2485338798f04a702bd", + "targetSha": "6773b1c128a2e7efa7571ff04b38238f00d06117", "metadataSourceSha": "de0941835cc8f0d01c982574ee258e110babdbeb", "profile": "runner-opencode", "model": "openrouter/deepseek/deepseek-v4.1-flash-1017", @@ -172,7 +172,7 @@ "baseTargetSha": "3b66c3c144360ddf32714a0742fd774b8ceafb16", "previousTargetSha": "ab2019e976076ec0eb60b026844f3d9211cb0208", "definitionSourceSha": "de0941835cc8f0d01c982574ee258e110babdbeb", - "candidateDescriptorSha256": "744a566faaa90b8048fff526e0a30b7475b86c2de88d697783f8da9e02f70e20", + "candidateDescriptorSha256": "aaf15ca6755fcfe550daa98bca1ed263e3d625159b53cd1e1f52fefbf9474778", "changedDefinitionFiles": [ { "path": "tests/runner-e2e/chat-scenarios/companies.yaml", @@ -193,9 +193,13 @@ { "path": "tests/runner-e2e/chat-scenarios/activity.yaml", "sha256": "33e98171de19c3151391164ebf266b53806f387164ad3c4e35f600044d4c1db2" + }, + { + "path": "tests/runner-e2e/chat-scenarios/questions-and-approvals.yaml", + "sha256": "91a9ddc4f5357f19a2cdc73ff8a090c741d14780e4ed502c856ebd557986f7e1" } ], - "conditions": "New candidate measurements only. Original definitions, prior failed attempts and source6 ab201 remain historical. This target retains that catalog and integrates merged runtime fixes #18711, #18713 and #18718, with the prior-run restoration condition adapted to its existing owner. It adds the reviewed previous-question locator, attributed human-only denial and conditional-outcome oracle, and actor-sensitive timeline narrative/recorded-reassignment fixes. Closed finish diagnostics retain an effective schema digest, hashed call correlation and bounded type/category enums only. Historical MCP timeout and continuation-shape causes remain unresolved; no new all-ten PASS is claimed. Queries, model, human/native counts and original clocks remain fixed; the earlier Studio144 explicit 2..3 accounting stays. Core PRs18717/18734/18742 are now merged. SDK115 completion remains UNKNOWN and the other five SDK cases remain held. Full244 runtime catalog is not merged to master and the196/244 gate is unchanged. This successor binds the seven reviewed scenario repairs plus framework evidence: canonical skill navigation, original announcement phase clock, attributed timeline context, bound hire revision history, credential warning and configuration-question admission, completed team visibility, and canonical private framework evidence. Twelve declaration fields across eleven cases change only oracle versions and skill source-file provenance. All original requests, setup, human/native bounds, model and clocks remain fixed. Historical failed attempts and missing-original UNKNOWN grades remain unchanged; no uniform-ten PASS is claimed. The existing SDK506/efa qualification is reused only for byte-identical registered source inputs. This next successor adds reviewed question-surface observation, grounded scenario grading and safe credential-guidance boundaries after the retained uniform attempt. Eleven existing oracle-version leaves advance; all sourceFiles declarations, requests, setup, counts, clocks, model, launcher and private protocol helpers remain fixed. Component qualification does not attest a new uniform-ten pass.", + "conditions": "Historical qualification conditions (retained; statements that all requests or sourceFiles remain fixed refer to those earlier successors): New candidate measurements only. Original definitions, prior failed attempts and source6 ab201 remain historical. This target retains that catalog and integrates merged runtime fixes #18711, #18713 and #18718, with the prior-run restoration condition adapted to its existing owner. It adds the reviewed previous-question locator, attributed human-only denial and conditional-outcome oracle, and actor-sensitive timeline narrative/recorded-reassignment fixes. Closed finish diagnostics retain an effective schema digest, hashed call correlation and bounded type/category enums only. Historical MCP timeout and continuation-shape causes remain unresolved; no new all-ten PASS is claimed. Queries, model, human/native counts and original clocks remain fixed; the earlier Studio144 explicit 2..3 accounting stays. Core PRs18717/18734/18742 are now merged. SDK115 completion remains UNKNOWN and the other five SDK cases remain held. Full244 runtime catalog is not merged to master and the196/244 gate is unchanged. This successor binds the seven reviewed scenario repairs plus framework evidence: canonical skill navigation, original announcement phase clock, attributed timeline context, bound hire revision history, credential warning and configuration-question admission, completed team visibility, and canonical private framework evidence. Twelve declaration fields across eleven cases change only oracle versions and skill source-file provenance. All original requests, setup, human/native bounds, model and clocks remain fixed. Historical failed attempts and missing-original UNKNOWN grades remain unchanged; no uniform-ten PASS is claimed. The existing SDK506/efa qualification is reused only for byte-identical registered source inputs. This next successor adds reviewed question-surface observation, grounded scenario grading and safe credential-guidance boundaries after the retained uniform attempt. Eleven existing oracle-version leaves advance; all sourceFiles declarations, requests, setup, counts, clocks, model, launcher and private protocol helpers remain fixed. Component qualification does not attest a new uniform-ten pass.\n\nCurrent successor conditions: This successor measures five reviewed repairs at the retained original model and bounds. Exactly one request is clarified: multipart53 now explicitly asks to walk through the visible form; its original broad request and failures4623/4717 remain historical. The previous inventory0cf530 is retained and the disclosed clarified inventory is814bb363. Other243 catalog requests are unchanged. The company-secret second human message now references the actual saved non-secret metadata instead of repeating the initial request; native continuation must remain attributed to the actual agent. Five oracle-version leaves and four source-file additions change; launcher, private helpers, SDK506/efa, identities, counts, clocks and the196/244 gate remain fixed. All five successor journeys remain unrun; component proof is not a PASS. For Team610, independently saved canonical skill/version/member proof establishes recovery; no required recovery-success phrase is imposed. A contradictory current denial still fails. If the required skill remains missing, truthful disclosure of that condition remains required.", "changedFields": [ { "key": "activity.work-timeline", @@ -232,7 +236,7 @@ "key": "activity.work-timeline", "field": "oracleVersion", "previous": "paperclip.chat-catalog.internal.v1", - "current": "paperclip.chat-catalog.work-timeline.v9" + "current": "paperclip.chat-catalog.work-timeline.v10" }, { "key": "activity.work-timeline", @@ -550,7 +554,7 @@ "key": "questions-and-approvals.multi-part-question", "field": "oracleVersion", "previous": "paperclip.chat-catalog.identity.v1", - "current": "paperclip.chat-catalog.required-question.v3" + "current": "paperclip.chat-catalog.required-question.v4" }, { "key": "questions-and-approvals.reject-and-revise", @@ -617,6 +621,9 @@ "current": [ "chat-catalog-domains/external-company-secret.ts", "chat-catalog-domains/external-secret-question.ts", + "chat-catalog-domains/external-secret-route.ts", + "chat-catalog-contract.ts", + "chat-catalog-flow.ts", "chat-catalog-domains/external-private-browser.ts", "chat-catalog-domains/external-private-browser-worker.ts", "chat-catalog-domains/external-credential-language.ts", @@ -635,7 +642,7 @@ "key": "secrets.configure-company-secret", "field": "oracleVersion", "previous": "paperclip.chat-catalog.external.v1", - "current": "paperclip.chat-catalog.company-secret-controller-baseline.v7" + "current": "paperclip.chat-catalog.company-secret-controller-baseline.v8" }, { "key": "secrets.personal-secret", @@ -667,6 +674,7 @@ "current": [ "chat-catalog-domains/external-personal-secret.ts", "chat-catalog-domains/external-secret-question.ts", + "chat-catalog-domains/external-secret-route.ts", "chat-catalog-domains/external-secret-endpoint.ts", "chat-catalog-domains/external-private-browser.ts", "chat-catalog-domains/external-private-browser-worker.ts", @@ -694,7 +702,7 @@ "key": "secrets.personal-secret", "field": "oracleVersion", "previous": "paperclip.chat-catalog.external.v1", - "current": "paperclip.chat-catalog.personal-secret.v8" + "current": "paperclip.chat-catalog.personal-secret.v9" }, { "key": "skills.author-skill", @@ -930,7 +938,13 @@ "key": "teams.install-team", "field": "oracleVersion", "previous": "paperclip.chat-catalog.external.v1", - "current": "paperclip.chat-catalog.team-install.v3" + "current": "paperclip.chat-catalog.team-install.v4" + }, + { + "key": "questions-and-approvals.multi-part-question", + "field": "originalQuery", + "previous": "Help me choose the details for the launch page before you build it.", + "current": "Walk me through the launch-page question form so I can choose the audience and tone and enter my own headline before you build anything." } ], "candidateDescriptorSnapshot": "fixtures/chat-repair-candidate-descriptors.json", @@ -970,7 +984,7 @@ "key": "activity.work-timeline", "field": "oracleVersion", "previous": "paperclip.chat-catalog.internal.v1", - "current": "paperclip.chat-catalog.work-timeline.v9" + "current": "paperclip.chat-catalog.work-timeline.v10" }, { "key": "agents.choose-primary", @@ -1258,7 +1272,7 @@ "key": "questions-and-approvals.multi-part-question", "field": "oracleVersion", "previous": "paperclip.chat-catalog.identity.v1", - "current": "paperclip.chat-catalog.required-question.v3" + "current": "paperclip.chat-catalog.required-question.v4" }, { "key": "questions-and-approvals.reject-and-revise", @@ -1325,6 +1339,9 @@ "current": [ "chat-catalog-domains/external-company-secret.ts", "chat-catalog-domains/external-secret-question.ts", + "chat-catalog-domains/external-secret-route.ts", + "chat-catalog-contract.ts", + "chat-catalog-flow.ts", "chat-catalog-domains/external-private-browser.ts", "chat-catalog-domains/external-private-browser-worker.ts", "chat-catalog-domains/external-credential-language.ts", @@ -1343,7 +1360,7 @@ "key": "secrets.configure-company-secret", "field": "oracleVersion", "previous": "paperclip.chat-catalog.external.v1", - "current": "paperclip.chat-catalog.company-secret-controller-baseline.v7" + "current": "paperclip.chat-catalog.company-secret-controller-baseline.v8" }, { "key": "secrets.personal-secret", @@ -1375,6 +1392,7 @@ "current": [ "chat-catalog-domains/external-personal-secret.ts", "chat-catalog-domains/external-secret-question.ts", + "chat-catalog-domains/external-secret-route.ts", "chat-catalog-domains/external-secret-endpoint.ts", "chat-catalog-domains/external-private-browser.ts", "chat-catalog-domains/external-private-browser-worker.ts", @@ -1413,7 +1431,7 @@ "key": "secrets.personal-secret", "field": "oracleVersion", "previous": "paperclip.chat-catalog.external.v1", - "current": "paperclip.chat-catalog.personal-secret.v8" + "current": "paperclip.chat-catalog.personal-secret.v9" }, { "key": "skills.author-skill", @@ -1693,7 +1711,7 @@ "key": "teams.install-team", "field": "oracleVersion", "previous": "paperclip.chat-catalog.external.v1", - "current": "paperclip.chat-catalog.team-install.v3" + "current": "paperclip.chat-catalog.team-install.v4" } ], "runtimeConditions": { @@ -1805,11 +1823,29 @@ ], "actualPostAssertionPGCasesPassed": 2 } + }, + "multipart53QuestionClarification": { + "key": "questions-and-approvals.multi-part-question", + "originalQuery": "Help me choose the details for the launch page before you build it.", + "clarifiedQuery": "Walk me through the launch-page question form so I can choose the audience and tone and enter my own headline before you build anything.", + "previousOracleVersion": "paperclip.chat-catalog.required-question.v3", + "oracleVersion": "paperclip.chat-catalog.required-question.v4", + "originalDefinitionSourceSha": "de0941835cc8f0d01c982574ee258e110babdbeb", + "previousInventoryDigest": "0cf530c11449508e91b5ce664d9375fc8e76466fa908255a1af0ff55169d0220", + "inventoryDigest": "814bb3633268cb0488563aed51fce2f987bd47650ddfb5a3302d0354292c2323", + "changedQueries": 1, + "unchangedOtherExecutableQueries": 241, + "unchangedOtherCatalogQueries": 243, + "historicalFailedResults": [ + 4623, + 4717 + ], + "condition": "Question-layer clarification names the already visible paged form; no supplied answers, APIs or hidden procedures. Prior broad-query failures remain FAILED. No original timeout, performance or reliability fix is claimed. Expected outcome, setup, validation, back retention, exact persisted answers, bound continuation, visible summary,300000ms and native/human bounds are unchanged. One targeted post-clarification measurement is planned; any later whole-ten regression is separate, with all attempts retained and no retry-until-green." } }, "candidateLauncherSnapshot": { "path": "fixtures/chat-repair-candidate-launcher.txt", - "targetSha": "5aef4bffa2b106f2066bf2485338798f04a702bd", + "targetSha": "6773b1c128a2e7efa7571ff04b38238f00d06117", "sourcePath": "tests/runner-e2e/launch.ts", "gitBlobOid": "c88df15f509b2ac8638af826647a97d45f16bd1f" } @@ -2352,7 +2388,7 @@ "activity/timeline" ], "checks": null, - "oracleVersion": "paperclip.chat-catalog.work-timeline.v9", + "oracleVersion": "paperclip.chat-catalog.work-timeline.v10", "sourceFiles": [ "chat-catalog-domains/internal-definition.ts", "chat-catalog-domains/internal-support.ts", @@ -9916,14 +9952,14 @@ "requiredCredentials": [], "externalInput": null, "installedSource": null, - "originalQuery": "Help me choose the details for the launch page before you build it.", + "originalQuery": "Walk me through the launch-page question form so I can choose the audience and tone and enter my own headline before you build anything.", "expectedOutcome": "The question flow retains earlier answers across pages, supports a custom answer and displays validation on missing required input. Only the submitted set resumes the work. Chat summarizes the chosen details accurately rather than treating a highlighted default or an unfinished page as consent.", "originalContext": "A bounded design request genuinely needs an audience, a tone and a custom headline; the fixture offers a paged form with choices and free text.", "capabilities": [ "questions-and-approvals/questions" ], "checks": null, - "oracleVersion": "paperclip.chat-catalog.required-question.v3", + "oracleVersion": "paperclip.chat-catalog.required-question.v4", "sourceFiles": [ "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", @@ -11144,10 +11180,13 @@ "connection-setup/credentials" ], "checks": null, - "oracleVersion": "paperclip.chat-catalog.company-secret-controller-baseline.v7", + "oracleVersion": "paperclip.chat-catalog.company-secret-controller-baseline.v8", "sourceFiles": [ "chat-catalog-domains/external-company-secret.ts", "chat-catalog-domains/external-secret-question.ts", + "chat-catalog-domains/external-secret-route.ts", + "chat-catalog-contract.ts", + "chat-catalog-flow.ts", "chat-catalog-domains/external-private-browser.ts", "chat-catalog-domains/external-private-browser-worker.ts", "chat-catalog-domains/external-credential-language.ts", @@ -11260,10 +11299,11 @@ "secrets/user-secrets" ], "checks": null, - "oracleVersion": "paperclip.chat-catalog.personal-secret.v8", + "oracleVersion": "paperclip.chat-catalog.personal-secret.v9", "sourceFiles": [ "chat-catalog-domains/external-personal-secret.ts", "chat-catalog-domains/external-secret-question.ts", + "chat-catalog-domains/external-secret-route.ts", "chat-catalog-domains/external-secret-endpoint.ts", "chat-catalog-domains/external-private-browser.ts", "chat-catalog-domains/external-private-browser-worker.ts", @@ -13240,7 +13280,7 @@ "agents/hire" ], "checks": null, - "oracleVersion": "paperclip.chat-catalog.team-install.v3", + "oracleVersion": "paperclip.chat-catalog.team-install.v4", "sourceFiles": [ "chat-catalog-domains/external-team-install.ts", "chat-catalog-domains/external-team-conditional.ts", diff --git a/tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs b/tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs index 75ad1a253b9..874d1fa427e 100644 --- a/tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs +++ b/tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs @@ -103,7 +103,7 @@ test('repair source metadata binds the corrected candidate and explicit changed const repair = manifest.source.repairMetadata; assert.equal(repair.baseTargetSha, '3b66c3c144360ddf32714a0742fd774b8ceafb16'); assert.equal(repair.previousTargetSha, 'ab2019e976076ec0eb60b026844f3d9211cb0208'); - assert.equal(manifest.targetSha, '5aef4bffa2b106f2066bf2485338798f04a702bd'); + assert.equal(manifest.targetSha, '6773b1c128a2e7efa7571ff04b38238f00d06117'); assert.equal(repair.definitionSourceSha, manifest.metadataSourceSha); assert.equal(repair.candidateDescriptorSnapshot, 'fixtures/chat-repair-candidate-descriptors.json'); const descriptorBytes = readFileSync(new URL(repair.candidateDescriptorSnapshot, import.meta.url)); @@ -315,12 +315,21 @@ test('repair source metadata binds the corrected candidate and explicit changed [ "teams.install-team", "oracleVersion" + ], + [ + "questions-and-approvals.multi-part-question", + "originalQuery" ] ]); for (const change of repair.changedFields) { assert.deepEqual(row(change.key)[change.field], change.current); assert.notDeepEqual(change.current, change.previous); } + assert.deepEqual(repair.changedFields.filter(change => change.key === 'questions-and-approvals.multi-part-question' && change.field === 'originalQuery'), [{ key: 'questions-and-approvals.multi-part-question', field: 'originalQuery', previous: "Help me choose the details for the launch page before you build it.", current: "Walk me through the launch-page question form so I can choose the audience and tone and enter my own headline before you build anything." }]); + assert.equal(repair.runtimeConditions.multipart53QuestionClarification.changedQueries, 1); + assert.equal(repair.runtimeConditions.multipart53QuestionClarification.unchangedOtherCatalogQueries, 243); + assert.equal(repair.runtimeConditions.multipart53QuestionClarification.previousInventoryDigest, '0cf530c11449508e91b5ce664d9375fc8e76466fa908255a1af0ff55169d0220'); + assert.equal(repair.runtimeConditions.multipart53QuestionClarification.inventoryDigest, '814bb3633268cb0488563aed51fce2f987bd47650ddfb5a3302d0354292c2323'); assert.equal(repair.descriptorChanges.length, 49); assert.equal(new Set(repair.descriptorChanges.map(change => change.key)).size, 33); assert.deepEqual([...new Set(repair.descriptorChanges.map(change => change.field))].sort(), ['expectedRunCount', 'minimumRunCount', 'oracleVersion', 'setupPolicy', 'sourceFiles']); @@ -332,7 +341,7 @@ test('repair source metadata binds the corrected candidate and explicit changed if (Object.hasOwn(row(change.key), change.field)) assert.deepEqual(row(change.key)[change.field], change.current); } assert.deepEqual(repair.changedDefinitionFiles.map(value => value.path), [ - 'tests/runner-e2e/chat-scenarios/companies.yaml', 'tests/runner-e2e/chat-scenarios/goals.yaml', 'tests/runner-e2e/chat-scenarios/skills.yaml', 'tests/runner-e2e/chat-scenarios/app-permissions.yaml', 'tests/runner-e2e/chat-scenarios/activity.yaml', + 'tests/runner-e2e/chat-scenarios/companies.yaml', 'tests/runner-e2e/chat-scenarios/goals.yaml', 'tests/runner-e2e/chat-scenarios/skills.yaml', 'tests/runner-e2e/chat-scenarios/app-permissions.yaml', 'tests/runner-e2e/chat-scenarios/activity.yaml', 'tests/runner-e2e/chat-scenarios/questions-and-approvals.yaml', ]); assert.ok(repair.changedDefinitionFiles.every(value => /^[a-f0-9]{64}$/.test(value.sha256))); for (const declaration of declarations) { diff --git a/tests/runner-e2e/fixtures/chat-repair-candidate-descriptors.json b/tests/runner-e2e/fixtures/chat-repair-candidate-descriptors.json index fbb91b3584b..51f901eb69c 100644 --- a/tests/runner-e2e/fixtures/chat-repair-candidate-descriptors.json +++ b/tests/runner-e2e/fixtures/chat-repair-candidate-descriptors.json @@ -258,7 +258,7 @@ "../../server/src/services/native-runtime/status-decision-committer.ts", "../../server/src/services/work-timeline.ts" ], - "oracleVersion": "paperclip.chat-catalog.work-timeline.v9", + "oracleVersion": "paperclip.chat-catalog.work-timeline.v10", "coverageGaps": [], "expectedRunCount": 3, "minimumRunCount": 3, @@ -4225,7 +4225,7 @@ "chat-catalog-domains/identity-questions-and-approvals.ts", "chat-scenarios/questions-and-approvals.yaml" ], - "oracleVersion": "paperclip.chat-catalog.required-question.v3", + "oracleVersion": "paperclip.chat-catalog.required-question.v4", "expectedRunCount": 2, "turnTimeoutMs": 120000, "attemptTimeoutMs": 300000, @@ -4892,6 +4892,9 @@ "sourceFiles": [ "chat-catalog-domains/external-company-secret.ts", "chat-catalog-domains/external-secret-question.ts", + "chat-catalog-domains/external-secret-route.ts", + "chat-catalog-contract.ts", + "chat-catalog-flow.ts", "chat-catalog-domains/external-private-browser.ts", "chat-catalog-domains/external-private-browser-worker.ts", "chat-catalog-domains/external-credential-language.ts", @@ -4905,7 +4908,7 @@ "chat-flow.ts", "chat-scenarios/secrets.yaml" ], - "oracleVersion": "paperclip.chat-catalog.company-secret-controller-baseline.v7", + "oracleVersion": "paperclip.chat-catalog.company-secret-controller-baseline.v8", "expectedRunCount": 2, "expectedHumanMessageCount": 2, "turnTimeoutMs": 120000, @@ -4959,6 +4962,7 @@ "sourceFiles": [ "chat-catalog-domains/external-personal-secret.ts", "chat-catalog-domains/external-secret-question.ts", + "chat-catalog-domains/external-secret-route.ts", "chat-catalog-domains/external-secret-endpoint.ts", "chat-catalog-domains/external-private-browser.ts", "chat-catalog-domains/external-private-browser-worker.ts", @@ -4981,7 +4985,7 @@ "../../ui/src/pages/secrets/SetMyUserSecretDialog.tsx", "chat-scenarios/secrets.yaml" ], - "oracleVersion": "paperclip.chat-catalog.personal-secret.v8", + "oracleVersion": "paperclip.chat-catalog.personal-secret.v9", "expectedRunCount": 3, "expectedHumanMessageCount": 2, "minimumRunCount": 3, @@ -6015,7 +6019,7 @@ "chat-flow.ts", "chat-scenarios/teams.yaml" ], - "oracleVersion": "paperclip.chat-catalog.team-install.v3", + "oracleVersion": "paperclip.chat-catalog.team-install.v4", "expectedRunCount": 2, "expectedHumanMessageCount": 2, "turnTimeoutMs": 120000,