From 90d4ecf22f6a7fb6d66202e74e56cdf9afaab175 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 2 Nov 2026 21:07:22 -0600 Subject: [PATCH] test(evals): pin synchronized chat repair declarations and definitions --- tests/runner-e2e/CHAT-FEATURES.md | 8 + .../chat-full-catalog-testops-manifest.json | 537 +++++++++++++++++- ...hat-full-catalog-testops-manifest.test.mjs | 41 +- 3 files changed, 557 insertions(+), 29 deletions(-) diff --git a/tests/runner-e2e/CHAT-FEATURES.md b/tests/runner-e2e/CHAT-FEATURES.md index c1ce3eca7e7..1b247330a0a 100644 --- a/tests/runner-e2e/CHAT-FEATURES.md +++ b/tests/runner-e2e/CHAT-FEATURES.md @@ -294,3 +294,11 @@ and credentials do not enter the public artifact path. Additional credentials reach only cases that declare them. External inputs use the existing selected-case private-input owner, disposable-resource checks and oracle commitments. Unconfigured inputs remain setup blockers. + +### Repair source 2 (candidate only) + +The catalog now pins candidate `36dff2eab3dfb6de6efb4fd2ce6c1a58f8a5f383`, succeeding `9ab0f951a5b5990c266ef4456610e0a0775aa334`. It carries the reviewed fixture, scoped observation, version-readback and evidence-readiness repairs. Static runtime descriptors are synchronized with the actual declarations; all 242 lazy factories must pass the real no-callback consistency check, together with the two legacy-cell controls, before each new repair-batch GO. The previous candidate's mismatched metadata caused launch 1690 to be cancelled before case/model work; it is an operational attempt, not a feature result. + +The manifest keeps the original definition source and values, with explicit scoped changes: Fast response names the requested product feature; goal-tree and authored-skill outcomes use the product-supported optional-owner and ordinary navigation contracts; question delivery/validation and company-secret setup declare their reviewed oracle versions. Each new measurement binds this candidate and the exact changed definition; earlier grades and the 244-case gate are unchanged. SDK scan-audit capture improves private failure diagnosis only: rejection, evidence requirements and privacy bounds remain in force, and the other SDK cases stay held until the finding is understood. + +Workflow, model, case clocks, native-run bounds, credentials, protected reporting and case identities do not change. The full harness TypeScript graph has not been qualified by local pure tests; source import/admission and the existing credential-free remote Runner build remain required before native queries. Static TestOps descriptions may still show the original definition until a separate idempotent metadata reconciliation at a safe source boundary. diff --git a/tests/runner-e2e/chat-full-catalog-testops-manifest.json b/tests/runner-e2e/chat-full-catalog-testops-manifest.json index ff93faaee37..4585cd6c58c 100644 --- a/tests/runner-e2e/chat-full-catalog-testops-manifest.json +++ b/tests/runner-e2e/chat-full-catalog-testops-manifest.json @@ -1,6 +1,6 @@ { "schema": "paperclip.chat-full-catalog-testops-manifest/v1", - "targetSha": "9ab0f951a5b5990c266ef4456610e0a0775aa334", + "targetSha": "36dff2eab3dfb6de6efb4fd2ce6c1a58f8a5f383", "metadataSourceSha": "de0941835cc8f0d01c982574ee258e110babdbeb", "profile": "runner-opencode", "model": "openrouter/deepseek/deepseek-v4.1-flash-1017", @@ -170,8 +170,304 @@ ], "repairMetadata": { "baseTargetSha": "3b66c3c144360ddf32714a0742fd774b8ceafb16", + "previousTargetSha": "9ab0f951a5b5990c266ef4456610e0a0775aa334", "definitionSourceSha": "de0941835cc8f0d01c982574ee258e110babdbeb", + "candidateDescriptorSha256": "5b098a30ed54c6021698b33b434145ca2e934ff2bb9c1e0738a8d8e5dc622797", + "changedDefinitionFiles": [ + { + "path": "tests/runner-e2e/chat-scenarios/companies.yaml", + "sha256": "fbaf17ac279d92127de4b58a3605edc5c7954dabfb050f49be210e1f85775d1d" + }, + { + "path": "tests/runner-e2e/chat-scenarios/goals.yaml", + "sha256": "b140b0e652a9945b7ead9e38e5bc01791f7dc9cd39b1f2b6f292544d50632dae" + }, + { + "path": "tests/runner-e2e/chat-scenarios/skills.yaml", + "sha256": "51f975d62540e77a48539821f443c6ab134c8ffb63e4c04bb2678a655de84a69" + } + ], + "conditions": "New candidate measurements only. Original 244 definitions, attempt history and old pass gate remain immutable. SDK audit custody remains diagnostic; scan rejection is unchanged. Explicit field overrides below retain original definition values. No model, clock, run-bound, workflow, credential or reporting authority expansion.", "changedFields": [ + { + "key": "agents.choose-primary", + "field": "sourceFiles", + "previous": [ + "chat-catalog-domains/runtime-primary.ts", + "chat-catalog-domains/runtime-support.ts", + "chat-scenarios/agents.yaml" + ], + "current": [ + "chat-catalog-domains/runtime-primary.ts", + "chat-catalog-domains/runtime-support.ts", + "chat-catalog-domains/identity-support.ts", + "chat-scenarios/agents.yaml" + ] + }, + { + "key": "companies.agent-observability", + "field": "sourceFiles", + "previous": [ + "chat-catalog-domains/identity-observability.ts", + "chat-catalog-domains/identity-private-browser.ts", + "chat-catalog-domains/identity-private-browser-worker.ts", + "chat-catalog-domains/identity-wizard.ts", + "chat-catalog-domains/identity-support.ts", + "chat-catalog-domains/internal-support.ts", + "chat-scenarios/companies.yaml" + ], + "current": [ + "chat-catalog-domains/identity-observability.ts", + "chat-catalog-domains/identity-company-observer.ts", + "chat-catalog-domains/identity-private-browser.ts", + "chat-catalog-domains/identity-private-browser-worker.ts", + "chat-catalog-domains/identity-wizard.ts", + "chat-catalog-domains/identity-support.ts", + "chat-catalog-domains/internal-support.ts", + "chat-scenarios/companies.yaml" + ] + }, + { + "key": "companies.configure-decision-model", + "field": "sourceFiles", + "previous": [ + "chat-catalog-domains/identity-company-models.ts", + "chat-catalog-domains/identity-support.ts", + "chat-catalog-domains/internal-support.ts", + "chat-scenarios/companies.yaml" + ], + "current": [ + "chat-catalog-domains/identity-company-models.ts", + "chat-catalog-domains/identity-company-observer.ts", + "chat-catalog-domains/identity-support.ts", + "chat-catalog-domains/internal-support.ts", + "chat-scenarios/companies.yaml" + ] + }, + { + "key": "companies.fast-responses", + "field": "sourceFiles", + "previous": [ + "chat-catalog-domains/identity-company-models.ts", + "chat-catalog-domains/identity-support.ts", + "chat-catalog-domains/internal-support.ts", + "chat-scenarios/companies.yaml" + ], + "current": [ + "chat-catalog-domains/identity-company-models.ts", + "chat-catalog-domains/identity-company-observer.ts", + "chat-catalog-domains/identity-support.ts", + "chat-catalog-domains/internal-support.ts", + "chat-scenarios/companies.yaml" + ] + }, + { + "key": "companies.fast-responses", + "field": "originalQuery", + "previous": "Make chat acknowledge my requests faster.", + "current": "Help me enable the company's Fast response feature so chat acknowledges my requests faster." + }, + { + "key": "companies.prompt-injection-guard", + "field": "sourceFiles", + "previous": [ + "chat-catalog-domains/identity-company-models.ts", + "chat-catalog-domains/identity-support.ts", + "chat-catalog-domains/internal-support.ts", + "chat-scenarios/companies.yaml" + ], + "current": [ + "chat-catalog-domains/identity-company-models.ts", + "chat-catalog-domains/identity-company-observer.ts", + "chat-catalog-domains/identity-support.ts", + "chat-catalog-domains/internal-support.ts", + "chat-scenarios/companies.yaml" + ] + }, + { + "key": "goals.create-goal-tree", + "field": "oracleVersion", + "previous": "paperclip.chat-catalog.internal.v1", + "current": "paperclip.chat-catalog.goal-hierarchy.v2" + }, + { + "key": "goals.create-goal-tree", + "field": "expectedOutcome", + "previous": "The saved goals have the requested levels, valid parent relationship and appropriate ownership or an explicit missing-owner question. Chat links to the hierarchy and does not invent progress metrics. A retry reuses the intended goals rather than creating another parallel tree.", + "current": "The saved goals have the requested levels and a valid same-company parent relationship. Ownership may remain unassigned when none was requested; any assigned owner belongs to the company. The saved hierarchy is inspectable through ordinary Goals navigation and public readback, without invented progress metrics. A retry reuses the intended goals rather than creating another parallel tree." + }, + { + "key": "questions-and-approvals.duplicate-answer", + "field": "oracleVersion", + "previous": "paperclip.chat-catalog.identity.v1", + "current": "paperclip.chat-catalog.answer-delivery.v2" + }, + { + "key": "questions-and-approvals.multi-part-question", + "field": "oracleVersion", + "previous": "paperclip.chat-catalog.identity.v1", + "current": "paperclip.chat-catalog.required-question.v2" + }, + { + "key": "secrets.configure-company-secret", + "field": "oracleVersion", + "previous": "paperclip.chat-catalog.external.v1", + "current": "paperclip.chat-catalog.company-secret-controller-baseline.v2" + }, + { + "key": "skills.author-skill", + "field": "sourceFiles", + "previous": [ + "chat-catalog-domains/integrator-support.ts", + "chat-catalog-domains/internal-support.ts", + "chat-catalog-domains/integrator-skills.ts", + "chat-scenarios/skills.yaml" + ], + "current": [ + "chat-catalog-domains/integrator-support.ts", + "chat-catalog-domains/internal-support.ts", + "chat-catalog-domains/integrator-skills.ts", + "chat-scenarios/skills.yaml", + "chat-catalog-domains/integrator-skill-versions.ts", + "chat-catalog-domains/identity-support.ts" + ] + }, + { + "key": "skills.author-skill", + "field": "oracleVersion", + "previous": "paperclip.chat-catalog.integrator.v1", + "current": "paperclip.chat-catalog.skill-authoring.v2" + }, + { + "key": "skills.author-skill", + "field": "expectedOutcome", + "previous": "A writable company skill captures the checklist with clear instructions and a saved revision. Chat links to the skill so the person can inspect or edit it. The agent preserves the actual checklist instead of merely posting a suggestion or pretending creation automatically enables the skill for every agent.", + "current": "A writable company skill captures the checklist with clear instructions and a saved revision. The person can discover it through ordinary Skills/My Skills navigation, open its editable Studio content and reload the same saved revision. No particular chat-link format is required. The agent preserves the actual checklist instead of merely posting a suggestion or pretending creation automatically enables the skill for every agent." + }, + { + "key": "skills.enable-for-agent", + "field": "sourceFiles", + "previous": [ + "chat-catalog-domains/integrator-skill-enable.ts", + "chat-catalog-domains/identity-support.ts", + "chat-catalog-domains/internal-definition.ts", + "chat-catalog-domains/internal-support.ts", + "chat-catalog-domains/integrator-support.ts", + "chat-scenarios/skills.yaml", + "../../server/src/routes/agents.ts", + "../../server/src/routes/company-skill-policy.ts", + "../../server/src/services/company-skills.ts", + "../../packages/shared/src/validators/skill-policy.ts" + ], + "current": [ + "chat-catalog-domains/integrator-skill-versions.ts", + "chat-catalog-domains/integrator-skill-enable.ts", + "chat-catalog-domains/identity-support.ts", + "chat-catalog-domains/internal-definition.ts", + "chat-catalog-domains/internal-support.ts", + "chat-catalog-domains/integrator-support.ts", + "chat-scenarios/skills.yaml", + "../../server/src/routes/agents.ts", + "../../server/src/routes/company-skill-policy.ts", + "../../server/src/services/company-skills.ts", + "../../packages/shared/src/validators/skill-policy.ts" + ] + }, + { + "key": "skills.explicit-restriction", + "field": "sourceFiles", + "previous": [ + "chat-catalog-domains/integrator-support.ts", + "chat-catalog-domains/internal-support.ts", + "chat-catalog-domains/integrator-skills.ts", + "chat-scenarios/skills.yaml" + ], + "current": [ + "chat-catalog-domains/integrator-support.ts", + "chat-catalog-domains/internal-support.ts", + "chat-catalog-domains/integrator-skills.ts", + "chat-scenarios/skills.yaml", + "chat-catalog-domains/integrator-skill-versions.ts" + ] + }, + { + "key": "skills.find-and-install", + "field": "sourceFiles", + "previous": [ + "chat-catalog-domains/integrator-skill-imports.ts", + "chat-catalog-domains/integrator-skill-source.ts", + "chat-catalog-domains/integrator-skills.ts", + "chat-catalog-domains/integrator-support.ts", + "chat-catalog-domains/internal-support.ts", + "chat-scenarios/skills.yaml" + ], + "current": [ + "chat-catalog-domains/integrator-skill-versions.ts", + "chat-catalog-domains/integrator-skill-imports.ts", + "chat-catalog-domains/integrator-skill-source.ts", + "chat-catalog-domains/integrator-skills.ts", + "chat-catalog-domains/integrator-support.ts", + "chat-catalog-domains/internal-support.ts", + "chat-scenarios/skills.yaml" + ] + }, + { + "key": "skills.fork-read-only", + "field": "sourceFiles", + "previous": [ + "chat-catalog-domains/integrator-skill-imports.ts", + "chat-catalog-domains/integrator-skill-source.ts", + "chat-catalog-domains/integrator-skills.ts", + "chat-catalog-domains/integrator-support.ts", + "chat-catalog-domains/internal-support.ts", + "chat-scenarios/skills.yaml" + ], + "current": [ + "chat-catalog-domains/integrator-skill-versions.ts", + "chat-catalog-domains/integrator-skill-imports.ts", + "chat-catalog-domains/integrator-skill-source.ts", + "chat-catalog-domains/integrator-skills.ts", + "chat-catalog-domains/integrator-support.ts", + "chat-catalog-domains/internal-support.ts", + "chat-scenarios/skills.yaml" + ] + }, + { + "key": "skills.review-update", + "field": "sourceFiles", + "previous": [ + "chat-catalog-domains/integrator-skill-update.ts", + "chat-catalog-domains/integrator-skill-source.ts", + "chat-catalog-domains/integrator-support.ts", + "chat-catalog-domains/internal-support.ts", + "chat-scenarios/skills.yaml" + ], + "current": [ + "chat-catalog-domains/integrator-skill-versions.ts", + "chat-catalog-domains/integrator-skill-update.ts", + "chat-catalog-domains/integrator-skill-source.ts", + "chat-catalog-domains/integrator-support.ts", + "chat-catalog-domains/internal-support.ts", + "chat-scenarios/skills.yaml" + ] + } + ], + "descriptorChanges": [ + { + "key": "agents.choose-primary", + "field": "sourceFiles", + "previous": [ + "chat-catalog-domains/runtime-primary.ts", + "chat-catalog-domains/runtime-support.ts", + "chat-scenarios/agents.yaml" + ], + "current": [ + "chat-catalog-domains/runtime-primary.ts", + "chat-catalog-domains/runtime-support.ts", + "chat-catalog-domains/identity-support.ts", + "chat-scenarios/agents.yaml" + ] + }, { "key": "companies.agent-observability", "field": "sourceFiles", @@ -246,20 +542,217 @@ "chat-scenarios/companies.yaml" ] }, + { + "key": "goals.create-goal-tree", + "field": "oracleVersion", + "previous": "paperclip.chat-catalog.internal.v1", + "current": "paperclip.chat-catalog.goal-hierarchy.v2" + }, + { + "key": "questions-and-approvals.duplicate-answer", + "field": "oracleVersion", + "previous": "paperclip.chat-catalog.identity.v1", + "current": "paperclip.chat-catalog.answer-delivery.v2" + }, + { + "key": "questions-and-approvals.multi-part-question", + "field": "oracleVersion", + "previous": "paperclip.chat-catalog.identity.v1", + "current": "paperclip.chat-catalog.required-question.v2" + }, { "key": "secrets.configure-company-secret", "field": "oracleVersion", "previous": "paperclip.chat-catalog.external.v1", "current": "paperclip.chat-catalog.company-secret-controller-baseline.v2" + }, + { + "key": "secrets.personal-secret", + "field": "setupPolicy", + "previous": null, + "current": { + "readiness": "preserve-scenario", + "invariantCheckIds": [ + "current-owner-missing-required-personal-value" + ] + } + }, + { + "key": "skills.author-skill", + "field": "sourceFiles", + "previous": [ + "chat-catalog-domains/integrator-support.ts", + "chat-catalog-domains/internal-support.ts", + "chat-catalog-domains/integrator-skills.ts", + "chat-scenarios/skills.yaml" + ], + "current": [ + "chat-catalog-domains/integrator-support.ts", + "chat-catalog-domains/internal-support.ts", + "chat-catalog-domains/integrator-skills.ts", + "chat-scenarios/skills.yaml", + "chat-catalog-domains/integrator-skill-versions.ts", + "chat-catalog-domains/identity-support.ts" + ] + }, + { + "key": "skills.author-skill", + "field": "oracleVersion", + "previous": "paperclip.chat-catalog.integrator.v1", + "current": "paperclip.chat-catalog.skill-authoring.v2" + }, + { + "key": "skills.enable-for-agent", + "field": "sourceFiles", + "previous": [ + "chat-catalog-domains/integrator-skill-enable.ts", + "chat-catalog-domains/identity-support.ts", + "chat-catalog-domains/internal-definition.ts", + "chat-catalog-domains/internal-support.ts", + "chat-catalog-domains/integrator-support.ts", + "chat-scenarios/skills.yaml", + "../../server/src/routes/agents.ts", + "../../server/src/routes/company-skill-policy.ts", + "../../server/src/services/company-skills.ts", + "../../packages/shared/src/validators/skill-policy.ts" + ], + "current": [ + "chat-catalog-domains/integrator-skill-versions.ts", + "chat-catalog-domains/integrator-skill-enable.ts", + "chat-catalog-domains/identity-support.ts", + "chat-catalog-domains/internal-definition.ts", + "chat-catalog-domains/internal-support.ts", + "chat-catalog-domains/integrator-support.ts", + "chat-scenarios/skills.yaml", + "../../server/src/routes/agents.ts", + "../../server/src/routes/company-skill-policy.ts", + "../../server/src/services/company-skills.ts", + "../../packages/shared/src/validators/skill-policy.ts" + ] + }, + { + "key": "skills.explicit-restriction", + "field": "sourceFiles", + "previous": [ + "chat-catalog-domains/integrator-support.ts", + "chat-catalog-domains/internal-support.ts", + "chat-catalog-domains/integrator-skills.ts", + "chat-scenarios/skills.yaml" + ], + "current": [ + "chat-catalog-domains/integrator-support.ts", + "chat-catalog-domains/internal-support.ts", + "chat-catalog-domains/integrator-skills.ts", + "chat-scenarios/skills.yaml", + "chat-catalog-domains/integrator-skill-versions.ts" + ] + }, + { + "key": "skills.find-and-install", + "field": "sourceFiles", + "previous": [ + "chat-catalog-domains/integrator-skill-imports.ts", + "chat-catalog-domains/integrator-skill-source.ts", + "chat-catalog-domains/integrator-skills.ts", + "chat-catalog-domains/integrator-support.ts", + "chat-catalog-domains/internal-support.ts", + "chat-scenarios/skills.yaml" + ], + "current": [ + "chat-catalog-domains/integrator-skill-versions.ts", + "chat-catalog-domains/integrator-skill-imports.ts", + "chat-catalog-domains/integrator-skill-source.ts", + "chat-catalog-domains/integrator-skills.ts", + "chat-catalog-domains/integrator-support.ts", + "chat-catalog-domains/internal-support.ts", + "chat-scenarios/skills.yaml" + ] + }, + { + "key": "skills.fork-read-only", + "field": "sourceFiles", + "previous": [ + "chat-catalog-domains/integrator-skill-imports.ts", + "chat-catalog-domains/integrator-skill-source.ts", + "chat-catalog-domains/integrator-skills.ts", + "chat-catalog-domains/integrator-support.ts", + "chat-catalog-domains/internal-support.ts", + "chat-scenarios/skills.yaml" + ], + "current": [ + "chat-catalog-domains/integrator-skill-versions.ts", + "chat-catalog-domains/integrator-skill-imports.ts", + "chat-catalog-domains/integrator-skill-source.ts", + "chat-catalog-domains/integrator-skills.ts", + "chat-catalog-domains/integrator-support.ts", + "chat-catalog-domains/internal-support.ts", + "chat-scenarios/skills.yaml" + ] + }, + { + "key": "skills.review-update", + "field": "sourceFiles", + "previous": [ + "chat-catalog-domains/integrator-skill-update.ts", + "chat-catalog-domains/integrator-skill-source.ts", + "chat-catalog-domains/integrator-support.ts", + "chat-catalog-domains/internal-support.ts", + "chat-scenarios/skills.yaml" + ], + "current": [ + "chat-catalog-domains/integrator-skill-versions.ts", + "chat-catalog-domains/integrator-skill-update.ts", + "chat-catalog-domains/integrator-skill-source.ts", + "chat-catalog-domains/integrator-support.ts", + "chat-catalog-domains/internal-support.ts", + "chat-scenarios/skills.yaml" + ] + }, + { + "key": "steering.stop-versus-pause", + "field": "setupPolicy", + "previous": { + "readiness": "preserve-scenario", + "invariantCheckIds": [ + "lead-ready", + "two-real-native-workers", + "independent-child-pause-budget" + ] + }, + "current": { + "readiness": "preserve-scenario", + "funding": "preserve-scenario", + "invariantCheckIds": [ + "lead-ready", + "two-real-native-workers", + "independent-child-pause-budget", + "active-branches-funded" + ] + } + }, + { + "key": "task-coordination.pause-tree", + "field": "setupPolicy", + "previous": { + "readiness": "preserve-scenario", + "invariantCheckIds": [ + "lead-ready", + "two-real-native-workers", + "independent-child-pause-budget" + ] + }, + "current": { + "readiness": "preserve-scenario", + "funding": "preserve-scenario", + "invariantCheckIds": [ + "lead-ready", + "two-real-native-workers", + "independent-child-pause-budget", + "active-branches-funded" + ] + } } - ], - "conditions": [ - "Initialize owned SDK evidence staging before retention.", - "Use real public scoped membership only for three incidental human fixtures.", - "Company settings observation uses the original setup-inclusive deadline; no larger case cap.", - "Company secret setup and v2 oracle permit only exact immutable controller-bootstrap secret references." - ], - "originalMeasurements": "Preserved. These are new candidate conditions, not retroactive passes." + ] } }, "cases": [ @@ -1262,6 +1755,7 @@ "sourceFiles": [ "chat-catalog-domains/runtime-primary.ts", "chat-catalog-domains/runtime-support.ts", + "chat-catalog-domains/identity-support.ts", "chat-scenarios/agents.yaml" ], "screenshot": "final-state.png", @@ -4088,7 +4582,7 @@ ], "externalInput": null, "installedSource": null, - "originalQuery": "Make chat acknowledge my requests faster.", + "originalQuery": "Help me enable the company's Fast response feature so chat acknowledges my requests faster.", "expectedOutcome": "The agent explains the offered fast-response setting and directs the human to its configuration and test surface. The journey checks a real acknowledgement and its failure or usage record. A quick acknowledgement is not presented as completion of the underlying task or evidence that the worker started.", "originalContext": "The fast-response experiment is available but disabled; settings and tests require a human connection manager.", "capabilities": [ @@ -6236,13 +6730,13 @@ "externalInput": null, "installedSource": null, "originalQuery": "Set a company goal of reaching a thousand signups, with a team goal for launching the website.", - "expectedOutcome": "The saved goals have the requested levels, valid parent relationship and appropriate ownership or an explicit missing-owner question. Chat links to the hierarchy and does not invent progress metrics. A retry reuses the intended goals rather than creating another parallel tree.", + "expectedOutcome": "The saved goals have the requested levels and a valid same-company parent relationship. Ownership may remain unassigned when none was requested; any assigned owner belongs to the company. The saved hierarchy is inspectable through ordinary Goals navigation and public readback, without invented progress metrics. A retry reuses the intended goals rather than creating another parallel tree.", "originalContext": "The company has a growth mission and no matching quarterly goal; the agent may create goals.", "capabilities": [ "goals/hierarchy" ], "checks": null, - "oracleVersion": "paperclip.chat-catalog.internal.v1", + "oracleVersion": "paperclip.chat-catalog.goal-hierarchy.v2", "sourceFiles": [ "chat-catalog-domains/internal-definition.ts", "chat-catalog-domains/internal-support.ts", @@ -8261,7 +8755,7 @@ "questions-and-approvals/continuation" ], "checks": null, - "oracleVersion": "paperclip.chat-catalog.identity.v1", + "oracleVersion": "paperclip.chat-catalog.answer-delivery.v2", "sourceFiles": [ "chat-catalog-domains/identity-confirmation-workflows.ts", "chat-catalog-domains/identity-support.ts", @@ -8347,7 +8841,7 @@ "questions-and-approvals/questions" ], "checks": null, - "oracleVersion": "paperclip.chat-catalog.identity.v1", + "oracleVersion": "paperclip.chat-catalog.required-question.v2", "sourceFiles": [ "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", @@ -9854,18 +10348,20 @@ "externalInput": null, "installedSource": null, "originalQuery": "Turn our website review checklist into a reusable skill.", - "expectedOutcome": "A writable company skill captures the checklist with clear instructions and a saved revision. Chat links to the skill so the person can inspect or edit it. The agent preserves the actual checklist instead of merely posting a suggestion or pretending creation automatically enables the skill for every agent.", + "expectedOutcome": "A writable company skill captures the checklist with clear instructions and a saved revision. The person can discover it through ordinary Skills/My Skills navigation, open its editable Studio content and reload the same saved revision. No particular chat-link format is required. The agent preserves the actual checklist instead of merely posting a suggestion or pretending creation automatically enables the skill for every agent.", "originalContext": "No matching skill exists; the CEO may create company skills.", "capabilities": [ "skills/authoring" ], "checks": null, - "oracleVersion": "paperclip.chat-catalog.integrator.v1", + "oracleVersion": "paperclip.chat-catalog.skill-authoring.v2", "sourceFiles": [ "chat-catalog-domains/integrator-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/integrator-skills.ts", - "chat-scenarios/skills.yaml" + "chat-scenarios/skills.yaml", + "chat-catalog-domains/integrator-skill-versions.ts", + "chat-catalog-domains/identity-support.ts" ], "screenshot": "final-state.png", "evidenceKind": "private-final-state-png", @@ -9905,6 +10401,7 @@ "checks": null, "oracleVersion": "paperclip.chat-catalog.skill-enable.standard-config.v2", "sourceFiles": [ + "chat-catalog-domains/integrator-skill-versions.ts", "chat-catalog-domains/integrator-skill-enable.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-definition.ts", @@ -9957,7 +10454,8 @@ "chat-catalog-domains/integrator-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/integrator-skills.ts", - "chat-scenarios/skills.yaml" + "chat-scenarios/skills.yaml", + "chat-catalog-domains/integrator-skill-versions.ts" ], "screenshot": "final-state.png", "evidenceKind": "private-final-state-png", @@ -9996,6 +10494,7 @@ "checks": null, "oracleVersion": "paperclip.chat-catalog.integrator.v1", "sourceFiles": [ + "chat-catalog-domains/integrator-skill-versions.ts", "chat-catalog-domains/integrator-skill-imports.ts", "chat-catalog-domains/integrator-skill-source.ts", "chat-catalog-domains/integrator-skills.ts", @@ -10041,6 +10540,7 @@ "checks": null, "oracleVersion": "paperclip.chat-catalog.integrator.v1", "sourceFiles": [ + "chat-catalog-domains/integrator-skill-versions.ts", "chat-catalog-domains/integrator-skill-imports.ts", "chat-catalog-domains/integrator-skill-source.ts", "chat-catalog-domains/integrator-skills.ts", @@ -10085,6 +10585,7 @@ "checks": null, "oracleVersion": "paperclip.chat-catalog.integrator.v1", "sourceFiles": [ + "chat-catalog-domains/integrator-skill-versions.ts", "chat-catalog-domains/integrator-skill-update.ts", "chat-catalog-domains/integrator-skill-source.ts", "chat-catalog-domains/integrator-support.ts", diff --git a/tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs b/tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs index afcd3ec2e09..1f09a993334 100644 --- a/tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs +++ b/tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs @@ -22,9 +22,13 @@ test('the manifest joins all244 source YAML scenarios without importing a domain assert.deepEqual(fullCatalogCases.map(value => value.key).sort(), scenarios.map(value => value.id).sort()); for (const scenario of scenarios) { const actual = row(scenario.id); - assert.equal(actual.originalQuery, scenario.query); + const override = field => manifest.source.repairMetadata.changedFields.find(change => change.key === scenario.id && change.field === field); + for (const [field, original] of [['originalQuery', scenario.query], ['expectedOutcome', scenario.expected_outcome]]) { + const change = override(field); + if (change) { assert.equal(change.previous, original); assert.equal(actual[field], change.current); } + else assert.equal(actual[field], original); + } assert.equal(actual.originalContext, scenario.context); - assert.equal(actual.expectedOutcome, scenario.expected_outcome); assert.ok(!Object.hasOwn(actual, 'status') && !Object.hasOwn(actual, 'passed')); } }); @@ -96,23 +100,38 @@ test('foreign identity, duplicated case and changed run-bound metadata are rejec }); -test('repair source metadata preserves original definitions and declares scoped observer and oracle changes', () => { +test('repair source metadata binds the corrected candidate and explicit changed definitions', () => { const repair = manifest.source.repairMetadata; assert.equal(repair.baseTargetSha, '3b66c3c144360ddf32714a0742fd774b8ceafb16'); - assert.equal(manifest.targetSha, '9ab0f951a5b5990c266ef4456610e0a0775aa334'); + assert.equal(repair.previousTargetSha, '9ab0f951a5b5990c266ef4456610e0a0775aa334'); + assert.equal(manifest.targetSha, '36dff2eab3dfb6de6efb4fd2ce6c1a58f8a5f383'); assert.equal(repair.definitionSourceSha, manifest.metadataSourceSha); + assert.equal(repair.candidateDescriptorSha256, '5b098a30ed54c6021698b33b434145ca2e934ff2bb9c1e0738a8d8e5dc622797'); assert.deepEqual(repair.changedFields.map(value => [value.key, value.field]), [ + ['agents.choose-primary', 'sourceFiles'], ['companies.agent-observability', 'sourceFiles'], ['companies.configure-decision-model', 'sourceFiles'], - ['companies.fast-responses', 'sourceFiles'], ['companies.prompt-injection-guard', 'sourceFiles'], + ['companies.fast-responses', 'sourceFiles'], ['companies.fast-responses', 'originalQuery'], + ['companies.prompt-injection-guard', 'sourceFiles'], + ['goals.create-goal-tree', 'oracleVersion'], ['goals.create-goal-tree', 'expectedOutcome'], + ['questions-and-approvals.duplicate-answer', 'oracleVersion'], ['questions-and-approvals.multi-part-question', 'oracleVersion'], ['secrets.configure-company-secret', 'oracleVersion'], + ['skills.author-skill', 'sourceFiles'], ['skills.author-skill', 'oracleVersion'], ['skills.author-skill', 'expectedOutcome'], + ['skills.enable-for-agent', 'sourceFiles'], ['skills.explicit-restriction', 'sourceFiles'], + ['skills.find-and-install', 'sourceFiles'], ['skills.fork-read-only', 'sourceFiles'], ['skills.review-update', 'sourceFiles'], ]); for (const change of repair.changedFields) { assert.deepEqual(row(change.key)[change.field], change.current); - if (change.field === 'sourceFiles') { - assert.deepEqual(change.current.filter(value => value !== 'chat-catalog-domains/identity-company-observer.ts'), change.previous); - } else { - assert.equal(change.previous, 'paperclip.chat-catalog.external.v1'); - assert.equal(change.current, 'paperclip.chat-catalog.company-secret-controller-baseline.v2'); - } + assert.notDeepEqual(change.current, change.previous); } + assert.equal(repair.descriptorChanges.length, 19); + assert.equal(new Set(repair.descriptorChanges.map(change => change.key)).size, 18); + assert.deepEqual([...new Set(repair.descriptorChanges.map(change => change.field))].sort(), ['oracleVersion', 'setupPolicy', 'sourceFiles']); + for (const change of repair.descriptorChanges) { + assert.notDeepEqual(change.current, change.previous); + if (change.field !== 'setupPolicy') assert.deepEqual(row(change.key)[change.field], change.current); + } + assert.deepEqual(repair.changedDefinitionFiles.map(value => value.path), [ + 'tests/runner-e2e/chat-scenarios/companies.yaml', 'tests/runner-e2e/chat-scenarios/goals.yaml', 'tests/runner-e2e/chat-scenarios/skills.yaml', + ]); + assert.ok(repair.changedDefinitionFiles.every(value => /^[a-f0-9]{64}$/.test(value.sha256))); });