test(evals): pin synchronized chat repair declarations and definitions

This commit is contained in:
Dotta committed 2026-11-02 21:07:22 -06:00
1 parent 09162fe893
commit 90d4ecf22f
3 files changed
+557 -29

No files matched your search

+8
View File
@@ -294,3 +294,11 @@ and credentials do not enter the public artifact path. Additional credentials
reach only cases that declare them. External inputs use the existing selected-case reach only cases that declare them. External inputs use the existing selected-case
private-input owner, disposable-resource checks and oracle commitments. private-input owner, disposable-resource checks and oracle commitments.
Unconfigured inputs remain setup blockers. Unconfigured inputs remain setup blockers.
### Repair source 2 (candidate only)
The catalog now pins candidate `36dff2eab3dfb6de6efb4fd2ce6c1a58f8a5f383`, succeeding `9ab0f951a5b5990c266ef4456610e0a0775aa334`. It carries the reviewed fixture, scoped observation, version-readback and evidence-readiness repairs. Static runtime descriptors are synchronized with the actual declarations; all 242 lazy factories must pass the real no-callback consistency check, together with the two legacy-cell controls, before each new repair-batch GO. The previous candidate's mismatched metadata caused launch 1690 to be cancelled before case/model work; it is an operational attempt, not a feature result.
The manifest keeps the original definition source and values, with explicit scoped changes: Fast response names the requested product feature; goal-tree and authored-skill outcomes use the product-supported optional-owner and ordinary navigation contracts; question delivery/validation and company-secret setup declare their reviewed oracle versions. Each new measurement binds this candidate and the exact changed definition; earlier grades and the 244-case gate are unchanged. SDK scan-audit capture improves private failure diagnosis only: rejection, evidence requirements and privacy bounds remain in force, and the other SDK cases stay held until the finding is understood.
Workflow, model, case clocks, native-run bounds, credentials, protected reporting and case identities do not change. The full harness TypeScript graph has not been qualified by local pure tests; source import/admission and the existing credential-free remote Runner build remain required before native queries. Static TestOps descriptions may still show the original definition until a separate idempotent metadata reconciliation at a safe source boundary.
@@ -1,6 +1,6 @@
{ {
"schema": "paperclip.chat-full-catalog-testops-manifest/v1", "schema": "paperclip.chat-full-catalog-testops-manifest/v1",
"targetSha": "9ab0f951a5b5990c266ef4456610e0a0775aa334", "targetSha": "36dff2eab3dfb6de6efb4fd2ce6c1a58f8a5f383",
"metadataSourceSha": "de0941835cc8f0d01c982574ee258e110babdbeb", "metadataSourceSha": "de0941835cc8f0d01c982574ee258e110babdbeb",
"profile": "runner-opencode", "profile": "runner-opencode",
"model": "openrouter/deepseek/deepseek-v4.1-flash-1017", "model": "openrouter/deepseek/deepseek-v4.1-flash-1017",
@@ -170,8 +170,304 @@
], ],
"repairMetadata": { "repairMetadata": {
"baseTargetSha": "3b66c3c144360ddf32714a0742fd774b8ceafb16", "baseTargetSha": "3b66c3c144360ddf32714a0742fd774b8ceafb16",
"previousTargetSha": "9ab0f951a5b5990c266ef4456610e0a0775aa334",
"definitionSourceSha": "de0941835cc8f0d01c982574ee258e110babdbeb", "definitionSourceSha": "de0941835cc8f0d01c982574ee258e110babdbeb",
"candidateDescriptorSha256": "5b098a30ed54c6021698b33b434145ca2e934ff2bb9c1e0738a8d8e5dc622797",
"changedDefinitionFiles": [
{
"path": "tests/runner-e2e/chat-scenarios/companies.yaml",
"sha256": "fbaf17ac279d92127de4b58a3605edc5c7954dabfb050f49be210e1f85775d1d"
},
{
"path": "tests/runner-e2e/chat-scenarios/goals.yaml",
"sha256": "b140b0e652a9945b7ead9e38e5bc01791f7dc9cd39b1f2b6f292544d50632dae"
},
{
"path": "tests/runner-e2e/chat-scenarios/skills.yaml",
"sha256": "51f975d62540e77a48539821f443c6ab134c8ffb63e4c04bb2678a655de84a69"
}
],
"conditions": "New candidate measurements only. Original 244 definitions, attempt history and old pass gate remain immutable. SDK audit custody remains diagnostic; scan rejection is unchanged. Explicit field overrides below retain original definition values. No model, clock, run-bound, workflow, credential or reporting authority expansion.",
"changedFields": [ "changedFields": [
{
"key": "agents.choose-primary",
"field": "sourceFiles",
"previous": [
"chat-catalog-domains/runtime-primary.ts",
"chat-catalog-domains/runtime-support.ts",
"chat-scenarios/agents.yaml"
],
"current": [
"chat-catalog-domains/runtime-primary.ts",
"chat-catalog-domains/runtime-support.ts",
"chat-catalog-domains/identity-support.ts",
"chat-scenarios/agents.yaml"
]
},
{
"key": "companies.agent-observability",
"field": "sourceFiles",
"previous": [
"chat-catalog-domains/identity-observability.ts",
"chat-catalog-domains/identity-private-browser.ts",
"chat-catalog-domains/identity-private-browser-worker.ts",
"chat-catalog-domains/identity-wizard.ts",
"chat-catalog-domains/identity-support.ts",
"chat-catalog-domains/internal-support.ts",
"chat-scenarios/companies.yaml"
],
"current": [
"chat-catalog-domains/identity-observability.ts",
"chat-catalog-domains/identity-company-observer.ts",
"chat-catalog-domains/identity-private-browser.ts",
"chat-catalog-domains/identity-private-browser-worker.ts",
"chat-catalog-domains/identity-wizard.ts",
"chat-catalog-domains/identity-support.ts",
"chat-catalog-domains/internal-support.ts",
"chat-scenarios/companies.yaml"
]
},
{
"key": "companies.configure-decision-model",
"field": "sourceFiles",
"previous": [
"chat-catalog-domains/identity-company-models.ts",
"chat-catalog-domains/identity-support.ts",
"chat-catalog-domains/internal-support.ts",
"chat-scenarios/companies.yaml"
],
"current": [
"chat-catalog-domains/identity-company-models.ts",
"chat-catalog-domains/identity-company-observer.ts",
"chat-catalog-domains/identity-support.ts",
"chat-catalog-domains/internal-support.ts",
"chat-scenarios/companies.yaml"
]
},
{
"key": "companies.fast-responses",
"field": "sourceFiles",
"previous": [
"chat-catalog-domains/identity-company-models.ts",
"chat-catalog-domains/identity-support.ts",
"chat-catalog-domains/internal-support.ts",
"chat-scenarios/companies.yaml"
],
"current": [
"chat-catalog-domains/identity-company-models.ts",
"chat-catalog-domains/identity-company-observer.ts",
"chat-catalog-domains/identity-support.ts",
"chat-catalog-domains/internal-support.ts",
"chat-scenarios/companies.yaml"
]
},
{
"key": "companies.fast-responses",
"field": "originalQuery",
"previous": "Make chat acknowledge my requests faster.",
"current": "Help me enable the company's Fast response feature so chat acknowledges my requests faster."
},
{
"key": "companies.prompt-injection-guard",
"field": "sourceFiles",
"previous": [
"chat-catalog-domains/identity-company-models.ts",
"chat-catalog-domains/identity-support.ts",
"chat-catalog-domains/internal-support.ts",
"chat-scenarios/companies.yaml"
],
"current": [
"chat-catalog-domains/identity-company-models.ts",
"chat-catalog-domains/identity-company-observer.ts",
"chat-catalog-domains/identity-support.ts",
"chat-catalog-domains/internal-support.ts",
"chat-scenarios/companies.yaml"
]
},
{
"key": "goals.create-goal-tree",
"field": "oracleVersion",
"previous": "paperclip.chat-catalog.internal.v1",
"current": "paperclip.chat-catalog.goal-hierarchy.v2"
},
{
"key": "goals.create-goal-tree",
"field": "expectedOutcome",
"previous": "The saved goals have the requested levels, valid parent relationship and appropriate ownership or an explicit missing-owner question. Chat links to the hierarchy and does not invent progress metrics. A retry reuses the intended goals rather than creating another parallel tree.",
"current": "The saved goals have the requested levels and a valid same-company parent relationship. Ownership may remain unassigned when none was requested; any assigned owner belongs to the company. The saved hierarchy is inspectable through ordinary Goals navigation and public readback, without invented progress metrics. A retry reuses the intended goals rather than creating another parallel tree."
},
{
"key": "questions-and-approvals.duplicate-answer",
"field": "oracleVersion",
"previous": "paperclip.chat-catalog.identity.v1",
"current": "paperclip.chat-catalog.answer-delivery.v2"
},
{
"key": "questions-and-approvals.multi-part-question",
"field": "oracleVersion",
"previous": "paperclip.chat-catalog.identity.v1",
"current": "paperclip.chat-catalog.required-question.v2"
},
{
"key": "secrets.configure-company-secret",
"field": "oracleVersion",
"previous": "paperclip.chat-catalog.external.v1",
"current": "paperclip.chat-catalog.company-secret-controller-baseline.v2"
},
{
"key": "skills.author-skill",
"field": "sourceFiles",
"previous": [
"chat-catalog-domains/integrator-support.ts",
"chat-catalog-domains/internal-support.ts",
"chat-catalog-domains/integrator-skills.ts",
"chat-scenarios/skills.yaml"
],
"current": [
"chat-catalog-domains/integrator-support.ts",
"chat-catalog-domains/internal-support.ts",
"chat-catalog-domains/integrator-skills.ts",
"chat-scenarios/skills.yaml",
"chat-catalog-domains/integrator-skill-versions.ts",
"chat-catalog-domains/identity-support.ts"
]
},
{
"key": "skills.author-skill",
"field": "oracleVersion",
"previous": "paperclip.chat-catalog.integrator.v1",
"current": "paperclip.chat-catalog.skill-authoring.v2"
},
{
"key": "skills.author-skill",
"field": "expectedOutcome",
"previous": "A writable company skill captures the checklist with clear instructions and a saved revision. Chat links to the skill so the person can inspect or edit it. The agent preserves the actual checklist instead of merely posting a suggestion or pretending creation automatically enables the skill for every agent.",
"current": "A writable company skill captures the checklist with clear instructions and a saved revision. The person can discover it through ordinary Skills/My Skills navigation, open its editable Studio content and reload the same saved revision. No particular chat-link format is required. The agent preserves the actual checklist instead of merely posting a suggestion or pretending creation automatically enables the skill for every agent."
},
{
"key": "skills.enable-for-agent",
"field": "sourceFiles",
"previous": [
"chat-catalog-domains/integrator-skill-enable.ts",
"chat-catalog-domains/identity-support.ts",
"chat-catalog-domains/internal-definition.ts",
"chat-catalog-domains/internal-support.ts",
"chat-catalog-domains/integrator-support.ts",
"chat-scenarios/skills.yaml",
"../../server/src/routes/agents.ts",
"../../server/src/routes/company-skill-policy.ts",
"../../server/src/services/company-skills.ts",
"../../packages/shared/src/validators/skill-policy.ts"
],
"current": [
"chat-catalog-domains/integrator-skill-versions.ts",
"chat-catalog-domains/integrator-skill-enable.ts",
"chat-catalog-domains/identity-support.ts",
"chat-catalog-domains/internal-definition.ts",
"chat-catalog-domains/internal-support.ts",
"chat-catalog-domains/integrator-support.ts",
"chat-scenarios/skills.yaml",
"../../server/src/routes/agents.ts",
"../../server/src/routes/company-skill-policy.ts",
"../../server/src/services/company-skills.ts",
"../../packages/shared/src/validators/skill-policy.ts"
]
},
{
"key": "skills.explicit-restriction",
"field": "sourceFiles",
"previous": [
"chat-catalog-domains/integrator-support.ts",
"chat-catalog-domains/internal-support.ts",
"chat-catalog-domains/integrator-skills.ts",
"chat-scenarios/skills.yaml"
],
"current": [
"chat-catalog-domains/integrator-support.ts",
"chat-catalog-domains/internal-support.ts",
"chat-catalog-domains/integrator-skills.ts",
"chat-scenarios/skills.yaml",
"chat-catalog-domains/integrator-skill-versions.ts"
]
},
{
"key": "skills.find-and-install",
"field": "sourceFiles",
"previous": [
"chat-catalog-domains/integrator-skill-imports.ts",
"chat-catalog-domains/integrator-skill-source.ts",
"chat-catalog-domains/integrator-skills.ts",
"chat-catalog-domains/integrator-support.ts",
"chat-catalog-domains/internal-support.ts",
"chat-scenarios/skills.yaml"
],
"current": [
"chat-catalog-domains/integrator-skill-versions.ts",
"chat-catalog-domains/integrator-skill-imports.ts",
"chat-catalog-domains/integrator-skill-source.ts",
"chat-catalog-domains/integrator-skills.ts",
"chat-catalog-domains/integrator-support.ts",
"chat-catalog-domains/internal-support.ts",
"chat-scenarios/skills.yaml"
]
},
{
"key": "skills.fork-read-only",
"field": "sourceFiles",
"previous": [
"chat-catalog-domains/integrator-skill-imports.ts",
"chat-catalog-domains/integrator-skill-source.ts",
"chat-catalog-domains/integrator-skills.ts",
"chat-catalog-domains/integrator-support.ts",
"chat-catalog-domains/internal-support.ts",
"chat-scenarios/skills.yaml"
],
"current": [
"chat-catalog-domains/integrator-skill-versions.ts",
"chat-catalog-domains/integrator-skill-imports.ts",
"chat-catalog-domains/integrator-skill-source.ts",
"chat-catalog-domains/integrator-skills.ts",
"chat-catalog-domains/integrator-support.ts",
"chat-catalog-domains/internal-support.ts",
"chat-scenarios/skills.yaml"
]
},
{
"key": "skills.review-update",
"field": "sourceFiles",
"previous": [
"chat-catalog-domains/integrator-skill-update.ts",
"chat-catalog-domains/integrator-skill-source.ts",
"chat-catalog-domains/integrator-support.ts",
"chat-catalog-domains/internal-support.ts",
"chat-scenarios/skills.yaml"
],
"current": [
"chat-catalog-domains/integrator-skill-versions.ts",
"chat-catalog-domains/integrator-skill-update.ts",
"chat-catalog-domains/integrator-skill-source.ts",
"chat-catalog-domains/integrator-support.ts",
"chat-catalog-domains/internal-support.ts",
"chat-scenarios/skills.yaml"
]
}
],
"descriptorChanges": [
{
"key": "agents.choose-primary",
"field": "sourceFiles",
"previous": [
"chat-catalog-domains/runtime-primary.ts",
"chat-catalog-domains/runtime-support.ts",
"chat-scenarios/agents.yaml"
],
"current": [
"chat-catalog-domains/runtime-primary.ts",
"chat-catalog-domains/runtime-support.ts",
"chat-catalog-domains/identity-support.ts",
"chat-scenarios/agents.yaml"
]
},
{ {
"key": "companies.agent-observability", "key": "companies.agent-observability",
"field": "sourceFiles", "field": "sourceFiles",
@@ -246,20 +542,217 @@
"chat-scenarios/companies.yaml" "chat-scenarios/companies.yaml"
] ]
}, },
{
"key": "goals.create-goal-tree",
"field": "oracleVersion",
"previous": "paperclip.chat-catalog.internal.v1",
"current": "paperclip.chat-catalog.goal-hierarchy.v2"
},
{
"key": "questions-and-approvals.duplicate-answer",
"field": "oracleVersion",
"previous": "paperclip.chat-catalog.identity.v1",
"current": "paperclip.chat-catalog.answer-delivery.v2"
},
{
"key": "questions-and-approvals.multi-part-question",
"field": "oracleVersion",
"previous": "paperclip.chat-catalog.identity.v1",
"current": "paperclip.chat-catalog.required-question.v2"
},
{ {
"key": "secrets.configure-company-secret", "key": "secrets.configure-company-secret",
"field": "oracleVersion", "field": "oracleVersion",
"previous": "paperclip.chat-catalog.external.v1", "previous": "paperclip.chat-catalog.external.v1",
"current": "paperclip.chat-catalog.company-secret-controller-baseline.v2" "current": "paperclip.chat-catalog.company-secret-controller-baseline.v2"
},
{
"key": "secrets.personal-secret",
"field": "setupPolicy",
"previous": null,
"current": {
"readiness": "preserve-scenario",
"invariantCheckIds": [
"current-owner-missing-required-personal-value"
]
}
},
{
"key": "skills.author-skill",
"field": "sourceFiles",
"previous": [
"chat-catalog-domains/integrator-support.ts",
"chat-catalog-domains/internal-support.ts",
"chat-catalog-domains/integrator-skills.ts",
"chat-scenarios/skills.yaml"
],
"current": [
"chat-catalog-domains/integrator-support.ts",
"chat-catalog-domains/internal-support.ts",
"chat-catalog-domains/integrator-skills.ts",
"chat-scenarios/skills.yaml",
"chat-catalog-domains/integrator-skill-versions.ts",
"chat-catalog-domains/identity-support.ts"
]
},
{
"key": "skills.author-skill",
"field": "oracleVersion",
"previous": "paperclip.chat-catalog.integrator.v1",
"current": "paperclip.chat-catalog.skill-authoring.v2"
},
{
"key": "skills.enable-for-agent",
"field": "sourceFiles",
"previous": [
"chat-catalog-domains/integrator-skill-enable.ts",
"chat-catalog-domains/identity-support.ts",
"chat-catalog-domains/internal-definition.ts",
"chat-catalog-domains/internal-support.ts",
"chat-catalog-domains/integrator-support.ts",
"chat-scenarios/skills.yaml",
"../../server/src/routes/agents.ts",
"../../server/src/routes/company-skill-policy.ts",
"../../server/src/services/company-skills.ts",
"../../packages/shared/src/validators/skill-policy.ts"
],
"current": [
"chat-catalog-domains/integrator-skill-versions.ts",
"chat-catalog-domains/integrator-skill-enable.ts",
"chat-catalog-domains/identity-support.ts",
"chat-catalog-domains/internal-definition.ts",
"chat-catalog-domains/internal-support.ts",
"chat-catalog-domains/integrator-support.ts",
"chat-scenarios/skills.yaml",
"../../server/src/routes/agents.ts",
"../../server/src/routes/company-skill-policy.ts",
"../../server/src/services/company-skills.ts",
"../../packages/shared/src/validators/skill-policy.ts"
]
},
{
"key": "skills.explicit-restriction",
"field": "sourceFiles",
"previous": [
"chat-catalog-domains/integrator-support.ts",
"chat-catalog-domains/internal-support.ts",
"chat-catalog-domains/integrator-skills.ts",
"chat-scenarios/skills.yaml"
],
"current": [
"chat-catalog-domains/integrator-support.ts",
"chat-catalog-domains/internal-support.ts",
"chat-catalog-domains/integrator-skills.ts",
"chat-scenarios/skills.yaml",
"chat-catalog-domains/integrator-skill-versions.ts"
]
},
{
"key": "skills.find-and-install",
"field": "sourceFiles",
"previous": [
"chat-catalog-domains/integrator-skill-imports.ts",
"chat-catalog-domains/integrator-skill-source.ts",
"chat-catalog-domains/integrator-skills.ts",
"chat-catalog-domains/integrator-support.ts",
"chat-catalog-domains/internal-support.ts",
"chat-scenarios/skills.yaml"
],
"current": [
"chat-catalog-domains/integrator-skill-versions.ts",
"chat-catalog-domains/integrator-skill-imports.ts",
"chat-catalog-domains/integrator-skill-source.ts",
"chat-catalog-domains/integrator-skills.ts",
"chat-catalog-domains/integrator-support.ts",
"chat-catalog-domains/internal-support.ts",
"chat-scenarios/skills.yaml"
]
},
{
"key": "skills.fork-read-only",
"field": "sourceFiles",
"previous": [
"chat-catalog-domains/integrator-skill-imports.ts",
"chat-catalog-domains/integrator-skill-source.ts",
"chat-catalog-domains/integrator-skills.ts",
"chat-catalog-domains/integrator-support.ts",
"chat-catalog-domains/internal-support.ts",
"chat-scenarios/skills.yaml"
],
"current": [
"chat-catalog-domains/integrator-skill-versions.ts",
"chat-catalog-domains/integrator-skill-imports.ts",
"chat-catalog-domains/integrator-skill-source.ts",
"chat-catalog-domains/integrator-skills.ts",
"chat-catalog-domains/integrator-support.ts",
"chat-catalog-domains/internal-support.ts",
"chat-scenarios/skills.yaml"
]
},
{
"key": "skills.review-update",
"field": "sourceFiles",
"previous": [
"chat-catalog-domains/integrator-skill-update.ts",
"chat-catalog-domains/integrator-skill-source.ts",
"chat-catalog-domains/integrator-support.ts",
"chat-catalog-domains/internal-support.ts",
"chat-scenarios/skills.yaml"
],
"current": [
"chat-catalog-domains/integrator-skill-versions.ts",
"chat-catalog-domains/integrator-skill-update.ts",
"chat-catalog-domains/integrator-skill-source.ts",
"chat-catalog-domains/integrator-support.ts",
"chat-catalog-domains/internal-support.ts",
"chat-scenarios/skills.yaml"
]
},
{
"key": "steering.stop-versus-pause",
"field": "setupPolicy",
"previous": {
"readiness": "preserve-scenario",
"invariantCheckIds": [
"lead-ready",
"two-real-native-workers",
"independent-child-pause-budget"
]
},
"current": {
"readiness": "preserve-scenario",
"funding": "preserve-scenario",
"invariantCheckIds": [
"lead-ready",
"two-real-native-workers",
"independent-child-pause-budget",
"active-branches-funded"
]
}
},
{
"key": "task-coordination.pause-tree",
"field": "setupPolicy",
"previous": {
"readiness": "preserve-scenario",
"invariantCheckIds": [
"lead-ready",
"two-real-native-workers",
"independent-child-pause-budget"
]
},
"current": {
"readiness": "preserve-scenario",
"funding": "preserve-scenario",
"invariantCheckIds": [
"lead-ready",
"two-real-native-workers",
"independent-child-pause-budget",
"active-branches-funded"
]
}
} }
], ]
"conditions": [
"Initialize owned SDK evidence staging before retention.",
"Use real public scoped membership only for three incidental human fixtures.",
"Company settings observation uses the original setup-inclusive deadline; no larger case cap.",
"Company secret setup and v2 oracle permit only exact immutable controller-bootstrap secret references."
],
"originalMeasurements": "Preserved. These are new candidate conditions, not retroactive passes."
} }
}, },
"cases": [ "cases": [
@@ -1262,6 +1755,7 @@
"sourceFiles": [ "sourceFiles": [
"chat-catalog-domains/runtime-primary.ts", "chat-catalog-domains/runtime-primary.ts",
"chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/runtime-support.ts",
"chat-catalog-domains/identity-support.ts",
"chat-scenarios/agents.yaml" "chat-scenarios/agents.yaml"
], ],
"screenshot": "final-state.png", "screenshot": "final-state.png",
@@ -4088,7 +4582,7 @@
], ],
"externalInput": null, "externalInput": null,
"installedSource": null, "installedSource": null,
"originalQuery": "Make chat acknowledge my requests faster.", "originalQuery": "Help me enable the company's Fast response feature so chat acknowledges my requests faster.",
"expectedOutcome": "The agent explains the offered fast-response setting and directs the human to its configuration and test surface. The journey checks a real acknowledgement and its failure or usage record. A quick acknowledgement is not presented as completion of the underlying task or evidence that the worker started.", "expectedOutcome": "The agent explains the offered fast-response setting and directs the human to its configuration and test surface. The journey checks a real acknowledgement and its failure or usage record. A quick acknowledgement is not presented as completion of the underlying task or evidence that the worker started.",
"originalContext": "The fast-response experiment is available but disabled; settings and tests require a human connection manager.", "originalContext": "The fast-response experiment is available but disabled; settings and tests require a human connection manager.",
"capabilities": [ "capabilities": [
@@ -6236,13 +6730,13 @@
"externalInput": null, "externalInput": null,
"installedSource": null, "installedSource": null,
"originalQuery": "Set a company goal of reaching a thousand signups, with a team goal for launching the website.", "originalQuery": "Set a company goal of reaching a thousand signups, with a team goal for launching the website.",
"expectedOutcome": "The saved goals have the requested levels, valid parent relationship and appropriate ownership or an explicit missing-owner question. Chat links to the hierarchy and does not invent progress metrics. A retry reuses the intended goals rather than creating another parallel tree.", "expectedOutcome": "The saved goals have the requested levels and a valid same-company parent relationship. Ownership may remain unassigned when none was requested; any assigned owner belongs to the company. The saved hierarchy is inspectable through ordinary Goals navigation and public readback, without invented progress metrics. A retry reuses the intended goals rather than creating another parallel tree.",
"originalContext": "The company has a growth mission and no matching quarterly goal; the agent may create goals.", "originalContext": "The company has a growth mission and no matching quarterly goal; the agent may create goals.",
"capabilities": [ "capabilities": [
"goals/hierarchy" "goals/hierarchy"
], ],
"checks": null, "checks": null,
"oracleVersion": "paperclip.chat-catalog.internal.v1", "oracleVersion": "paperclip.chat-catalog.goal-hierarchy.v2",
"sourceFiles": [ "sourceFiles": [
"chat-catalog-domains/internal-definition.ts", "chat-catalog-domains/internal-definition.ts",
"chat-catalog-domains/internal-support.ts", "chat-catalog-domains/internal-support.ts",
@@ -8261,7 +8755,7 @@
"questions-and-approvals/continuation" "questions-and-approvals/continuation"
], ],
"checks": null, "checks": null,
"oracleVersion": "paperclip.chat-catalog.identity.v1", "oracleVersion": "paperclip.chat-catalog.answer-delivery.v2",
"sourceFiles": [ "sourceFiles": [
"chat-catalog-domains/identity-confirmation-workflows.ts", "chat-catalog-domains/identity-confirmation-workflows.ts",
"chat-catalog-domains/identity-support.ts", "chat-catalog-domains/identity-support.ts",
@@ -8347,7 +8841,7 @@
"questions-and-approvals/questions" "questions-and-approvals/questions"
], ],
"checks": null, "checks": null,
"oracleVersion": "paperclip.chat-catalog.identity.v1", "oracleVersion": "paperclip.chat-catalog.required-question.v2",
"sourceFiles": [ "sourceFiles": [
"chat-catalog-domains/identity-support.ts", "chat-catalog-domains/identity-support.ts",
"chat-catalog-domains/internal-support.ts", "chat-catalog-domains/internal-support.ts",
@@ -9854,18 +10348,20 @@
"externalInput": null, "externalInput": null,
"installedSource": null, "installedSource": null,
"originalQuery": "Turn our website review checklist into a reusable skill.", "originalQuery": "Turn our website review checklist into a reusable skill.",
"expectedOutcome": "A writable company skill captures the checklist with clear instructions and a saved revision. Chat links to the skill so the person can inspect or edit it. The agent preserves the actual checklist instead of merely posting a suggestion or pretending creation automatically enables the skill for every agent.", "expectedOutcome": "A writable company skill captures the checklist with clear instructions and a saved revision. The person can discover it through ordinary Skills/My Skills navigation, open its editable Studio content and reload the same saved revision. No particular chat-link format is required. The agent preserves the actual checklist instead of merely posting a suggestion or pretending creation automatically enables the skill for every agent.",
"originalContext": "No matching skill exists; the CEO may create company skills.", "originalContext": "No matching skill exists; the CEO may create company skills.",
"capabilities": [ "capabilities": [
"skills/authoring" "skills/authoring"
], ],
"checks": null, "checks": null,
"oracleVersion": "paperclip.chat-catalog.integrator.v1", "oracleVersion": "paperclip.chat-catalog.skill-authoring.v2",
"sourceFiles": [ "sourceFiles": [
"chat-catalog-domains/integrator-support.ts", "chat-catalog-domains/integrator-support.ts",
"chat-catalog-domains/internal-support.ts", "chat-catalog-domains/internal-support.ts",
"chat-catalog-domains/integrator-skills.ts", "chat-catalog-domains/integrator-skills.ts",
"chat-scenarios/skills.yaml" "chat-scenarios/skills.yaml",
"chat-catalog-domains/integrator-skill-versions.ts",
"chat-catalog-domains/identity-support.ts"
], ],
"screenshot": "final-state.png", "screenshot": "final-state.png",
"evidenceKind": "private-final-state-png", "evidenceKind": "private-final-state-png",
@@ -9905,6 +10401,7 @@
"checks": null, "checks": null,
"oracleVersion": "paperclip.chat-catalog.skill-enable.standard-config.v2", "oracleVersion": "paperclip.chat-catalog.skill-enable.standard-config.v2",
"sourceFiles": [ "sourceFiles": [
"chat-catalog-domains/integrator-skill-versions.ts",
"chat-catalog-domains/integrator-skill-enable.ts", "chat-catalog-domains/integrator-skill-enable.ts",
"chat-catalog-domains/identity-support.ts", "chat-catalog-domains/identity-support.ts",
"chat-catalog-domains/internal-definition.ts", "chat-catalog-domains/internal-definition.ts",
@@ -9957,7 +10454,8 @@
"chat-catalog-domains/integrator-support.ts", "chat-catalog-domains/integrator-support.ts",
"chat-catalog-domains/internal-support.ts", "chat-catalog-domains/internal-support.ts",
"chat-catalog-domains/integrator-skills.ts", "chat-catalog-domains/integrator-skills.ts",
"chat-scenarios/skills.yaml" "chat-scenarios/skills.yaml",
"chat-catalog-domains/integrator-skill-versions.ts"
], ],
"screenshot": "final-state.png", "screenshot": "final-state.png",
"evidenceKind": "private-final-state-png", "evidenceKind": "private-final-state-png",
@@ -9996,6 +10494,7 @@
"checks": null, "checks": null,
"oracleVersion": "paperclip.chat-catalog.integrator.v1", "oracleVersion": "paperclip.chat-catalog.integrator.v1",
"sourceFiles": [ "sourceFiles": [
"chat-catalog-domains/integrator-skill-versions.ts",
"chat-catalog-domains/integrator-skill-imports.ts", "chat-catalog-domains/integrator-skill-imports.ts",
"chat-catalog-domains/integrator-skill-source.ts", "chat-catalog-domains/integrator-skill-source.ts",
"chat-catalog-domains/integrator-skills.ts", "chat-catalog-domains/integrator-skills.ts",
@@ -10041,6 +10540,7 @@
"checks": null, "checks": null,
"oracleVersion": "paperclip.chat-catalog.integrator.v1", "oracleVersion": "paperclip.chat-catalog.integrator.v1",
"sourceFiles": [ "sourceFiles": [
"chat-catalog-domains/integrator-skill-versions.ts",
"chat-catalog-domains/integrator-skill-imports.ts", "chat-catalog-domains/integrator-skill-imports.ts",
"chat-catalog-domains/integrator-skill-source.ts", "chat-catalog-domains/integrator-skill-source.ts",
"chat-catalog-domains/integrator-skills.ts", "chat-catalog-domains/integrator-skills.ts",
@@ -10085,6 +10585,7 @@
"checks": null, "checks": null,
"oracleVersion": "paperclip.chat-catalog.integrator.v1", "oracleVersion": "paperclip.chat-catalog.integrator.v1",
"sourceFiles": [ "sourceFiles": [
"chat-catalog-domains/integrator-skill-versions.ts",
"chat-catalog-domains/integrator-skill-update.ts", "chat-catalog-domains/integrator-skill-update.ts",
"chat-catalog-domains/integrator-skill-source.ts", "chat-catalog-domains/integrator-skill-source.ts",
"chat-catalog-domains/integrator-support.ts", "chat-catalog-domains/integrator-support.ts",
@@ -22,9 +22,13 @@ test('the manifest joins all244 source YAML scenarios without importing a domain
assert.deepEqual(fullCatalogCases.map(value => value.key).sort(), scenarios.map(value => value.id).sort()); assert.deepEqual(fullCatalogCases.map(value => value.key).sort(), scenarios.map(value => value.id).sort());
for (const scenario of scenarios) { for (const scenario of scenarios) {
const actual = row(scenario.id); const actual = row(scenario.id);
assert.equal(actual.originalQuery, scenario.query); const override = field => manifest.source.repairMetadata.changedFields.find(change => change.key === scenario.id && change.field === field);
for (const [field, original] of [['originalQuery', scenario.query], ['expectedOutcome', scenario.expected_outcome]]) {
const change = override(field);
if (change) { assert.equal(change.previous, original); assert.equal(actual[field], change.current); }
else assert.equal(actual[field], original);
}
assert.equal(actual.originalContext, scenario.context); assert.equal(actual.originalContext, scenario.context);
assert.equal(actual.expectedOutcome, scenario.expected_outcome);
assert.ok(!Object.hasOwn(actual, 'status') && !Object.hasOwn(actual, 'passed')); assert.ok(!Object.hasOwn(actual, 'status') && !Object.hasOwn(actual, 'passed'));
} }
}); });
@@ -96,23 +100,38 @@ test('foreign identity, duplicated case and changed run-bound metadata are rejec
}); });
test('repair source metadata preserves original definitions and declares scoped observer and oracle changes', () => { test('repair source metadata binds the corrected candidate and explicit changed definitions', () => {
const repair = manifest.source.repairMetadata; const repair = manifest.source.repairMetadata;
assert.equal(repair.baseTargetSha, '3b66c3c144360ddf32714a0742fd774b8ceafb16'); assert.equal(repair.baseTargetSha, '3b66c3c144360ddf32714a0742fd774b8ceafb16');
assert.equal(manifest.targetSha, '9ab0f951a5b5990c266ef4456610e0a0775aa334'); assert.equal(repair.previousTargetSha, '9ab0f951a5b5990c266ef4456610e0a0775aa334');
assert.equal(manifest.targetSha, '36dff2eab3dfb6de6efb4fd2ce6c1a58f8a5f383');
assert.equal(repair.definitionSourceSha, manifest.metadataSourceSha); assert.equal(repair.definitionSourceSha, manifest.metadataSourceSha);
assert.equal(repair.candidateDescriptorSha256, '5b098a30ed54c6021698b33b434145ca2e934ff2bb9c1e0738a8d8e5dc622797');
assert.deepEqual(repair.changedFields.map(value => [value.key, value.field]), [ assert.deepEqual(repair.changedFields.map(value => [value.key, value.field]), [
['agents.choose-primary', 'sourceFiles'],
['companies.agent-observability', 'sourceFiles'], ['companies.configure-decision-model', 'sourceFiles'], ['companies.agent-observability', 'sourceFiles'], ['companies.configure-decision-model', 'sourceFiles'],
['companies.fast-responses', 'sourceFiles'], ['companies.prompt-injection-guard', 'sourceFiles'], ['companies.fast-responses', 'sourceFiles'], ['companies.fast-responses', 'originalQuery'],
['companies.prompt-injection-guard', 'sourceFiles'],
['goals.create-goal-tree', 'oracleVersion'], ['goals.create-goal-tree', 'expectedOutcome'],
['questions-and-approvals.duplicate-answer', 'oracleVersion'], ['questions-and-approvals.multi-part-question', 'oracleVersion'],
['secrets.configure-company-secret', 'oracleVersion'], ['secrets.configure-company-secret', 'oracleVersion'],
['skills.author-skill', 'sourceFiles'], ['skills.author-skill', 'oracleVersion'], ['skills.author-skill', 'expectedOutcome'],
['skills.enable-for-agent', 'sourceFiles'], ['skills.explicit-restriction', 'sourceFiles'],
['skills.find-and-install', 'sourceFiles'], ['skills.fork-read-only', 'sourceFiles'], ['skills.review-update', 'sourceFiles'],
]); ]);
for (const change of repair.changedFields) { for (const change of repair.changedFields) {
assert.deepEqual(row(change.key)[change.field], change.current); assert.deepEqual(row(change.key)[change.field], change.current);
if (change.field === 'sourceFiles') { assert.notDeepEqual(change.current, change.previous);
assert.deepEqual(change.current.filter(value => value !== 'chat-catalog-domains/identity-company-observer.ts'), change.previous);
} else {
assert.equal(change.previous, 'paperclip.chat-catalog.external.v1');
assert.equal(change.current, 'paperclip.chat-catalog.company-secret-controller-baseline.v2');
}
} }
assert.equal(repair.descriptorChanges.length, 19);
assert.equal(new Set(repair.descriptorChanges.map(change => change.key)).size, 18);
assert.deepEqual([...new Set(repair.descriptorChanges.map(change => change.field))].sort(), ['oracleVersion', 'setupPolicy', 'sourceFiles']);
for (const change of repair.descriptorChanges) {
assert.notDeepEqual(change.current, change.previous);
if (change.field !== 'setupPolicy') assert.deepEqual(row(change.key)[change.field], change.current);
}
assert.deepEqual(repair.changedDefinitionFiles.map(value => value.path), [
'tests/runner-e2e/chat-scenarios/companies.yaml', 'tests/runner-e2e/chat-scenarios/goals.yaml', 'tests/runner-e2e/chat-scenarios/skills.yaml',
]);
assert.ok(repair.changedDefinitionFiles.every(value => /^[a-f0-9]{64}$/.test(value.sha256)));
}); });