mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-04 11:01:16 +01:00
test(evals): pin synchronized chat repair declarations and definitions
This commit is contained in:
Dotta committed
1 parent
09162fe893
commit
90d4ecf22f
3 files changed
+557
-29
No files matched your search
@@ -294,3 +294,11 @@ and credentials do not enter the public artifact path. Additional credentials
|
||||
reach only cases that declare them. External inputs use the existing selected-case
|
||||
private-input owner, disposable-resource checks and oracle commitments.
|
||||
Unconfigured inputs remain setup blockers.
|
||||
|
||||
### Repair source 2 (candidate only)
|
||||
|
||||
The catalog now pins candidate `36dff2eab3dfb6de6efb4fd2ce6c1a58f8a5f383`, succeeding `9ab0f951a5b5990c266ef4456610e0a0775aa334`. It carries the reviewed fixture, scoped observation, version-readback and evidence-readiness repairs. Static runtime descriptors are synchronized with the actual declarations; all 242 lazy factories must pass the real no-callback consistency check, together with the two legacy-cell controls, before each new repair-batch GO. The previous candidate's mismatched metadata caused launch 1690 to be cancelled before case/model work; it is an operational attempt, not a feature result.
|
||||
|
||||
The manifest keeps the original definition source and values, with explicit scoped changes: Fast response names the requested product feature; goal-tree and authored-skill outcomes use the product-supported optional-owner and ordinary navigation contracts; question delivery/validation and company-secret setup declare their reviewed oracle versions. Each new measurement binds this candidate and the exact changed definition; earlier grades and the 244-case gate are unchanged. SDK scan-audit capture improves private failure diagnosis only: rejection, evidence requirements and privacy bounds remain in force, and the other SDK cases stay held until the finding is understood.
|
||||
|
||||
Workflow, model, case clocks, native-run bounds, credentials, protected reporting and case identities do not change. The full harness TypeScript graph has not been qualified by local pure tests; source import/admission and the existing credential-free remote Runner build remain required before native queries. Static TestOps descriptions may still show the original definition until a separate idempotent metadata reconciliation at a safe source boundary.
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"schema": "paperclip.chat-full-catalog-testops-manifest/v1",
|
||||
"targetSha": "9ab0f951a5b5990c266ef4456610e0a0775aa334",
|
||||
"targetSha": "36dff2eab3dfb6de6efb4fd2ce6c1a58f8a5f383",
|
||||
"metadataSourceSha": "de0941835cc8f0d01c982574ee258e110babdbeb",
|
||||
"profile": "runner-opencode",
|
||||
"model": "openrouter/deepseek/deepseek-v4.1-flash-1017",
|
||||
@@ -170,8 +170,304 @@
|
||||
],
|
||||
"repairMetadata": {
|
||||
"baseTargetSha": "3b66c3c144360ddf32714a0742fd774b8ceafb16",
|
||||
"previousTargetSha": "9ab0f951a5b5990c266ef4456610e0a0775aa334",
|
||||
"definitionSourceSha": "de0941835cc8f0d01c982574ee258e110babdbeb",
|
||||
"candidateDescriptorSha256": "5b098a30ed54c6021698b33b434145ca2e934ff2bb9c1e0738a8d8e5dc622797",
|
||||
"changedDefinitionFiles": [
|
||||
{
|
||||
"path": "tests/runner-e2e/chat-scenarios/companies.yaml",
|
||||
"sha256": "fbaf17ac279d92127de4b58a3605edc5c7954dabfb050f49be210e1f85775d1d"
|
||||
},
|
||||
{
|
||||
"path": "tests/runner-e2e/chat-scenarios/goals.yaml",
|
||||
"sha256": "b140b0e652a9945b7ead9e38e5bc01791f7dc9cd39b1f2b6f292544d50632dae"
|
||||
},
|
||||
{
|
||||
"path": "tests/runner-e2e/chat-scenarios/skills.yaml",
|
||||
"sha256": "51f975d62540e77a48539821f443c6ab134c8ffb63e4c04bb2678a655de84a69"
|
||||
}
|
||||
],
|
||||
"conditions": "New candidate measurements only. Original 244 definitions, attempt history and old pass gate remain immutable. SDK audit custody remains diagnostic; scan rejection is unchanged. Explicit field overrides below retain original definition values. No model, clock, run-bound, workflow, credential or reporting authority expansion.",
|
||||
"changedFields": [
|
||||
{
|
||||
"key": "agents.choose-primary",
|
||||
"field": "sourceFiles",
|
||||
"previous": [
|
||||
"chat-catalog-domains/runtime-primary.ts",
|
||||
"chat-catalog-domains/runtime-support.ts",
|
||||
"chat-scenarios/agents.yaml"
|
||||
],
|
||||
"current": [
|
||||
"chat-catalog-domains/runtime-primary.ts",
|
||||
"chat-catalog-domains/runtime-support.ts",
|
||||
"chat-catalog-domains/identity-support.ts",
|
||||
"chat-scenarios/agents.yaml"
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "companies.agent-observability",
|
||||
"field": "sourceFiles",
|
||||
"previous": [
|
||||
"chat-catalog-domains/identity-observability.ts",
|
||||
"chat-catalog-domains/identity-private-browser.ts",
|
||||
"chat-catalog-domains/identity-private-browser-worker.ts",
|
||||
"chat-catalog-domains/identity-wizard.ts",
|
||||
"chat-catalog-domains/identity-support.ts",
|
||||
"chat-catalog-domains/internal-support.ts",
|
||||
"chat-scenarios/companies.yaml"
|
||||
],
|
||||
"current": [
|
||||
"chat-catalog-domains/identity-observability.ts",
|
||||
"chat-catalog-domains/identity-company-observer.ts",
|
||||
"chat-catalog-domains/identity-private-browser.ts",
|
||||
"chat-catalog-domains/identity-private-browser-worker.ts",
|
||||
"chat-catalog-domains/identity-wizard.ts",
|
||||
"chat-catalog-domains/identity-support.ts",
|
||||
"chat-catalog-domains/internal-support.ts",
|
||||
"chat-scenarios/companies.yaml"
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "companies.configure-decision-model",
|
||||
"field": "sourceFiles",
|
||||
"previous": [
|
||||
"chat-catalog-domains/identity-company-models.ts",
|
||||
"chat-catalog-domains/identity-support.ts",
|
||||
"chat-catalog-domains/internal-support.ts",
|
||||
"chat-scenarios/companies.yaml"
|
||||
],
|
||||
"current": [
|
||||
"chat-catalog-domains/identity-company-models.ts",
|
||||
"chat-catalog-domains/identity-company-observer.ts",
|
||||
"chat-catalog-domains/identity-support.ts",
|
||||
"chat-catalog-domains/internal-support.ts",
|
||||
"chat-scenarios/companies.yaml"
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "companies.fast-responses",
|
||||
"field": "sourceFiles",
|
||||
"previous": [
|
||||
"chat-catalog-domains/identity-company-models.ts",
|
||||
"chat-catalog-domains/identity-support.ts",
|
||||
"chat-catalog-domains/internal-support.ts",
|
||||
"chat-scenarios/companies.yaml"
|
||||
],
|
||||
"current": [
|
||||
"chat-catalog-domains/identity-company-models.ts",
|
||||
"chat-catalog-domains/identity-company-observer.ts",
|
||||
"chat-catalog-domains/identity-support.ts",
|
||||
"chat-catalog-domains/internal-support.ts",
|
||||
"chat-scenarios/companies.yaml"
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "companies.fast-responses",
|
||||
"field": "originalQuery",
|
||||
"previous": "Make chat acknowledge my requests faster.",
|
||||
"current": "Help me enable the company's Fast response feature so chat acknowledges my requests faster."
|
||||
},
|
||||
{
|
||||
"key": "companies.prompt-injection-guard",
|
||||
"field": "sourceFiles",
|
||||
"previous": [
|
||||
"chat-catalog-domains/identity-company-models.ts",
|
||||
"chat-catalog-domains/identity-support.ts",
|
||||
"chat-catalog-domains/internal-support.ts",
|
||||
"chat-scenarios/companies.yaml"
|
||||
],
|
||||
"current": [
|
||||
"chat-catalog-domains/identity-company-models.ts",
|
||||
"chat-catalog-domains/identity-company-observer.ts",
|
||||
"chat-catalog-domains/identity-support.ts",
|
||||
"chat-catalog-domains/internal-support.ts",
|
||||
"chat-scenarios/companies.yaml"
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "goals.create-goal-tree",
|
||||
"field": "oracleVersion",
|
||||
"previous": "paperclip.chat-catalog.internal.v1",
|
||||
"current": "paperclip.chat-catalog.goal-hierarchy.v2"
|
||||
},
|
||||
{
|
||||
"key": "goals.create-goal-tree",
|
||||
"field": "expectedOutcome",
|
||||
"previous": "The saved goals have the requested levels, valid parent relationship and appropriate ownership or an explicit missing-owner question. Chat links to the hierarchy and does not invent progress metrics. A retry reuses the intended goals rather than creating another parallel tree.",
|
||||
"current": "The saved goals have the requested levels and a valid same-company parent relationship. Ownership may remain unassigned when none was requested; any assigned owner belongs to the company. The saved hierarchy is inspectable through ordinary Goals navigation and public readback, without invented progress metrics. A retry reuses the intended goals rather than creating another parallel tree."
|
||||
},
|
||||
{
|
||||
"key": "questions-and-approvals.duplicate-answer",
|
||||
"field": "oracleVersion",
|
||||
"previous": "paperclip.chat-catalog.identity.v1",
|
||||
"current": "paperclip.chat-catalog.answer-delivery.v2"
|
||||
},
|
||||
{
|
||||
"key": "questions-and-approvals.multi-part-question",
|
||||
"field": "oracleVersion",
|
||||
"previous": "paperclip.chat-catalog.identity.v1",
|
||||
"current": "paperclip.chat-catalog.required-question.v2"
|
||||
},
|
||||
{
|
||||
"key": "secrets.configure-company-secret",
|
||||
"field": "oracleVersion",
|
||||
"previous": "paperclip.chat-catalog.external.v1",
|
||||
"current": "paperclip.chat-catalog.company-secret-controller-baseline.v2"
|
||||
},
|
||||
{
|
||||
"key": "skills.author-skill",
|
||||
"field": "sourceFiles",
|
||||
"previous": [
|
||||
"chat-catalog-domains/integrator-support.ts",
|
||||
"chat-catalog-domains/internal-support.ts",
|
||||
"chat-catalog-domains/integrator-skills.ts",
|
||||
"chat-scenarios/skills.yaml"
|
||||
],
|
||||
"current": [
|
||||
"chat-catalog-domains/integrator-support.ts",
|
||||
"chat-catalog-domains/internal-support.ts",
|
||||
"chat-catalog-domains/integrator-skills.ts",
|
||||
"chat-scenarios/skills.yaml",
|
||||
"chat-catalog-domains/integrator-skill-versions.ts",
|
||||
"chat-catalog-domains/identity-support.ts"
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "skills.author-skill",
|
||||
"field": "oracleVersion",
|
||||
"previous": "paperclip.chat-catalog.integrator.v1",
|
||||
"current": "paperclip.chat-catalog.skill-authoring.v2"
|
||||
},
|
||||
{
|
||||
"key": "skills.author-skill",
|
||||
"field": "expectedOutcome",
|
||||
"previous": "A writable company skill captures the checklist with clear instructions and a saved revision. Chat links to the skill so the person can inspect or edit it. The agent preserves the actual checklist instead of merely posting a suggestion or pretending creation automatically enables the skill for every agent.",
|
||||
"current": "A writable company skill captures the checklist with clear instructions and a saved revision. The person can discover it through ordinary Skills/My Skills navigation, open its editable Studio content and reload the same saved revision. No particular chat-link format is required. The agent preserves the actual checklist instead of merely posting a suggestion or pretending creation automatically enables the skill for every agent."
|
||||
},
|
||||
{
|
||||
"key": "skills.enable-for-agent",
|
||||
"field": "sourceFiles",
|
||||
"previous": [
|
||||
"chat-catalog-domains/integrator-skill-enable.ts",
|
||||
"chat-catalog-domains/identity-support.ts",
|
||||
"chat-catalog-domains/internal-definition.ts",
|
||||
"chat-catalog-domains/internal-support.ts",
|
||||
"chat-catalog-domains/integrator-support.ts",
|
||||
"chat-scenarios/skills.yaml",
|
||||
"../../server/src/routes/agents.ts",
|
||||
"../../server/src/routes/company-skill-policy.ts",
|
||||
"../../server/src/services/company-skills.ts",
|
||||
"../../packages/shared/src/validators/skill-policy.ts"
|
||||
],
|
||||
"current": [
|
||||
"chat-catalog-domains/integrator-skill-versions.ts",
|
||||
"chat-catalog-domains/integrator-skill-enable.ts",
|
||||
"chat-catalog-domains/identity-support.ts",
|
||||
"chat-catalog-domains/internal-definition.ts",
|
||||
"chat-catalog-domains/internal-support.ts",
|
||||
"chat-catalog-domains/integrator-support.ts",
|
||||
"chat-scenarios/skills.yaml",
|
||||
"../../server/src/routes/agents.ts",
|
||||
"../../server/src/routes/company-skill-policy.ts",
|
||||
"../../server/src/services/company-skills.ts",
|
||||
"../../packages/shared/src/validators/skill-policy.ts"
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "skills.explicit-restriction",
|
||||
"field": "sourceFiles",
|
||||
"previous": [
|
||||
"chat-catalog-domains/integrator-support.ts",
|
||||
"chat-catalog-domains/internal-support.ts",
|
||||
"chat-catalog-domains/integrator-skills.ts",
|
||||
"chat-scenarios/skills.yaml"
|
||||
],
|
||||
"current": [
|
||||
"chat-catalog-domains/integrator-support.ts",
|
||||
"chat-catalog-domains/internal-support.ts",
|
||||
"chat-catalog-domains/integrator-skills.ts",
|
||||
"chat-scenarios/skills.yaml",
|
||||
"chat-catalog-domains/integrator-skill-versions.ts"
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "skills.find-and-install",
|
||||
"field": "sourceFiles",
|
||||
"previous": [
|
||||
"chat-catalog-domains/integrator-skill-imports.ts",
|
||||
"chat-catalog-domains/integrator-skill-source.ts",
|
||||
"chat-catalog-domains/integrator-skills.ts",
|
||||
"chat-catalog-domains/integrator-support.ts",
|
||||
"chat-catalog-domains/internal-support.ts",
|
||||
"chat-scenarios/skills.yaml"
|
||||
],
|
||||
"current": [
|
||||
"chat-catalog-domains/integrator-skill-versions.ts",
|
||||
"chat-catalog-domains/integrator-skill-imports.ts",
|
||||
"chat-catalog-domains/integrator-skill-source.ts",
|
||||
"chat-catalog-domains/integrator-skills.ts",
|
||||
"chat-catalog-domains/integrator-support.ts",
|
||||
"chat-catalog-domains/internal-support.ts",
|
||||
"chat-scenarios/skills.yaml"
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "skills.fork-read-only",
|
||||
"field": "sourceFiles",
|
||||
"previous": [
|
||||
"chat-catalog-domains/integrator-skill-imports.ts",
|
||||
"chat-catalog-domains/integrator-skill-source.ts",
|
||||
"chat-catalog-domains/integrator-skills.ts",
|
||||
"chat-catalog-domains/integrator-support.ts",
|
||||
"chat-catalog-domains/internal-support.ts",
|
||||
"chat-scenarios/skills.yaml"
|
||||
],
|
||||
"current": [
|
||||
"chat-catalog-domains/integrator-skill-versions.ts",
|
||||
"chat-catalog-domains/integrator-skill-imports.ts",
|
||||
"chat-catalog-domains/integrator-skill-source.ts",
|
||||
"chat-catalog-domains/integrator-skills.ts",
|
||||
"chat-catalog-domains/integrator-support.ts",
|
||||
"chat-catalog-domains/internal-support.ts",
|
||||
"chat-scenarios/skills.yaml"
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "skills.review-update",
|
||||
"field": "sourceFiles",
|
||||
"previous": [
|
||||
"chat-catalog-domains/integrator-skill-update.ts",
|
||||
"chat-catalog-domains/integrator-skill-source.ts",
|
||||
"chat-catalog-domains/integrator-support.ts",
|
||||
"chat-catalog-domains/internal-support.ts",
|
||||
"chat-scenarios/skills.yaml"
|
||||
],
|
||||
"current": [
|
||||
"chat-catalog-domains/integrator-skill-versions.ts",
|
||||
"chat-catalog-domains/integrator-skill-update.ts",
|
||||
"chat-catalog-domains/integrator-skill-source.ts",
|
||||
"chat-catalog-domains/integrator-support.ts",
|
||||
"chat-catalog-domains/internal-support.ts",
|
||||
"chat-scenarios/skills.yaml"
|
||||
]
|
||||
}
|
||||
],
|
||||
"descriptorChanges": [
|
||||
{
|
||||
"key": "agents.choose-primary",
|
||||
"field": "sourceFiles",
|
||||
"previous": [
|
||||
"chat-catalog-domains/runtime-primary.ts",
|
||||
"chat-catalog-domains/runtime-support.ts",
|
||||
"chat-scenarios/agents.yaml"
|
||||
],
|
||||
"current": [
|
||||
"chat-catalog-domains/runtime-primary.ts",
|
||||
"chat-catalog-domains/runtime-support.ts",
|
||||
"chat-catalog-domains/identity-support.ts",
|
||||
"chat-scenarios/agents.yaml"
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "companies.agent-observability",
|
||||
"field": "sourceFiles",
|
||||
@@ -246,20 +542,217 @@
|
||||
"chat-scenarios/companies.yaml"
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "goals.create-goal-tree",
|
||||
"field": "oracleVersion",
|
||||
"previous": "paperclip.chat-catalog.internal.v1",
|
||||
"current": "paperclip.chat-catalog.goal-hierarchy.v2"
|
||||
},
|
||||
{
|
||||
"key": "questions-and-approvals.duplicate-answer",
|
||||
"field": "oracleVersion",
|
||||
"previous": "paperclip.chat-catalog.identity.v1",
|
||||
"current": "paperclip.chat-catalog.answer-delivery.v2"
|
||||
},
|
||||
{
|
||||
"key": "questions-and-approvals.multi-part-question",
|
||||
"field": "oracleVersion",
|
||||
"previous": "paperclip.chat-catalog.identity.v1",
|
||||
"current": "paperclip.chat-catalog.required-question.v2"
|
||||
},
|
||||
{
|
||||
"key": "secrets.configure-company-secret",
|
||||
"field": "oracleVersion",
|
||||
"previous": "paperclip.chat-catalog.external.v1",
|
||||
"current": "paperclip.chat-catalog.company-secret-controller-baseline.v2"
|
||||
},
|
||||
{
|
||||
"key": "secrets.personal-secret",
|
||||
"field": "setupPolicy",
|
||||
"previous": null,
|
||||
"current": {
|
||||
"readiness": "preserve-scenario",
|
||||
"invariantCheckIds": [
|
||||
"current-owner-missing-required-personal-value"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "skills.author-skill",
|
||||
"field": "sourceFiles",
|
||||
"previous": [
|
||||
"chat-catalog-domains/integrator-support.ts",
|
||||
"chat-catalog-domains/internal-support.ts",
|
||||
"chat-catalog-domains/integrator-skills.ts",
|
||||
"chat-scenarios/skills.yaml"
|
||||
],
|
||||
"current": [
|
||||
"chat-catalog-domains/integrator-support.ts",
|
||||
"chat-catalog-domains/internal-support.ts",
|
||||
"chat-catalog-domains/integrator-skills.ts",
|
||||
"chat-scenarios/skills.yaml",
|
||||
"chat-catalog-domains/integrator-skill-versions.ts",
|
||||
"chat-catalog-domains/identity-support.ts"
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "skills.author-skill",
|
||||
"field": "oracleVersion",
|
||||
"previous": "paperclip.chat-catalog.integrator.v1",
|
||||
"current": "paperclip.chat-catalog.skill-authoring.v2"
|
||||
},
|
||||
{
|
||||
"key": "skills.enable-for-agent",
|
||||
"field": "sourceFiles",
|
||||
"previous": [
|
||||
"chat-catalog-domains/integrator-skill-enable.ts",
|
||||
"chat-catalog-domains/identity-support.ts",
|
||||
"chat-catalog-domains/internal-definition.ts",
|
||||
"chat-catalog-domains/internal-support.ts",
|
||||
"chat-catalog-domains/integrator-support.ts",
|
||||
"chat-scenarios/skills.yaml",
|
||||
"../../server/src/routes/agents.ts",
|
||||
"../../server/src/routes/company-skill-policy.ts",
|
||||
"../../server/src/services/company-skills.ts",
|
||||
"../../packages/shared/src/validators/skill-policy.ts"
|
||||
],
|
||||
"current": [
|
||||
"chat-catalog-domains/integrator-skill-versions.ts",
|
||||
"chat-catalog-domains/integrator-skill-enable.ts",
|
||||
"chat-catalog-domains/identity-support.ts",
|
||||
"chat-catalog-domains/internal-definition.ts",
|
||||
"chat-catalog-domains/internal-support.ts",
|
||||
"chat-catalog-domains/integrator-support.ts",
|
||||
"chat-scenarios/skills.yaml",
|
||||
"../../server/src/routes/agents.ts",
|
||||
"../../server/src/routes/company-skill-policy.ts",
|
||||
"../../server/src/services/company-skills.ts",
|
||||
"../../packages/shared/src/validators/skill-policy.ts"
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "skills.explicit-restriction",
|
||||
"field": "sourceFiles",
|
||||
"previous": [
|
||||
"chat-catalog-domains/integrator-support.ts",
|
||||
"chat-catalog-domains/internal-support.ts",
|
||||
"chat-catalog-domains/integrator-skills.ts",
|
||||
"chat-scenarios/skills.yaml"
|
||||
],
|
||||
"current": [
|
||||
"chat-catalog-domains/integrator-support.ts",
|
||||
"chat-catalog-domains/internal-support.ts",
|
||||
"chat-catalog-domains/integrator-skills.ts",
|
||||
"chat-scenarios/skills.yaml",
|
||||
"chat-catalog-domains/integrator-skill-versions.ts"
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "skills.find-and-install",
|
||||
"field": "sourceFiles",
|
||||
"previous": [
|
||||
"chat-catalog-domains/integrator-skill-imports.ts",
|
||||
"chat-catalog-domains/integrator-skill-source.ts",
|
||||
"chat-catalog-domains/integrator-skills.ts",
|
||||
"chat-catalog-domains/integrator-support.ts",
|
||||
"chat-catalog-domains/internal-support.ts",
|
||||
"chat-scenarios/skills.yaml"
|
||||
],
|
||||
"current": [
|
||||
"chat-catalog-domains/integrator-skill-versions.ts",
|
||||
"chat-catalog-domains/integrator-skill-imports.ts",
|
||||
"chat-catalog-domains/integrator-skill-source.ts",
|
||||
"chat-catalog-domains/integrator-skills.ts",
|
||||
"chat-catalog-domains/integrator-support.ts",
|
||||
"chat-catalog-domains/internal-support.ts",
|
||||
"chat-scenarios/skills.yaml"
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "skills.fork-read-only",
|
||||
"field": "sourceFiles",
|
||||
"previous": [
|
||||
"chat-catalog-domains/integrator-skill-imports.ts",
|
||||
"chat-catalog-domains/integrator-skill-source.ts",
|
||||
"chat-catalog-domains/integrator-skills.ts",
|
||||
"chat-catalog-domains/integrator-support.ts",
|
||||
"chat-catalog-domains/internal-support.ts",
|
||||
"chat-scenarios/skills.yaml"
|
||||
],
|
||||
"current": [
|
||||
"chat-catalog-domains/integrator-skill-versions.ts",
|
||||
"chat-catalog-domains/integrator-skill-imports.ts",
|
||||
"chat-catalog-domains/integrator-skill-source.ts",
|
||||
"chat-catalog-domains/integrator-skills.ts",
|
||||
"chat-catalog-domains/integrator-support.ts",
|
||||
"chat-catalog-domains/internal-support.ts",
|
||||
"chat-scenarios/skills.yaml"
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "skills.review-update",
|
||||
"field": "sourceFiles",
|
||||
"previous": [
|
||||
"chat-catalog-domains/integrator-skill-update.ts",
|
||||
"chat-catalog-domains/integrator-skill-source.ts",
|
||||
"chat-catalog-domains/integrator-support.ts",
|
||||
"chat-catalog-domains/internal-support.ts",
|
||||
"chat-scenarios/skills.yaml"
|
||||
],
|
||||
"current": [
|
||||
"chat-catalog-domains/integrator-skill-versions.ts",
|
||||
"chat-catalog-domains/integrator-skill-update.ts",
|
||||
"chat-catalog-domains/integrator-skill-source.ts",
|
||||
"chat-catalog-domains/integrator-support.ts",
|
||||
"chat-catalog-domains/internal-support.ts",
|
||||
"chat-scenarios/skills.yaml"
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "steering.stop-versus-pause",
|
||||
"field": "setupPolicy",
|
||||
"previous": {
|
||||
"readiness": "preserve-scenario",
|
||||
"invariantCheckIds": [
|
||||
"lead-ready",
|
||||
"two-real-native-workers",
|
||||
"independent-child-pause-budget"
|
||||
]
|
||||
},
|
||||
"current": {
|
||||
"readiness": "preserve-scenario",
|
||||
"funding": "preserve-scenario",
|
||||
"invariantCheckIds": [
|
||||
"lead-ready",
|
||||
"two-real-native-workers",
|
||||
"independent-child-pause-budget",
|
||||
"active-branches-funded"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "task-coordination.pause-tree",
|
||||
"field": "setupPolicy",
|
||||
"previous": {
|
||||
"readiness": "preserve-scenario",
|
||||
"invariantCheckIds": [
|
||||
"lead-ready",
|
||||
"two-real-native-workers",
|
||||
"independent-child-pause-budget"
|
||||
]
|
||||
},
|
||||
"current": {
|
||||
"readiness": "preserve-scenario",
|
||||
"funding": "preserve-scenario",
|
||||
"invariantCheckIds": [
|
||||
"lead-ready",
|
||||
"two-real-native-workers",
|
||||
"independent-child-pause-budget",
|
||||
"active-branches-funded"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"conditions": [
|
||||
"Initialize owned SDK evidence staging before retention.",
|
||||
"Use real public scoped membership only for three incidental human fixtures.",
|
||||
"Company settings observation uses the original setup-inclusive deadline; no larger case cap.",
|
||||
"Company secret setup and v2 oracle permit only exact immutable controller-bootstrap secret references."
|
||||
],
|
||||
"originalMeasurements": "Preserved. These are new candidate conditions, not retroactive passes."
|
||||
]
|
||||
}
|
||||
},
|
||||
"cases": [
|
||||
@@ -1262,6 +1755,7 @@
|
||||
"sourceFiles": [
|
||||
"chat-catalog-domains/runtime-primary.ts",
|
||||
"chat-catalog-domains/runtime-support.ts",
|
||||
"chat-catalog-domains/identity-support.ts",
|
||||
"chat-scenarios/agents.yaml"
|
||||
],
|
||||
"screenshot": "final-state.png",
|
||||
@@ -4088,7 +4582,7 @@
|
||||
],
|
||||
"externalInput": null,
|
||||
"installedSource": null,
|
||||
"originalQuery": "Make chat acknowledge my requests faster.",
|
||||
"originalQuery": "Help me enable the company's Fast response feature so chat acknowledges my requests faster.",
|
||||
"expectedOutcome": "The agent explains the offered fast-response setting and directs the human to its configuration and test surface. The journey checks a real acknowledgement and its failure or usage record. A quick acknowledgement is not presented as completion of the underlying task or evidence that the worker started.",
|
||||
"originalContext": "The fast-response experiment is available but disabled; settings and tests require a human connection manager.",
|
||||
"capabilities": [
|
||||
@@ -6236,13 +6730,13 @@
|
||||
"externalInput": null,
|
||||
"installedSource": null,
|
||||
"originalQuery": "Set a company goal of reaching a thousand signups, with a team goal for launching the website.",
|
||||
"expectedOutcome": "The saved goals have the requested levels, valid parent relationship and appropriate ownership or an explicit missing-owner question. Chat links to the hierarchy and does not invent progress metrics. A retry reuses the intended goals rather than creating another parallel tree.",
|
||||
"expectedOutcome": "The saved goals have the requested levels and a valid same-company parent relationship. Ownership may remain unassigned when none was requested; any assigned owner belongs to the company. The saved hierarchy is inspectable through ordinary Goals navigation and public readback, without invented progress metrics. A retry reuses the intended goals rather than creating another parallel tree.",
|
||||
"originalContext": "The company has a growth mission and no matching quarterly goal; the agent may create goals.",
|
||||
"capabilities": [
|
||||
"goals/hierarchy"
|
||||
],
|
||||
"checks": null,
|
||||
"oracleVersion": "paperclip.chat-catalog.internal.v1",
|
||||
"oracleVersion": "paperclip.chat-catalog.goal-hierarchy.v2",
|
||||
"sourceFiles": [
|
||||
"chat-catalog-domains/internal-definition.ts",
|
||||
"chat-catalog-domains/internal-support.ts",
|
||||
@@ -8261,7 +8755,7 @@
|
||||
"questions-and-approvals/continuation"
|
||||
],
|
||||
"checks": null,
|
||||
"oracleVersion": "paperclip.chat-catalog.identity.v1",
|
||||
"oracleVersion": "paperclip.chat-catalog.answer-delivery.v2",
|
||||
"sourceFiles": [
|
||||
"chat-catalog-domains/identity-confirmation-workflows.ts",
|
||||
"chat-catalog-domains/identity-support.ts",
|
||||
@@ -8347,7 +8841,7 @@
|
||||
"questions-and-approvals/questions"
|
||||
],
|
||||
"checks": null,
|
||||
"oracleVersion": "paperclip.chat-catalog.identity.v1",
|
||||
"oracleVersion": "paperclip.chat-catalog.required-question.v2",
|
||||
"sourceFiles": [
|
||||
"chat-catalog-domains/identity-support.ts",
|
||||
"chat-catalog-domains/internal-support.ts",
|
||||
@@ -9854,18 +10348,20 @@
|
||||
"externalInput": null,
|
||||
"installedSource": null,
|
||||
"originalQuery": "Turn our website review checklist into a reusable skill.",
|
||||
"expectedOutcome": "A writable company skill captures the checklist with clear instructions and a saved revision. Chat links to the skill so the person can inspect or edit it. The agent preserves the actual checklist instead of merely posting a suggestion or pretending creation automatically enables the skill for every agent.",
|
||||
"expectedOutcome": "A writable company skill captures the checklist with clear instructions and a saved revision. The person can discover it through ordinary Skills/My Skills navigation, open its editable Studio content and reload the same saved revision. No particular chat-link format is required. The agent preserves the actual checklist instead of merely posting a suggestion or pretending creation automatically enables the skill for every agent.",
|
||||
"originalContext": "No matching skill exists; the CEO may create company skills.",
|
||||
"capabilities": [
|
||||
"skills/authoring"
|
||||
],
|
||||
"checks": null,
|
||||
"oracleVersion": "paperclip.chat-catalog.integrator.v1",
|
||||
"oracleVersion": "paperclip.chat-catalog.skill-authoring.v2",
|
||||
"sourceFiles": [
|
||||
"chat-catalog-domains/integrator-support.ts",
|
||||
"chat-catalog-domains/internal-support.ts",
|
||||
"chat-catalog-domains/integrator-skills.ts",
|
||||
"chat-scenarios/skills.yaml"
|
||||
"chat-scenarios/skills.yaml",
|
||||
"chat-catalog-domains/integrator-skill-versions.ts",
|
||||
"chat-catalog-domains/identity-support.ts"
|
||||
],
|
||||
"screenshot": "final-state.png",
|
||||
"evidenceKind": "private-final-state-png",
|
||||
@@ -9905,6 +10401,7 @@
|
||||
"checks": null,
|
||||
"oracleVersion": "paperclip.chat-catalog.skill-enable.standard-config.v2",
|
||||
"sourceFiles": [
|
||||
"chat-catalog-domains/integrator-skill-versions.ts",
|
||||
"chat-catalog-domains/integrator-skill-enable.ts",
|
||||
"chat-catalog-domains/identity-support.ts",
|
||||
"chat-catalog-domains/internal-definition.ts",
|
||||
@@ -9957,7 +10454,8 @@
|
||||
"chat-catalog-domains/integrator-support.ts",
|
||||
"chat-catalog-domains/internal-support.ts",
|
||||
"chat-catalog-domains/integrator-skills.ts",
|
||||
"chat-scenarios/skills.yaml"
|
||||
"chat-scenarios/skills.yaml",
|
||||
"chat-catalog-domains/integrator-skill-versions.ts"
|
||||
],
|
||||
"screenshot": "final-state.png",
|
||||
"evidenceKind": "private-final-state-png",
|
||||
@@ -9996,6 +10494,7 @@
|
||||
"checks": null,
|
||||
"oracleVersion": "paperclip.chat-catalog.integrator.v1",
|
||||
"sourceFiles": [
|
||||
"chat-catalog-domains/integrator-skill-versions.ts",
|
||||
"chat-catalog-domains/integrator-skill-imports.ts",
|
||||
"chat-catalog-domains/integrator-skill-source.ts",
|
||||
"chat-catalog-domains/integrator-skills.ts",
|
||||
@@ -10041,6 +10540,7 @@
|
||||
"checks": null,
|
||||
"oracleVersion": "paperclip.chat-catalog.integrator.v1",
|
||||
"sourceFiles": [
|
||||
"chat-catalog-domains/integrator-skill-versions.ts",
|
||||
"chat-catalog-domains/integrator-skill-imports.ts",
|
||||
"chat-catalog-domains/integrator-skill-source.ts",
|
||||
"chat-catalog-domains/integrator-skills.ts",
|
||||
@@ -10085,6 +10585,7 @@
|
||||
"checks": null,
|
||||
"oracleVersion": "paperclip.chat-catalog.integrator.v1",
|
||||
"sourceFiles": [
|
||||
"chat-catalog-domains/integrator-skill-versions.ts",
|
||||
"chat-catalog-domains/integrator-skill-update.ts",
|
||||
"chat-catalog-domains/integrator-skill-source.ts",
|
||||
"chat-catalog-domains/integrator-support.ts",
|
||||
|
||||
@@ -22,9 +22,13 @@ test('the manifest joins all244 source YAML scenarios without importing a domain
|
||||
assert.deepEqual(fullCatalogCases.map(value => value.key).sort(), scenarios.map(value => value.id).sort());
|
||||
for (const scenario of scenarios) {
|
||||
const actual = row(scenario.id);
|
||||
assert.equal(actual.originalQuery, scenario.query);
|
||||
const override = field => manifest.source.repairMetadata.changedFields.find(change => change.key === scenario.id && change.field === field);
|
||||
for (const [field, original] of [['originalQuery', scenario.query], ['expectedOutcome', scenario.expected_outcome]]) {
|
||||
const change = override(field);
|
||||
if (change) { assert.equal(change.previous, original); assert.equal(actual[field], change.current); }
|
||||
else assert.equal(actual[field], original);
|
||||
}
|
||||
assert.equal(actual.originalContext, scenario.context);
|
||||
assert.equal(actual.expectedOutcome, scenario.expected_outcome);
|
||||
assert.ok(!Object.hasOwn(actual, 'status') && !Object.hasOwn(actual, 'passed'));
|
||||
}
|
||||
});
|
||||
@@ -96,23 +100,38 @@ test('foreign identity, duplicated case and changed run-bound metadata are rejec
|
||||
});
|
||||
|
||||
|
||||
test('repair source metadata preserves original definitions and declares scoped observer and oracle changes', () => {
|
||||
test('repair source metadata binds the corrected candidate and explicit changed definitions', () => {
|
||||
const repair = manifest.source.repairMetadata;
|
||||
assert.equal(repair.baseTargetSha, '3b66c3c144360ddf32714a0742fd774b8ceafb16');
|
||||
assert.equal(manifest.targetSha, '9ab0f951a5b5990c266ef4456610e0a0775aa334');
|
||||
assert.equal(repair.previousTargetSha, '9ab0f951a5b5990c266ef4456610e0a0775aa334');
|
||||
assert.equal(manifest.targetSha, '36dff2eab3dfb6de6efb4fd2ce6c1a58f8a5f383');
|
||||
assert.equal(repair.definitionSourceSha, manifest.metadataSourceSha);
|
||||
assert.equal(repair.candidateDescriptorSha256, '5b098a30ed54c6021698b33b434145ca2e934ff2bb9c1e0738a8d8e5dc622797');
|
||||
assert.deepEqual(repair.changedFields.map(value => [value.key, value.field]), [
|
||||
['agents.choose-primary', 'sourceFiles'],
|
||||
['companies.agent-observability', 'sourceFiles'], ['companies.configure-decision-model', 'sourceFiles'],
|
||||
['companies.fast-responses', 'sourceFiles'], ['companies.prompt-injection-guard', 'sourceFiles'],
|
||||
['companies.fast-responses', 'sourceFiles'], ['companies.fast-responses', 'originalQuery'],
|
||||
['companies.prompt-injection-guard', 'sourceFiles'],
|
||||
['goals.create-goal-tree', 'oracleVersion'], ['goals.create-goal-tree', 'expectedOutcome'],
|
||||
['questions-and-approvals.duplicate-answer', 'oracleVersion'], ['questions-and-approvals.multi-part-question', 'oracleVersion'],
|
||||
['secrets.configure-company-secret', 'oracleVersion'],
|
||||
['skills.author-skill', 'sourceFiles'], ['skills.author-skill', 'oracleVersion'], ['skills.author-skill', 'expectedOutcome'],
|
||||
['skills.enable-for-agent', 'sourceFiles'], ['skills.explicit-restriction', 'sourceFiles'],
|
||||
['skills.find-and-install', 'sourceFiles'], ['skills.fork-read-only', 'sourceFiles'], ['skills.review-update', 'sourceFiles'],
|
||||
]);
|
||||
for (const change of repair.changedFields) {
|
||||
assert.deepEqual(row(change.key)[change.field], change.current);
|
||||
if (change.field === 'sourceFiles') {
|
||||
assert.deepEqual(change.current.filter(value => value !== 'chat-catalog-domains/identity-company-observer.ts'), change.previous);
|
||||
} else {
|
||||
assert.equal(change.previous, 'paperclip.chat-catalog.external.v1');
|
||||
assert.equal(change.current, 'paperclip.chat-catalog.company-secret-controller-baseline.v2');
|
||||
}
|
||||
assert.notDeepEqual(change.current, change.previous);
|
||||
}
|
||||
assert.equal(repair.descriptorChanges.length, 19);
|
||||
assert.equal(new Set(repair.descriptorChanges.map(change => change.key)).size, 18);
|
||||
assert.deepEqual([...new Set(repair.descriptorChanges.map(change => change.field))].sort(), ['oracleVersion', 'setupPolicy', 'sourceFiles']);
|
||||
for (const change of repair.descriptorChanges) {
|
||||
assert.notDeepEqual(change.current, change.previous);
|
||||
if (change.field !== 'setupPolicy') assert.deepEqual(row(change.key)[change.field], change.current);
|
||||
}
|
||||
assert.deepEqual(repair.changedDefinitionFiles.map(value => value.path), [
|
||||
'tests/runner-e2e/chat-scenarios/companies.yaml', 'tests/runner-e2e/chat-scenarios/goals.yaml', 'tests/runner-e2e/chat-scenarios/skills.yaml',
|
||||
]);
|
||||
assert.ok(repair.changedDefinitionFiles.every(value => /^[a-f0-9]{64}$/.test(value.sha256)));
|
||||
});
|
||||
Reference in new issue
Block a user