diff --git a/doc/plans/2026-10-02-legacy-document-skill-repair.json b/doc/plans/2026-10-02-legacy-document-skill-repair.json new file mode 100644 index 0000000000..b215bc08b8 --- /dev/null +++ b/doc/plans/2026-10-02-legacy-document-skill-repair.json @@ -0,0 +1,1187 @@ +{ + "schema": "paperclip.legacy-document-skill-repair.safe-comparison/v1", + "asOf": "2026-10-02T21:22:00Z", + "partial": false, + "expectedCellsPerVariant": 4, + "expectedProviderTurnsPerVariant": 4, + "candidateSha": "abd0b628ca642c09a54a4edc56a5227402f6686e", + "baselineSha": "bc83fe030234439ac51279502a28803958963e2e", + "originalMachinePairedDelta": { + "newFailures": 0, + "newPasses": 1, + "unchangedPasses": 1, + "unchangedFailures": 2, + "pending": 0 + }, + "observedBehavioralDeteriorations": [ + "OpenCode explicit case: candidate lacks a clickable link, while baseline provided a clickable API document URL. Both fail original UI-only oracle; equal machine verdicts hide this presentation deterioration." + ], + "baseline": [ + { + "executionId": "stock-harness.legacy-claude.local.assigned-skill-explicit-invocation", + "profileId": "legacy-claude", + "caseId": "assigned-skill-explicit-invocation", + "status": "failed", + "model": "claude-sonnet-4-6", + "durationMs": 34592, + "cleanup": "passed", + "billing": { + "llm": { + "runCount": 1, + "runsWithTokenUsage": 1, + "runsWithReportedCost": 1, + "inputTokens": 28439, + "outputTokens": 1137, + "cachedInputTokens": 143829, + "totalTokens": 173405, + "reportedCostUsd": 0.16683944999999997, + "costStatus": "reported" + }, + "runtime": { + "provider": "local", + "agentRunDurationMs": 23032, + "leaseDurationMs": null, + "leaseCount": 0, + "costStatus": "not_metered", + "costSource": "local_not_metered" + }, + "reportedCostUsd": 0.16683944999999997, + "estimatedRuntimeCostUsd": 0, + "observedAndEstimatedCostUsd": 0.16683944999999997, + "complete": true + }, + "sourceSha": "bc83fe030234439ac51279502a28803958963e2e", + "providerDurationMs": 23032, + "resultSha256": "67bcbe152852280b58fec97754601c57c416c2cb7965de55557240b8448dca63", + "contextEvidenceSha256": "9bc59db9678a3f5cfc207d1aeeec1f868f94429a7a10d8d214621f286532a1b7", + "failedBehaviorChecks": [ + "single-durable-output" + ], + "publicDocumentCount": 0, + "savedRevisionEvidence": false, + "runPurposes": [ + "assignment" + ], + "prerequisites": { + "passed": true, + "sourceSha": "bc83fe030234439ac51279502a28803958963e2e", + "sourceFingerprint": "27fbbc1792dca136c9c72dd492ff16c18bce0638ce4979985d798cde85175201", + "providerCallsBeforeAdmission": 0, + "executedChecks": 571 + } + }, + { + "executionId": "stock-harness.legacy-claude.local.assigned-skill-paperclip-document", + "profileId": "legacy-claude", + "caseId": "assigned-skill-paperclip-document", + "status": "passed", + "model": "claude-sonnet-4-6", + "durationMs": 50778, + "cleanup": "passed", + "billing": { + "llm": { + "runCount": 1, + "runsWithTokenUsage": 1, + "runsWithReportedCost": 1, + "inputTokens": 27675, + "outputTokens": 2062, + "cachedInputTokens": 216852, + "totalTokens": 246589, + "reportedCostUsd": 0.19975709999999997, + "costStatus": "reported" + }, + "runtime": { + "provider": "local", + "agentRunDurationMs": 34999, + "leaseDurationMs": null, + "leaseCount": 0, + "costStatus": "not_metered", + "costSource": "local_not_metered" + }, + "reportedCostUsd": 0.19975709999999997, + "estimatedRuntimeCostUsd": 0, + "observedAndEstimatedCostUsd": 0.19975709999999997, + "complete": true + }, + "sourceSha": "bc83fe030234439ac51279502a28803958963e2e", + "providerDurationMs": 34999, + "resultSha256": "bca45a760fc5ec30ae8f154d9ec52ab4073b3ce2375256b8718f91200cb79d4e", + "contextEvidenceSha256": "473f3b417e71ee823ae9aad2596c7f1379b3fd5cf8d8fd540c30a4f3a2e10e21", + "failedBehaviorChecks": [], + "publicDocumentCount": 1, + "savedRevisionEvidence": true, + "runPurposes": [ + "assignment" + ], + "prerequisites": { + "passed": true, + "sourceSha": "bc83fe030234439ac51279502a28803958963e2e", + "sourceFingerprint": "27fbbc1792dca136c9c72dd492ff16c18bce0638ce4979985d798cde85175201", + "providerCallsBeforeAdmission": 0, + "executedChecks": 571 + } + }, + { + "executionId": "stock-harness.legacy-opencode.local.assigned-skill-explicit-invocation", + "profileId": "legacy-opencode", + "caseId": "assigned-skill-explicit-invocation", + "status": "failed", + "model": "openrouter/deepseek/deepseek-v4-flash-0731", + "durationMs": 47625, + "cleanup": "passed", + "billing": { + "llm": { + "runCount": 1, + "runsWithTokenUsage": 1, + "runsWithReportedCost": 1, + "inputTokens": 24068, + "outputTokens": 1960, + "cachedInputTokens": 149248, + "totalTokens": 175276, + "reportedCostUsd": 0.0033927116000000003, + "costStatus": "reported" + }, + "runtime": { + "provider": "local", + "agentRunDurationMs": 36770, + "leaseDurationMs": null, + "leaseCount": 0, + "costStatus": "not_metered", + "costSource": "local_not_metered" + }, + "reportedCostUsd": 0.0033927116000000003, + "estimatedRuntimeCostUsd": 0, + "observedAndEstimatedCostUsd": 0.0033927116000000003, + "complete": true + }, + "sourceSha": "bc83fe030234439ac51279502a28803958963e2e", + "providerDurationMs": 36770, + "resultSha256": "a322e292fd7688c714cc8e57e7f64ca70c19f0784dc242b3e809a6e4b0be224e", + "contextEvidenceSha256": "3e295ec42b9c7208a7e4b17bf8329d9e1f01ba984c4a586539d7a703ffd59bf6", + "failedBehaviorChecks": [ + "single-durable-output" + ], + "publicDocumentCount": 0, + "savedRevisionEvidence": false, + "runPurposes": [ + "assignment" + ], + "prerequisites": { + "passed": true, + "sourceSha": "bc83fe030234439ac51279502a28803958963e2e", + "sourceFingerprint": "27fbbc1792dca136c9c72dd492ff16c18bce0638ce4979985d798cde85175201", + "providerCallsBeforeAdmission": 0, + "executedChecks": 571 + } + }, + { + "executionId": "stock-harness.legacy-opencode.local.assigned-skill-paperclip-document", + "profileId": "legacy-opencode", + "caseId": "assigned-skill-paperclip-document", + "status": "failed", + "model": "openrouter/deepseek/deepseek-v4-flash-0731", + "durationMs": 217620, + "cleanup": "passed", + "billing": { + "llm": { + "runCount": 1, + "runsWithTokenUsage": 1, + "runsWithReportedCost": 1, + "inputTokens": 78795, + "outputTokens": 4331, + "cachedInputTokens": 846848, + "totalTokens": 929974, + "reportedCostUsd": 0.0102644593, + "costStatus": "reported" + }, + "runtime": { + "provider": "local", + "agentRunDurationMs": 206191, + "leaseDurationMs": null, + "leaseCount": 0, + "costStatus": "not_metered", + "costSource": "local_not_metered" + }, + "reportedCostUsd": 0.0102644593, + "estimatedRuntimeCostUsd": 0, + "observedAndEstimatedCostUsd": 0.0102644593, + "complete": true + }, + "sourceSha": "bc83fe030234439ac51279502a28803958963e2e", + "providerDurationMs": 206191, + "resultSha256": "60dfb068ea80d0155d93fdace77a29fd19d64bdb0740e6e6b63f7e83ccfb5ded", + "contextEvidenceSha256": "23eed2766b5e8db9cea0ba5c76da235cf2d4222fa5718420cc60d6e13c86fa06", + "failedBehaviorChecks": [ + "saved-document-link" + ], + "publicDocumentCount": 1, + "savedRevisionEvidence": true, + "runPurposes": [ + "assignment" + ], + "prerequisites": { + "passed": true, + "sourceSha": "bc83fe030234439ac51279502a28803958963e2e", + "sourceFingerprint": "27fbbc1792dca136c9c72dd492ff16c18bce0638ce4979985d798cde85175201", + "providerCallsBeforeAdmission": 0, + "executedChecks": 571 + }, + "observedHandoff": "clickable API document URL; rejected by UI-only link oracle" + } + ], + "candidate": [ + { + "executionId": "stock-harness.legacy-claude.local.assigned-skill-explicit-invocation", + "profileId": "legacy-claude", + "caseId": "assigned-skill-explicit-invocation", + "status": "passed", + "model": "claude-sonnet-4-6", + "durationMs": 46702, + "cleanup": "passed", + "billing": { + "llm": { + "runCount": 1, + "runsWithTokenUsage": 1, + "runsWithReportedCost": 1, + "inputTokens": 40330, + "outputTokens": 1336, + "cachedInputTokens": 185090, + "totalTokens": 226756, + "reportedCostUsd": 0.22679475, + "costStatus": "reported" + }, + "runtime": { + "provider": "local", + "agentRunDurationMs": 27702, + "leaseDurationMs": null, + "leaseCount": 0, + "costStatus": "not_metered", + "costSource": "local_not_metered" + }, + "reportedCostUsd": 0.22679475, + "estimatedRuntimeCostUsd": 0, + "observedAndEstimatedCostUsd": 0.22679475, + "complete": true + }, + "sourceSha": "abd0b628ca642c09a54a4edc56a5227402f6686e", + "providerDurationMs": 27702, + "resultSha256": "2f66f62dc2f16756400a0f5b0b0d55e5c9f5f7cb2590d84ba05f3d4ee6b4ad66", + "contextEvidenceSha256": "c5619df489213873db64d35424b08f9852b0099bab07e92d3a71f993008de389", + "failedBehaviorChecks": [], + "publicDocumentCount": 1, + "savedRevisionEvidence": true, + "runPurposes": [ + "assignment" + ], + "prerequisites": { + "passed": true, + "sourceSha": "abd0b628ca642c09a54a4edc56a5227402f6686e", + "sourceFingerprint": "441999ae2618f1f1e9f430c099bbef63d2948495599f0d8b342d8e5f027a7f4c", + "providerCallsBeforeAdmission": 0, + "executedChecks": 571 + } + }, + { + "executionId": "stock-harness.legacy-claude.local.assigned-skill-paperclip-document", + "profileId": "legacy-claude", + "caseId": "assigned-skill-paperclip-document", + "status": "passed", + "model": "claude-sonnet-4-6", + "durationMs": 56928, + "cleanup": "passed", + "billing": { + "llm": { + "runCount": 1, + "runsWithTokenUsage": 1, + "runsWithReportedCost": 1, + "inputTokens": 42017, + "outputTokens": 2468, + "cachedInputTokens": 205226, + "totalTokens": 249711, + "reportedCostUsd": 0.25614254999999997, + "costStatus": "reported" + }, + "runtime": { + "provider": "local", + "agentRunDurationMs": 40513, + "leaseDurationMs": null, + "leaseCount": 0, + "costStatus": "not_metered", + "costSource": "local_not_metered" + }, + "reportedCostUsd": 0.25614254999999997, + "estimatedRuntimeCostUsd": 0, + "observedAndEstimatedCostUsd": 0.25614254999999997, + "complete": true + }, + "sourceSha": "abd0b628ca642c09a54a4edc56a5227402f6686e", + "providerDurationMs": 40513, + "resultSha256": "8a0ef693477a03925d8166d1872b5db4f5547ae48eb9df934beabcce9eccf147", + "contextEvidenceSha256": "f7aba5c9653567a0c3ee252d01fbcee20c81c432b4402d76c7e00221b6917f2e", + "failedBehaviorChecks": [], + "publicDocumentCount": 1, + "savedRevisionEvidence": true, + "runPurposes": [ + "assignment" + ], + "prerequisites": { + "passed": true, + "sourceSha": "abd0b628ca642c09a54a4edc56a5227402f6686e", + "sourceFingerprint": "441999ae2618f1f1e9f430c099bbef63d2948495599f0d8b342d8e5f027a7f4c", + "providerCallsBeforeAdmission": 0, + "executedChecks": 571 + } + }, + { + "executionId": "stock-harness.legacy-opencode.local.assigned-skill-explicit-invocation", + "profileId": "legacy-opencode", + "caseId": "assigned-skill-explicit-invocation", + "status": "failed", + "model": "openrouter/deepseek/deepseek-v4-flash-0731", + "durationMs": 79578, + "cleanup": "passed", + "billing": { + "llm": { + "runCount": 2, + "runsWithTokenUsage": 2, + "runsWithReportedCost": 2, + "inputTokens": 54946, + "outputTokens": 2264, + "cachedInputTokens": 169472, + "totalTokens": 226682, + "reportedCostUsd": 0.0047148437000000005, + "costStatus": "reported" + }, + "runtime": { + "provider": "local", + "agentRunDurationMs": 68032, + "leaseDurationMs": null, + "leaseCount": 0, + "costStatus": "not_metered", + "costSource": "local_not_metered" + }, + "reportedCostUsd": 0.0047148437000000005, + "estimatedRuntimeCostUsd": 0, + "observedAndEstimatedCostUsd": 0.0047148437000000005, + "complete": true + }, + "sourceSha": "abd0b628ca642c09a54a4edc56a5227402f6686e", + "providerDurationMs": 68032, + "resultSha256": "f807c235250d8675d29aa34fc97d0e89be2c7ea885edfc2f3845627037552c0b", + "contextEvidenceSha256": "0dc98ad8ef1f22d69c3a3d7344082a6d5a0effc8d9c73c5d4c1267c92800b57a", + "failedBehaviorChecks": [ + "single-durable-output" + ], + "publicDocumentCount": 0, + "savedRevisionEvidence": false, + "runPurposes": [ + "assignment", + "issue_disposition_repair" + ], + "prerequisites": { + "passed": true, + "sourceSha": "abd0b628ca642c09a54a4edc56a5227402f6686e", + "sourceFingerprint": "441999ae2618f1f1e9f430c099bbef63d2948495599f0d8b342d8e5f027a7f4c", + "providerCallsBeforeAdmission": 0, + "executedChecks": 571 + } + }, + { + "executionId": "stock-harness.legacy-opencode.local.assigned-skill-paperclip-document", + "profileId": "legacy-opencode", + "caseId": "assigned-skill-paperclip-document", + "status": "failed", + "model": "openrouter/deepseek/deepseek-v4-flash-0731", + "durationMs": 59941, + "cleanup": "passed", + "billing": { + "llm": { + "runCount": 1, + "runsWithTokenUsage": 1, + "runsWithReportedCost": 1, + "inputTokens": 33724, + "outputTokens": 2597, + "cachedInputTokens": 147200, + "totalTokens": 183521, + "reportedCostUsd": 0.0042468724, + "costStatus": "reported" + }, + "runtime": { + "provider": "local", + "agentRunDurationMs": 46300, + "leaseDurationMs": null, + "leaseCount": 0, + "costStatus": "not_metered", + "costSource": "local_not_metered" + }, + "reportedCostUsd": 0.0042468724, + "estimatedRuntimeCostUsd": 0, + "observedAndEstimatedCostUsd": 0.0042468724, + "complete": true + }, + "sourceSha": "abd0b628ca642c09a54a4edc56a5227402f6686e", + "providerDurationMs": 46300, + "resultSha256": "cff6e5bf4b6b646ae6f9b5a6a2934c444c37a14f50b591f5226ccac0e4de1dda", + "contextEvidenceSha256": "6c943d78291c1e493cb29ca3a52dcb96d186fd18ed51e4cfb17055649162f6f2", + "failedBehaviorChecks": [ + "saved-document-link" + ], + "publicDocumentCount": 1, + "savedRevisionEvidence": true, + "runPurposes": [ + "assignment" + ], + "prerequisites": { + "passed": true, + "sourceSha": "abd0b628ca642c09a54a4edc56a5227402f6686e", + "sourceFingerprint": "441999ae2618f1f1e9f430c099bbef63d2948495599f0d8b342d8e5f027a7f4c", + "providerCallsBeforeAdmission": 0, + "executedChecks": 571 + }, + "observedHandoff": "code-formatted prefix-less UI document path; no clickable anchor" + } + ], + "matchedProfileProof": { + "schema": "paperclip.legacy-document-skill-repair.fixture-match-proof/v1", + "cells": [ + { + "executionId": "stock-harness.legacy-claude.local.assigned-skill-explicit-invocation", + "fixtureConfigSha256": "6c21be9bb3fcec6274372fc586d88e42d82bc0d438e26a69ef614f8e6fcb8519", + "model": "claude-sonnet-4-6" + }, + { + "executionId": "stock-harness.legacy-claude.local.assigned-skill-paperclip-document", + "fixtureConfigSha256": "ea1cd67b0f68ef3830c011768d1a1133b5828c602d5ae3e021889dcd72708344", + "model": "claude-sonnet-4-6" + }, + { + "executionId": "stock-harness.legacy-opencode.local.assigned-skill-explicit-invocation", + "fixtureConfigSha256": "dd673f469cab583d26bc2e4b484f18b27884ec14a59542177026389181e2da2c", + "model": "openrouter/deepseek/deepseek-v4-flash-0731" + }, + { + "executionId": "stock-harness.legacy-opencode.local.assigned-skill-paperclip-document", + "fixtureConfigSha256": "b5173358d664ab6b3daa67d8758f18d59ec99cb9c485c02ccf953a9f8375e01e", + "model": "openrouter/deepseek/deepseek-v4-flash-0731" + } + ] + }, + "identicalSourceCount": 203, + "skillSources": [ + { + "path": "skills/paperclip/SKILL.md", + "candidate": { + "present": true, + "sha256": "65e379cab319bfef5911c0f6c9423cb262c9b8f1fef3fd08d45c1e4f32f8971a" + }, + "baseline": { + "present": true, + "sha256": "77aafc0c7b16a82254af642f0ebff723dcef7d7fbb6d3354efc0c11032e0344d" + } + }, + { + "path": "skills/paperclip/references/issue-documents.md", + "candidate": { + "present": true, + "sha256": "353845a406b1b4948487c348c86eb7b8c89a55ae67afe203100eee10aee46336" + }, + "baseline": { + "present": false, + "sha256": null + } + } + ], + "toolDeliveryAudit": { + "schema": "paperclip.legacy-document-repair.tool-audit/v1", + "rows": [ + { + "variant": "candidate", + "case": "assigned-skill-paperclip-document", + "sourceSha": "abd0b628ca642c09a54a4edc56a5227402f6686e", + "runOrdinal": 1, + "runPurpose": "assignment", + "startedAt": "2026-10-02T20:37:40.215Z", + "contextEvidenceSha256": "6c943d78291c1e493cb29ca3a52dcb96d186fd18ed51e4cfb17055649162f6f2", + "toolCalls": [ + { + "tool": "skill", + "skill": "context-integrity-output-7b55f897b3b41", + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": false + }, + { + "tool": "skill", + "skill": "paperclip", + "operationalSkillLoad": true, + "earlyRecipeVisible": true, + "newReferencePointerVisible": true, + "newReferenceRead": false, + "outputTruncated": true, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": false + }, + { + "tool": "bash", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": false + }, + { + "tool": "read", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": true, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": false + }, + { + "tool": "bash", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": true, + "statusApiCall": false + }, + { + "tool": "bash", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": false + }, + { + "tool": "bash", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": false + }, + { + "tool": "bash", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": true + } + ] + }, + { + "variant": "candidate", + "case": "assigned-skill-explicit-invocation", + "sourceSha": "abd0b628ca642c09a54a4edc56a5227402f6686e", + "runOrdinal": 1, + "runPurpose": "assignment", + "startedAt": "2026-10-02T20:37:37.587Z", + "contextEvidenceSha256": "0dc98ad8ef1f22d69c3a3d7344082a6d5a0effc8d9c73c5d4c1267c92800b57a", + "toolCalls": [ + { + "tool": "skill", + "skill": "context-integrity-output-60e03959432b1", + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": false + }, + { + "tool": "bash", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": false + }, + { + "tool": "write", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": true, + "documentApiCall": false, + "statusApiCall": false + } + ] + }, + { + "variant": "candidate", + "case": "assigned-skill-explicit-invocation", + "sourceSha": "abd0b628ca642c09a54a4edc56a5227402f6686e", + "runOrdinal": 2, + "runPurpose": "issue_disposition_repair", + "startedAt": "2026-10-02T20:37:52.592Z", + "contextEvidenceSha256": "0dc98ad8ef1f22d69c3a3d7344082a6d5a0effc8d9c73c5d4c1267c92800b57a", + "toolCalls": [ + { + "tool": "skill", + "skill": "paperclip", + "operationalSkillLoad": true, + "earlyRecipeVisible": true, + "newReferencePointerVisible": true, + "newReferenceRead": false, + "outputTruncated": true, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": false + }, + { + "tool": "bash", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": false + }, + { + "tool": "bash", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": false + }, + { + "tool": "bash", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": false + }, + { + "tool": "bash", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": false + }, + { + "tool": "bash", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": false + }, + { + "tool": "bash", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": false + }, + { + "tool": "bash", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": true + } + ] + }, + { + "variant": "baseline", + "case": "assigned-skill-explicit-invocation", + "sourceSha": "bc83fe030234439ac51279502a28803958963e2e", + "runOrdinal": 1, + "runPurpose": "assignment", + "startedAt": "2026-10-02T20:37:57.110Z", + "contextEvidenceSha256": "3e295ec42b9c7208a7e4b17bf8329d9e1f01ba984c4a586539d7a703ffd59bf6", + "toolCalls": [ + { + "tool": "skill", + "skill": "context-integrity-output-816e6b4cdd6a1", + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": false + }, + { + "tool": "bash", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": false + }, + { + "tool": "write", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": true, + "documentApiCall": false, + "statusApiCall": false + }, + { + "tool": "skill", + "skill": "paperclip", + "operationalSkillLoad": true, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": true, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": false + }, + { + "tool": "bash", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": false + }, + { + "tool": "bash", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": false + }, + { + "tool": "bash", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": true + }, + { + "tool": "write", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": false + }, + { + "tool": "bash", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": true + } + ] + }, + { + "variant": "baseline", + "case": "assigned-skill-paperclip-document", + "sourceSha": "bc83fe030234439ac51279502a28803958963e2e", + "runOrdinal": 1, + "runPurpose": "assignment", + "startedAt": "2026-10-02T20:38:02.031Z", + "contextEvidenceSha256": "23eed2766b5e8db9cea0ba5c76da235cf2d4222fa5718420cc60d6e13c86fa06", + "toolCalls": [ + { + "tool": "skill", + "skill": "context-integrity-output-06ce9804653e1", + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": false + }, + { + "tool": "skill", + "skill": "paperclip", + "operationalSkillLoad": true, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": true, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": false + }, + { + "tool": "bash", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": false + }, + { + "tool": "bash", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": false + }, + { + "tool": "bash", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": false + }, + { + "tool": "bash", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": false + }, + { + "tool": "bash", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": false + }, + { + "tool": "read", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": false + }, + { + "tool": "bash", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": false + }, + { + "tool": "bash", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": false + }, + { + "tool": "bash", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": false + }, + { + "tool": "bash", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": false + }, + { + "tool": "bash", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": true, + "statusApiCall": false + }, + { + "tool": "bash", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": false + }, + { + "tool": "read", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": true, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": false + }, + { + "tool": "bash", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": false + }, + { + "tool": "bash", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": false + }, + { + "tool": "bash", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": false + }, + { + "tool": "read", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": false + }, + { + "tool": "bash", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": false + }, + { + "tool": "bash", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": true, + "statusApiCall": false + }, + { + "tool": "bash", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": true, + "statusApiCall": false + }, + { + "tool": "bash", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": false, + "statusApiCall": false + }, + { + "tool": "bash", + "skill": null, + "operationalSkillLoad": false, + "earlyRecipeVisible": false, + "newReferencePointerVisible": false, + "newReferenceRead": false, + "outputTruncated": false, + "localDocumentWrite": false, + "documentApiCall": true, + "statusApiCall": true + } + ] + } + ] + }, + "limitations": [ + "Tiny manual/shared prompts and merged native Codex base fix remain fixed; only the two operational skill sources differ.", + "Original task-document wording is storage-ambiguous; explicit case requests a usable link but its oracle requires the canonical UI route, so the baseline API-link failure has an oracle/request ambiguity.", + "Provider-free source review confirms a board-readable GET document endpoint and an unprefixed UI redirect preserving the hash; actual authenticated browser navigation in the cleaned-up measured instance was not replayed.", + "One paired trial cannot establish broad performance, causality, timing or spending improvement.", + "Candidate OpenCode original has an additional automatic disposition-repair run; all nine actual runs and costs are retained.", + "Local runtime is unmetered; reported LLM cost is not a statement of actual external billing.", + "Subsequent generated capability metadata and pre-provider drift checks are derived maintenance after these measured frozen revisions, not a new provider qualification." + ] +} diff --git a/doc/plans/2026-10-02-legacy-document-skill-repair.md b/doc/plans/2026-10-02-legacy-document-skill-repair.md index ff8cc3c9ab..728e0b7c48 100644 --- a/doc/plans/2026-10-02-legacy-document-skill-repair.md +++ b/doc/plans/2026-10-02-legacy-document-skill-repair.md @@ -1,18 +1,51 @@ # Legacy document skill repair qualification — 2026-10-02 -**TL;DR: no repair performance result yet.** All four paired cases are pending. The original prompt-removal comparison observed two new classic Claude/OpenCode document-delivery failures; its retained verdicts remain unchanged. This focused comparison measures a small operational-skill repair while holding the tiny manual/shared prompts fixed, rather than remeasuring the combined instruction removal. +**TL;DR: one newly passing case (Claude original document delivery), one unchanged pass (Claude explicit storage), two unchanged OpenCode failures, and no pending pairs or new overall machine failures. OpenCode's explicit-case clickable handoff nevertheless got worse:** baseline supplied a clickable API document URL, while candidate supplied only a code-formatted path. Both fail the original UI-only link oracle, whose intended route was narrower than the request's "usable link" wording. This result does not establish general non-regression. -The approved repair adds early API-runtime document PUT/receipt/link guidance and a short generic issue-document reference. A valid saved-revision write receipt is sufficient; GET is used for existing-document updates or unclear/conflicting receipts. Explicit destinations, downloadable files, and native document-tool boundaries remain intact. The default manual stays eight words. +This focused repair keeps the eight-word manual, reduced shared prompts and merged Codex base fix #14920 fixed. It varies only the early operational Paperclip skill recipe and the presence of the new `issue-documents.md` reference. The earlier combined prompt-removal comparison retains its two new classic Claude/OpenCode storage failures; this trial repairs Claude's original case but leaves OpenCode's original regression unresolved. -| Classic profile | Case | Pre-fix skill | Repaired skill | +| Classic profile | Case | Pre-fix skill → repaired skill | Retained evidence | | --- | --- | --- | --- | -| Claude | Original assigned skill | Pending | Pending | -| Claude | Explicit Paperclip document | Pending | Pending | -| OpenCode | Original assigned skill | Pending | Pending | -| OpenCode | Explicit Paperclip document | Pending | Pending | +| Claude | Original assigned skill | Fail → Pass | Baseline lacks a durable task document; candidate saves one containing the skill marker. | +| Claude | Explicit Paperclip document | Pass → Pass | Both save the document/revision and provide a canonical UI link. | +| OpenCode | Original assigned skill | Fail → Fail | Both write a workspace file rather than a public task document. Candidate requires automatic disposition recovery. | +| OpenCode | Explicit Paperclip document | Fail → Fail | Both save the document/revision. Baseline provides a clickable API URL; candidate gives a code-formatted UI path without an anchor. Original UI-link checks fail in both. | -The two variants select these four cells only, eight expected provider turns total. All model, effort, auth, tool, permission, budget, fixture and behavioral-grader sources match; only `skills/paperclip/SKILL.md` and the presence of `skills/paperclip/references/issue-documents.md` differ. The absent historical reference is explicitly fingerprinted as absent; its new content is not copied into the baseline. +Original machine verdicts and evidence hashes are retained in the [safe evidence projection](2026-10-02-legacy-document-skill-repair.json). No model was rerun to improve these results. -The original request and pinned output procedure are preserved verbatim. The added explicit case independently reads the public task document and its saved revision/content and requires an agent comment linking the exact same-app document. Correct relative and same-app absolute URLs pass. Workspace-only output, missing revisions, wrong content, wrong document keys, other-origin URLs and user-only links fail calibration. +## Frozen sources and public reports -Local preparation: 909 E2E support tests and E2E typecheck pass; the narrow selector discovers exactly these four cells. Initial sandbox listener failures and two obsolete catalog-count assertions are retained separately; the unchanged behavioral oracles were not weakened. Exact-head prerequisites, immutable source freeze and protected GitHub dispatch are next. Actual costs and outcome deltas remain unknown until retained results are inspected. Existing Claude chat-memory and legacy ACP credential/receipt failures remain separate unresolved findings. +- [Repaired skill campaign](https://github.com/paperclipai/paperclip/actions/runs/37060885547), source `abd0b628ca642c09a54a4edc56a5227402f6686e`, branch `codex/legacy-document-repaired-skill`. [Published candidate report](https://d1p6rlowie26tp.cloudfront.net/runner-e2e/campaigns/gha-37060885547-1/index.html). +- [Pre-fix skill campaign](https://github.com/paperclipai/paperclip/actions/runs/37060888047), source `bc83fe030234439ac51279502a28803958963e2e`, branch `codex/legacy-document-pre-fix-skill`. [Published baseline report](https://d1p6rlowie26tp.cloudfront.net/runner-e2e/campaigns/gha-37060888047-1/index.html). +- Both trusted default-branch workflows ran concurrently on separate concurrency targets. All eight cells pass their exact-source prerequisite gate before providers; all eight clean up successfully. Failed campaigns are published successfully and remain failed. +- The four model/effort/auth/tool/permission/environment configurations match. Models are `claude-sonnet-4-6` and `openrouter/deepseek/deepseek-v4-flash-0731`. The comparison records 203 byte-identical fixture/behavior sources and explicit present/absent skill-source fingerprints. The new reference is absent from the baseline, rather than silently introduced into it. +- Local admission preparation passed 909 E2E support tests, E2E typecheck and four-cell discovery; the candidate exact-source gate passed 571 checks, including one Rust check. Initial sandbox and obsolete catalog-count attempts remain retained. The original request and behavioral oracle were not weakened. + +## What OpenCode actually received + +The repaired original assignment invokes only its assigned Context integrity output skill, then writes `task-document.md`. There is no operational Paperclip skill load or issue-document reference read before that local write. A separate automatic `issue_disposition_repair` run then loads Paperclip. Its returned content contains the new early saved-document paragraph and reference pointer even though the overall skill result is marked truncated; it never reads the reference or saves a document, and only finalizes the issue's status. + +This is evidence of operational-skill selection after work, not evidence that the assignment saw and ignored the new recipe. The pre-fix original run also writes locally before loading the old Paperclip skill. Whether the adapter should activate operational Paperclip guidance before work is a production-delivery question for review; expanding a skill the first assignment never opened would not address the observed sequence. + +The explicit candidate loads the assigned skill, then Paperclip, receives the new early paragraph, reads `issue-documents.md`, and calls the document API successfully. Its saved document has a persisted revision. The remaining defect is its comment's non-clickable path. Baseline supplied an actual Markdown link to the issue-document API GET endpoint. Read-only source review confirms that this endpoint returns the document in board context, and the unprefixed UI issue route redirects through the selected company's prefix while preserving the document hash. Actual authenticated browser navigation against these cleaned-up measured instances was not replayed, so neither URL is classified as a proven broken route. Candidate's missing clickable anchor is directly observable. + +A future explicit fixture should name a clickable Paperclip UI document link if that is the intended contract. This clarification would not retroactively pass or fail either original result. A short canonical Markdown UI-link example can be considered separately from operational-skill delivery. No further production instruction change or paid rerun was made for this diagnosis. + +## Runs, timing and cost + +Eight provider turns were expected; nine actual runs are retained. Candidate OpenCode's original assignment succeeds but leaves the task `in_progress`, triggering automatic disposition recovery. The recovery succeeds and marks it done, without creating the missing document. Both runs, their time and cost are counted rather than selecting a better attempt. + +| Profile | Case | Baseline provider seconds | Candidate provider seconds | Baseline reported USD | Candidate reported USD | +| --- | --- | ---: | ---: | ---: | ---: | +| legacy-claude | Original | 23.032 | 27.702 | 0.1668394500 | 0.2267947500 | +| legacy-claude | Explicit storage | 34.999 | 40.513 | 0.1997571000 | 0.2561425500 | +| legacy-opencode | Original | 36.770 | 68.032 | 0.0033927116 | 0.0047148437 | +| legacy-opencode | Explicit storage | 206.191 | 46.300 | 0.0102644593 | 0.0042468724 | + +Reported LLM totals are $0.3802537209 baseline and $0.4918990161 candidate. All four baseline and five candidate ledger rows contain token usage and reported cost. Local runtime is unmetered and actual external billing is not established. Single trials, the additional recovery and baseline OpenCode's long explicit-case execution prevent a general timing or spending conclusion. + +## PR maintenance and remaining qualifications + +The skill insertion shifted generated capability heading anchors. General PR CI detected stale `capabilities.yaml`; the paid workflow's narrower build and every exact-source admission still passed before provider access. Both canonical capability metadata inventories are regenerated after measurement, include the new reference where declared, and adds a credential-free admission test that rejects stale/missing manifests. These maintenance changes do not modify the frozen measured branches or the skill bytes used in this comparison. + +Draft [PR #14948](https://github.com/paperclipai/paperclip/pull/14948) keeps this repair and the tiny manual; draft [PR #14961](https://github.com/paperclipai/paperclip/pull/14961) measures native completion documentation separately. OpenCode original delivery remains unresolved, and existing Claude chat-memory and legacy ACP credential/receipt findings remain separate. No broad matrix rerun was launched, and merging remains user-controlled. diff --git a/doc/plans/2026-10-02-stock-harness-paperclip-checklist.md b/doc/plans/2026-10-02-stock-harness-paperclip-checklist.md index ee4d44b940..4f0f7f5827 100644 --- a/doc/plans/2026-10-02-stock-harness-paperclip-checklist.md +++ b/doc/plans/2026-10-02-stock-harness-paperclip-checklist.md @@ -414,15 +414,17 @@ will address them. Record intentional behavior explicitly rather than as a bug. | 2026-10-02 | Dotta requested executable eval coverage for everything implemented so far and all subsequent changes before continuing. | Added SH-1–SH-3 coverage map, credential-free prerequisite, and 24 production-default-hire cells. 478 prerequisite and 860 support tests passed; E2E typecheck/discovery passed. Live provider results remain `not_run`; item 2.2/2.3 unchanged. | | 2026-10-02 | Dotta requested PRs and GitHub-runner before/after qualification while discussing subsequent work separately. | Draft [PR #14948](https://github.com/paperclipai/paperclip/pull/14948); native Codex diagnostic [passed](https://github.com/paperclipai/paperclip/actions/runs/37034213743), 34.745 s provider / 56.660 s cell. Comparison restores only the prior manual/shared prompts and holds #14920 constant; no general coding-quality claim. | | 2026-10-02 | Full candidate cold setup failed before provider admission; stopped and retained the attempt. | [Run 37037105491](https://github.com/paperclipai/paperclip/actions/runs/37037105491), target `36e987246`: prerequisite imports lacked the plugin SDK build. Added ordinary dependency setup before credential-free prerequisites and exact-source coverage of connection guidance. Cold pilot and full matched campaigns pending. | - -For each completed item, add the chosen behavior, changed paths, verification -results, remaining exceptions, and follow-ups here before checking it off. | 2026-10-02 | Dotta deferred 2.2 additional legacy carriers and approved the first 2.3 native tool-description slice separately. | Keep wrappers open; native finish/block documentation must not be supplied to legacy skill/API completion paths. | | 2026-10-02 | Matched default-manual/shared-prompt campaigns measured failures, with #14920 constant. | Candidate `f02d8d0df`: 15/24 pass. Historical `12c5433c6`: 15/24 pass after the one runner-shutdown recovery also timed out. Classic Claude/OpenCode Paperclip document delivery regressed in observed trials; keep PR #14948 draft. [Live report](2026-10-02-stock-harness-live-comparison.md). | | 2026-10-02 | Cold prerequisite and packaging faults were repaired without weakening admission or behavioral graders. | Three setup attempts stopped before providers. Current `1eb5ba420` pilot passes 557 prerequisite checks and protected report publication; source/hash/cost evidence retained. Candidate cancellation and historical runner shutdown recovered only for missing cells. | | 2026-10-02 | Final bounded recovery completed; no further model reruns. | Both matched cohorts have all 24 results, 15 pass and nine fail. Two classic skill deliveries regress; two OpenCode ordered cases pass only with reduced instructions. Overall parity is not behavioral equivalence. All 48 retained result/receipt projections are hashed and sanitized; original interruptions and partial unknown spend remain recorded. | -| 2026-10-02 | Dotta approved the measured legacy delivery repair and narrow follow-up qualification. | Early operational skill PUT/receipt/link guidance plus generic issue-document reference; eight-word manual retained. Focused original + explicit Paperclip-storage cases on classic Claude/OpenCode, with only the two skill sources varied. Source freeze/admission pending; no full matrix rerun. Claude chat-memory (F15) and ACP credential/receipt failures (F16) remain separate and unresolved. | -| 2026-10-02 | Native tool-description comparison completed all six paired cases. | Zero newly failing cases, three unchanged completion passes, three unchanged blocker failures. Claude/OpenCode blocker API matchers pass in both variants; UI matcher wrongly demanded marker-only replies. Codex's exact-action punctuation failure is unchanged. Original failures retained; provider-free grader correction/calibration in progress, no native fixed-prompt removal measured. | +| 2026-10-02 | Dotta approved the measured legacy delivery repair and narrow follow-up qualification. | Early operational skill PUT/receipt/link guidance plus generic issue-document reference; eight-word manual retained. Focused original + explicit Paperclip-storage cases on classic Claude/OpenCode, with only the two skill sources varied. All four pairs completed: Claude original Fail → Pass, Claude explicit Pass → Pass, both OpenCode cases Fail → Fail. Explicit OpenCode handoff worsened beneath the unchanged machine grade (clickable API URL → code-formatted path). [Repair report](2026-10-02-legacy-document-skill-repair.md); no full matrix rerun. Claude chat-memory (F15) and ACP credential/receipt failures (F16) remain separate and unresolved. | +| 2026-10-02 | Native tool-description comparison completed all six paired cases. | Zero newly failing cases, three unchanged completion passes, three unchanged blocker failures. Claude/OpenCode blocker API matchers pass in both variants; UI matcher wrongly demanded marker-only replies. Codex's exact-action punctuation failure is unchanged. Original failures retained; corrected 10-case browser calibration and separate retained-DOM replay pass all six visible replies. Exact Codex action failures remain. No native fixed-prompt removal measured. | - F18: Native blocker browser assertion required a marker-only reply despite asking for owner/action/reason. Correct marker-plus-explanation checks symmetrically, calibrate contradictory and future-condition replies, retain original verdicts. -- F19: Both failed classic agents loaded Paperclip; existing OpenCode skill truncation omits the late document recipe. Early skill delivery recipe approved; discovery pointers and restoring the manual are unnecessary for this observed issue. +- F19: Original prompt-removal cohorts loaded Paperclip; OpenCode skill truncation omitted the late API recipe. The early repair fixed Claude in one paired trial. In the repaired original OpenCode assignment, Paperclip was first loaded only during disposition recovery after a local-file write. Shared legacy operational-skill delivery/selection needs review before more recipe expansion. +- F20: Explicit repaired OpenCode reads the early recipe/reference and saves a public document/revision, but hands off a code-formatted path rather than a clickable anchor. Baseline provides a clickable API URL, rejected by the UI-only oracle. Preserve both grades; clarify future clickable UI-link fixture and review the smallest link example separately. +- F21: Skill heading insertion shifted both generated capability inventories. General CI caught stale metadata after the frozen repair campaign’s narrower build admitted providers. Regenerate canonical derived files and require stale-manifest/inventory checks before future provider admission. + +For each completed item, add the chosen behavior, changed paths, verification +results, remaining exceptions, and follow-ups here before checking it off. diff --git a/packages/paperclip-runner/docs/capability-contract.md b/packages/paperclip-runner/docs/capability-contract.md index 15e6d7d4c1..9bc3d85c41 100644 --- a/packages/paperclip-runner/docs/capability-contract.md +++ b/packages/paperclip-runner/docs/capability-contract.md @@ -43,35 +43,35 @@ The skill/reference inventory and eval cases are the only normative behavior sou | --- | --- | --- | | skill:skills/paperclip/SKILL.md:paperclip-skill:10 | optional_agent_tool | skills/paperclip/SKILL.md:10 | | skill:skills/paperclip/SKILL.md:terminology:14 | optional_agent_tool | skills/paperclip/SKILL.md:14 | -| skill:skills/paperclip/SKILL.md:authentication:18 | control_plane_owned | skills/paperclip/SKILL.md:18 | -| skill:skills/paperclip/SKILL.md:conversation-tasks:30 | optional_agent_tool | skills/paperclip/SKILL.md:30 | -| skill:skills/paperclip/SKILL.md:server-verified-external-chat-turns:47 | control_plane_owned | skills/paperclip/SKILL.md:47 | -| skill:skills/paperclip/SKILL.md:the-heartbeat-procedure:87 | optional_agent_tool | skills/paperclip/SKILL.md:87 | -| skill:skills/paperclip/SKILL.md:generated-artifacts-and-work-products:196 | always_agent_tool | skills/paperclip/SKILL.md:196 | -| skill:skills/paperclip/SKILL.md:status-quick-guide:244 | control_plane_owned | skills/paperclip/SKILL.md:244 | -| skill:skills/paperclip/SKILL.md:monitors-and-watchers-say-only-what-you-actually-scheduled:254 | optional_agent_tool | skills/paperclip/SKILL.md:254 | -| skill:skills/paperclip/SKILL.md:delegating-review-tasks:267 | always_agent_tool | skills/paperclip/SKILL.md:267 | -| skill:skills/paperclip/SKILL.md:managing-a-user-s-inbox:278 | control_plane_owned | skills/paperclip/SKILL.md:278 | -| skill:skills/paperclip/SKILL.md:issue-dependencies-blockers:286 | control_plane_owned | skills/paperclip/SKILL.md:286 | -| skill:skills/paperclip/SKILL.md:requesting-board-approval:311 | optional_agent_tool | skills/paperclip/SKILL.md:311 | -| skill:skills/paperclip/SKILL.md:issue-thread-interactions:332 | optional_agent_tool | skills/paperclip/SKILL.md:332 | -| skill:skills/paperclip/SKILL.md:standalone-decisions:361 | optional_agent_tool | skills/paperclip/SKILL.md:361 | -| skill:skills/paperclip/SKILL.md:mcp-tool-approval-gates:465 | optional_agent_tool | skills/paperclip/SKILL.md:465 | -| skill:skills/paperclip/SKILL.md:niche-workflow-pointers:507 | optional_agent_tool | skills/paperclip/SKILL.md:507 | -| skill:skills/paperclip/SKILL.md:cases:517 | optional_agent_tool | skills/paperclip/SKILL.md:517 | -| skill:skills/paperclip/SKILL.md:company-skills-workflow:522 | optional_agent_tool | skills/paperclip/SKILL.md:522 | -| skill:skills/paperclip/SKILL.md:routines:533 | optional_agent_tool | skills/paperclip/SKILL.md:533 | -| skill:skills/paperclip/SKILL.md:issue-workspace-runtime-controls:544 | optional_agent_tool | skills/paperclip/SKILL.md:544 | -| skill:skills/paperclip/SKILL.md:proposing-credentials-safely:551 | optional_agent_tool | skills/paperclip/SKILL.md:551 | -| skill:skills/paperclip/SKILL.md:reading-granted-secrets:558 | optional_agent_tool | skills/paperclip/SKILL.md:558 | -| skill:skills/paperclip/SKILL.md:critical-rules:584 | optional_agent_tool | skills/paperclip/SKILL.md:584 | -| skill:skills/paperclip/SKILL.md:comment-style-required:605 | always_agent_tool | skills/paperclip/SKILL.md:605 | -| skill:skills/paperclip/SKILL.md:update:637 | optional_agent_tool | skills/paperclip/SKILL.md:637 | -| skill:skills/paperclip/SKILL.md:planning-required-when-planning-requested:647 | optional_agent_tool | skills/paperclip/SKILL.md:647 | -| skill:skills/paperclip/SKILL.md:key-endpoints-hot-routes:680 | optional_agent_tool | skills/paperclip/SKILL.md:680 | -| skill:skills/paperclip/SKILL.md:searching-issues:709 | optional_agent_tool | skills/paperclip/SKILL.md:709 | -| skill:skills/paperclip/SKILL.md:full-reference:719 | optional_agent_tool | skills/paperclip/SKILL.md:719 | -| skill:skills/paperclip/SKILL.md:conversational-confirmation-answers:723 | always_agent_tool | skills/paperclip/SKILL.md:723 | +| skill:skills/paperclip/SKILL.md:authentication:20 | control_plane_owned | skills/paperclip/SKILL.md:20 | +| skill:skills/paperclip/SKILL.md:conversation-tasks:32 | optional_agent_tool | skills/paperclip/SKILL.md:32 | +| skill:skills/paperclip/SKILL.md:server-verified-external-chat-turns:49 | control_plane_owned | skills/paperclip/SKILL.md:49 | +| skill:skills/paperclip/SKILL.md:the-heartbeat-procedure:89 | optional_agent_tool | skills/paperclip/SKILL.md:89 | +| skill:skills/paperclip/SKILL.md:generated-artifacts-and-work-products:198 | always_agent_tool | skills/paperclip/SKILL.md:198 | +| skill:skills/paperclip/SKILL.md:status-quick-guide:246 | control_plane_owned | skills/paperclip/SKILL.md:246 | +| skill:skills/paperclip/SKILL.md:monitors-and-watchers-say-only-what-you-actually-scheduled:256 | optional_agent_tool | skills/paperclip/SKILL.md:256 | +| skill:skills/paperclip/SKILL.md:delegating-review-tasks:269 | always_agent_tool | skills/paperclip/SKILL.md:269 | +| skill:skills/paperclip/SKILL.md:managing-a-user-s-inbox:280 | control_plane_owned | skills/paperclip/SKILL.md:280 | +| skill:skills/paperclip/SKILL.md:issue-dependencies-blockers:288 | control_plane_owned | skills/paperclip/SKILL.md:288 | +| skill:skills/paperclip/SKILL.md:requesting-board-approval:313 | optional_agent_tool | skills/paperclip/SKILL.md:313 | +| skill:skills/paperclip/SKILL.md:issue-thread-interactions:334 | optional_agent_tool | skills/paperclip/SKILL.md:334 | +| skill:skills/paperclip/SKILL.md:standalone-decisions:363 | optional_agent_tool | skills/paperclip/SKILL.md:363 | +| skill:skills/paperclip/SKILL.md:mcp-tool-approval-gates:467 | optional_agent_tool | skills/paperclip/SKILL.md:467 | +| skill:skills/paperclip/SKILL.md:niche-workflow-pointers:509 | optional_agent_tool | skills/paperclip/SKILL.md:509 | +| skill:skills/paperclip/SKILL.md:cases:519 | optional_agent_tool | skills/paperclip/SKILL.md:519 | +| skill:skills/paperclip/SKILL.md:company-skills-workflow:524 | optional_agent_tool | skills/paperclip/SKILL.md:524 | +| skill:skills/paperclip/SKILL.md:routines:535 | optional_agent_tool | skills/paperclip/SKILL.md:535 | +| skill:skills/paperclip/SKILL.md:issue-workspace-runtime-controls:546 | optional_agent_tool | skills/paperclip/SKILL.md:546 | +| skill:skills/paperclip/SKILL.md:proposing-credentials-safely:553 | optional_agent_tool | skills/paperclip/SKILL.md:553 | +| skill:skills/paperclip/SKILL.md:reading-granted-secrets:560 | optional_agent_tool | skills/paperclip/SKILL.md:560 | +| skill:skills/paperclip/SKILL.md:critical-rules:586 | optional_agent_tool | skills/paperclip/SKILL.md:586 | +| skill:skills/paperclip/SKILL.md:comment-style-required:607 | always_agent_tool | skills/paperclip/SKILL.md:607 | +| skill:skills/paperclip/SKILL.md:update:639 | optional_agent_tool | skills/paperclip/SKILL.md:639 | +| skill:skills/paperclip/SKILL.md:planning-required-when-planning-requested:649 | optional_agent_tool | skills/paperclip/SKILL.md:649 | +| skill:skills/paperclip/SKILL.md:key-endpoints-hot-routes:682 | optional_agent_tool | skills/paperclip/SKILL.md:682 | +| skill:skills/paperclip/SKILL.md:searching-issues:711 | optional_agent_tool | skills/paperclip/SKILL.md:711 | +| skill:skills/paperclip/SKILL.md:full-reference:721 | optional_agent_tool | skills/paperclip/SKILL.md:721 | +| skill:skills/paperclip/SKILL.md:conversational-confirmation-answers:725 | always_agent_tool | skills/paperclip/SKILL.md:725 | | skill:skills/paperclip/references/artifacts.md:generated-artifacts-and-work-products:1 | always_agent_tool | skills/paperclip/references/artifacts.md:1 | | skill:skills/paperclip/references/artifacts.md:workspace-only-file-references:15 | optional_agent_tool | skills/paperclip/references/artifacts.md:15 | | skill:skills/paperclip/references/cases.md:cases:1 | optional_agent_tool | skills/paperclip/references/cases.md:1 | diff --git a/packages/paperclip-runner/generated/capability/capabilities.yaml b/packages/paperclip-runner/generated/capability/capabilities.yaml index 61bba278fd..2142c6e634 100644 --- a/packages/paperclip-runner/generated/capability/capabilities.yaml +++ b/packages/paperclip-runner/generated/capability/capabilities.yaml @@ -20,261 +20,261 @@ "expectedMockState": "runtime_decision_record" }, { - "id": "skill:skills/paperclip/SKILL.md:18", + "id": "skill:skills/paperclip/SKILL.md:20", "kind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md#L18:authentication", + "sourceAnchor": "skills/paperclip/SKILL.md#L20:authentication", "heading": "Authentication", "primaryDisposition": "control_plane_owned", "semanticOperation": "runtime_reconciliation", "expectedMockState": "runtime_decision_record" }, { - "id": "skill:skills/paperclip/SKILL.md:30", + "id": "skill:skills/paperclip/SKILL.md:32", "kind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md#L30:conversation-tasks", + "sourceAnchor": "skills/paperclip/SKILL.md#L32:conversation-tasks", "heading": "Conversation tasks", "primaryDisposition": "control_plane_owned", "semanticOperation": "runtime_reconciliation", "expectedMockState": "runtime_decision_record" }, { - "id": "skill:skills/paperclip/SKILL.md:47", + "id": "skill:skills/paperclip/SKILL.md:49", "kind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md#L47:server-verified-external-chat-turns", + "sourceAnchor": "skills/paperclip/SKILL.md#L49:server-verified-external-chat-turns", "heading": "Server-Verified External Chat Turns", "primaryDisposition": "control_plane_owned", "semanticOperation": "runtime_reconciliation", "expectedMockState": "runtime_decision_record" }, { - "id": "skill:skills/paperclip/SKILL.md:87", + "id": "skill:skills/paperclip/SKILL.md:89", "kind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md#L87:the-heartbeat-procedure", + "sourceAnchor": "skills/paperclip/SKILL.md#L89:the-heartbeat-procedure", "heading": "The Heartbeat Procedure", "primaryDisposition": "control_plane_owned", "semanticOperation": "runtime_reconciliation", "expectedMockState": "runtime_decision_record" }, { - "id": "skill:skills/paperclip/SKILL.md:196", + "id": "skill:skills/paperclip/SKILL.md:198", "kind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md#L196:generated-artifacts-and-work-products", + "sourceAnchor": "skills/paperclip/SKILL.md#L198:generated-artifacts-and-work-products", "heading": "Generated Artifacts and Work Products", "primaryDisposition": "always_agent_tool", "semanticOperation": "register_deliverable", "expectedMockState": "operation_result" }, { - "id": "skill:skills/paperclip/SKILL.md:244", + "id": "skill:skills/paperclip/SKILL.md:246", "kind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md#L244:status-quick-guide", + "sourceAnchor": "skills/paperclip/SKILL.md#L246:status-quick-guide", "heading": "Status Quick Guide", "primaryDisposition": "control_plane_owned", "semanticOperation": "runtime_reconciliation", "expectedMockState": "runtime_decision_record" }, { - "id": "skill:skills/paperclip/SKILL.md:254", + "id": "skill:skills/paperclip/SKILL.md:256", "kind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md#L254:monitors-and-watchers-say-only-what-you-actually-scheduled", + "sourceAnchor": "skills/paperclip/SKILL.md#L256:monitors-and-watchers-say-only-what-you-actually-scheduled", "heading": "Monitors and Watchers (say only what you actually scheduled)", "primaryDisposition": "control_plane_owned", "semanticOperation": "runtime_reconciliation", "expectedMockState": "runtime_decision_record" }, { - "id": "skill:skills/paperclip/SKILL.md:267", + "id": "skill:skills/paperclip/SKILL.md:269", "kind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md#L267:delegating-review-tasks", + "sourceAnchor": "skills/paperclip/SKILL.md#L269:delegating-review-tasks", "heading": "Delegating review tasks", "primaryDisposition": "control_plane_owned", "semanticOperation": "runtime_reconciliation", "expectedMockState": "runtime_decision_record" }, { - "id": "skill:skills/paperclip/SKILL.md:278", + "id": "skill:skills/paperclip/SKILL.md:280", "kind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md#L278:managing-a-user-s-inbox", + "sourceAnchor": "skills/paperclip/SKILL.md#L280:managing-a-user-s-inbox", "heading": "Managing A User's Inbox", "primaryDisposition": "control_plane_owned", "semanticOperation": "runtime_reconciliation", "expectedMockState": "runtime_decision_record" }, { - "id": "skill:skills/paperclip/SKILL.md:286", + "id": "skill:skills/paperclip/SKILL.md:288", "kind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md#L286:issue-dependencies-blockers", + "sourceAnchor": "skills/paperclip/SKILL.md#L288:issue-dependencies-blockers", "heading": "Issue Dependencies (Blockers)", "primaryDisposition": "control_plane_owned", "semanticOperation": "runtime_reconciliation", "expectedMockState": "runtime_decision_record" }, { - "id": "skill:skills/paperclip/SKILL.md:311", + "id": "skill:skills/paperclip/SKILL.md:313", "kind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md#L311:requesting-board-approval", + "sourceAnchor": "skills/paperclip/SKILL.md#L313:requesting-board-approval", "heading": "Requesting Board Approval", "primaryDisposition": "optional_agent_tool", "semanticOperation": "scoped_discovery", "expectedMockState": "operation_result" }, { - "id": "skill:skills/paperclip/SKILL.md:332", + "id": "skill:skills/paperclip/SKILL.md:334", "kind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md#L332:issue-thread-interactions", + "sourceAnchor": "skills/paperclip/SKILL.md#L334:issue-thread-interactions", "heading": "Issue-Thread Interactions", "primaryDisposition": "always_agent_tool", "semanticOperation": "request_human_input", "expectedMockState": "operation_result" }, { - "id": "skill:skills/paperclip/SKILL.md:361", + "id": "skill:skills/paperclip/SKILL.md:363", "kind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md#L361:standalone-decisions", + "sourceAnchor": "skills/paperclip/SKILL.md#L363:standalone-decisions", "heading": "Standalone Decisions", "primaryDisposition": "control_plane_owned", "semanticOperation": "runtime_reconciliation", "expectedMockState": "runtime_decision_record" }, { - "id": "skill:skills/paperclip/SKILL.md:465", + "id": "skill:skills/paperclip/SKILL.md:467", "kind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md#L465:mcp-tool-approval-gates", + "sourceAnchor": "skills/paperclip/SKILL.md#L467:mcp-tool-approval-gates", "heading": "MCP Tool Approval Gates", "primaryDisposition": "optional_agent_tool", "semanticOperation": "scoped_discovery", "expectedMockState": "operation_result" }, { - "id": "skill:skills/paperclip/SKILL.md:507", + "id": "skill:skills/paperclip/SKILL.md:509", "kind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md#L507:niche-workflow-pointers", + "sourceAnchor": "skills/paperclip/SKILL.md#L509:niche-workflow-pointers", "heading": "Niche Workflow Pointers", "primaryDisposition": "control_plane_owned", "semanticOperation": "runtime_reconciliation", "expectedMockState": "runtime_decision_record" }, { - "id": "skill:skills/paperclip/SKILL.md:517", + "id": "skill:skills/paperclip/SKILL.md:519", "kind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md#L517:cases", + "sourceAnchor": "skills/paperclip/SKILL.md#L519:cases", "heading": "Cases", "primaryDisposition": "optional_agent_tool", "semanticOperation": "scoped_discovery", "expectedMockState": "operation_result" }, { - "id": "skill:skills/paperclip/SKILL.md:522", + "id": "skill:skills/paperclip/SKILL.md:524", "kind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md#L522:company-skills-workflow", + "sourceAnchor": "skills/paperclip/SKILL.md#L524:company-skills-workflow", "heading": "Company Skills Workflow", "primaryDisposition": "optional_agent_tool", "semanticOperation": "scoped_discovery", "expectedMockState": "operation_result" }, { - "id": "skill:skills/paperclip/SKILL.md:533", + "id": "skill:skills/paperclip/SKILL.md:535", "kind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md#L533:routines", + "sourceAnchor": "skills/paperclip/SKILL.md#L535:routines", "heading": "Routines", "primaryDisposition": "optional_agent_tool", "semanticOperation": "scoped_discovery", "expectedMockState": "operation_result" }, { - "id": "skill:skills/paperclip/SKILL.md:544", + "id": "skill:skills/paperclip/SKILL.md:546", "kind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md#L544:issue-workspace-runtime-controls", + "sourceAnchor": "skills/paperclip/SKILL.md#L546:issue-workspace-runtime-controls", "heading": "Issue Workspace Runtime Controls", "primaryDisposition": "optional_agent_tool", "semanticOperation": "scoped_discovery", "expectedMockState": "operation_result" }, { - "id": "skill:skills/paperclip/SKILL.md:551", + "id": "skill:skills/paperclip/SKILL.md:553", "kind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md#L551:proposing-credentials-safely", + "sourceAnchor": "skills/paperclip/SKILL.md#L553:proposing-credentials-safely", "heading": "Proposing Credentials Safely", "primaryDisposition": "control_plane_owned", "semanticOperation": "runtime_reconciliation", "expectedMockState": "runtime_decision_record" }, { - "id": "skill:skills/paperclip/SKILL.md:558", + "id": "skill:skills/paperclip/SKILL.md:560", "kind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md#L558:reading-granted-secrets", + "sourceAnchor": "skills/paperclip/SKILL.md#L560:reading-granted-secrets", "heading": "Reading Granted Secrets", "primaryDisposition": "optional_agent_tool", "semanticOperation": "scoped_discovery", "expectedMockState": "operation_result" }, { - "id": "skill:skills/paperclip/SKILL.md:584", + "id": "skill:skills/paperclip/SKILL.md:586", "kind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md#L584:critical-rules", + "sourceAnchor": "skills/paperclip/SKILL.md#L586:critical-rules", "heading": "Critical Rules", "primaryDisposition": "control_plane_owned", "semanticOperation": "runtime_reconciliation", "expectedMockState": "runtime_decision_record" }, { - "id": "skill:skills/paperclip/SKILL.md:605", + "id": "skill:skills/paperclip/SKILL.md:607", "kind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md#L605:comment-style-required", + "sourceAnchor": "skills/paperclip/SKILL.md#L607:comment-style-required", "heading": "Comment Style (Required)", "primaryDisposition": "always_agent_tool", "semanticOperation": "report_progress", "expectedMockState": "operation_result" }, { - "id": "skill:skills/paperclip/SKILL.md:637", + "id": "skill:skills/paperclip/SKILL.md:639", "kind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md#L637:update", + "sourceAnchor": "skills/paperclip/SKILL.md#L639:update", "heading": "Update", "primaryDisposition": "control_plane_owned", "semanticOperation": "runtime_reconciliation", "expectedMockState": "runtime_decision_record" }, { - "id": "skill:skills/paperclip/SKILL.md:647", + "id": "skill:skills/paperclip/SKILL.md:649", "kind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md#L647:planning-required-when-planning-requested", + "sourceAnchor": "skills/paperclip/SKILL.md#L649:planning-required-when-planning-requested", "heading": "Planning (Required when planning requested)", "primaryDisposition": "always_agent_tool", "semanticOperation": "write_document", "expectedMockState": "operation_result" }, { - "id": "skill:skills/paperclip/SKILL.md:680", + "id": "skill:skills/paperclip/SKILL.md:682", "kind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md#L680:key-endpoints-hot-routes", + "sourceAnchor": "skills/paperclip/SKILL.md#L682:key-endpoints-hot-routes", "heading": "Key Endpoints (Hot Routes)", "primaryDisposition": "control_plane_owned", "semanticOperation": "runtime_reconciliation", "expectedMockState": "runtime_decision_record" }, { - "id": "skill:skills/paperclip/SKILL.md:709", + "id": "skill:skills/paperclip/SKILL.md:711", "kind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md#L709:searching-issues", + "sourceAnchor": "skills/paperclip/SKILL.md#L711:searching-issues", "heading": "Searching Issues", "primaryDisposition": "control_plane_owned", "semanticOperation": "runtime_reconciliation", "expectedMockState": "runtime_decision_record" }, { - "id": "skill:skills/paperclip/SKILL.md:719", + "id": "skill:skills/paperclip/SKILL.md:721", "kind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md#L719:full-reference", + "sourceAnchor": "skills/paperclip/SKILL.md#L721:full-reference", "heading": "Full Reference", "primaryDisposition": "control_plane_owned", "semanticOperation": "runtime_reconciliation", "expectedMockState": "runtime_decision_record" }, { - "id": "skill:skills/paperclip/SKILL.md:723", + "id": "skill:skills/paperclip/SKILL.md:725", "kind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md#L723:conversational-confirmation-answers", + "sourceAnchor": "skills/paperclip/SKILL.md#L725:conversational-confirmation-answers", "heading": "Conversational confirmation answers", "primaryDisposition": "always_agent_tool", "semanticOperation": "call_api", @@ -1162,6 +1162,33 @@ "semanticOperation": "runtime_reconciliation", "expectedMockState": "runtime_decision_record" }, + { + "id": "skill:skills/paperclip/references/issue-documents.md:1", + "kind": "skill_heading", + "sourceAnchor": "skills/paperclip/references/issue-documents.md#L1:issue-documents-through-the-api", + "heading": "Issue documents through the API", + "primaryDisposition": "always_agent_tool", + "semanticOperation": "write_document", + "expectedMockState": "operation_result" + }, + { + "id": "skill:skills/paperclip/references/issue-documents.md:8", + "kind": "skill_heading", + "sourceAnchor": "skills/paperclip/references/issue-documents.md#L8:create-a-document", + "heading": "Create a document", + "primaryDisposition": "always_agent_tool", + "semanticOperation": "write_document", + "expectedMockState": "operation_result" + }, + { + "id": "skill:skills/paperclip/references/issue-documents.md:40", + "kind": "skill_heading", + "sourceAnchor": "skills/paperclip/references/issue-documents.md#L40:update-or-resolve-an-unclear-write", + "heading": "Update or resolve an unclear write", + "primaryDisposition": "control_plane_owned", + "semanticOperation": "runtime_reconciliation", + "expectedMockState": "runtime_decision_record" + }, { "id": "skill:skills/paperclip/references/issue-workspaces.md:1", "kind": "skill_heading", diff --git a/packages/paperclip-runner/generated/capability/capability-contract.md b/packages/paperclip-runner/generated/capability/capability-contract.md index 907bf749bd..2c201ef5cd 100644 --- a/packages/paperclip-runner/generated/capability/capability-contract.md +++ b/packages/paperclip-runner/generated/capability/capability-contract.md @@ -2,9 +2,9 @@ Generated by `scripts/generate-capability-contract.mjs`; do not edit generated files. -- Skill/reference headings: 158 +- Skill/reference headings: 161 - Legacy MCP tools: 42 - Eval cases: 106 across 16 groups -- Deterministic content SHA-256: `2707de482cfbe6c7a483f6dee3bf4a2fc66e2b944bb50c3818ca6d5ef777b1de` +- Deterministic content SHA-256: `02d00ef8016554867dddb8200a2f0035dbad387945e4f49ef5fd875318e69599` Every row has exactly one primary disposition, a source anchor, a semantic operation, and a mock-state expectation. diff --git a/packages/paperclip-runner/scripts/generate-capability-contract.d.mts b/packages/paperclip-runner/scripts/generate-capability-contract.d.mts new file mode 100644 index 0000000000..401886022f --- /dev/null +++ b/packages/paperclip-runner/scripts/generate-capability-contract.d.mts @@ -0,0 +1 @@ +export function buildContract(): Promise>; diff --git a/packages/paperclip-runner/scripts/generate-capability-contract.mjs b/packages/paperclip-runner/scripts/generate-capability-contract.mjs index 840b94521c..7903efd7ca 100644 --- a/packages/paperclip-runner/scripts/generate-capability-contract.mjs +++ b/packages/paperclip-runner/scripts/generate-capability-contract.mjs @@ -159,7 +159,7 @@ function renderHandoff() { ].join("\n") + "\n"; } -async function buildContract() { +export async function buildContract() { const contract = JSON.parse(await readFile(contractPath, "utf8")); const capabilities = await readSkillHeadings(contract.skillSources); const discoveredTools = await readLegacyTools(); diff --git a/packages/paperclip-runner/spec/capability/capabilities.yaml b/packages/paperclip-runner/spec/capability/capabilities.yaml index 5e07d65358..fc458728c5 100644 --- a/packages/paperclip-runner/spec/capability/capabilities.yaml +++ b/packages/paperclip-runner/spec/capability/capabilities.yaml @@ -44,9 +44,9 @@ ] }, { - "id": "skill:skills/paperclip/SKILL.md:authentication:18", + "id": "skill:skills/paperclip/SKILL.md:authentication:20", "sourceKind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md:18", + "sourceAnchor": "skills/paperclip/SKILL.md:20", "title": "Authentication", "expectedSemantics": "Skill guidance headed “Authentication”.", "primaryDisposition": "control_plane_owned", @@ -55,13 +55,13 @@ "control_plane_invariant" ], "evidenceIds": [ - "skill:skills/paperclip/SKILL.md:18" + "skill:skills/paperclip/SKILL.md:20" ] }, { - "id": "skill:skills/paperclip/SKILL.md:conversation-tasks:30", + "id": "skill:skills/paperclip/SKILL.md:conversation-tasks:32", "sourceKind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md:30", + "sourceAnchor": "skills/paperclip/SKILL.md:32", "title": "Conversation tasks", "expectedSemantics": "Skill guidance headed “Conversation tasks”.", "primaryDisposition": "optional_agent_tool", @@ -70,13 +70,13 @@ "control_plane_invariant" ], "evidenceIds": [ - "skill:skills/paperclip/SKILL.md:30" + "skill:skills/paperclip/SKILL.md:32" ] }, { - "id": "skill:skills/paperclip/SKILL.md:server-verified-external-chat-turns:47", + "id": "skill:skills/paperclip/SKILL.md:server-verified-external-chat-turns:49", "sourceKind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md:47", + "sourceAnchor": "skills/paperclip/SKILL.md:49", "title": "Server-Verified External Chat Turns", "expectedSemantics": "Skill guidance headed “Server-Verified External Chat Turns”.", "primaryDisposition": "control_plane_owned", @@ -85,13 +85,13 @@ "control_plane_invariant" ], "evidenceIds": [ - "skill:skills/paperclip/SKILL.md:47" + "skill:skills/paperclip/SKILL.md:49" ] }, { - "id": "skill:skills/paperclip/SKILL.md:the-heartbeat-procedure:87", + "id": "skill:skills/paperclip/SKILL.md:the-heartbeat-procedure:89", "sourceKind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md:87", + "sourceAnchor": "skills/paperclip/SKILL.md:89", "title": "The Heartbeat Procedure", "expectedSemantics": "Skill guidance headed “The Heartbeat Procedure”.", "primaryDisposition": "optional_agent_tool", @@ -100,13 +100,13 @@ "control_plane_invariant" ], "evidenceIds": [ - "skill:skills/paperclip/SKILL.md:87" + "skill:skills/paperclip/SKILL.md:89" ] }, { - "id": "skill:skills/paperclip/SKILL.md:generated-artifacts-and-work-products:196", + "id": "skill:skills/paperclip/SKILL.md:generated-artifacts-and-work-products:198", "sourceKind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md:196", + "sourceAnchor": "skills/paperclip/SKILL.md:198", "title": "Generated Artifacts and Work Products", "expectedSemantics": "Skill guidance headed “Generated Artifacts and Work Products”.", "primaryDisposition": "always_agent_tool", @@ -115,13 +115,13 @@ "control_plane_invariant" ], "evidenceIds": [ - "skill:skills/paperclip/SKILL.md:196" + "skill:skills/paperclip/SKILL.md:198" ] }, { - "id": "skill:skills/paperclip/SKILL.md:status-quick-guide:244", + "id": "skill:skills/paperclip/SKILL.md:status-quick-guide:246", "sourceKind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md:244", + "sourceAnchor": "skills/paperclip/SKILL.md:246", "title": "Status Quick Guide", "expectedSemantics": "Skill guidance headed “Status Quick Guide”.", "primaryDisposition": "control_plane_owned", @@ -130,13 +130,13 @@ "control_plane_invariant" ], "evidenceIds": [ - "skill:skills/paperclip/SKILL.md:244" + "skill:skills/paperclip/SKILL.md:246" ] }, { - "id": "skill:skills/paperclip/SKILL.md:monitors-and-watchers-say-only-what-you-actually-scheduled:254", + "id": "skill:skills/paperclip/SKILL.md:monitors-and-watchers-say-only-what-you-actually-scheduled:256", "sourceKind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md:254", + "sourceAnchor": "skills/paperclip/SKILL.md:256", "title": "Monitors and Watchers (say only what you actually scheduled)", "expectedSemantics": "Skill guidance headed “Monitors and Watchers (say only what you actually scheduled)”.", "primaryDisposition": "optional_agent_tool", @@ -145,13 +145,13 @@ "control_plane_invariant" ], "evidenceIds": [ - "skill:skills/paperclip/SKILL.md:254" + "skill:skills/paperclip/SKILL.md:256" ] }, { - "id": "skill:skills/paperclip/SKILL.md:delegating-review-tasks:267", + "id": "skill:skills/paperclip/SKILL.md:delegating-review-tasks:269", "sourceKind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md:267", + "sourceAnchor": "skills/paperclip/SKILL.md:269", "title": "Delegating review tasks", "expectedSemantics": "Skill guidance headed “Delegating review tasks”.", "primaryDisposition": "always_agent_tool", @@ -160,13 +160,13 @@ "control_plane_invariant" ], "evidenceIds": [ - "skill:skills/paperclip/SKILL.md:267" + "skill:skills/paperclip/SKILL.md:269" ] }, { - "id": "skill:skills/paperclip/SKILL.md:managing-a-user-s-inbox:278", + "id": "skill:skills/paperclip/SKILL.md:managing-a-user-s-inbox:280", "sourceKind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md:278", + "sourceAnchor": "skills/paperclip/SKILL.md:280", "title": "Managing A User's Inbox", "expectedSemantics": "Skill guidance headed “Managing A User's Inbox”.", "primaryDisposition": "control_plane_owned", @@ -175,13 +175,13 @@ "control_plane_invariant" ], "evidenceIds": [ - "skill:skills/paperclip/SKILL.md:278" + "skill:skills/paperclip/SKILL.md:280" ] }, { - "id": "skill:skills/paperclip/SKILL.md:issue-dependencies-blockers:286", + "id": "skill:skills/paperclip/SKILL.md:issue-dependencies-blockers:288", "sourceKind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md:286", + "sourceAnchor": "skills/paperclip/SKILL.md:288", "title": "Issue Dependencies (Blockers)", "expectedSemantics": "Skill guidance headed “Issue Dependencies (Blockers)”.", "primaryDisposition": "control_plane_owned", @@ -190,13 +190,13 @@ "control_plane_invariant" ], "evidenceIds": [ - "skill:skills/paperclip/SKILL.md:286" + "skill:skills/paperclip/SKILL.md:288" ] }, { - "id": "skill:skills/paperclip/SKILL.md:requesting-board-approval:311", + "id": "skill:skills/paperclip/SKILL.md:requesting-board-approval:313", "sourceKind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md:311", + "sourceAnchor": "skills/paperclip/SKILL.md:313", "title": "Requesting Board Approval", "expectedSemantics": "Skill guidance headed “Requesting Board Approval”.", "primaryDisposition": "optional_agent_tool", @@ -205,13 +205,13 @@ "control_plane_invariant" ], "evidenceIds": [ - "skill:skills/paperclip/SKILL.md:311" + "skill:skills/paperclip/SKILL.md:313" ] }, { - "id": "skill:skills/paperclip/SKILL.md:issue-thread-interactions:332", + "id": "skill:skills/paperclip/SKILL.md:issue-thread-interactions:334", "sourceKind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md:332", + "sourceAnchor": "skills/paperclip/SKILL.md:334", "title": "Issue-Thread Interactions", "expectedSemantics": "Skill guidance headed “Issue-Thread Interactions”.", "primaryDisposition": "optional_agent_tool", @@ -220,13 +220,13 @@ "control_plane_invariant" ], "evidenceIds": [ - "skill:skills/paperclip/SKILL.md:332" + "skill:skills/paperclip/SKILL.md:334" ] }, { - "id": "skill:skills/paperclip/SKILL.md:standalone-decisions:361", + "id": "skill:skills/paperclip/SKILL.md:standalone-decisions:363", "sourceKind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md:361", + "sourceAnchor": "skills/paperclip/SKILL.md:363", "title": "Standalone Decisions", "expectedSemantics": "Skill guidance headed “Standalone Decisions”.", "primaryDisposition": "optional_agent_tool", @@ -235,13 +235,13 @@ "control_plane_invariant" ], "evidenceIds": [ - "skill:skills/paperclip/SKILL.md:361" + "skill:skills/paperclip/SKILL.md:363" ] }, { - "id": "skill:skills/paperclip/SKILL.md:mcp-tool-approval-gates:465", + "id": "skill:skills/paperclip/SKILL.md:mcp-tool-approval-gates:467", "sourceKind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md:465", + "sourceAnchor": "skills/paperclip/SKILL.md:467", "title": "MCP Tool Approval Gates", "expectedSemantics": "Skill guidance headed “MCP Tool Approval Gates”.", "primaryDisposition": "optional_agent_tool", @@ -250,13 +250,13 @@ "control_plane_invariant" ], "evidenceIds": [ - "skill:skills/paperclip/SKILL.md:465" + "skill:skills/paperclip/SKILL.md:467" ] }, { - "id": "skill:skills/paperclip/SKILL.md:niche-workflow-pointers:507", + "id": "skill:skills/paperclip/SKILL.md:niche-workflow-pointers:509", "sourceKind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md:507", + "sourceAnchor": "skills/paperclip/SKILL.md:509", "title": "Niche Workflow Pointers", "expectedSemantics": "Skill guidance headed “Niche Workflow Pointers”.", "primaryDisposition": "optional_agent_tool", @@ -265,13 +265,13 @@ "control_plane_invariant" ], "evidenceIds": [ - "skill:skills/paperclip/SKILL.md:507" + "skill:skills/paperclip/SKILL.md:509" ] }, { - "id": "skill:skills/paperclip/SKILL.md:cases:517", + "id": "skill:skills/paperclip/SKILL.md:cases:519", "sourceKind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md:517", + "sourceAnchor": "skills/paperclip/SKILL.md:519", "title": "Cases", "expectedSemantics": "Skill guidance headed “Cases”.", "primaryDisposition": "optional_agent_tool", @@ -280,13 +280,13 @@ "control_plane_invariant" ], "evidenceIds": [ - "skill:skills/paperclip/SKILL.md:517" + "skill:skills/paperclip/SKILL.md:519" ] }, { - "id": "skill:skills/paperclip/SKILL.md:company-skills-workflow:522", + "id": "skill:skills/paperclip/SKILL.md:company-skills-workflow:524", "sourceKind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md:522", + "sourceAnchor": "skills/paperclip/SKILL.md:524", "title": "Company Skills Workflow", "expectedSemantics": "Skill guidance headed “Company Skills Workflow”.", "primaryDisposition": "optional_agent_tool", @@ -295,13 +295,13 @@ "control_plane_invariant" ], "evidenceIds": [ - "skill:skills/paperclip/SKILL.md:522" + "skill:skills/paperclip/SKILL.md:524" ] }, { - "id": "skill:skills/paperclip/SKILL.md:routines:533", + "id": "skill:skills/paperclip/SKILL.md:routines:535", "sourceKind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md:533", + "sourceAnchor": "skills/paperclip/SKILL.md:535", "title": "Routines", "expectedSemantics": "Skill guidance headed “Routines”.", "primaryDisposition": "optional_agent_tool", @@ -310,13 +310,13 @@ "control_plane_invariant" ], "evidenceIds": [ - "skill:skills/paperclip/SKILL.md:533" + "skill:skills/paperclip/SKILL.md:535" ] }, { - "id": "skill:skills/paperclip/SKILL.md:issue-workspace-runtime-controls:544", + "id": "skill:skills/paperclip/SKILL.md:issue-workspace-runtime-controls:546", "sourceKind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md:544", + "sourceAnchor": "skills/paperclip/SKILL.md:546", "title": "Issue Workspace Runtime Controls", "expectedSemantics": "Skill guidance headed “Issue Workspace Runtime Controls”.", "primaryDisposition": "optional_agent_tool", @@ -325,13 +325,13 @@ "control_plane_invariant" ], "evidenceIds": [ - "skill:skills/paperclip/SKILL.md:544" + "skill:skills/paperclip/SKILL.md:546" ] }, { - "id": "skill:skills/paperclip/SKILL.md:proposing-credentials-safely:551", + "id": "skill:skills/paperclip/SKILL.md:proposing-credentials-safely:553", "sourceKind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md:551", + "sourceAnchor": "skills/paperclip/SKILL.md:553", "title": "Proposing Credentials Safely", "expectedSemantics": "Skill guidance headed “Proposing Credentials Safely”.", "primaryDisposition": "optional_agent_tool", @@ -340,13 +340,13 @@ "control_plane_invariant" ], "evidenceIds": [ - "skill:skills/paperclip/SKILL.md:551" + "skill:skills/paperclip/SKILL.md:553" ] }, { - "id": "skill:skills/paperclip/SKILL.md:reading-granted-secrets:558", + "id": "skill:skills/paperclip/SKILL.md:reading-granted-secrets:560", "sourceKind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md:558", + "sourceAnchor": "skills/paperclip/SKILL.md:560", "title": "Reading Granted Secrets", "expectedSemantics": "Skill guidance headed “Reading Granted Secrets”.", "primaryDisposition": "optional_agent_tool", @@ -355,13 +355,13 @@ "control_plane_invariant" ], "evidenceIds": [ - "skill:skills/paperclip/SKILL.md:558" + "skill:skills/paperclip/SKILL.md:560" ] }, { - "id": "skill:skills/paperclip/SKILL.md:critical-rules:584", + "id": "skill:skills/paperclip/SKILL.md:critical-rules:586", "sourceKind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md:584", + "sourceAnchor": "skills/paperclip/SKILL.md:586", "title": "Critical Rules", "expectedSemantics": "Skill guidance headed “Critical Rules”.", "primaryDisposition": "optional_agent_tool", @@ -370,13 +370,13 @@ "control_plane_invariant" ], "evidenceIds": [ - "skill:skills/paperclip/SKILL.md:584" + "skill:skills/paperclip/SKILL.md:586" ] }, { - "id": "skill:skills/paperclip/SKILL.md:comment-style-required:605", + "id": "skill:skills/paperclip/SKILL.md:comment-style-required:607", "sourceKind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md:605", + "sourceAnchor": "skills/paperclip/SKILL.md:607", "title": "Comment Style (Required)", "expectedSemantics": "Skill guidance headed “Comment Style (Required)”.", "primaryDisposition": "always_agent_tool", @@ -385,13 +385,13 @@ "control_plane_invariant" ], "evidenceIds": [ - "skill:skills/paperclip/SKILL.md:605" + "skill:skills/paperclip/SKILL.md:607" ] }, { - "id": "skill:skills/paperclip/SKILL.md:update:637", + "id": "skill:skills/paperclip/SKILL.md:update:639", "sourceKind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md:637", + "sourceAnchor": "skills/paperclip/SKILL.md:639", "title": "Update", "expectedSemantics": "Skill guidance headed “Update”.", "primaryDisposition": "optional_agent_tool", @@ -400,13 +400,13 @@ "control_plane_invariant" ], "evidenceIds": [ - "skill:skills/paperclip/SKILL.md:637" + "skill:skills/paperclip/SKILL.md:639" ] }, { - "id": "skill:skills/paperclip/SKILL.md:planning-required-when-planning-requested:647", + "id": "skill:skills/paperclip/SKILL.md:planning-required-when-planning-requested:649", "sourceKind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md:647", + "sourceAnchor": "skills/paperclip/SKILL.md:649", "title": "Planning (Required when planning requested)", "expectedSemantics": "Skill guidance headed “Planning (Required when planning requested)”.", "primaryDisposition": "optional_agent_tool", @@ -415,13 +415,13 @@ "control_plane_invariant" ], "evidenceIds": [ - "skill:skills/paperclip/SKILL.md:647" + "skill:skills/paperclip/SKILL.md:649" ] }, { - "id": "skill:skills/paperclip/SKILL.md:key-endpoints-hot-routes:680", + "id": "skill:skills/paperclip/SKILL.md:key-endpoints-hot-routes:682", "sourceKind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md:680", + "sourceAnchor": "skills/paperclip/SKILL.md:682", "title": "Key Endpoints (Hot Routes)", "expectedSemantics": "Skill guidance headed “Key Endpoints (Hot Routes)”.", "primaryDisposition": "optional_agent_tool", @@ -430,13 +430,13 @@ "control_plane_invariant" ], "evidenceIds": [ - "skill:skills/paperclip/SKILL.md:680" + "skill:skills/paperclip/SKILL.md:682" ] }, { - "id": "skill:skills/paperclip/SKILL.md:searching-issues:709", + "id": "skill:skills/paperclip/SKILL.md:searching-issues:711", "sourceKind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md:709", + "sourceAnchor": "skills/paperclip/SKILL.md:711", "title": "Searching Issues", "expectedSemantics": "Skill guidance headed “Searching Issues”.", "primaryDisposition": "optional_agent_tool", @@ -445,13 +445,13 @@ "control_plane_invariant" ], "evidenceIds": [ - "skill:skills/paperclip/SKILL.md:709" + "skill:skills/paperclip/SKILL.md:711" ] }, { - "id": "skill:skills/paperclip/SKILL.md:full-reference:719", + "id": "skill:skills/paperclip/SKILL.md:full-reference:721", "sourceKind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md:719", + "sourceAnchor": "skills/paperclip/SKILL.md:721", "title": "Full Reference", "expectedSemantics": "Skill guidance headed “Full Reference”.", "primaryDisposition": "optional_agent_tool", @@ -460,13 +460,13 @@ "control_plane_invariant" ], "evidenceIds": [ - "skill:skills/paperclip/SKILL.md:719" + "skill:skills/paperclip/SKILL.md:721" ] }, { - "id": "skill:skills/paperclip/SKILL.md:conversational-confirmation-answers:723", + "id": "skill:skills/paperclip/SKILL.md:conversational-confirmation-answers:725", "sourceKind": "skill_heading", - "sourceAnchor": "skills/paperclip/SKILL.md:723", + "sourceAnchor": "skills/paperclip/SKILL.md:725", "title": "Conversational confirmation answers", "expectedSemantics": "Skill guidance headed “Conversational confirmation answers”.", "primaryDisposition": "always_agent_tool", @@ -475,7 +475,7 @@ "control_plane_invariant" ], "evidenceIds": [ - "skill:skills/paperclip/SKILL.md:723" + "skill:skills/paperclip/SKILL.md:725" ] }, { diff --git a/packages/paperclip-runner/spec/capability/source-contract.json b/packages/paperclip-runner/spec/capability/source-contract.json index ebc924bad3..5ff5ae8ce5 100644 --- a/packages/paperclip-runner/spec/capability/source-contract.json +++ b/packages/paperclip-runner/spec/capability/source-contract.json @@ -7,6 +7,7 @@ "skills/paperclip/references/artifacts.md", "skills/paperclip/references/cases.md", "skills/paperclip/references/company-skills.md", + "skills/paperclip/references/issue-documents.md", "skills/paperclip/references/issue-workspaces.md", "skills/paperclip/references/routines.md", "skills/paperclip/references/workflows.md" diff --git a/packages/paperclip-runner/test/capability-contract.test.mjs b/packages/paperclip-runner/test/capability-contract.test.mjs index 234259a91f..39bc5d3e02 100644 --- a/packages/paperclip-runner/test/capability-contract.test.mjs +++ b/packages/paperclip-runner/test/capability-contract.test.mjs @@ -18,7 +18,8 @@ test("generated Capability inventory has full source coverage", async () => { readRows("eval-traceability.yaml"), ]); - assert.equal(capabilities.length, 158); + assert.equal(capabilities.length, 161); + assert.equal(capabilities.filter(row => row.sourceAnchor.startsWith("skills/paperclip/references/issue-documents.md#")).length, 3); assert.equal(tools.length, 42); assert.equal(evals.length, 106); assert.equal(new Set(evals.map((row) => row.group)).size, 16); diff --git a/tests/runner-e2e/stock-harness-checks.mjs b/tests/runner-e2e/stock-harness-checks.mjs index da7722f553..03d557d35d 100644 --- a/tests/runner-e2e/stock-harness-checks.mjs +++ b/tests/runner-e2e/stock-harness-checks.mjs @@ -40,10 +40,10 @@ export const stockHarnessGates = [ required: ["renders standard assignment wake with task authority", "renders scoped planning wake authority"] }, { id: "SH-eval", name: "Independent oracle and qualification admission", cwd: ".", config: "tests/runner-e2e/vitest.config.ts", - files: ["tests/runner-e2e/paperclip-document.test.ts", "tests/runner-e2e/stock-harness.test.ts", "tests/runner-e2e/stock-harness-checks.test.mjs", + files: ["tests/runner-e2e/stock-harness-manifest.test.ts", "tests/runner-e2e/paperclip-document.test.ts", "tests/runner-e2e/stock-harness.test.ts", "tests/runner-e2e/stock-harness-checks.test.mjs", "tests/runner-e2e/stock-harness-admission.test.ts", "tests/runner-e2e/stock-harness-digest.test.ts", "tests/runner-e2e/select-rerun-artifacts.test.ts"], - required: ["rejects old SHA before providers", "allows toolchain paths and excludes every present or future credential", + required: ["requires generated capability manifests for the current skill sources", "rejects old SHA before providers", "allows toolchain paths and excludes every present or future credential", "changes when the evaluated server/src/onboarding-assets/default/AGENTS.md changes", "changes when the evaluated packages/adapter-utils/src/server-utils.ts changes", "changes when the evaluated packages/shared/src/connection-intent-guidance.ts changes", @@ -71,6 +71,14 @@ export function sourceFingerprint() { "tests/runner-e2e/stock-harness.ts", "tests/runner-e2e/stock-harness-checks.mjs", "tests/runner-e2e/catalog.ts", "tests/runner-e2e/stock-harness-admission.ts", "tests/runner-e2e/launch.ts", "tests/runner-e2e/runner.spec.ts", "tests/runner-e2e/context-integrity-cases.ts", "tests/runner-e2e/context-integrity-flow.ts", "tests/runner-e2e/context-integrity-scoring.ts", + "tests/runner-e2e/stock-harness-manifest.ts", "packages/paperclip-runner/scripts/generate-capability-contract.mjs", + "packages/paperclip-runner/spec/capability/source-contract.json", + "packages/paperclip-runner/scripts/check-capability-inventory.mjs", "packages/paperclip-runner/scripts/lib/capability-inventory.mjs", + "packages/paperclip-runner/spec/capability/capabilities.yaml", "packages/paperclip-runner/spec/capability/eval-traceability.yaml", + "packages/paperclip-runner/spec/capability/mcp-tool-map.yaml", "packages/paperclip-runner/spec/capability/inventory.schema.json", + "packages/paperclip-runner/src/generated/capability-contract.ts", "packages/paperclip-runner/docs/capability-contract.md", + ...["capabilities.yaml", "mcp-tool-map.yaml", "eval-traceability.yaml", "capability-contract.md", "downstream-handoff.md"] + .map(file => `packages/paperclip-runner/generated/capability/${file}`), "packages/adapter-utils/src/server-utils.ts", "packages/shared/src/connection-intent-guidance.ts", "server/src/onboarding-assets/default/AGENTS.md", "server/src/routes/agents.ts", "scripts/ensure-plugin-build-deps.mjs", "packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts", diff --git a/tests/runner-e2e/stock-harness-digest.test.ts b/tests/runner-e2e/stock-harness-digest.test.ts index 454e7355f5..e5d882e56a 100644 --- a/tests/runner-e2e/stock-harness-digest.test.ts +++ b/tests/runner-e2e/stock-harness-digest.test.ts @@ -5,7 +5,7 @@ import { stockHarnessSourceDigest, stockHarnessSkillSources } from "./stock-harn vi.mock("node:fs", async importOriginal => ({ ...await importOriginal(), readFileSync: vi.fn() })); describe("stock harness instruction revision", () => { - it.each(["server/src/onboarding-assets/default/AGENTS.md", "packages/adapter-utils/src/server-utils.ts", "packages/shared/src/connection-intent-guidance.ts", "skills/paperclip/SKILL.md", "skills/paperclip/references/issue-documents.md"])( + it.each(["server/src/onboarding-assets/default/AGENTS.md", "packages/adapter-utils/src/server-utils.ts", "packages/shared/src/connection-intent-guidance.ts", "skills/paperclip/SKILL.md", "skills/paperclip/references/issue-documents.md", "packages/paperclip-runner/generated/capability/capabilities.yaml", "packages/paperclip-runner/spec/capability/capabilities.yaml", "tests/runner-e2e/stock-harness-manifest.ts"])( "changes when the evaluated %s changes", source => { vi.mocked(readFileSync).mockImplementation(() => Buffer.from("unchanged")); const original = stockHarnessSourceDigest(); diff --git a/tests/runner-e2e/stock-harness-manifest.test.ts b/tests/runner-e2e/stock-harness-manifest.test.ts new file mode 100644 index 0000000000..e8ce32b645 --- /dev/null +++ b/tests/runner-e2e/stock-harness-manifest.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from "vitest"; +import { readFileSync } from "node:fs"; +import { verifyCapabilityManifest } from "./stock-harness-manifest.js"; + +describe("stock harness generated instruction metadata", () => { + it("requires generated capability manifests for the current skill sources", async () => { + expect(await verifyCapabilityManifest()).toBe(true); + }); + it("rejects a stale manifest before provider admission", async () => { + await expect(verifyCapabilityManifest(path => path.endsWith("capabilities.yaml") + ? "stale skill anchors" : readFileSync(path, "utf8"))).rejects.toThrow("manifest drift"); + }); + it("rejects unavailable generated evidence", async () => { + await expect(verifyCapabilityManifest(() => { throw new Error("missing generated evidence"); })) + .rejects.toThrow("missing generated evidence"); + }); + it("rejects a stale source inventory before provider admission", async () => { + await expect(verifyCapabilityManifest(undefined, () => { throw new Error("source inventory drift"); })) + .rejects.toThrow("source inventory drift"); + }); +}); diff --git a/tests/runner-e2e/stock-harness-manifest.ts b/tests/runner-e2e/stock-harness-manifest.ts new file mode 100644 index 0000000000..dbc260923b --- /dev/null +++ b/tests/runner-e2e/stock-harness-manifest.ts @@ -0,0 +1,19 @@ +import { readFileSync } from "node:fs"; +import { execFileSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { buildContract } from "../../packages/paperclip-runner/scripts/generate-capability-contract.mjs"; + +/** Canonical derived metadata must match the instruction sources before providers. */ +export async function verifyCapabilityManifest( + read = (path: string) => readFileSync(path, "utf8"), + checkInventory = () => execFileSync(process.execPath, [fileURLToPath(new URL( + "../../packages/paperclip-runner/scripts/check-capability-inventory.mjs", import.meta.url, + ))], { stdio: "pipe" }), +) { + const expected = await buildContract(); + for (const [path, content] of Object.entries(expected)) { + if (read(path) !== content) throw new Error(`Generated capability manifest drift: ${path}`); + } + checkInventory(); + return true; +} diff --git a/tests/runner-e2e/stock-harness.ts b/tests/runner-e2e/stock-harness.ts index 128701cf00..bb68599f61 100644 --- a/tests/runner-e2e/stock-harness.ts +++ b/tests/runner-e2e/stock-harness.ts @@ -123,7 +123,19 @@ export function stockHarnessSourceDigest() { for (const source of [ "stock-harness.ts", "context-integrity-cases.ts", "context-integrity-scoring.ts", "context-integrity-flow.ts", "chat-cases.ts", "chat-flow.ts", "live-fixtures.ts", "runner.spec.ts", - "stock-harness-checks.mjs", "stock-harness-admission.ts", + "stock-harness-checks.mjs", "stock-harness-admission.ts", "stock-harness-manifest.ts", + "../../packages/paperclip-runner/scripts/generate-capability-contract.mjs", + "../../packages/paperclip-runner/scripts/check-capability-inventory.mjs", + "../../packages/paperclip-runner/scripts/lib/capability-inventory.mjs", + "../../packages/paperclip-runner/spec/capability/source-contract.json", + "../../packages/paperclip-runner/spec/capability/capabilities.yaml", + "../../packages/paperclip-runner/spec/capability/eval-traceability.yaml", + "../../packages/paperclip-runner/spec/capability/mcp-tool-map.yaml", + "../../packages/paperclip-runner/spec/capability/inventory.schema.json", + "../../packages/paperclip-runner/src/generated/capability-contract.ts", + "../../packages/paperclip-runner/docs/capability-contract.md", + ...["capabilities.yaml", "mcp-tool-map.yaml", "eval-traceability.yaml", "capability-contract.md", "downstream-handoff.md"] + .map(file => `../../packages/paperclip-runner/generated/capability/${file}`), "../../server/src/onboarding-assets/default/AGENTS.md", "../../packages/adapter-utils/src/server-utils.ts", "../../packages/shared/src/connection-intent-guidance.ts",