From 4bd9a581fc53547a20fc52f9b7a315c6e086ddc6 Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 8 Oct 2026 13:34:25 -0500 Subject: [PATCH] Record current Copilot v33 qualification gates without relabeling history --- ...2026-10-03-copilot-production-readiness.md | 21 +- .../2026-10-08-copilot-qualification-v33.json | 319 ++++++++++++++++++ 2 files changed, 339 insertions(+), 1 deletion(-) create mode 100644 doc/plans/2026-10-08-copilot-qualification-v33.json diff --git a/doc/plans/2026-10-03-copilot-production-readiness.md b/doc/plans/2026-10-03-copilot-production-readiness.md index 3e02301a29a..b3ff37985ed 100644 --- a/doc/plans/2026-10-03-copilot-production-readiness.md +++ b/doc/plans/2026-10-03-copilot-production-readiness.md @@ -2,7 +2,26 @@ Current release checklist, updated 2026-10-08 (America/Chicago). -## Current release gate: v32 +## Current release gate: v33 + +The frozen application source is `024e1422f9c7b43ff1b083e7d8a25e1969dc4acf`; native artifacts come from `941fbcb664a885eea713e732edc10ac133c62238`. Copilot **1.0.88**, exact **gpt-5.6-luna**, profile **v33**, and `sha256:482c4e997b4690c7e4a5375dd035560aaad69000f8f5291904d76f8c3a88284a` identify this campaign. Master baseline is `0ac194450a48a407450921a16c3ef8684dcb85ca`. Earlier results below remain historical. + +The reviewed server follow-up is `50d54d78e552869cae154d88ade68063191a547e`. It preserves explicitly selected remote environments under managed-only policy and resolves forced Kubernetes before Copilot admission. All 63 focused regressions, server typecheck and server build pass. It changes neither qualified runtime/profile/image inputs nor the campaign's local and Daytona execution policies. Its ordinary installed canary is a separate remaining gate; existing live results retain their measured source revision. + +| Current gate | Result | +|---|---| +| Runner protocol | **7/7 pass**, 46 assertions, context-before-action included, cleanup confirmed | +| Product local | **10/10 pass**, including attached command, denial, pending-permission Stop, provider death, warm continuity and all five core workflows; saved-token grants and exact profile/model verified | +| Product Daytona | **0/10 current passes**; exact v33 publication approval pending; attached-command cell runs first | +| Platform install/startup | Actual installed ARM64, x64 under Rosetta, and Linux x64 under Docker emulation initialize and exit cleanly; native Daytona still pending | +| Authenticated metadata | Packaged runtime discovers 19 models including the exact qualification model; no prompt | +| Ordinary user path | Fresh normal package installation and local/Daytona canaries on the reviewed source remain required, without qualification overrides | +| Current image | Credential scan covered 225,852 files with zero selected credential matches. Tag `copilot-qualification-22fd2a8ae3414eee317e`; OCI index `sha256:65fc4f30c15b3d744534ce3d7240dfbf3d3ff081da6d51180bd89f2802bd6e56`; not published | +| CI / review / shipping | Runtime and regression heads passed CI and fresh 5/5 review. Setup and qualification follow-ups require current-head CI/review. Separate code-owner approval and shipped canary remain required | + +No automatic behavior retries occurred. Retained failures include the v32 permission-target defect, pre-provider Vite startup failures, and the private eval admission mismatch. Their original grades remain unchanged. The reconciled allowance had **$39.86 remaining** after the local campaign; provider USD stays unknown and attributed to GitHub. No budget or billing settings were reset. The current machine-readable checklist is [2026-10-08-copilot-qualification-v33.json](2026-10-08-copilot-qualification-v33.json). + +## Historical v32 release attempt Fresh review identified two release blockers: arbitrary unavailable Copilot models could be saved, and independently decoding transport chunks could corrupt multibyte provider text. Creation, hiring, edits and environment changes now require the existing bounded authenticated metadata probe; the UI accepts only discovered models. The three protocol readers now retain a UTF-8 decoder across chunks. diff --git a/doc/plans/2026-10-08-copilot-qualification-v33.json b/doc/plans/2026-10-08-copilot-qualification-v33.json new file mode 100644 index 00000000000..4890501f328 --- /dev/null +++ b/doc/plans/2026-10-08-copilot-qualification-v33.json @@ -0,0 +1,319 @@ +{ + "schema": "paperclip.copilot.production-qualification/v33", + "updatedAt": "2026-10-08T18:33:47.962516+00:00", + "qualifiedApplicationSource": "024e1422f9c7b43ff1b083e7d8a25e1969dc4acf", + "reviewedApplicationSource": "50d54d78e552869cae154d88ade68063191a547e", + "runtimeArtifactSource": "941fbcb664a885eea713e732edc10ac133c62238", + "masterBaseline": "0ac194450a48a407450921a16c3ef8684dcb85ca", + "copilotVersion": "1.0.88", + "profileVersion": 33, + "profileDigest": "sha256:482c4e997b4690c7e4a5375dd035560aaad69000f8f5291904d76f8c3a88284a", + "model": "gpt-5.6-luna", + "protocol": { + "passed": 7, + "selected": 7, + "assertionsPassed": 46, + "caseResults": [ + { + "runId": "copilot-v33-final-protocol-get-task-context-02", + "sourceRevision": "024e1422f9c7b43ff1b083e7d8a25e1969dc4acf", + "profileDigest": "sha256:482c4e997b4690c7e4a5375dd035560aaad69000f8f5291904d76f8c3a88284a", + "model": "gpt-5.6-luna", + "passed": true, + "checksPassed": 4, + "artifactSha256": "sha256:752da200cbe3e852fbd446fe75b2a45f39a98e15c2f1ac5cdc161243111328a6", + "scoreSha256": "sha256:00e58b1292581e2645b738af7da692859cb76ce638a02de09bda910ee1456ae9", + "cleanupConfirmed": true, + "automaticRetries": 0 + }, + { + "runId": "copilot-v33-final-protocol-context-before-action-01", + "sourceRevision": "024e1422f9c7b43ff1b083e7d8a25e1969dc4acf", + "profileDigest": "sha256:482c4e997b4690c7e4a5375dd035560aaad69000f8f5291904d76f8c3a88284a", + "model": "gpt-5.6-luna", + "passed": true, + "checksPassed": 7, + "artifactSha256": "sha256:97df880f40aab985297981bc51aaa83947bebcb26d8a739b08c7162822815709", + "scoreSha256": "sha256:9f98e4aae60990e10d8f7a273ae616be50a295e401c7ba3f27bc9e7b8162cffa", + "cleanupConfirmed": true, + "automaticRetries": 0 + }, + { + "runId": "copilot-v33-final-protocol-create-task-document-01", + "sourceRevision": "024e1422f9c7b43ff1b083e7d8a25e1969dc4acf", + "profileDigest": "sha256:482c4e997b4690c7e4a5375dd035560aaad69000f8f5291904d76f8c3a88284a", + "model": "gpt-5.6-luna", + "passed": true, + "checksPassed": 7, + "artifactSha256": "sha256:1e1f408c9bcf797769acc0942deb42ef7b276bd6080a0a427c634460b3224e1f", + "scoreSha256": "sha256:c0a3e17db2db039962a4759f597937d1bb0d5fe3ac78948aae31ea110e24d72d", + "cleanupConfirmed": true, + "automaticRetries": 0 + }, + { + "runId": "copilot-v33-final-protocol-finish-task-01", + "sourceRevision": "024e1422f9c7b43ff1b083e7d8a25e1969dc4acf", + "profileDigest": "sha256:482c4e997b4690c7e4a5375dd035560aaad69000f8f5291904d76f8c3a88284a", + "model": "gpt-5.6-luna", + "passed": true, + "checksPassed": 5, + "artifactSha256": "sha256:e7a893d3954190b6839a49275a1ee74052aa8d5f51c07142906c304a986da839", + "scoreSha256": "sha256:e26558f48ea16349a4c80fa0e786aeea2c054d3acc98f1f251507b7ae7d12125", + "cleanupConfirmed": true, + "automaticRetries": 0 + }, + { + "runId": "copilot-v33-final-protocol-request-human-confirmation-01", + "sourceRevision": "024e1422f9c7b43ff1b083e7d8a25e1969dc4acf", + "profileDigest": "sha256:482c4e997b4690c7e4a5375dd035560aaad69000f8f5291904d76f8c3a88284a", + "model": "gpt-5.6-luna", + "passed": true, + "checksPassed": 5, + "artifactSha256": "sha256:c30a0539a3f078e77f85a4309d479412f55277c773f82217e56aa149e179867c", + "scoreSha256": "sha256:295027e137ec4a40c9b7b7c8c956fe33a1adc53256d3c941006a5209a02cd0b8", + "cleanupConfirmed": true, + "automaticRetries": 0 + }, + { + "runId": "copilot-v33-final-protocol-workflow-context-document-progress-01", + "sourceRevision": "024e1422f9c7b43ff1b083e7d8a25e1969dc4acf", + "profileDigest": "sha256:482c4e997b4690c7e4a5375dd035560aaad69000f8f5291904d76f8c3a88284a", + "model": "gpt-5.6-luna", + "passed": true, + "checksPassed": 9, + "artifactSha256": "sha256:14c8a4ebbde4b0eb059ea49ec0803463eec736c00fbacaeff10baed4d7987e28", + "scoreSha256": "sha256:e2a5999981ac6d71a472c01644e53bc46b1348b525b729dc344d57e90513dc40", + "cleanupConfirmed": true, + "automaticRetries": 0 + }, + { + "runId": "copilot-v33-final-protocol-workflow-governed-wait-01", + "sourceRevision": "024e1422f9c7b43ff1b083e7d8a25e1969dc4acf", + "profileDigest": "sha256:482c4e997b4690c7e4a5375dd035560aaad69000f8f5291904d76f8c3a88284a", + "model": "gpt-5.6-luna", + "passed": true, + "checksPassed": 9, + "artifactSha256": "sha256:b503d260edc3b08555e1cb1fe79367d779a919a27b3cfc516e3456f9f3043334", + "scoreSha256": "sha256:e067915a6adc859b5d2ff1e5a4a63a1ad3b0b52382f9f5410fbc366b59ba0b5e", + "cleanupConfirmed": true, + "automaticRetries": 0 + } + ] + }, + "product": { + "localPassed": 10, + "localSelected": 10, + "daytonaPassed": 0, + "daytonaSelected": 10, + "cells": [ + { + "selector": "native-active-stop.runner-acpx-copilot.local.pending-permission-stop", + "status": "passed", + "sourceRevision": "024e1422f9c7b43ff1b083e7d8a25e1969dc4acf", + "resultSha256": "sha256:a1a8568c2b6e3e0fb83e429fc7516f954577562ea861a8c034c7f690df57cb23", + "cleanupConfirmed": true + }, + { + "selector": "native-active-stop.runner-acpx-copilot.daytona.pending-permission-stop", + "status": "pending", + "sourceRevision": null, + "resultSha256": null, + "cleanupConfirmed": null + }, + { + "selector": "copilot-provider-death.runner-acpx-copilot.local.pending-permission-provider-death", + "status": "passed", + "sourceRevision": "024e1422f9c7b43ff1b083e7d8a25e1969dc4acf", + "resultSha256": "sha256:2915c499e9c93bc465867e71fdb5c8afe3f003d6de1dae3fc606f2dc697a84a9", + "cleanupConfirmed": true + }, + { + "selector": "copilot-provider-death.runner-acpx-copilot.daytona.pending-permission-provider-death", + "status": "pending", + "sourceRevision": null, + "resultSha256": null, + "cleanupConfirmed": null + }, + { + "selector": "copilot-protection.runner-acpx-copilot.local.native-permission-deny-write", + "status": "passed", + "sourceRevision": "024e1422f9c7b43ff1b083e7d8a25e1969dc4acf", + "resultSha256": "sha256:1d13e096ef032acc0294705d1da4580e0ed4a8589ba8c5138bd8361568fcfb0e", + "cleanupConfirmed": true + }, + { + "selector": "copilot-protection.runner-acpx-copilot.local.attached-async-settlement", + "status": "passed", + "sourceRevision": "024e1422f9c7b43ff1b083e7d8a25e1969dc4acf", + "resultSha256": "sha256:75a19460e29997e4f5ad45c857d4b752f3452b6e1b36270ace95d201dfbfe058", + "cleanupConfirmed": true + }, + { + "selector": "copilot-protection.runner-acpx-copilot.daytona.native-permission-deny-write", + "status": "pending", + "sourceRevision": null, + "resultSha256": null, + "cleanupConfirmed": null + }, + { + "selector": "copilot-protection.runner-acpx-copilot.daytona.attached-async-settlement", + "status": "pending", + "sourceRevision": null, + "resultSha256": null, + "cleanupConfirmed": null + }, + { + "selector": "rich-acp-warm-continuity.runner-acpx-copilot.local.warm-three-turn", + "status": "passed", + "sourceRevision": "024e1422f9c7b43ff1b083e7d8a25e1969dc4acf", + "resultSha256": "sha256:78dd1f84f77ca88923bf3d36c7d0c00c84e3c79385e9e19d6125c47b14cb68fc", + "cleanupConfirmed": true + }, + { + "selector": "rich-acp-warm-continuity.runner-acpx-copilot.daytona.warm-three-turn", + "status": "pending", + "sourceRevision": null, + "resultSha256": null, + "cleanupConfirmed": null + }, + { + "selector": "extended-harnesses.runner-acpx-copilot.local.hello-complete", + "status": "passed", + "sourceRevision": "024e1422f9c7b43ff1b083e7d8a25e1969dc4acf", + "resultSha256": "sha256:67cd97fa41f68e40035e258e567f091168f7b29f63f90d7a8899cba84db06053", + "cleanupConfirmed": true + }, + { + "selector": "extended-harnesses.runner-acpx-copilot.local.question-resume-complete", + "status": "passed", + "sourceRevision": "024e1422f9c7b43ff1b083e7d8a25e1969dc4acf", + "resultSha256": "sha256:01bbc56ee046539e7cae842c3f27c1e99c6f2a2dbc908bc6a0c52de78328ac10", + "cleanupConfirmed": true + }, + { + "selector": "extended-harnesses.runner-acpx-copilot.local.plan-approve-complete", + "status": "passed", + "sourceRevision": "024e1422f9c7b43ff1b083e7d8a25e1969dc4acf", + "resultSha256": "sha256:1603983a6b7b75ceb8adace86a52beaac0c32da6dd08af21b04dec2070061715", + "cleanupConfirmed": true + }, + { + "selector": "extended-harnesses.runner-acpx-copilot.local.structured-question-restart-resume", + "status": "passed", + "sourceRevision": "024e1422f9c7b43ff1b083e7d8a25e1969dc4acf", + "resultSha256": "sha256:0274dde5c70b5ab5d08d858ca7c7667399a70a7015d8a2d0937a94a1692f7d30", + "cleanupConfirmed": true + }, + { + "selector": "extended-harnesses.runner-acpx-copilot.local.file-edit-validate", + "status": "passed", + "sourceRevision": "024e1422f9c7b43ff1b083e7d8a25e1969dc4acf", + "resultSha256": "sha256:1322c4f56b80b0fd3020e3f49aafaa9ecfadbba8d4ff2aadc119a1386348c354", + "cleanupConfirmed": true + }, + { + "selector": "extended-harnesses.runner-acpx-copilot.daytona.hello-complete", + "status": "pending", + "sourceRevision": null, + "resultSha256": null, + "cleanupConfirmed": null + }, + { + "selector": "extended-harnesses.runner-acpx-copilot.daytona.question-resume-complete", + "status": "pending", + "sourceRevision": null, + "resultSha256": null, + "cleanupConfirmed": null + }, + { + "selector": "extended-harnesses.runner-acpx-copilot.daytona.plan-approve-complete", + "status": "pending", + "sourceRevision": null, + "resultSha256": null, + "cleanupConfirmed": null + }, + { + "selector": "extended-harnesses.runner-acpx-copilot.daytona.structured-question-restart-resume", + "status": "pending", + "sourceRevision": null, + "resultSha256": null, + "cleanupConfirmed": null + }, + { + "selector": "extended-harnesses.runner-acpx-copilot.daytona.file-edit-validate", + "status": "pending", + "sourceRevision": null, + "resultSha256": null, + "cleanupConfirmed": null + } + ] + }, + "image": { + "tag": "ghcr.io/paperclipai/paperclip-daytona-runner:copilot-qualification-22fd2a8ae3414eee317e", + "indexDigest": "sha256:65fc4f30c15b3d744534ce3d7240dfbf3d3ff081da6d51180bd89f2802bd6e56", + "knownCredentialMatches": 0, + "filesScanned": 225852, + "published": false, + "exactApprovalPending": true, + "reviewedServerDeltaChangesImage": false + }, + "installedPlatformSmokes": { + "darwin-arm64": "passed", + "darwin-x64": "passed under Rosetta", + "linux-x64": "passed under Docker emulation; native Daytona qualification pending" + }, + "reviewedSetupDelta": { + "source": "50d54d78e552869cae154d88ade68063191a547e", + "changedFiles": [ + "server/src/services/copilot-connection-probe.ts", + "server/src/routes/agents.ts", + "server/src/__tests__/copilot-connection-probe.test.ts", + "server/src/__tests__/agent-test-environment-routes.test.ts", + "server/src/__tests__/ai-connections.test.ts" + ], + "focusedRegressionsPassed": 63, + "serverTypecheckPassed": true, + "serverBuildPassed": true, + "qualifiedProfileChanged": false, + "selectedLocalPolicyChanged": false, + "selectedDaytonaPolicyChanged": false, + "ordinaryInstallCanary": "pending on reviewed source" + }, + "canaries": { + "ordinaryLocal": "pending", + "ordinaryDaytona": "pending" + }, + "budget": { + "remainingConservativeAllowanceUsd": 39.86, + "nativeUsd": null, + "billingEntity": "github", + "budgetReset": false, + "historicalAttemptsRetained": true + }, + "retainedFailedAttempts": [ + { + "attempt": "v32 local pending-permission Stop", + "cause": "Actual target hidden by lexical /tmp versus /private/tmp workspace alias check", + "runtimeRepair": "profile v33 safe canonical target mapping" + }, + { + "attempt": "v33 pre-provider Product startup", + "cause": "Vite Connect non-router stack and route metadata", + "runtimeRepair": "server router-stack guards; credential-free startup and all ten local cells pass" + }, + { + "attempt": "v33 protocol admission attempt 1", + "cause": "private eval closed allowlist lacked exact v33 profile; no provider session", + "runtimeRepair": "private roster admission synchronized; all seven cases pass" + } + ], + "releaseGates": [ + "Exact v33 public image publication approval", + "Ten exact native Daytona Product cells", + "Ordinary installed local and Daytona canaries on reviewed source", + "Current-head CI and fresh clean review", + "Separate code-owner approval, normal merge and shipped canary" + ], + "productionReady": false, + "shipped": false +}