From 5630481a38e5af7c67bc5f20e986b51ffcb91ab9 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 29 Sep 2026 10:34:51 -0500 Subject: [PATCH] test(copilot): preserve final question failure and offline recovery proof Co-Authored-By: Paperclip --- .../runner-copilot-capabilities.md | 33 +++ ...opilot-product-v3-question-2026-09-29.json | 78 ++++++ ...t-runtime-context-recovery-2026-09-29.json | 248 ++++++++++++++++++ 3 files changed, 359 insertions(+) create mode 100644 packages/paperclip-runner/test/fixtures/copilot-product-v3-question-2026-09-29.json create mode 100644 packages/paperclip-runner/test/fixtures/copilot-runtime-context-recovery-2026-09-29.json diff --git a/doc/architecture/runner-copilot-capabilities.md b/doc/architecture/runner-copilot-capabilities.md index 2ff06fdf21e..a942ca2950c 100644 --- a/doc/architecture/runner-copilot-capabilities.md +++ b/doc/architecture/runner-copilot-capabilities.md @@ -70,6 +70,39 @@ answered Cobalt choice, and warm-session continuation. Copilot instead supplied `[terminal marker]` to `paperclip_finish`. Neither the grader nor the terminal marker requirement changed. A final-runtime rerun remains required. +### Final v3 Product question attempt + +The [final v3 question receipt](../../packages/paperclip-runner/test/fixtures/copilot-product-v3-question-2026-09-29.json) +retains the single reserved attempt at source `3d0c45920`, with the exact frozen +controller distribution, native assets, pack, Node, daemon and sidecar hashes. +The first turn produced the structured question and the board answered Cobalt. +Continuation failed during native `session.open` recovery, before continuation +usage was reported, with an unclassified sidecar rejection. The 72.086-second +case failed; cleanup passed and no retry occurred. This is a recovery/admission +failure, distinct from the earlier literal-marker behavior failure. Neither is +removed from qualification evidence. + +One run reports GitHub/unpriced usage: 28,545 input, 13,818 cached input and 525 +output tokens. Provider USD and upstream model-request count remain unknown. +The refreshed account counter moved from 5 to 6 of 1,500 included credits; +additional usage remained disabled with a $0 budget and $0 account cash charges. +This account-level delta is not a per-turn USD allocation. The Product aggregate's +zero reported cost with `unpriced`/incomplete coverage is not an authoritative +zero-cost receipt. No additional live attempt is authorized by this result. + +A [credential-free native recovery reproduction](../../packages/paperclip-runner/test/fixtures/copilot-runtime-context-recovery-2026-09-29.json) +closes the provider, deletes the prior registered instruction copy, and reproduces +`ENOENT` at `bindAcpxAgentFiles`. Reopening with the current registered copy +succeeds with the same native session. A second explicit fixture turn proves +actual native `session/load` and reaches `end_turn`. The fixture uses exactly two +loopback responses, a synthetic credential and test-only model metadata; +it makes no paid calls and does not qualify authenticated model selection. This +supports the stale durable runtime-context diagnosis; the failed Product run did +not retain the underlying wire error. Cold rotated restoration now takes the +current authenticated runtime context. Live reconnect and pending warm-transition +receipts keep their existing context; replacing that context in an active provider +is a separate lifecycle operation. + ### Native detached work is explicitly unsupported The broader 2026-09-29 deterministic probe reproduces early completion for diff --git a/packages/paperclip-runner/test/fixtures/copilot-product-v3-question-2026-09-29.json b/packages/paperclip-runner/test/fixtures/copilot-product-v3-question-2026-09-29.json new file mode 100644 index 00000000000..7687a9510b2 --- /dev/null +++ b/packages/paperclip-runner/test/fixtures/copilot-product-v3-question-2026-09-29.json @@ -0,0 +1,78 @@ +{ + "schema": "paperclip.copilot-product-v3-question.v1", + "centralReservationId": "copilot-v3-local-question-20260929-01", + "attempt": "copilot-final-question-20260929t151802z", + "model": "gpt-5.6-luna", + "providerVersion": "1.0.88", + "profileVersion": 3, + "build": { + "sourceRevision": "3d0c45920c326821c2f5ff48dc985ef8158b46e4", + "sourceRoot": "/private/tmp/paperclip-copilot-frozen-3d0c45920/source", + "sourceArchiveSha256": "7f802e5fed2b5dede8b6bfc23affafb7f2c542e98dc09b5e3d111ede0d2e33c7", + "originalLockSha256": "e0c928a494f90ddad3c00791e83f09315ee8c82df2a0418a809dcf93649a8ab3", + "resolvedLockSha256": "60ae61439844519b104cdf014f0f7b7972804cd1c04a10a8a8777cc8da7dc2f2", + "actualControllerPackageRoot": "/private/tmp/paperclip-copilot-frozen-3d0c45920/source/packages/paperclip-runner", + "providerPackDigest": "sha256:977d464241d8a8f36bc879fd1cdddf1e675f72fd6f1f2d3fe2f8e273fafc160e", + "nodeBinary": "/private/tmp/paperclip-pi-v5-node-20260928/darwin-arm64/node", + "nodeSha256": "e4b5a3af0e05c75de2eae013904145f40fe7fc2a6e6f17510128bf45cca4e79b", + "runnerBinary": "/private/tmp/paperclip-copilot-frozen-3d0c45920/paperclip-runnerd", + "runnerSha256": "2b51c48198c3724255756241cddc1b5efea87dab1c49a6ef7eb897513c7c043b", + "sidecarSha256": "7cfee6ca612b3d7bcc266fdd36f2e77d7bb027169c56e0c9c36b9676232f33a4", + "acpxPatchSha256": "a64405199b381688ffd72fdbb88396e423394cb407813d7b6f1db43bae9c3933", + "frozenBuildRecordSha256": "0a756acca0cc44534465757c4dd01b7039fbe9320a613c7df0e373f96bad7644" + }, + "status": "failed", + "failureClass": "candidate_failure", + "failureStage": "native session recovery after answered question", + "diagnosis": "Product retained an unclassified session.open rejection. Credential-free exact-native recovery reproduces ENOENT when the persisted descriptor references the collected prior-run instruction directory, and succeeds with the current registered directory. This is the supported cause, not directly retained Product wire evidence. No paid retry performed.", + "durationMs": 72086, + "outerDeadlineSeconds": 300, + "activeProviderDeadlineSeconds": 120, + "automaticRetries": 0, + "cleanup": "passed", + "runs": [ + { + "status": "failed", + "providerUsage": null, + "errorCode": "native_session_interrupted" + }, + { + "status": "succeeded", + "providerUsage": { + "provider": "github", + "model": "gpt-5.6-luna", + "costStatus": "unpriced", + "inputTokens": 28545, + "outputTokens": 525, + "cachedInputTokens": 13818, + "sessionReused": false + }, + "errorCode": null + } + ], + "interactionStatuses": [ + { + "kind": "ask_user_questions", + "status": "answered" + } + ], + "providerCostUsd": null, + "upstreamModelRequestCount": null, + "githubReconciliation": { + "baselineIncludedCredits": 5, + "afterIncludedCredits": 6, + "accountCreditDelta": 1, + "includedCreditTotal": 1500, + "additionalUsageEnabled": false, + "additionalCashBudgetUsd": 0, + "accountCashChargesUsd": 0, + "snapshotSource": "Lead observed refreshed GitHub billing UI after attempt, approximately 2026-09-29 15:24 UTC", + "recordedAt": "2026-09-29T15:26:24.763193+00:00", + "perRunUsdAllocation": null + }, + "evidence": { + "result.json": "e67192baff115744fdb122f275e3e1f9d0405898327d984986ed774545ebfef4", + "snapshots/api-state.json": "044f6d79d1c99a4fb4da89e80b82f6ac2de0c7f536148a5f6d701b149f76d6a3", + "evidence-manifest.json": "6da9b164991c6c4b0005f9e1ad94c790018254c5c78be6049111c0a4ff7d695d" + } +} diff --git a/packages/paperclip-runner/test/fixtures/copilot-runtime-context-recovery-2026-09-29.json b/packages/paperclip-runner/test/fixtures/copilot-runtime-context-recovery-2026-09-29.json new file mode 100644 index 00000000000..2e88c783d04 --- /dev/null +++ b/packages/paperclip-runner/test/fixtures/copilot-runtime-context-recovery-2026-09-29.json @@ -0,0 +1,248 @@ +{ + "schema": "paperclip.copilot-runtime-context-recovery.v1", + "recordedAt": "2026-09-29T15:32:50.799156+00:00", + "sourceRevision": "3d0c45920c326821c2f5ff48dc985ef8158b46e4", + "providerVersion": "1.0.88", + "providerBinarySha256": "a9ff8babb10b7e443182ae96a8bc50a9c826ef1c773e1344c396eb5bf7f512c3", + "paidProviderCalls": 0, + "loopbackModelResponses": 2, + "fixtureMechanism": "env -i; synthetic token; COPILOT_OFFLINE=true; loopback OpenAI fixture. Exact native binary and actual AcpxRuntimeHost; second explicit turn forces lazy native session/load. Test-only gpt-4.1 model option injected into guard metadata because offline native initialization omits model options; actual native mode/allow_all remain unchanged. Not authenticated model qualification.", + "modelMetadataFixtureInjected": true, + "probeScriptSha256": "e2cf49568c42d8cb0dc3404cd263adfebac5507a6e694faa8f6db948c85ba4e1", + "rawPrivateWireSha256": "7d19d4f7dc0c2ad4265a6bd17bb8f25f572b31e409c7745279529b6fc4d9840d", + "steps": [ + "Open native provider with old registered agent-files directory", + "Run one deterministic loopback response", + "Close native host, delete old directory", + "Reopen with old context: ENOENT at bindAcpxAgentFiles lstat", + "Reopen with current registered directory: passes with original native session ID", + "Start a second explicit fixture turn: native session/load observed, then session/prompt completes end_turn", + "Close host; exactly two requested fixture turns and two loopback model responses" + ], + "staleContextError": { + "code": "ENOENT", + "phase": "reopen", + "operation": "lstat", + "path": "/agent-copy-old" + }, + "restoredSessionIdentityUnchanged": true, + "restoredPhase": "reopen-current-copy", + "wireProjection": [ + { + "direction": "outbound", + "id": 0, + "method": "initialize" + }, + { + "direction": "inbound", + "id": 0 + }, + { + "direction": "outbound", + "id": 1, + "method": "session/new" + }, + { + "direction": "inbound", + "id": 1, + "configOptions": [ + { + "type": "select", + "id": "mode", + "name": "Mode", + "currentValue": "https://agentclientprotocol.com/protocol/session-modes#agent", + "options": [ + { + "value": "https://agentclientprotocol.com/protocol/session-modes#agent", + "name": "Agent", + "description": "Default agent mode for conversational interactions" + }, + { + "value": "https://agentclientprotocol.com/protocol/session-modes#plan", + "name": "Plan", + "description": "Plan mode for creating and executing multi-step plans" + }, + { + "value": "https://agentclientprotocol.com/protocol/session-modes#autopilot", + "name": "Autopilot", + "description": "Autonomous mode that enables allow-all and runs until task completion without user interaction (experimental)" + } + ], + "category": "mode", + "description": "Controls how Copilot responds: a conversational agent, planning multi-step work, or autonomous autopilot." + }, + { + "type": "select", + "id": "allow_all", + "name": "Allow All", + "currentValue": "off", + "options": [ + { + "value": "on", + "name": "On", + "description": "Automatically approve all tool, path, and URL requests" + }, + { + "value": "off", + "name": "Off", + "description": "Require approval for tool, path, and URL requests" + } + ], + "category": "permissions", + "description": "Controls whether Copilot prompts for approval before using tools, accessing paths, or fetching URLs." + } + ] + }, + { + "direction": "inbound", + "method": "session/update", + "sessionUpdate": "available_commands_update" + }, + { + "direction": "inbound", + "method": "session/update", + "sessionUpdate": "available_commands_update" + }, + { + "direction": "outbound", + "id": 2, + "method": "session/prompt" + }, + { + "direction": "inbound", + "method": "session/update", + "sessionUpdate": "config_option_update" + }, + { + "direction": "inbound", + "method": "session/update", + "sessionUpdate": "session_info_update" + }, + { + "direction": "inbound", + "method": "session/update", + "sessionUpdate": "usage_update" + }, + { + "direction": "inbound", + "method": "session/update", + "sessionUpdate": "agent_message_chunk" + }, + { + "direction": "inbound", + "id": 2 + }, + { + "direction": "outbound", + "id": 0, + "method": "initialize" + }, + { + "direction": "inbound", + "id": 0 + }, + { + "direction": "outbound", + "id": 1, + "method": "session/load" + }, + { + "direction": "inbound", + "method": "session/update", + "sessionUpdate": "user_message_chunk" + }, + { + "direction": "inbound", + "method": "session/update", + "sessionUpdate": "agent_message_chunk" + }, + { + "direction": "inbound", + "id": 1, + "configOptions": [ + { + "type": "select", + "id": "mode", + "name": "Mode", + "currentValue": "https://agentclientprotocol.com/protocol/session-modes#agent", + "options": [ + { + "value": "https://agentclientprotocol.com/protocol/session-modes#agent", + "name": "Agent", + "description": "Default agent mode for conversational interactions" + }, + { + "value": "https://agentclientprotocol.com/protocol/session-modes#plan", + "name": "Plan", + "description": "Plan mode for creating and executing multi-step plans" + }, + { + "value": "https://agentclientprotocol.com/protocol/session-modes#autopilot", + "name": "Autopilot", + "description": "Autonomous mode that enables allow-all and runs until task completion without user interaction (experimental)" + } + ], + "category": "mode", + "description": "Controls how Copilot responds: a conversational agent, planning multi-step work, or autonomous autopilot." + }, + { + "type": "select", + "id": "allow_all", + "name": "Allow All", + "currentValue": "off", + "options": [ + { + "value": "on", + "name": "On", + "description": "Automatically approve all tool, path, and URL requests" + }, + { + "value": "off", + "name": "Off", + "description": "Require approval for tool, path, and URL requests" + } + ], + "category": "permissions", + "description": "Controls whether Copilot prompts for approval before using tools, accessing paths, or fetching URLs." + } + ] + }, + { + "direction": "inbound", + "method": "session/update", + "sessionUpdate": "available_commands_update" + }, + { + "direction": "inbound", + "method": "session/update", + "sessionUpdate": "available_commands_update" + }, + { + "direction": "outbound", + "id": 2, + "method": "session/prompt" + }, + { + "direction": "inbound", + "method": "session/update", + "sessionUpdate": "usage_update" + }, + { + "direction": "inbound", + "method": "session/update", + "sessionUpdate": "agent_message_chunk" + }, + { + "direction": "inbound", + "id": 2 + } + ], + "limits": [ + "This reproduces the suspected Product recovery cause; failed Product cleanup did not retain raw session.open cause.", + "Live provider context replacement and root retention until active work settles are distinct from closed-provider restoration." + ], + "continuationResult": { + "status": "completed", + "stopReason": "end_turn" + } +}