mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-29 04:25:52 +01:00
test(copilot): preserve final question failure and offline recovery proof
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed
1 parent
e08bb92aa2
commit
5630481a38
3 files changed
+359
No files matched your search
@@ -70,6 +70,39 @@ answered Cobalt choice, and warm-session continuation. Copilot instead supplied
|
||||
`[terminal marker]` to `paperclip_finish`. Neither the grader nor the terminal
|
||||
marker requirement changed. A final-runtime rerun remains required.
|
||||
|
||||
### Final v3 Product question attempt
|
||||
|
||||
The [final v3 question receipt](../../packages/paperclip-runner/test/fixtures/copilot-product-v3-question-2026-09-29.json)
|
||||
retains the single reserved attempt at source `3d0c45920`, with the exact frozen
|
||||
controller distribution, native assets, pack, Node, daemon and sidecar hashes.
|
||||
The first turn produced the structured question and the board answered Cobalt.
|
||||
Continuation failed during native `session.open` recovery, before continuation
|
||||
usage was reported, with an unclassified sidecar rejection. The 72.086-second
|
||||
case failed; cleanup passed and no retry occurred. This is a recovery/admission
|
||||
failure, distinct from the earlier literal-marker behavior failure. Neither is
|
||||
removed from qualification evidence.
|
||||
|
||||
One run reports GitHub/unpriced usage: 28,545 input, 13,818 cached input and 525
|
||||
output tokens. Provider USD and upstream model-request count remain unknown.
|
||||
The refreshed account counter moved from 5 to 6 of 1,500 included credits;
|
||||
additional usage remained disabled with a $0 budget and $0 account cash charges.
|
||||
This account-level delta is not a per-turn USD allocation. The Product aggregate's
|
||||
zero reported cost with `unpriced`/incomplete coverage is not an authoritative
|
||||
zero-cost receipt. No additional live attempt is authorized by this result.
|
||||
|
||||
A [credential-free native recovery reproduction](../../packages/paperclip-runner/test/fixtures/copilot-runtime-context-recovery-2026-09-29.json)
|
||||
closes the provider, deletes the prior registered instruction copy, and reproduces
|
||||
`ENOENT` at `bindAcpxAgentFiles`. Reopening with the current registered copy
|
||||
succeeds with the same native session. A second explicit fixture turn proves
|
||||
actual native `session/load` and reaches `end_turn`. The fixture uses exactly two
|
||||
loopback responses, a synthetic credential and test-only model metadata;
|
||||
it makes no paid calls and does not qualify authenticated model selection. This
|
||||
supports the stale durable runtime-context diagnosis; the failed Product run did
|
||||
not retain the underlying wire error. Cold rotated restoration now takes the
|
||||
current authenticated runtime context. Live reconnect and pending warm-transition
|
||||
receipts keep their existing context; replacing that context in an active provider
|
||||
is a separate lifecycle operation.
|
||||
|
||||
### Native detached work is explicitly unsupported
|
||||
|
||||
The broader 2026-09-29 deterministic probe reproduces early completion for
|
||||
|
||||
+78
@@ -0,0 +1,78 @@
|
||||
{
|
||||
"schema": "paperclip.copilot-product-v3-question.v1",
|
||||
"centralReservationId": "copilot-v3-local-question-20260929-01",
|
||||
"attempt": "copilot-final-question-20260929t151802z",
|
||||
"model": "gpt-5.6-luna",
|
||||
"providerVersion": "1.0.88",
|
||||
"profileVersion": 3,
|
||||
"build": {
|
||||
"sourceRevision": "3d0c45920c326821c2f5ff48dc985ef8158b46e4",
|
||||
"sourceRoot": "/private/tmp/paperclip-copilot-frozen-3d0c45920/source",
|
||||
"sourceArchiveSha256": "7f802e5fed2b5dede8b6bfc23affafb7f2c542e98dc09b5e3d111ede0d2e33c7",
|
||||
"originalLockSha256": "e0c928a494f90ddad3c00791e83f09315ee8c82df2a0418a809dcf93649a8ab3",
|
||||
"resolvedLockSha256": "60ae61439844519b104cdf014f0f7b7972804cd1c04a10a8a8777cc8da7dc2f2",
|
||||
"actualControllerPackageRoot": "/private/tmp/paperclip-copilot-frozen-3d0c45920/source/packages/paperclip-runner",
|
||||
"providerPackDigest": "sha256:977d464241d8a8f36bc879fd1cdddf1e675f72fd6f1f2d3fe2f8e273fafc160e",
|
||||
"nodeBinary": "/private/tmp/paperclip-pi-v5-node-20260928/darwin-arm64/node",
|
||||
"nodeSha256": "e4b5a3af0e05c75de2eae013904145f40fe7fc2a6e6f17510128bf45cca4e79b",
|
||||
"runnerBinary": "/private/tmp/paperclip-copilot-frozen-3d0c45920/paperclip-runnerd",
|
||||
"runnerSha256": "2b51c48198c3724255756241cddc1b5efea87dab1c49a6ef7eb897513c7c043b",
|
||||
"sidecarSha256": "7cfee6ca612b3d7bcc266fdd36f2e77d7bb027169c56e0c9c36b9676232f33a4",
|
||||
"acpxPatchSha256": "a64405199b381688ffd72fdbb88396e423394cb407813d7b6f1db43bae9c3933",
|
||||
"frozenBuildRecordSha256": "0a756acca0cc44534465757c4dd01b7039fbe9320a613c7df0e373f96bad7644"
|
||||
},
|
||||
"status": "failed",
|
||||
"failureClass": "candidate_failure",
|
||||
"failureStage": "native session recovery after answered question",
|
||||
"diagnosis": "Product retained an unclassified session.open rejection. Credential-free exact-native recovery reproduces ENOENT when the persisted descriptor references the collected prior-run instruction directory, and succeeds with the current registered directory. This is the supported cause, not directly retained Product wire evidence. No paid retry performed.",
|
||||
"durationMs": 72086,
|
||||
"outerDeadlineSeconds": 300,
|
||||
"activeProviderDeadlineSeconds": 120,
|
||||
"automaticRetries": 0,
|
||||
"cleanup": "passed",
|
||||
"runs": [
|
||||
{
|
||||
"status": "failed",
|
||||
"providerUsage": null,
|
||||
"errorCode": "native_session_interrupted"
|
||||
},
|
||||
{
|
||||
"status": "succeeded",
|
||||
"providerUsage": {
|
||||
"provider": "github",
|
||||
"model": "gpt-5.6-luna",
|
||||
"costStatus": "unpriced",
|
||||
"inputTokens": 28545,
|
||||
"outputTokens": 525,
|
||||
"cachedInputTokens": 13818,
|
||||
"sessionReused": false
|
||||
},
|
||||
"errorCode": null
|
||||
}
|
||||
],
|
||||
"interactionStatuses": [
|
||||
{
|
||||
"kind": "ask_user_questions",
|
||||
"status": "answered"
|
||||
}
|
||||
],
|
||||
"providerCostUsd": null,
|
||||
"upstreamModelRequestCount": null,
|
||||
"githubReconciliation": {
|
||||
"baselineIncludedCredits": 5,
|
||||
"afterIncludedCredits": 6,
|
||||
"accountCreditDelta": 1,
|
||||
"includedCreditTotal": 1500,
|
||||
"additionalUsageEnabled": false,
|
||||
"additionalCashBudgetUsd": 0,
|
||||
"accountCashChargesUsd": 0,
|
||||
"snapshotSource": "Lead observed refreshed GitHub billing UI after attempt, approximately 2026-09-29 15:24 UTC",
|
||||
"recordedAt": "2026-09-29T15:26:24.763193+00:00",
|
||||
"perRunUsdAllocation": null
|
||||
},
|
||||
"evidence": {
|
||||
"result.json": "e67192baff115744fdb122f275e3e1f9d0405898327d984986ed774545ebfef4",
|
||||
"snapshots/api-state.json": "044f6d79d1c99a4fb4da89e80b82f6ac2de0c7f536148a5f6d701b149f76d6a3",
|
||||
"evidence-manifest.json": "6da9b164991c6c4b0005f9e1ad94c790018254c5c78be6049111c0a4ff7d695d"
|
||||
}
|
||||
}
|
||||
+248
@@ -0,0 +1,248 @@
|
||||
{
|
||||
"schema": "paperclip.copilot-runtime-context-recovery.v1",
|
||||
"recordedAt": "2026-09-29T15:32:50.799156+00:00",
|
||||
"sourceRevision": "3d0c45920c326821c2f5ff48dc985ef8158b46e4",
|
||||
"providerVersion": "1.0.88",
|
||||
"providerBinarySha256": "a9ff8babb10b7e443182ae96a8bc50a9c826ef1c773e1344c396eb5bf7f512c3",
|
||||
"paidProviderCalls": 0,
|
||||
"loopbackModelResponses": 2,
|
||||
"fixtureMechanism": "env -i; synthetic token; COPILOT_OFFLINE=true; loopback OpenAI fixture. Exact native binary and actual AcpxRuntimeHost; second explicit turn forces lazy native session/load. Test-only gpt-4.1 model option injected into guard metadata because offline native initialization omits model options; actual native mode/allow_all remain unchanged. Not authenticated model qualification.",
|
||||
"modelMetadataFixtureInjected": true,
|
||||
"probeScriptSha256": "e2cf49568c42d8cb0dc3404cd263adfebac5507a6e694faa8f6db948c85ba4e1",
|
||||
"rawPrivateWireSha256": "7d19d4f7dc0c2ad4265a6bd17bb8f25f572b31e409c7745279529b6fc4d9840d",
|
||||
"steps": [
|
||||
"Open native provider with old registered agent-files directory",
|
||||
"Run one deterministic loopback response",
|
||||
"Close native host, delete old directory",
|
||||
"Reopen with old context: ENOENT at bindAcpxAgentFiles lstat",
|
||||
"Reopen with current registered directory: passes with original native session ID",
|
||||
"Start a second explicit fixture turn: native session/load observed, then session/prompt completes end_turn",
|
||||
"Close host; exactly two requested fixture turns and two loopback model responses"
|
||||
],
|
||||
"staleContextError": {
|
||||
"code": "ENOENT",
|
||||
"phase": "reopen",
|
||||
"operation": "lstat",
|
||||
"path": "<owned-fixture>/agent-copy-old"
|
||||
},
|
||||
"restoredSessionIdentityUnchanged": true,
|
||||
"restoredPhase": "reopen-current-copy",
|
||||
"wireProjection": [
|
||||
{
|
||||
"direction": "outbound",
|
||||
"id": 0,
|
||||
"method": "initialize"
|
||||
},
|
||||
{
|
||||
"direction": "inbound",
|
||||
"id": 0
|
||||
},
|
||||
{
|
||||
"direction": "outbound",
|
||||
"id": 1,
|
||||
"method": "session/new"
|
||||
},
|
||||
{
|
||||
"direction": "inbound",
|
||||
"id": 1,
|
||||
"configOptions": [
|
||||
{
|
||||
"type": "select",
|
||||
"id": "mode",
|
||||
"name": "Mode",
|
||||
"currentValue": "https://agentclientprotocol.com/protocol/session-modes#agent",
|
||||
"options": [
|
||||
{
|
||||
"value": "https://agentclientprotocol.com/protocol/session-modes#agent",
|
||||
"name": "Agent",
|
||||
"description": "Default agent mode for conversational interactions"
|
||||
},
|
||||
{
|
||||
"value": "https://agentclientprotocol.com/protocol/session-modes#plan",
|
||||
"name": "Plan",
|
||||
"description": "Plan mode for creating and executing multi-step plans"
|
||||
},
|
||||
{
|
||||
"value": "https://agentclientprotocol.com/protocol/session-modes#autopilot",
|
||||
"name": "Autopilot",
|
||||
"description": "Autonomous mode that enables allow-all and runs until task completion without user interaction (experimental)"
|
||||
}
|
||||
],
|
||||
"category": "mode",
|
||||
"description": "Controls how Copilot responds: a conversational agent, planning multi-step work, or autonomous autopilot."
|
||||
},
|
||||
{
|
||||
"type": "select",
|
||||
"id": "allow_all",
|
||||
"name": "Allow All",
|
||||
"currentValue": "off",
|
||||
"options": [
|
||||
{
|
||||
"value": "on",
|
||||
"name": "On",
|
||||
"description": "Automatically approve all tool, path, and URL requests"
|
||||
},
|
||||
{
|
||||
"value": "off",
|
||||
"name": "Off",
|
||||
"description": "Require approval for tool, path, and URL requests"
|
||||
}
|
||||
],
|
||||
"category": "permissions",
|
||||
"description": "Controls whether Copilot prompts for approval before using tools, accessing paths, or fetching URLs."
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"direction": "inbound",
|
||||
"method": "session/update",
|
||||
"sessionUpdate": "available_commands_update"
|
||||
},
|
||||
{
|
||||
"direction": "inbound",
|
||||
"method": "session/update",
|
||||
"sessionUpdate": "available_commands_update"
|
||||
},
|
||||
{
|
||||
"direction": "outbound",
|
||||
"id": 2,
|
||||
"method": "session/prompt"
|
||||
},
|
||||
{
|
||||
"direction": "inbound",
|
||||
"method": "session/update",
|
||||
"sessionUpdate": "config_option_update"
|
||||
},
|
||||
{
|
||||
"direction": "inbound",
|
||||
"method": "session/update",
|
||||
"sessionUpdate": "session_info_update"
|
||||
},
|
||||
{
|
||||
"direction": "inbound",
|
||||
"method": "session/update",
|
||||
"sessionUpdate": "usage_update"
|
||||
},
|
||||
{
|
||||
"direction": "inbound",
|
||||
"method": "session/update",
|
||||
"sessionUpdate": "agent_message_chunk"
|
||||
},
|
||||
{
|
||||
"direction": "inbound",
|
||||
"id": 2
|
||||
},
|
||||
{
|
||||
"direction": "outbound",
|
||||
"id": 0,
|
||||
"method": "initialize"
|
||||
},
|
||||
{
|
||||
"direction": "inbound",
|
||||
"id": 0
|
||||
},
|
||||
{
|
||||
"direction": "outbound",
|
||||
"id": 1,
|
||||
"method": "session/load"
|
||||
},
|
||||
{
|
||||
"direction": "inbound",
|
||||
"method": "session/update",
|
||||
"sessionUpdate": "user_message_chunk"
|
||||
},
|
||||
{
|
||||
"direction": "inbound",
|
||||
"method": "session/update",
|
||||
"sessionUpdate": "agent_message_chunk"
|
||||
},
|
||||
{
|
||||
"direction": "inbound",
|
||||
"id": 1,
|
||||
"configOptions": [
|
||||
{
|
||||
"type": "select",
|
||||
"id": "mode",
|
||||
"name": "Mode",
|
||||
"currentValue": "https://agentclientprotocol.com/protocol/session-modes#agent",
|
||||
"options": [
|
||||
{
|
||||
"value": "https://agentclientprotocol.com/protocol/session-modes#agent",
|
||||
"name": "Agent",
|
||||
"description": "Default agent mode for conversational interactions"
|
||||
},
|
||||
{
|
||||
"value": "https://agentclientprotocol.com/protocol/session-modes#plan",
|
||||
"name": "Plan",
|
||||
"description": "Plan mode for creating and executing multi-step plans"
|
||||
},
|
||||
{
|
||||
"value": "https://agentclientprotocol.com/protocol/session-modes#autopilot",
|
||||
"name": "Autopilot",
|
||||
"description": "Autonomous mode that enables allow-all and runs until task completion without user interaction (experimental)"
|
||||
}
|
||||
],
|
||||
"category": "mode",
|
||||
"description": "Controls how Copilot responds: a conversational agent, planning multi-step work, or autonomous autopilot."
|
||||
},
|
||||
{
|
||||
"type": "select",
|
||||
"id": "allow_all",
|
||||
"name": "Allow All",
|
||||
"currentValue": "off",
|
||||
"options": [
|
||||
{
|
||||
"value": "on",
|
||||
"name": "On",
|
||||
"description": "Automatically approve all tool, path, and URL requests"
|
||||
},
|
||||
{
|
||||
"value": "off",
|
||||
"name": "Off",
|
||||
"description": "Require approval for tool, path, and URL requests"
|
||||
}
|
||||
],
|
||||
"category": "permissions",
|
||||
"description": "Controls whether Copilot prompts for approval before using tools, accessing paths, or fetching URLs."
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"direction": "inbound",
|
||||
"method": "session/update",
|
||||
"sessionUpdate": "available_commands_update"
|
||||
},
|
||||
{
|
||||
"direction": "inbound",
|
||||
"method": "session/update",
|
||||
"sessionUpdate": "available_commands_update"
|
||||
},
|
||||
{
|
||||
"direction": "outbound",
|
||||
"id": 2,
|
||||
"method": "session/prompt"
|
||||
},
|
||||
{
|
||||
"direction": "inbound",
|
||||
"method": "session/update",
|
||||
"sessionUpdate": "usage_update"
|
||||
},
|
||||
{
|
||||
"direction": "inbound",
|
||||
"method": "session/update",
|
||||
"sessionUpdate": "agent_message_chunk"
|
||||
},
|
||||
{
|
||||
"direction": "inbound",
|
||||
"id": 2
|
||||
}
|
||||
],
|
||||
"limits": [
|
||||
"This reproduces the suspected Product recovery cause; failed Product cleanup did not retain raw session.open cause.",
|
||||
"Live provider context replacement and root retention until active work settles are distinct from closed-provider restoration."
|
||||
],
|
||||
"continuationResult": {
|
||||
"status": "completed",
|
||||
"stopReason": "end_turn"
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user