mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-21 00:05:49 +02:00
test(copilot): record native system instruction delivery evidence
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
a93e8c03f6
commit
751f59f26c
2 files changed
+90
No files matched your search
@@ -103,6 +103,24 @@ current authenticated runtime context. Live reconnect and pending warm-transitio
|
||||
receipts keep their existing context; replacing that context in an active provider
|
||||
is a separate lifecycle operation.
|
||||
|
||||
### Native instruction delivery
|
||||
|
||||
The [native model-request probe](../../packages/paperclip-runner/test/fixtures/copilot-native-instruction-delivery-2026-09-29.json)
|
||||
retains a separate instruction-delivery failure: Copilot 1.0.88 ignores generic
|
||||
ACP `_meta.systemPrompt` on new sessions, and ACPX does not include it on load.
|
||||
Actual loopback model requests contained neither the Paperclip prompt nor the
|
||||
registered directory guidance. Passing a refreshed descriptor alone is insufficient.
|
||||
|
||||
A credential-free follow-up wrote the composed text to the isolated provider
|
||||
home's `copilot-instructions.md`, using Copilot's native personal-instruction
|
||||
mechanism. The first request's system message contained the old directory and
|
||||
original entry. After closing the provider, deleting the old directory and
|
||||
reopening with fresh instructions, actual `session/load` and a second explicit
|
||||
turn completed. The second model request contained the current directory and
|
||||
updated entry, with neither old value in its system message. Both attempts used
|
||||
two synthetic loopback responses and no paid calls. This probe is not a final
|
||||
built-production or authenticated qualification result.
|
||||
|
||||
### Native detached work is explicitly unsupported
|
||||
|
||||
The broader 2026-09-29 deterministic probe reproduces early completion for
|
||||
|
||||
Vendored
+72
@@ -0,0 +1,72 @@
|
||||
{
|
||||
"schema": "paperclip.copilot-native-instruction-delivery.v1",
|
||||
"date": "2026-09-29",
|
||||
"providerVersion": "1.0.88",
|
||||
"providerBinarySha256": "a9ff8babb10b7e443182ae96a8bc50a9c826ef1c773e1344c396eb5bf7f512c3",
|
||||
"runtimeSourceRevision": "3d0c45920c326821c2f5ff48dc985ef8158b46e4",
|
||||
"fixtureMechanism": "Credential-free env -i / COPILOT_OFFLINE=true with synthetic credential and loopback OpenAI fixture. Actual native binary and AcpxRuntimeHost. Test-only offline model metadata injection retained. Second explicit requested turn forces native session/load. File-mechanism probe writes the proposed instruction file in prepareSandbox; it is not a final built-production proof.",
|
||||
"documentation": "https://docs.github.com/en/copilot/how-tos/copilot-cli/customize-copilot/add-custom-instructions",
|
||||
"attempts": [
|
||||
{
|
||||
"mechanism": "generic ACP metadata only",
|
||||
"rawPrivateEvidenceSha256": "c2a0944f325cbbd73e3238d3aa93b5d1305ec924686b96e5933c2a1b41dc166e",
|
||||
"probeScriptSha256": "ba346a6ce5652f189ca95b993902570c121142d9c69a85cb2ad88427bdf47360",
|
||||
"loopbackModelResponses": 2,
|
||||
"paidProviderCalls": 0,
|
||||
"nativeLoadObserved": true,
|
||||
"continuationResult": {
|
||||
"status": "completed",
|
||||
"stopReason": "end_turn"
|
||||
},
|
||||
"modelRequestProjection": [
|
||||
{
|
||||
"request": 1,
|
||||
"paperclipPromptInSystem": false,
|
||||
"oldRegisteredRootInSystem": false,
|
||||
"currentRegisteredRootInSystem": false,
|
||||
"originalCustomEntryInSystem": false,
|
||||
"updatedCustomEntryInSystem": false
|
||||
},
|
||||
{
|
||||
"request": 2,
|
||||
"paperclipPromptInSystem": false,
|
||||
"oldRegisteredRootInSystem": false,
|
||||
"currentRegisteredRootInSystem": false,
|
||||
"originalCustomEntryInSystem": false,
|
||||
"updatedCustomEntryInSystem": false
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"mechanism": "isolated COPILOT_HOME/copilot-instructions.md refreshed before admission",
|
||||
"rawPrivateEvidenceSha256": "abe3317c36f178c7b4f512070b290ed5fc2c066ca19b52152a5bebb4ff8ba195",
|
||||
"probeScriptSha256": "85752c52ff53f99e4a7a8d6fe40b41fbea1d55b07efbcd4c4062674946089850",
|
||||
"loopbackModelResponses": 2,
|
||||
"paidProviderCalls": 0,
|
||||
"nativeLoadObserved": true,
|
||||
"continuationResult": {
|
||||
"status": "completed",
|
||||
"stopReason": "end_turn"
|
||||
},
|
||||
"modelRequestProjection": [
|
||||
{
|
||||
"request": 1,
|
||||
"paperclipPromptInSystem": true,
|
||||
"oldRegisteredRootInSystem": true,
|
||||
"currentRegisteredRootInSystem": false,
|
||||
"originalCustomEntryInSystem": true,
|
||||
"updatedCustomEntryInSystem": false
|
||||
},
|
||||
{
|
||||
"request": 2,
|
||||
"paperclipPromptInSystem": true,
|
||||
"oldRegisteredRootInSystem": false,
|
||||
"currentRegisteredRootInSystem": true,
|
||||
"originalCustomEntryInSystem": false,
|
||||
"updatedCustomEntryInSystem": true
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"conclusion": "Pinned Copilot ignores generic systemPrompt metadata. Native personal instruction file supplies fresh composed text to actual model system message after load, replacing old root/custom-entry text. Both attempts retained; final production build and authenticated qualification remain required."
|
||||
}
|
||||
Reference in new issue
Block a user