From fcb142168559e1be499afeeb5ee023f2867d55ab Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 29 Sep 2026 09:51:04 -0500 Subject: [PATCH] Retain Copilot detached-command early completion regression Co-Authored-By: Paperclip --- .../runner-copilot-capabilities.md | 21 ++++++++++++-- .../scripts/probe-copilot-acp.py | 21 +++++++++----- ...copilot-policy-conformance-2026-09-29.json | 29 ++++++++++++++++++- 3 files changed, 60 insertions(+), 11 deletions(-) diff --git a/doc/architecture/runner-copilot-capabilities.md b/doc/architecture/runner-copilot-capabilities.md index ac06f60ce46..54162551324 100644 --- a/doc/architecture/runner-copilot-capabilities.md +++ b/doc/architecture/runner-copilot-capabilities.md @@ -41,7 +41,7 @@ answer cannot be delivered. This narrows the blocking-input question for the admitted pinned mode without claiming an upstream question responder exists. The [offline conformance record](../../packages/paperclip-runner/test/fixtures/copilot-policy-conformance-2026-09-29.json) -retains eight attempts and their evidence digests. A protected file outside the +retains ten attempts and their evidence digests. A protected file outside the working directory was denied before its contents reached the fixture model. A durable conversation was closed and loaded, then requested permission for a shell write; `reject_once` prevented the file. The initial attempt to load an @@ -69,6 +69,23 @@ answered Cobalt choice, and warm-session continuation. Copilot instead supplied `[terminal marker]` to `paperclip_finish`. Neither the grader nor the terminal marker requirement changed. A final-runtime rerun remains required. +### Detached background work remains a production blocker + +The broader 2026-09-29 deterministic probe reproduces early completion for +`bash` with `mode: async` and `detach: true`. After an allowed finite two-second +command, Copilot sends a completed tool update naming a detached shell ID, then +`session.idle` and `end_turn` before the marker file exists. A separate diagnostic +keeps observing for three seconds after terminal and proves the marker appears +late. Both attempts retain their failed settlement assertion. + +The preceding standard tool-call notification includes `detach: true`, but the +permission request carries only the command. Native background-task change +notices have empty payloads and expose no task-list or settle control. Neither +notification counts nor a fixed delay establish settlement. The prior live +finite-detached pass exercised a model that waited; it does not qualify this +native lifetime. A governed runner-owned background interface or a versioned +upstream settlement fix is required to support detached work safely. + ## Evidence and scope The audit inspected the exact `@github/copilot@1.0.88` platform archives, the @@ -211,7 +228,7 @@ Daytona claims require the separate live product qualification. | Task file changes / diffs | Standard tools carry locations/diff content for create/edit/str_replace/apply_patch. | Shared ACP tool activity retains bounded `rawOutput`, `inputUpdated` and the first validated relative location. Structured tool `content` diffs/images, `rawInput` and secondary locations are dropped; no complete diff presentation is claimed. | Real denied-create wire contains a diff, but wire presence is not runner/UI preservation. P1 add typed, bounded diff/image content and all validated locations with tool/session provenance; Product file-edit/validation and downloadable-file presentation pass; rich diff rendering remains unqualified. | | Provider workspace files | `session.workspace_file_changed.path` is relative to provider session workspace files, not task cwd. | Validated reference tagged `provider_session_workspace`, resolution required. | Source + traversal tests; P1 safe file retrieval/upload. | | Images / binary artifacts | Prompt image input; native content-addressed binary_asset base64. | Hash/length-validated metadata references; bytes not blindly read from disk or emitted in activity. | Source + unit digest tests; P1 durable artifact storage; >32 KiB payload provider omission remains. | -| Background settlement | Standard prompt waits for idle in tested attached async-shell case; lossy native idle/receipt also exist. | ACP terminal result remains authoritative; raw event cannot end turn. | Offline attached and real-service finite detached commands completed before end_turn; marker verified through cleanup. Arbitrary background lifetimes remain unqualified; P0 broader settlement coverage. | +| Background settlement | Standard prompt waits for idle in tested attached async-shell case; lossy native idle/receipt also exist. | ACP terminal result remains authoritative; raw event cannot end turn. | Offline attached and real-service finite detached commands completed before end_turn; marker verified through cleanup. A deterministic detach:true case now reproduces end_turn before the finite command completes; see the 2026-09-29 blocker. P0 actual background settlement is required. | | Compaction/context | Native compaction lifecycle/token counts/context git metadata. | Safe bounded counters/status, immutable workspace binding. | Source + projection tests; raw summary/private custom instructions omitted. | | Goals / remote/schedules | Native autopilot/objectives/remote/schedule facilities; no qualified ACP goal protocol. | Unsupported through this profile; remote disabled. | Source; P2 separate governance review before control exposure. | diff --git a/packages/paperclip-runner/scripts/probe-copilot-acp.py b/packages/paperclip-runner/scripts/probe-copilot-acp.py index dc15bf7b351..7a5546e17f8 100644 --- a/packages/paperclip-runner/scripts/probe-copilot-acp.py +++ b/packages/paperclip-runner/scripts/probe-copilot-acp.py @@ -5,7 +5,7 @@ This proves a protocol fixture, never GitHub/model/Daytona qualification. import argparse,json,subprocess,tempfile,pathlib,os,select,time,threading,http.server,hashlib,shutil,contextlib,sys parser=argparse.ArgumentParser(description=__doc__) parser.add_argument('--package-root',required=True) -parser.add_argument('--scenario',choices=['deny-write','deny-shell','deny-read','attached-shell','discover-inputs','native-question','native-plan'],default='deny-write') +parser.add_argument('--scenario',choices=['deny-write','deny-shell','deny-read','attached-shell','detached-shell','discover-inputs','native-question','native-plan'],default='deny-write') parser.add_argument('--mode',choices=['agent','plan','autopilot'],default='agent') resume_group=parser.add_mutually_exclusive_group() resume_group.add_argument('--resume',action='store_true',help='Close and load the same native session before prompting') @@ -47,8 +47,8 @@ with contextlib.ExitStack() as cleanup: outside=root.parent/(root.name+'-protected.txt') outside.write_text('PRIVATE_FIXTURE_SENTINEL') cleanup.callback(outside.unlink,missing_ok=True) - tool_name={'deny-write':'create','deny-shell':'bash','deny-read':'view','attached-shell':'bash','native-question':'ask_user','native-plan':'exit_plan_mode'}.get(args.scenario) - tool_arguments={'deny-write':{'path':str(root/'denied.txt'),'file_text':'MUST NOT EXIST'},'deny-shell':{'command':'printf MUST_NOT_EXIST > denied.txt','description':'Denied command fixture'},'deny-read':{'path':str(outside)},'attached-shell':{'command':command,'description':'Bounded settlement fixture','mode':'async','detach':False},'native-question':{'question':'Choose one','choices':['A','B'],'allowFreeform':False},'native-plan':{'summary':'Fixture plan','planContent':'Do fixture work'}}.get(args.scenario) + tool_name={'deny-write':'create','deny-shell':'bash','deny-read':'view','attached-shell':'bash','detached-shell':'bash','native-question':'ask_user','native-plan':'exit_plan_mode'}.get(args.scenario) + tool_arguments={'deny-write':{'path':str(root/'denied.txt'),'file_text':'MUST NOT EXIST'},'deny-shell':{'command':'printf MUST_NOT_EXIST > denied.txt','description':'Denied command fixture'},'deny-read':{'path':str(outside)},'attached-shell':{'command':command,'description':'Bounded settlement fixture','mode':'async','detach':False},'detached-shell':{'command':command,'description':'Bounded detached settlement fixture','mode':'async','detach':True},'native-question':{'question':'Choose one','choices':['A','B'],'allowFreeform':False},'native-plan':{'summary':'Fixture plan','planContent':'Do fixture work'}}.get(args.scenario) marker_at_prompt_result=False class Handler(http.server.BaseHTTPRequestHandler): def log_message(self,*args):pass @@ -97,7 +97,7 @@ with contextlib.ExitStack() as cleanup: if 'id' in message and 'method' not in message:responses[message['id']]=message if message.get('method')=='session/request_permission': requested_command=message['params'].get('toolCall',{}).get('rawInput',{}).get('command') - allow=(args.scenario=='attached-shell' and requested_command==command) or (seeding and requested_command==seed_command) + allow=(args.scenario in ('attached-shell','detached-shell') and requested_command==command) or (seeding and requested_command==seed_command) reject=next((o for o in message['params']['options'] if o['kind']==('allow_once' if allow else 'reject_once')),None) outcome={'outcome':'selected','optionId':reject['optionId']} if reject else {'outcome':'cancelled'} permission_responses.append({'id':message['id'],'outcome':outcome}) @@ -136,12 +136,17 @@ with contextlib.ExitStack() as cleanup: if 'error' in session:raise RuntimeError('Copilot rejected session load') final_prompt_id=nextid+1 request('session/prompt',{'sessionId':session_id,'prompt':[{'type':'text','text':'Reply with Fixture complete.'}]}) - marker_at_prompt_result=(root/('settlement.txt' if args.scenario=='attached-shell' else 'denied.txt')).exists() - for _ in range(3):pump(.1) + marker_at_prompt_result=(root/('settlement.txt' if args.scenario in ('attached-shell','detached-shell') else 'denied.txt')).exists() + if args.scenario=='detached-shell': + # Observe the bounded child after terminal; never turn a late marker into success. + observation_deadline=time.monotonic()+3 + while time.monotonic()