diff --git a/doc/architecture/runner-rich-acp-capabilities.md b/doc/architecture/runner-rich-acp-capabilities.md index 7c788248bb5..cd19e11ee4f 100644 --- a/doc/architecture/runner-rich-acp-capabilities.md +++ b/doc/architecture/runner-rich-acp-capabilities.md @@ -132,6 +132,27 @@ Its installed Linux provider-pack digest is `sha256:08ad9f6a6fb9c14c87e3bc5b20d01876986178c768433d709a45540e8d40a4be`. An image build and manifest check are not paid remote qualification. +The [sanitized qualification checkpoint](runner-rich-acp-qualification-2026-09-29.json) +retains current attempts and their original source/profile identities. Pi's +corrected `bd4cc29c3` Daytona hello passed six matchers with one run; the sandbox +was absent throughout the full cleanup window and all owned processes retired. +Three delayed reads stabilized at $0.000630644 OpenRouter usage plus $0.0046878842 +sandbox analytics ($0.0053185282 combined, provisional provider accounting). + +Further live tests are paused because a credential-free reproduction confirmed +host semaphore exhaustion during embedded PostgreSQL bootstrap (`semget` returned +`ENOSPC`). Copilot's diagnostic local and first Daytona attempts failed before +provider execution. The Daytona invocation never reached fixture creation or a +cloud-write path; its machine cleanup flag remains `unresolved_scope_not_captured` +and is retained separately from that source-order assessment. This host failure +does not explain the earlier Copilot provider-session startup timeout. The +remaining Pi local/native campaign is prepared but has not launched. + +GitHub billing coverage was also audited: public standard-runner and self-hosted +GitHub minute charges do not add a per-minute charge, but external fleet costs +and private evals CI incremental charges remain unmetered. The $100 live-test +ceiling is not evidence of complete infrastructure cost attribution. + ## Registered persistent agent files Candidate ACP processes receive `AGENT_HOME` only from the authenticated diff --git a/doc/architecture/runner-rich-acp-qualification-2026-09-29.json b/doc/architecture/runner-rich-acp-qualification-2026-09-29.json new file mode 100644 index 00000000000..029453890c2 --- /dev/null +++ b/doc/architecture/runner-rich-acp-qualification-2026-09-29.json @@ -0,0 +1,278 @@ +{ + "schema": "paperclip.rich-acp-qualification-checkpoint.v1", + "updatedAt": "2026-09-29T18:26:17.961300+00:00", + "qualificationStatus": "pending", + "runtimeSource": "bd4cc29c3017ed5e1484423e842fd48e3b2f49f3", + "modelReceiptPolicy": "Native per-run USD remains null/unpriced when absent. Included subscription credits, exclusive API-key charges and sandbox analytics are separate evidence. Unknown costs are never assumed zero.", + "evidenceScope": "Sanitized checkpoint of retained private receipts; historical reports retain earlier attempts. This file is not an executable conformance result or production qualification certificate.", + "currentProfilePaidCoverage": { + "cursor4": "No current-profile paid Product receipt. Historical local successes used Cursor2. Account spend-limit decision pending.", + "copilot4": "bd4 local hello, semantic question, semantic plan and controller-restart passed. File startup timeout unresolved; two diagnostic attempts failed before provider launch because private DB initialization failed. No successful Daytona Product run.", + "pi7": "807 local hello and bd4 Daytona hello passed. Current bd4 local8 campaign is reserved but not launched." + }, + "hostBlocker": "Credential-free exact initdb reproduced semget ENOSPC; kernel semaphore pool nearly exhausted. No global IPC changes performed. Paid launches paused.", + "ciCostCoverage": "Public standard GitHub-hosted minutes and self-hosted GitHub runner charges are zero; external self-hosted fleet infrastructure costs and private evals CI incremental charges remain unmetered.", + "attempts": [ + { + "id": "copilot-v4-local-question-20260929-01", + "provider": "copilot", + "status": "failed_continuation", + "cell": "extended-harnesses.runner-acpx-copilot.local.question-resume-complete", + "sourceRevision": "807feaecf2b6c6e2a52f5c18dc868b05d7883054", + "profileVersion": 4, + "profileDigest": "sha256:a119e436b4ad13ee70e1f58bedad4a0f20761b5fd982b752981f0cb1bc65f797", + "providerPackDigest": "sha256:03b5c9f9520e70c77e63dd2aafbd56864569dbdb6d50b0519a904b00d37e4218", + "model": "gpt-5.6-luna", + "createdAt": "2026-09-29T17:23:42.459281+00:00", + "reconciledAt": "2026-09-29T17:26:32.220513+00:00", + "actualCostUsd": null, + "costCoverage": "Included-credit counter6\u21927; additionalusage remainsdisabled$0;nativeperrunUSDunknown", + "failure": "Question UI reached; run.attach rejected same settled ACPX provider profile and session during warm recovery", + "attemptCount": 1, + "providerInferenceTurns": 1, + "ownedProcessesRetired": true, + "ownedProcessCount": 59, + "maxRetries": 0, + "privateEvidenceName": "copilot-v4-807-question-01" + }, + { + "id": "pi-v7-local-hello-20260929-01", + "provider": "pi", + "status": "passed", + "cell": "extended-harnesses.runner-acpx-pi.local.hello-complete", + "sourceRevision": "807feaecf2b6c6e2a52f5c18dc868b05d7883054", + "profileVersion": 7, + "profileDigest": "sha256:fec5b1448f4135710887133a9192747c476a825a56c255b52b17b1780ccb3761", + "providerPackDigest": "sha256:03b5c9f9520e70c77e63dd2aafbd56864569dbdb6d50b0519a904b00d37e4218", + "model": "openrouter/deepseek/deepseek-v4-flash-0731", + "createdAt": "2026-09-29T17:25:01.741345+00:00", + "reconciledAt": "2026-09-29T17:28:37.702173+00:00", + "actualCostUsd": 0.000646464, + "costCoverage": "exclusive OpenRouter key delayed delta; native per-run USD unpriced", + "attemptCount": 1, + "providerInferenceTurns": 1, + "ownedProcessesRetired": true, + "ownedProcessCount": 46, + "observedAutomaticRetries": 0, + "maxRetries": 0, + "durationMs": 56780, + "privateEvidenceName": "pi-v7-local-hello-20260929-01" + }, + { + "id": "pi-v7-daytona-hello-20260929-01", + "provider": "pi", + "status": "failed_before_inference", + "cell": "extended-harnesses.runner-acpx-pi.daytona.hello-complete", + "sourceRevision": "807feaecf2b6c6e2a52f5c18dc868b05d7883054", + "profileVersion": 7, + "profileDigest": "sha256:fec5b1448f4135710887133a9192747c476a825a56c255b52b17b1780ccb3761", + "model": "openrouter/deepseek/deepseek-v4-flash-0731", + "image": "ghcr.io/paperclipai/paperclip-daytona-runner@sha256:340ab3df00cfba6e78c9602bb3b79d023da6ddcc00737661324e3647083cb4f6", + "createdAt": "2026-09-29T17:45:40.146562+00:00", + "reconciledAt": "2026-09-29T17:55:55.859885+00:00", + "actualCostUsd": 0.00040716400000000006, + "costCoverage": "Sandbox-specific usage interval and exclusive OpenRouter delta; conservative720secondinfraupperbound retained pendingdelayedconfirmation", + "costFinality": "provisional sandbox analytics; explicit closed interval, delayed observation pending", + "failure": "Controller rejected missing build-owned remote provider-pack/binary env before inference; supervisor additionally treated removed teardown config as ambiguous", + "attemptCount": 1, + "productRunCount": 1, + "providerInferenceTurns": 0, + "ownedProcessesRetired": true, + "ownedSandboxCount": 1, + "cleanupStatus": "observed_absent", + "maxRetries": 0, + "inferenceKeyDeltaUsd": 0, + "privateEvidenceName": "pi-v7-daytona-hello-20260929-01" + }, + { + "id": "copilot-v4-local-question-20260929-02", + "provider": "copilot", + "status": "failed_before_inference", + "cell": "extended-harnesses.runner-acpx-copilot.local.question-resume-complete", + "sourceRevision": "bd4cc29c3017ed5e1484423e842fd48e3b2f49f3", + "profileVersion": 4, + "profileDigest": "sha256:a119e436b4ad13ee70e1f58bedad4a0f20761b5fd982b752981f0cb1bc65f797", + "providerPackDigest": "sha256:c19135b9175fffe9b18c06ca735ba1610e6cb2e36305c760d763c1f41fa67d81", + "model": "gpt-5.6-luna", + "createdAt": "2026-09-29T17:49:21.114937+00:00", + "reconciledAt": "2026-09-29T17:52:56.200598+00:00", + "actualCostUsd": 0, + "costCoverage": "No provider startup/inference; refreshed account remains7/1500includedcredits and additional0/0; nativeUSDreceipt remainsnull", + "failure": "Private embedded Postgres package postinstall was skipped; initdb aborted for missing dylib alias before server/provider startup", + "attemptCount": 1, + "productRunCount": 0, + "providerInferenceTurns": 0, + "ownedProcessesRetired": true, + "ownedProcessCount": 10, + "maxRetries": 0, + "receiptMetadataDiscrepancy": { + "reservedBuildProvenanceSha256": "a6b3a4f3933aa5b78b784bd7a2561886c4fa47a4ae80038fb05c425719318a6a", + "actualSnapshotSha256": "6ff74bde30217858c1c4ec87d2f1a07fb389678ed7e592e647c685df1ecf03f5", + "reason": "Agent added private Git excludes/Vitest graph metadata before launch after reservation; actualruntimeinventories unchanged and pre/post verified. Preserve discrepancy; subsequent reservations bind finalreceipt." + }, + "privateEvidenceName": "copilot-v4-bd4-question-01" + }, + { + "id": "copilot-v4-local-question-20260929-03", + "provider": "copilot", + "status": "passed", + "cell": "extended-harnesses.runner-acpx-copilot.local.question-resume-complete", + "sourceRevision": "bd4cc29c3017ed5e1484423e842fd48e3b2f49f3", + "profileVersion": 4, + "profileDigest": "sha256:a119e436b4ad13ee70e1f58bedad4a0f20761b5fd982b752981f0cb1bc65f797", + "providerPackDigest": "sha256:c19135b9175fffe9b18c06ca735ba1610e6cb2e36305c760d763c1f41fa67d81", + "model": "gpt-5.6-luna", + "createdAt": "2026-09-29T17:53:59.166487+00:00", + "reconciledAt": "2026-09-29T17:56:55.359304+00:00", + "actualCostUsd": null, + "costCoverage": "GitHub account includedcreditdelta1, additionalusage disabled0/0; native perrunUSD unpriced", + "attemptCount": 1, + "productRunCount": 2, + "ownedProcessesRetired": true, + "ownedProcessCount": 59, + "matcherPassCount": 6, + "matcherCount": 6, + "invariantFailures": 0, + "postRunAuthorityPassed": true, + "maxRetries": 0, + "additionalUsageUsd": 0, + "includedCreditDelta": 1, + "providerSessionReused": true, + "privateEvidenceName": "copilot-v4-bd4-question-02" + }, + { + "id": "copilot-v4-local-remaining-20260929-01", + "provider": "copilot", + "status": "failed_last_cell", + "cells": [ + "extended-harnesses.runner-acpx-copilot.local.hello-complete", + "extended-harnesses.runner-acpx-copilot.local.plan-approve-complete", + "extended-harnesses.runner-acpx-copilot.local.structured-question-restart-resume", + "extended-harnesses.runner-acpx-copilot.local.file-edit-validate" + ], + "sourceRevision": "bd4cc29c3017ed5e1484423e842fd48e3b2f49f3", + "profileVersion": 4, + "profileDigest": "sha256:a119e436b4ad13ee70e1f58bedad4a0f20761b5fd982b752981f0cb1bc65f797", + "providerPackDigest": "sha256:c19135b9175fffe9b18c06ca735ba1610e6cb2e36305c760d763c1f41fa67d81", + "model": "gpt-5.6-luna", + "createdAt": "2026-09-29T17:59:30.648559+00:00", + "reconciledAt": "2026-09-29T18:08:08.105294+00:00", + "actualCostUsd": null, + "costCoverage": "Refreshed GitHub account included credits8\u219211; extra usage disabled$0/$0; native per-run USD unpriced", + "failure": "file-edit-validate: session.open timed out; earlier hello, plan and controller-restart cases passed", + "attemptCount": 4, + "ownedProcessesRetired": true, + "postRunAuthorityPassed": true, + "observedAutomaticRetries": 0, + "maxRetries": 0, + "additionalUsageUsd": 0, + "includedCreditDelta": 3, + "privateEvidenceName": "copilot-v4-bd4-remaining-01" + }, + { + "id": "pi-v7-daytona-hello-20260929-02", + "provider": "pi", + "status": "passed", + "cell": "extended-harnesses.runner-acpx-pi.daytona.hello-complete", + "sourceRevision": "bd4cc29c3017ed5e1484423e842fd48e3b2f49f3", + "profileVersion": 7, + "profileDigest": "sha256:fec5b1448f4135710887133a9192747c476a825a56c255b52b17b1780ccb3761", + "model": "openrouter/deepseek/deepseek-v4-flash-0731", + "image": "ghcr.io/paperclipai/paperclip-daytona-runner@sha256:bff4c3f291087a0eeae37e4c20dd51857b92833eaf73ba3aca4157f37de1e109", + "createdAt": "2026-09-29T18:11:17.415830+00:00", + "reconciledAt": "2026-09-29T18:25:33.859370+00:00", + "actualCostUsd": 0.0053185282, + "costCoverage": "Exclusive OpenRouter key delta plus exact owned sandbox-specific delayed analytics; conservative infra lifetime bound retained", + "costFinality": "provisional provider analytics, stable at three delayed observations", + "attemptCount": 1, + "productRunCount": 1, + "providerInferenceTurns": 1, + "ownedProcessesRetired": true, + "ownedSandboxCount": 1, + "cleanupStatus": "observed_absent", + "matcherPassCount": 6, + "matcherCount": 6, + "postRunAuthorityPassed": true, + "inferenceKeyDeltaUsd": 0.000630644, + "daytonaObservedCostUsd": 0.0046878842, + "privateEvidenceName": "qualification-summary.json", + "privateEvidenceSha256": "1311b63488d5741a79c7a12a462e733f9eb3165d2f0ed429bab42f80e749cc82" + }, + { + "id": "copilot-v4-local-file-20260929-01", + "provider": "copilot", + "status": "failed_before_inference", + "cell": "extended-harnesses.runner-acpx-copilot.local.file-edit-validate", + "sourceRevision": "bd4cc29c3017ed5e1484423e842fd48e3b2f49f3", + "profileVersion": 4, + "profileDigest": "sha256:a119e436b4ad13ee70e1f58bedad4a0f20761b5fd982b752981f0cb1bc65f797", + "providerPackDigest": "sha256:c19135b9175fffe9b18c06ca735ba1610e6cb2e36305c760d763c1f41fa67d81", + "model": "gpt-5.6-luna", + "createdAt": "2026-09-29T18:15:20.477112+00:00", + "reconciledAt": "2026-09-29T18:17:17.885254+00:00", + "actualCostUsd": 0, + "costCoverage": "No provider launch; GitHubaccount11\u219211 and extra0/0. No nativeUSDreceipt.", + "failure": "Embedded PostgreSQL bootstrap exited1 before webserver/API/provider startup; does not diagnose original PRP session.open timeout", + "attemptCount": 1, + "productRunCount": 0, + "providerInferenceTurns": 0, + "ownedProcessesRetired": true, + "ownedProcessCount": 20, + "productCleanupStatus": "not_started", + "postRunAuthorityPassed": true, + "maxRetries": 0, + "additionalUsageUsd": 0, + "includedCreditDelta": 0, + "privateEvidenceName": "verified-outcome-summary.json", + "privateEvidenceSha256": "a812a668d3acb37e1b9553adca95fbce6863f4998ba233c2545b2b5859c6a39e" + }, + { + "id": "copilot-v4-daytona-hello-20260929-01", + "provider": "copilot", + "status": "failed_before_inference", + "cell": "extended-harnesses.runner-acpx-copilot.daytona.hello-complete", + "sourceRevision": "bd4cc29c3017ed5e1484423e842fd48e3b2f49f3", + "profileVersion": 4, + "profileDigest": "sha256:a119e436b4ad13ee70e1f58bedad4a0f20761b5fd982b752981f0cb1bc65f797", + "model": "gpt-5.6-luna", + "image": "ghcr.io/paperclipai/paperclip-daytona-runner@sha256:bff4c3f291087a0eeae37e4c20dd51857b92833eaf73ba3aca4157f37de1e109", + "createdAt": "2026-09-29T18:21:51.168258+00:00", + "reconciledAt": "2026-09-29T18:25:33.859370+00:00", + "actualCostUsd": 0, + "costCoverage": "Source-order proof: webserver initdb failed before health readiness, fixture creation and cloud/provider execution. Supervisor API admission only read account/analytics. No scope existed; retain machine unknown flag rather than claim a sandbox was deleted.", + "failure": "Embedded PostgreSQL bootstrap failed before fixture/task; credential-free exact initdb reproduction confirmed semget ENOSPC host semaphore exhaustion", + "attemptCount": 1, + "productRunCount": 0, + "providerInferenceTurns": 0, + "ownedProcessesRetired": true, + "machineCleanupStatus": "unresolved_scope_not_captured", + "productCleanupStatus": "not_started", + "cloudWritePathReached": false, + "postRunAuthorityPassed": true, + "privateEvidenceName": "verified-outcome-summary.json", + "privateEvidenceSha256": "f82411ec1b3c9ee9486e7d2d2c5dc81ecf8e603304a28d911e020341cda9b503" + }, + { + "id": "pi-v7-local-bd4-campaign-20260929-01", + "provider": "pi", + "status": "reserved_waiting_for_copilot_product_shutdown", + "cells": [ + "extended-harnesses.runner-acpx-pi.local.hello-complete", + "pi-native.runner-acpx-pi.local.native-questions", + "extended-harnesses.runner-acpx-pi.local.question-resume-complete", + "extended-harnesses.runner-acpx-pi.local.plan-approve-complete", + "extended-harnesses.runner-acpx-pi.local.structured-question-restart-resume", + "extended-harnesses.runner-acpx-pi.local.file-edit-validate", + "pi-native.runner-acpx-pi.local.agent-files-fresh-run", + "pi-native.runner-acpx-pi.local.restrictive-denial" + ], + "sourceRevision": "bd4cc29c3017ed5e1484423e842fd48e3b2f49f3", + "profileVersion": 7, + "profileDigest": "sha256:fec5b1448f4135710887133a9192747c476a825a56c255b52b17b1780ccb3761", + "model": "openrouter/deepseek/deepseek-v4-flash-0731", + "createdAt": "2026-09-29T18:23:16.046245+00:00", + "actualCostUsd": null, + "maxRetries": 0 + } + ] +}