mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-25 02:26:12 +02:00
Record green accounting verification and stabilize test fixtures
This commit is contained in:
1 parent
841b836be2
commit
d65c320472
10 files changed
+14457
-8
No files matched your search
@@ -122,9 +122,9 @@ Keep this section even if this slice incidentally resolves some assertions.
|
||||
| Title/description words select work mode | 4 | Heuristic removed in [explicit work-mode follow-up](2026-09-22-explicit-work-mode-authority.md). Clarification: this was a liveness diagnostic exemption, not a stored mode mutation. Deterministic checks and 4/4 corrected live wording cases pass; both explicit planning controls pass |
|
||||
| Commentary counts as progress | 1 | Retain for progress-evidence audit; no comment-count budget resets in this slice |
|
||||
| Wording selects liveness versus handoff path in heartbeat | 1 | Retain separately in results; must be covered while closing the current shared authority boundary |
|
||||
| Native autonomous continuation compatibility probes | 2 | Replace with reachable public-contract tests; `same_agent` injection is not a current model-facing defect |
|
||||
| Productive continuation versus repair/failure budgets | Three new findings | [Accounting baseline](2026-09-22-continuation-accounting-baseline.md) exposes delayed repair spending infrastructure allowance, infrastructure retries spending productive allowance, and delayed repair cancellation during promotion. Test-only slice; production fixes remain open |
|
||||
| Blank task route after reload | Repeated live observation | Investigate browser bootstrap/rendering; campaigns 35744074887 and 35747200170, plus accounting campaign 35813099816 after two correct legacy steps with a pending question. Empty screenshot retained; full quiet productive case remains unverified |
|
||||
| Native autonomous continuation compatibility probes | 2 | Replaced with reachable question/response continuation tests in the [September 23 follow-up](../../tests/lifecycle-baseline/CONTINUATION-ACCOUNTING-FIXES-2026-09-23.md), including no execution before an answer and duplicate-delivery checks. `same_agent` injection was not a current model-facing defect |
|
||||
| Productive continuation versus repair/failure budgets | Three new findings | Fixed with separate persisted allowances and episode-aware delayed repair promotion. All original intended-behavior failures pass; [fresh verification and retained baseline](../../tests/lifecycle-baseline/CONTINUATION-ACCOUNTING-FIXES-2026-09-23.md) distinguish deterministic and live evidence |
|
||||
| Blank task route after reload | Repeated live observation | Development-worker interception of Vite module revalidation removed. The [September 23 report](../../tests/lifecycle-baseline/CONTINUATION-ACCOUNTING-FIXES-2026-09-23.md) retains the trace limitation and shows the previously failing legacy journey completing all five steps, plus both native reload journeys. The original empty screenshots and failed measurements remain preserved |
|
||||
| Duplicate legacy response | One later live observation | Campaign 35805477715 legacy work-mode neutral: provider issued two successful PATCH calls, second escaping an underscore. Retain as model behavior evidence; exact-once matcher rejected it |
|
||||
| Missing-comment policy ownership | Coverage boundary | Existing typed retry policy remains; review separately if consolidating all post-run compliance into one disposition contract |
|
||||
| Maintain CI and prepare App/Evals review | Pending | Promote fixed invariants into maintained cheap gates; paid suites remain explicit |
|
||||
@@ -134,4 +134,5 @@ Reference reports: `tests/lifecycle-baseline/BASELINE-2026-09-21.md`,
|
||||
Live follow-up: 40/40 Product E2E passed on App `331e89bb3`; original protocol
|
||||
correction: 8/8 passed. Those results do not establish prose-free authority.
|
||||
|
||||
Accounting test-first follow-up: [scenario matrix and executable layers](2026-09-22-continuation-accounting-baseline.md). Production fixes for newly exposed cross-lane accounting failures remain a separate slice.
|
||||
Accounting test-first follow-up: [scenario matrix and executable layers](2026-09-22-continuation-accounting-baseline.md).
|
||||
Production follow-up: [September 23 implementation and verification](../../tests/lifecycle-baseline/CONTINUATION-ACCOUNTING-FIXES-2026-09-23.md).
|
||||
@@ -433,6 +433,8 @@ describe("sandbox callback bridge", () => {
|
||||
const queueDir = path.posix.join(rootDir, "queue");
|
||||
const directories = sandboxCallbackBridgeDirectories(queueDir);
|
||||
const processed: string[] = [];
|
||||
let signalStarted!: () => void;
|
||||
const started = new Promise<void>(resolve => { signalStarted = resolve; });
|
||||
|
||||
const worker = await startSandboxCallbackBridgeWorker({
|
||||
client: createFileSystemSandboxCallbackBridgeQueueClient(),
|
||||
@@ -440,6 +442,7 @@ describe("sandbox callback bridge", () => {
|
||||
authorizeRequest: async () => null,
|
||||
handleRequest: async (request) => {
|
||||
processed.push(request.id);
|
||||
signalStarted();
|
||||
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||
return {
|
||||
status: 200,
|
||||
@@ -475,9 +478,9 @@ describe("sandbox callback bridge", () => {
|
||||
"utf8",
|
||||
);
|
||||
|
||||
for (let attempt = 0; attempt < 50 && processed.length === 0; attempt += 1) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 5));
|
||||
}
|
||||
// Begin the short drain deadline only after the first handler has started.
|
||||
// A fixed 250ms polling window can expire during a loaded test run.
|
||||
await started;
|
||||
|
||||
await worker.stop({ drainTimeoutMs: 10 });
|
||||
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
# Continuation accounting fixes — September 23, 2026
|
||||
|
||||
Follow-up to the [September 22 baseline](CONTINUATION-ACCOUNTING-2026-09-22.md).
|
||||
Original failed measurements remain unchanged. The production and test changes
|
||||
are committed as `cfbf03adee36ff8c3235858d046147c2d3c078d4`.
|
||||
|
||||
## Changes
|
||||
|
||||
- Infrastructure retries and max-turn continuations have separate persisted
|
||||
counters. Resource waits and disposition repairs carry both counts forward;
|
||||
neither spend nor replenish them. Repair slots remain in their durable episode.
|
||||
- Delayed legacy repairs use the episode fingerprint that created them, with a
|
||||
successful source run bound to the same company, task and agent. Invalid source,
|
||||
changed episode and exhausted slot are rejected. Existing late gates still apply.
|
||||
- The two unsupported native `same_agent` compatibility probes now exercise the
|
||||
public question/response tool contract, including misleading prose, no execution
|
||||
before a response, duplicate delivery and five successive productive turns.
|
||||
- The unstamped development service worker no longer intercepts Vite module
|
||||
revalidation. The retained blank-page trace showed a bodyless module `304` through
|
||||
the worker before React mounted. The browser regression proves repeated conditional
|
||||
module reloads bypass that worker; it does not reproduce the entire original hang.
|
||||
Stamped production offline caching retains its existing privacy checks.
|
||||
|
||||
## Deterministic verification
|
||||
|
||||
[Saved measurement](baselines/2026-09-23-continuation-accounting-fixed.json):
|
||||
**1,073 / 1,073 passing**, no skipped tests or missing inventory evidence.
|
||||
|
||||
| Layer | Passed / total |
|
||||
|---|---:|
|
||||
| Unit | 423 / 423 |
|
||||
| Runner | 184 / 184 |
|
||||
| Database/heartbeat integration | 381 / 381 |
|
||||
| Grading | 85 / 85 |
|
||||
|
||||
The original baseline was 1,053 / 1,059. All four newly failing intended-behavior
|
||||
assertions pass unchanged. Two obsolete native probes were replaced as described
|
||||
above. The new total additionally includes nine accounting helper assertions and
|
||||
five database regressions for alternation/restart, repair debit preservation and
|
||||
invalid delayed repair sources.
|
||||
|
||||
Measurement provenance: the saved runner metadata records parent `c0b5ee8b3` plus
|
||||
working-tree fingerprint `1ef24dd199eff085dd4f7897398e08473c9ff618c23182246648589ab5239cd0`.
|
||||
Those exact source/test changes were then committed as `cfbf03ade`; the raw local
|
||||
measurement is `.lifecycle-baseline/2026-09-23T14-21-33.472Z/`, including `source.diff`.
|
||||
The saved JSON is copied unchanged rather than relabeled as a clean-commit run.
|
||||
|
||||
Other completed checks:
|
||||
|
||||
- Product E2E support: 475 / 475 after the comment-format calibration follow-up
|
||||
(474 / 474 before it). An initial sandboxed run could not bind fixture
|
||||
sockets; the normal-permission rerun passes.
|
||||
- Cheap real-browser support: 5 / 5, including three conditional reloads using the
|
||||
actual development worker. The new regression fails on the previous worker
|
||||
because it intercepts all three module fetches.
|
||||
- Accounting helper, worker cache/privacy and build stamping checks: 33 / 33.
|
||||
- Baseline inventory/report support: 4 / 4.
|
||||
- E2E and baseline typechecks, recursive repository typecheck, token gates and
|
||||
repository build pass.
|
||||
- The [grading-only follow-up](baselines/2026-09-23-continuation-accounting-grading-followup.json)
|
||||
passes 86 / 86, adding a test that rejects the exact serialized-array mistake
|
||||
observed live. Combined with the unchanged unit/runner/integration layers,
|
||||
the current inventory has 1,074 passing assertions. The full 1,073-test
|
||||
measurement above remains an unmodified record of its earlier execution.
|
||||
|
||||
The full repository test first passed its general server phase (12,902 passed,
|
||||
83 pre-existing skips), then stopped on three offline fallback tests that loaded
|
||||
the development worker while expecting production behavior. Their fixture now
|
||||
uses a production build stamp, preserving all assertions. The full UI rerun
|
||||
passes 6,503 / 6,503; CLI passes 502 / 502.
|
||||
|
||||
The remaining supported repository groups ran separately, avoiding repetition
|
||||
of the passing general server phase. Two route shards initially exceeded their
|
||||
15-second cold-import test deadline while the UI group was running; unchanged
|
||||
reruns at lower load are retained separately. Broad package verification also
|
||||
exposed a pre-existing callback-drain fixture assumption that a handler starts
|
||||
within a 250ms polling window. That test now awaits an explicit start signal;
|
||||
its original drain assertions and all 61 tests in the file pass. These follow-up
|
||||
changes affect test setup only, not the product source measured by the live run.
|
||||
|
||||
Final repository verification: **25,944 passed, 0 failed, 102 existing skips**
|
||||
across all groups of `scripts/run-vitest-stable.mjs`. All 148 serialized test
|
||||
files are accounted for exactly once in the final three shard results. The
|
||||
[verification record](baselines/2026-09-23-continuation-accounting-repository-verification.json)
|
||||
retains the original failures, successful group counts and log hashes. Recursive
|
||||
typecheck and build pass; the two test-only follow-ups also pass their package
|
||||
typechecks and token gates. This is a complete resumed verification, not a claim
|
||||
that the first monolithic `pnpm test:run` invocation exited successfully.
|
||||
|
||||
## Live verification
|
||||
|
||||
[Campaign 35873996790](https://github.com/paperclipai/paperclip/actions/runs/35873996790)
|
||||
ran the same eight explicit `continuation-accounting` cells against `cfbf03ade`.
|
||||
The qualified Codex `gpt-5.6-sol` profiles, local isolated fixtures, real provider calls and
|
||||
Chromium browser remain the same. No expected outcome or live grader was weakened.
|
||||
|
||||
Result: **7 / 8**, no retries, all evidence packages valid and cleanup successful.
|
||||
All three previously failing live journeys pass: both bounded exhaustion cases
|
||||
and the legacy quiet productive workflow, including repeated task reloads. Its
|
||||
final screenshot shows Done and five document cards at revision 1. Stop, approval,
|
||||
legacy noisy productivity and native noisy productivity also pass.
|
||||
|
||||
Native quiet productivity completed all five correct documents and four answered
|
||||
questions with no repair/failure debt. Its sole failed assertion is `perturbation`:
|
||||
the model posted `["ACCOUNTINGQUIET..."]` instead of `ACCOUNTINGQUIET...` on each
|
||||
turn. This is a completed model instruction-following failure, with an ambiguous
|
||||
JSON-array presentation in the fixture prompt; it is not a lifecycle accounting
|
||||
failure or missing evidence. The original machine grade remains failed.
|
||||
|
||||
- [Published 7/8 report](https://d1p6rlowie26tp.cloudfront.net/runner-e2e/campaigns/gha-35873996790-1/index.html)
|
||||
- [Immutable first post-fix measurement](baselines/2026-09-23-continuation-accounting-fixed-initial-live.json)
|
||||
|
||||
Commit `841b836be232e6624776ee4e3f7641271d954914` presents each exact comment body
|
||||
as a separate fenced literal and explicitly excludes brackets/quotation marks.
|
||||
The exact-match grader is unchanged, and the added calibration rejects serialized
|
||||
array bodies. [Final campaign 35876252684](https://github.com/paperclipai/paperclip/actions/runs/35876252684)
|
||||
reran all eight cells against that source. The previous campaign's tests, merged
|
||||
report and S3 publication finished before dispatch; only its queued secondary
|
||||
GitHub Pages copy could be superseded by the workflow's same-branch concurrency.
|
||||
|
||||
Billing for the first post-fix run is partial: 31/32 run records have token usage,
|
||||
10 have reported cost, and one is cancelled before dispatch. Reported zero cost
|
||||
does not mean zero actual spending. The saved measurement retains all coverage
|
||||
fields and definition fingerprints.
|
||||
|
||||
Final result: **8 / 8 passed**, no retries, no incomplete cells, all eight evidence
|
||||
packages valid and cleanup successful. Both native and legacy productive wording
|
||||
pairs complete five exactly-once revision-one records through four real responses.
|
||||
Both legacy exhaustion variants reach the expected bounded stop; the late Stop
|
||||
and approval workflows retain their authoritative behavior.
|
||||
|
||||
- [Final live report](https://d1p6rlowie26tp.cloudfront.net/runner-e2e/campaigns/gha-35876252684-1/index.html)
|
||||
- [Immutable final measurement](baselines/2026-09-23-continuation-accounting-fixed-live.json)
|
||||
|
||||
The final measurement also has partial billing: 31/32 run records report usage,
|
||||
10 report cost, and one is cancelled before provider execution. No zero-spend
|
||||
claim is made. Both post-fix campaigns and the September 22 failures are retained.
|
||||
@@ -152,3 +152,16 @@ September 22 follow-up: [legacy continuation implementation and verification](LE
|
||||
The [continuation accounting matrix](../../doc/plans/2026-09-22-continuation-accounting-baseline.md) adds ACCT-01 through ACCT-04 for separate allowances, false progress, late gates and restart/replay. Its real-provider companion is the explicit-only `continuation-accounting` Product E2E suite.
|
||||
The [September 22 measurement](CONTINUATION-ACCOUNTING-2026-09-22.md) records the
|
||||
enabled failures and preserves both initial and corrected live campaigns.
|
||||
The [September 23 fixes and fresh verification](CONTINUATION-ACCOUNTING-FIXES-2026-09-23.md)
|
||||
retain the original measurements and cover separate persisted allowances, delayed
|
||||
repair promotion and the current native question/response continuation contract.
|
||||
|
||||
The inexpensive browser regressions use real Chromium without a provider or
|
||||
Paperclip instance. They check screenshot readiness and development service-worker
|
||||
module revalidation across repeated reloads:
|
||||
|
||||
```sh
|
||||
pnpm exec playwright test --config tests/runner-e2e/playwright-support.config.ts
|
||||
```
|
||||
|
||||
Set `PAPERCLIP_PLAYWRIGHT_CHANNEL=chrome` to use an installed Chrome browser.
|
||||
+2856
File diff suppressed because it is too large.
Load diff
+2846
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
+1773
File diff suppressed because it is too large.
Load diff
+119
@@ -0,0 +1,119 @@
|
||||
{
|
||||
"schema": "paperclip.repository-verification/v1",
|
||||
"measuredAt": "2026-09-23T15:04:23.646010+00:00",
|
||||
"productionCommit": "cfbf03adee36ff8c3235858d046147c2d3c078d4",
|
||||
"fixtureCommit": "841b836be232e6624776ee4e3f7641271d954914",
|
||||
"scope": "All groups of scripts/run-vitest-stable.mjs, resumed after the initial pnpm test:run failure. Two test-only fixture fixes were verified before this record; no production change after the live campaign source.",
|
||||
"counts": {
|
||||
"passed": 25944,
|
||||
"failed": 0,
|
||||
"skipped": 102
|
||||
},
|
||||
"serializedCoverage": {
|
||||
"expected": 148,
|
||||
"executed": 148,
|
||||
"unique": 148,
|
||||
"missing": []
|
||||
},
|
||||
"groups": [
|
||||
{
|
||||
"group": "general-server",
|
||||
"status": "passed",
|
||||
"counts": {
|
||||
"passed": 12902,
|
||||
"failed": 0,
|
||||
"skipped": 83
|
||||
},
|
||||
"completedInvocations": 1,
|
||||
"logPath": "/private/tmp/accounting-fixed-full-tests.log",
|
||||
"logSha256": "4424a8ed0eab9dd363d225a152b22954c661db1833e66cfb2a555b975f618592"
|
||||
},
|
||||
{
|
||||
"group": "workspaces-a",
|
||||
"status": "passed",
|
||||
"counts": {
|
||||
"passed": 7005,
|
||||
"failed": 0,
|
||||
"skipped": 0
|
||||
},
|
||||
"completedInvocations": 2,
|
||||
"logPath": "/private/tmp/accounting-fixed-workspaces-a.log",
|
||||
"logSha256": "350e65148379100561e2ae367383c58b850899bc6f40cb7bb833b2069e7000ae"
|
||||
},
|
||||
{
|
||||
"group": "workspaces-b",
|
||||
"status": "passed",
|
||||
"counts": {
|
||||
"passed": 3398,
|
||||
"failed": 0,
|
||||
"skipped": 19
|
||||
},
|
||||
"completedInvocations": 12,
|
||||
"logPath": "/private/tmp/accounting-fixed-workspaces-b-final.log",
|
||||
"logSha256": "9f76c4968ce972fc47c503065f967e66bea2321b213108884df641d7c9905fd5"
|
||||
},
|
||||
{
|
||||
"group": "serialized-0",
|
||||
"status": "passed",
|
||||
"counts": {
|
||||
"passed": 760,
|
||||
"failed": 0,
|
||||
"skipped": 0
|
||||
},
|
||||
"completedInvocations": 47,
|
||||
"logPath": "/private/tmp/accounting-fixed-serialized-0-retry.log",
|
||||
"logSha256": "26bbb8c43423f7b3d4a0f2b2cab31434891aa9d5a226bdb070c50f280993edb2"
|
||||
},
|
||||
{
|
||||
"group": "serialized-1",
|
||||
"status": "passed",
|
||||
"counts": {
|
||||
"passed": 938,
|
||||
"failed": 0,
|
||||
"skipped": 0
|
||||
},
|
||||
"completedInvocations": 50,
|
||||
"logPath": "/private/tmp/accounting-fixed-serialized-1.log",
|
||||
"logSha256": "3892df72f6d484c71508d9900acb06fae8ba7854e9981827ffbd8c6adc1069c3"
|
||||
},
|
||||
{
|
||||
"group": "serialized-2",
|
||||
"status": "passed",
|
||||
"counts": {
|
||||
"passed": 941,
|
||||
"failed": 0,
|
||||
"skipped": 0
|
||||
},
|
||||
"completedInvocations": 51,
|
||||
"logPath": "/private/tmp/accounting-fixed-serialized-2-retry.log",
|
||||
"logSha256": "1a2174825d988a152b066fe83a4ce7398bcec7617e8a0d2dfa2eadcb111a78db"
|
||||
}
|
||||
],
|
||||
"initialFailures": [
|
||||
{
|
||||
"log": "/private/tmp/accounting-fixed-full-tests.log",
|
||||
"reason": "Three production offline-fallback assertions loaded an unstamped development worker; fixture now stamps a production build, unchanged assertions pass."
|
||||
},
|
||||
{
|
||||
"logs": [
|
||||
"/private/tmp/accounting-fixed-serialized-0.log",
|
||||
"/private/tmp/accounting-fixed-serialized-2.log"
|
||||
],
|
||||
"reason": "First route cold imports exceeded the 15s test deadline during concurrent UI execution; full unchanged shards rerun successfully after UI completed."
|
||||
},
|
||||
{
|
||||
"log": "/private/tmp/accounting-fixed-workspaces-b.log",
|
||||
"reason": "Callback drain fixture could stop before its handler started because its polling window was only 250ms. Test now awaits a handler-start signal; full workspace group passes."
|
||||
}
|
||||
],
|
||||
"checks": {
|
||||
"recursiveTypecheck": "passed",
|
||||
"build": "passed",
|
||||
"followupPackageTypechecks": "passed",
|
||||
"tokenGates": "passed",
|
||||
"baseline": "1073/1073 plus grading follow-up 86/86 (one added calibration, 1074 current assertions)",
|
||||
"e2eSupport": "475/475",
|
||||
"browserSupport": "5/5",
|
||||
"live": "8/8, no retries, all evidence valid, cleanup passed"
|
||||
}
|
||||
}
|
||||
@@ -29,7 +29,10 @@ function loadServiceWorkerFetchListener(overrides: {
|
||||
keys: vi.fn(async () => []),
|
||||
delete: vi.fn(async () => true),
|
||||
};
|
||||
const code = readFileSync(resolve(uiRoot, "public/sw.js"), "utf8");
|
||||
// Offline fallback belongs to a stamped production build. Development
|
||||
// leaves Vite requests to the browser instead of intercepting them.
|
||||
const code = readFileSync(resolve(uiRoot, "public/sw.js"), "utf8")
|
||||
.replace("__PAPERCLIP_BUILD_ID__", "fixture-production");
|
||||
new Function("self", "caches", "fetch", "Response", "URL", code)(
|
||||
swSelf,
|
||||
caches,
|
||||
|
||||
Reference in new issue
Block a user