From 01f764fe61bea76f5fdcbe70c9ef300747747cfb Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Fri, 13 Nov 2026 19:41:04 -0600 Subject: [PATCH] test(evals): pin the case clock in the original-deadline rejection tests (#21501) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The Runner support suite tests the chat catalog observers that grade agent work > - One observer test fails at random on unrelated pull requests > - The test simulates a late poll result with a fixed clock jump from the fixture time. The observer computes its deadline from a later clock read. If 2 ms or more pass between the two reads, the jump does not pass the deadline. > - This pull request pins the clock to the fixture time for the deadline rejection cases, as the sibling deadline tests do > - The benefit is a deterministic test that still fails when the late-deadline guard is removed ## Linked Issues or Issue Description **What happened?** `tests/runner-e2e/chat-catalog-domains/identity-plan-decision-observer.test.ts` > "rejects late await original deadline without resetting it" failed in the "ci / Runner support suite" job of PR #21418 (run 43503383519, 2026-11-13 20:38Z). The error was: `expected [Function] to throw error including 'CONTROL_SOURCE_NOT_SETTLED' but got 'CONTROL_AFTER_SOURCE_SETTLED'`. PR #21418 changes only a plan document and a benchmark script. **Expected behavior** The test passes every time. The result must not depend on how fast the test worker runs. **Steps to reproduce** 1. Add `await new Promise(r => setTimeout(r, 2));` before the `act` call in the "late await" case. 2. Run `node node_modules/vitest/vitest.mjs run --config tests/runner-e2e/vitest.config.ts chat-catalog-domains/identity-plan-decision-observer.test.ts`. 3. The test fails with the CI error. With a gap of 0 or 1 ms it passes. **Paperclip version or commit** `master` at 2a8b485140. **Relevant logs or output** The test sets the clock to `f.now + 180_001` after the poll loads. `f.now` is the fixture's `Date.now()`. The observer sets `deadlineAt = Date.now() + 180_000` later, in `act`. The accept check is `Date.now() < deadlineAt`. It is true when `act` reads the clock 2 ms or more after the fixture. A loaded CI worker can create that gap. ## What Changed - In the four "rejects … original deadline without resetting it" cases, `Date.now` now returns the fixture time until the test jumps it. The "late await" jump is then exactly 1 ms past the observer's deadline. The other three cases fail at the budget check as before. ## Verification - Red: with a 2 ms or 5 ms gap before `act`, the unchanged test fails with the CI error (2 of 2 runs). With a 0 or 1 ms gap, it passes. - Green: with the fix, the test passes with gaps of 0, 2, 5 and 50 ms (39/39 tests each). - Mutation: with the fix, I removed `Date.now() < deadlineAt &&` from the accept check in `identity-plan-review.ts`. The "late await" test fails. The assertion still detects the regression that it guards. - `pnpm verify` selected broad CI coverage (exit 2) because the file is a global CI input. Required CI supplies that coverage. ## Risks - Low risk. The change affects only one test file. It does not change product code or assertions. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), in Claude Code with tool use and shell execution. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 5.5 --- .../identity-plan-decision-observer.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/runner-e2e/chat-catalog-domains/identity-plan-decision-observer.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-plan-decision-observer.test.ts index 180c7883ee0..4f54cb0b066 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-plan-decision-observer.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-plan-decision-observer.test.ts @@ -124,7 +124,7 @@ it("retains the original180-second phase when the case has more time remaining", expect(port.deadline).toBe(f.now + 180_000); }); it.each(["missing", "invalid", "expired", "late await"])("rejects %s original deadline without resetting it", async fault => { - const f = fixture("questions-and-approvals.review-plan", fault === "invalid" ? NaN : fault === "expired" ? -1 : 600_000); + const f = fixture("questions-and-approvals.review-plan", fault === "invalid" ? NaN : fault === "expired" ? -1 : 600_000); vi.spyOn(Date, "now").mockReturnValue(f.now); const ctx: ChatCatalogContext = fault === "missing" ? { ...f.ctx, observationDeadlineAt: undefined } : f.ctx; if (fault === "late await") port.afterLoadNow = f.now + 180_001; await expect(planReviewIdentityCases[0]!.act!(ctx, f.state)).rejects.toThrow(fault === "late await" ? "CONTROL_SOURCE_NOT_SETTLED" : "Original plan-review observation budget");