From b5c53a996b071855d7b833f6a65c849a863a723f Mon Sep 17 00:00:00 2001 From: paperclipbugbot Date: Fri, 20 Nov 2026 04:37:39 -0800 Subject: [PATCH] fix(native): stop a repeated startup rejection when a run resumes its predecessor's provider session (#22393) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - Native runs on a remote sandbox start a Runner, and the Runner opens the provider session with `session.open` > - When the provider refuses `session.open` at startup, no provider work began. The executor records a startup failure signature, so the next attempt can stop a repeat of the same refusal (#20874) > - Only bootstrap retries recorded that signature. A run that continues its predecessor's provider session (for example a provider-limit Retry) recovers as an exact checkpoint resume, so it never recorded one > - Such a run retried every 30 seconds against the same provider rate limit, spent all 3 attempts, and ended as `provider_session_open_rejected`. That outcome is reported as a run failure > - This pull request records the signature for every remote startup rejection > - The benefit is that the second identical refusal ends the run at once. With #22386, a repeated rate-limit refusal then becomes the usage-limit hold that retries the task at the reset ## Linked Issues or Issue Description Refs #22386 Refs #20874 **What happened?** A staging run resumed its predecessor's provider session. The provider refused `session.open` with HTTP 429 ("would exceed your account's rate limit") on each attempt. The recovery mode was `exact_checkpoint_resume`, so no startup failure signature was saved. Each attempt looked new. The run made 3 attempts in about 3 minutes and ended as `provider_session_open_rejected` (retry budget spent). **Expected behavior** The second identical startup refusal stops the retries, as it does for a bootstrap retry. A rate-limit refusal waits for the reset (with #22386) and does not use the full retry budget. **Steps to reproduce** 1. Give a remote native run a session checkpoint with an established provider session. 2. Make the provider refuse `session.open` at startup with the same error on two attempts, with the runner exit proven. 3. See that the second attempt is retried again, and the run ends only when the attempt budget is spent. **Paperclip version or commit** `master` at 201abacd6e. ## What Changed - `native-session-executor.ts`: record `startupFailureSignature` in the recovery history for any remote `session.open` startup rejection, not only for `bootstrap_retry`. - `native-startup-rejected-retry.integration.test.ts`: add a test for a run that resumes its predecessor's provider session. The first attempt records the signature. The second identical refusal ends as `native_provider_startup_rejected`. ## Verification - `cd server && npx vitest run src/services/native-runtime/native-startup-rejected-retry.integration.test.ts`: 9/9 pass. The new test fails on `master` (the history has no `startupFailureSignature`) and passes with the change. - Related suites pass: `native-startup-retry`, `native-remote-retry-admission`, `native-runner-restart-recovery`, `heartbeat-provider-limit-wait`, `native-provider-capacity`, `native-session-executor` (1912 pass). One test in `native-session-executor.test.ts` ("compares export process compatibility…") fails, and it also fails on unchanged `master`. - I merged this branch with #22386 locally without conflicts. A scratch test with an Anthropic 429 refusal on an exact checkpoint resume ended as `native_provider_usage_limit` on the second attempt. I did not commit that scratch test. ## Risks - Low risk. The signature only changes behavior for a remote `provider_session_open_rejected` startup failure. A resumed run that meets the same startup refusal twice now stops after the second attempt and not after the third. A different refusal still retries. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), agentic coding with tool use and code execution. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [ ] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: paperclipbugbot <356415228+paperclipbugbot@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- .../native-runtime/native-session-executor.ts | 2 +- ...startup-rejected-retry.integration.test.ts | 20 +++++++++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index bdf7d3ae00c..328ebbab345 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -13400,7 +13400,7 @@ async function executePaperclipNativeSessionAttempt( ? sourceFailureCode : disposition.failureCode; const providerFailure = providerFailureFromError(error); - const startupFailureSignature = recoveryEvidence.recoveryMode === "bootstrap_retry" + const startupFailureSignature = recoveryEvidence.recoveryMode === "bootstrap_retry" || remoteStartupRejection ? nativeStartupFailureSignature(providerFailure) : null; const startupRejectedAction = repeatedRemoteStartupFailure && providerFailure ? nativeRepeatedStartupFailureAction(providerFailure.message) : NATIVE_PROVIDER_STARTUP_REJECTED_ACTION; diff --git a/server/src/services/native-runtime/native-startup-rejected-retry.integration.test.ts b/server/src/services/native-runtime/native-startup-rejected-retry.integration.test.ts index b2421734634..44605b8f4ba 100644 --- a/server/src/services/native-runtime/native-startup-rejected-retry.integration.test.ts +++ b/server/src/services/native-runtime/native-startup-rejected-retry.integration.test.ts @@ -46,7 +46,8 @@ describe("native session.open the provider rejected before any provider work", ( message: "error creating thread: failed to load AGENTS.md instructions for environment `local`: fs sandbox helper failed with status exit status: 1: bwrap: loopback: Failed RTM_NEWADDR: Operation not permitted", ...fields }); - async function rejectSessionOpen(failure: Error, attempt = 0, options: { remote?: boolean; recoveryHistory?: Record[] } = {}) { + async function rejectSessionOpen(failure: Error, attempt = 0, + options: { remote?: boolean; recoveryHistory?: Record[]; resumesProviderSession?: boolean } = {}) { const issueId = randomUUID(); await db.insert(issues).values({ id: issueId, companyId, title: "Plan the change", status: "in_progress", workMode: "standard", assigneeAgentId: agentId, createdByUserId: "board-user", responsibleUserId: "board-user" }); @@ -71,7 +72,10 @@ describe("native session.open the provider rejected before any provider work", ( contract: buildNativeCompletionContract(issue, { revision: contract.revision }) }, interactionResponses: [], credentialBindings: [], }; - await db.update(heartbeatRuns).set({ runnerProfileJson: { nativeExecutionInput: execution } }).where(eq(heartbeatRuns.id, runId)); + const sessionCheckpoint = options.resumesProviderSession + ? { sessionId: native.normalizedSessionId, providerSessionId: randomUUID(), terminal: null } : undefined; + await db.update(heartbeatRuns).set({ runnerProfileJson: { nativeExecutionInput: execution, + ...(sessionCheckpoint ? { sessionCheckpoint } : {}) } }).where(eq(heartbeatRuns.id, runId)); provider.execute.mockReset().mockRejectedValue(failure); await executePaperclipNativeSession({ db, execution, runnerInstanceId: native.runnerInstanceId, ...(options.remote ? { runnerExecutionTarget: { kind: "remote", transport: "sandbox", providerKey: "test", remoteCwd: "/workspace", @@ -149,4 +153,16 @@ describe("native session.open the provider rejected before any provider work", ( const [issue] = await db.select().from(issues).where(eq(issues.id, issueId)); await expect(taskInterruptionPresentation(db, issue)).resolves.toMatchObject({ state: "failed", canRetry: true }); }, 120_000); + + it("stops a repeated remote startup rejection on a run that resumes its predecessor's provider session", async () => { + const first = await rejectSessionOpen(exitProven(rejection(startupFailure())), 0, { remote: true, resumesProviderSession: true }); + expect(first.coordinator).toMatchObject({ phase: "retryable_failure", + failureDetail: expect.objectContaining({ recoveryMode: "exact_checkpoint_resume", originalFailureCode: "provider_session_open_rejected" }) }); + const firstHistory = first.coordinator.recoveryHistory as Record[]; + expect(firstHistory.at(-1)).toMatchObject({ disposition: "retryable_failure", startupFailureSignature: expect.any(String) }); + const { coordinator } = await rejectSessionOpen(exitProven(rejection(startupFailure())), 1, { remote: true, + resumesProviderSession: true, recoveryHistory: firstHistory }); + expect(coordinator).toMatchObject({ phase: "terminal_failure", failureCode: NATIVE_PROVIDER_STARTUP_REJECTED, nextAttemptAt: null, + failureDetail: expect.objectContaining({ nextAction: expect.stringContaining("bwrap: loopback: Failed RTM_NEWADDR") }) }); + }, 120_000); });