From cfbf03adee36ff8c3235858d046147c2d3c078d4 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 23 Sep 2026 07:24:23 -0700 Subject: [PATCH] Separate continuation allowances and preserve delayed repair authority --- ...026-09-23-continuation-accounting-fixes.md | 41 ++++++++++ .../heartbeat-retry-scheduling.test.ts | 36 +++++++++ .../legacy-continuation-authority.test.ts | 34 +++++++++ .../modules/run-dispatch/adapters/postgres.ts | 23 +++++- .../execution-recovery-attempt.test.ts | 20 ++++- .../services/execution-recovery-attempt.ts | 74 ++++++++++++++++--- server/src/services/heartbeat.ts | 17 ++--- server/src/services/recovery/service.ts | 4 +- .../lifecycle-heartbeat.test.ts | 48 +++++++++++- tests/lifecycle-baseline/inventory.mjs | 4 +- tests/runner-e2e/playwright-support.config.ts | 2 +- .../runner-e2e/service-worker-reload.spec.ts | 50 +++++++++++++ ui/public/sw.js | 4 + .../lib/service-worker-cache-policy.test.ts | 14 +++- 14 files changed, 338 insertions(+), 33 deletions(-) create mode 100644 doc/plans/2026-09-23-continuation-accounting-fixes.md create mode 100644 tests/runner-e2e/service-worker-reload.spec.ts diff --git a/doc/plans/2026-09-23-continuation-accounting-fixes.md b/doc/plans/2026-09-23-continuation-accounting-fixes.md new file mode 100644 index 00000000000..dbdb67ad77e --- /dev/null +++ b/doc/plans/2026-09-23-continuation-accounting-fixes.md @@ -0,0 +1,41 @@ +# Continuation accounting fixes — September 23, 2026 + +Follow the [test-first accounting baseline](2026-09-22-continuation-accounting-baseline.md) +and preserve its failed measurements. Repair the product against those assertions; +do not change their intended outcomes. + +## Implementation + +1. Persist independent infrastructure-failure and max-turn continuation counts in + server-owned run context. Repair slots remain in the durable disposition episode. + Carry both counts across waits, repairs and controller restart. A change of retry + reason cannot erase prior debt or charge another lane. Historical ambiguous + counters remain conservative; known repair/productive counters are not failures. +2. Promote delayed legacy repairs against their persisted episode fingerprint and + successful source run, checking company, issue, agent and bounded repair slot. + Retain the existing late Stop, wait/approval, ownership, pause and spending gates. +3. Replace the two obsolete scripted native `same_agent` probes with the reachable + public question/response contract: create a durable question, wait for its answer, + deliver once, and reject duplicate delivery. Keep the misleading-prose pair and + the sequence longer than the infrastructure retry allowance. +4. Keep the development service worker out of Vite's module revalidation. The failed + browser trace contains a bodyless module `304` through the worker before React + mounts. A small real-browser regression checks repeated conditional reloads with + the actual worker; production cache/privacy tests continue using a stamped build. + The live rerun must establish whether the observed blank-page failure is resolved. + +## Verification + +- Existing failing unit, scheduler and promotion assertions must pass unchanged. +- Exercise alternating failure/productive/resource-wait retries through the actual + scheduler, recreate the controller each time, and exhaust each allowance separately. +- Verify repairs preserve both debits, invalid source/episode/slot cannot promote, + and late gates cancel delayed repairs without another debit. +- Run all deterministic lifecycle layers, E2E support, the cheap browser suite, + relevant typechecks, then the required repository checks. +- Rerun all eight explicit `continuation-accounting` real-provider/browser cells in + parallel GitHub Actions jobs. Retain original grades, source/definition fingerprints, + cleanup, retries, usage/cost coverage and failure evidence. Save the new measurement + separately from September 22; do not regrade the baseline. + +Other retained findings remain in the [backlog](2026-09-22-legacy-continuation-authority.md). diff --git a/server/src/__tests__/heartbeat-retry-scheduling.test.ts b/server/src/__tests__/heartbeat-retry-scheduling.test.ts index b3c7d033273..5360d5216cc 100644 --- a/server/src/__tests__/heartbeat-retry-scheduling.test.ts +++ b/server/src/__tests__/heartbeat-retry-scheduling.test.ts @@ -662,6 +662,42 @@ describeEmbeddedPostgres("heartbeat bounded retry scheduling", () => { expect(scheduled).toMatchObject({ outcome: "scheduled", attempt: 1 }); }); + it("ACCT-01 alternating failures, productive continuations and waits preserve both allowances after restart", async () => { + const f = await seedMaxTurnFixture(); + let current = (await heartbeat.getRun(f.runId))!; + for (const [reason, attempt, failures, continuations] of [ + ["transient_failure", 1, 1, 0], + [MAX_TURN_CONTINUATION_RETRY_REASON, 1, 1, 1], + ["workspace_busy", 1, 1, 1], + ["transient_failure", 2, 2, 1], + [MAX_TURN_CONTINUATION_RETRY_REASON, 2, 2, 2], + ["ai_connection_busy", 1, 2, 2], + ] as const) { + // Exercise the typed terminal outcome that requests each lane. + const waiting = reason === "workspace_busy" || reason === "ai_connection_busy"; + await db.update(heartbeatRuns).set({ + status: waiting ? "cancelled" : "failed", errorCode: waiting ? reason : "adapter_failed", + resultJson: { ...(reason === MAX_TURN_CONTINUATION_RETRY_REASON ? { stopReason: "max_turns_exhausted" } : {}), + executionRecovery: { kind: waiting ? (reason === "workspace_busy" ? "workspace_wait" : "ai_connection_wait") : "bootstrap", providerWorkStarted: false } }, + }).where(eq(heartbeatRuns.id, current.id)); + const restarted = heartbeatService(db); + const result = await restarted.scheduleBoundedRetry(current.id, { now: f.now, retryReason: reason, maxAttempts: 2, delayMs: 1 }); + expect(result, JSON.stringify({ reason, result })).toMatchObject({ outcome: "scheduled", attempt }); + if (result.outcome !== "scheduled") throw new Error("Missing successor"); + expect(result.run.contextSnapshot?.executionRetryAccounting).toEqual({ version: 1, failureRetries: failures, maxTurnContinuations: continuations }); + await db.update(heartbeatRuns).set({ status: "failed", finishedAt: f.now, + resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }, + }).where(eq(heartbeatRuns.id, result.run.id)); + await db.update(issues).set({ executionRunId: result.run.id }).where(eq(issues.id, f.issueId)); + current = (await restarted.getRun(result.run.id))!; + } + for (const retryReason of ["transient_failure", MAX_TURN_CONTINUATION_RETRY_REASON]) { + expect(await heartbeatService(db).scheduleBoundedRetry(current.id, { now: f.now, retryReason, maxAttempts: 2, delayMs: 1 })) + .toMatchObject({ outcome: "retry_exhausted", attempt: 3 }); + } + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(7); + }); + it.each(["silent", "comments-and-tools"])("ACCT-02 exhausted infrastructure retries remain exhausted after %s and restart", async variant => { const f = await seedMaxTurnFixture({ scheduledRetryAttempt: BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS.length }); await db.update(heartbeatRuns).set({ scheduledRetryReason: "transient_failure", diff --git a/server/src/__tests__/legacy-continuation-authority.test.ts b/server/src/__tests__/legacy-continuation-authority.test.ts index 9906160ce88..9d8bbd35dc9 100644 --- a/server/src/__tests__/legacy-continuation-authority.test.ts +++ b/server/src/__tests__/legacy-continuation-authority.test.ts @@ -101,6 +101,8 @@ describe("legacy continuation persisted authority", () => { if (gate === "pause") await db.update(agents).set({ status: "paused" }).where(eq(agents.id, f.agentId)); if (gate === "reassigned") await db.update(issues).set({ assigneeAgentId: null }).where(eq(issues.id, f.issueId)); expect(await f.createRecovery().legacyRepairDispatchBlock(second.id)).not.toBeNull(); + expect((await heartbeatService(db).promoteDueScheduledRetries(new Date(second.scheduledRetryAt!.getTime() + 1))).runIds).not.toContain(second.id); + expect((await f.runs()).find(r => r.id === second.id)?.status).toBe("cancelled"); await f.createRecovery().reconcileLegacyContinuation(first.id); expect(await f.runs()).toHaveLength(3); expect((await f.actions())[0].attemptCount).toBe(2); @@ -121,6 +123,38 @@ describe("legacy continuation persisted authority", () => { expect((await f.actions())[0].attemptCount).toBe(2); }); + it("ACCT-01 repairs preserve prior infrastructure and productive debits", async () => { + const f = await fixture(); + const accounting = { version: 1, failureRetries: 2, maxTurnContinuations: 1 }; + await db.update(heartbeatRuns).set({ contextSnapshot: { issueId: f.issueId, executionRetryAccounting: accounting } }).where(eq(heartbeatRuns.id, f.runId)); + await f.createRecovery().reconcileLegacyContinuation(f.runId); + const first = (await f.runs()).find(r => r.id !== f.runId)!; + expect(first.contextSnapshot?.executionRetryAccounting).toEqual(accounting); + await f.finish(first); + await f.createRecovery().reconcileLegacyContinuation(first.id); + expect((await f.runs()).find(r => r.status === "scheduled_retry")?.contextSnapshot?.executionRetryAccounting).toEqual(accounting); + }); + + it.each(["foreign-source", "changed-episode", "exhausted-slot"])("ACCT-04 rejects a delayed repair with %s", async mutation => { + const f = await fixture(); + await f.createRecovery().reconcileLegacyContinuation(f.runId); + const first = (await f.runs()).find(r => r.id !== f.runId)!; + await f.finish(first); + await f.createRecovery().reconcileLegacyContinuation(first.id); + const second = (await f.runs()).find(r => r.status === "scheduled_retry")!; + const context = structuredClone(second.contextSnapshot!) as Record; + if (mutation === "foreign-source") { + const foreign = await fixture(); + context.dispositionRepairSourceRunId = foreign.runId; + await db.update(issues).set({ status: "done" }).where(eq(issues.id, foreign.issueId)); + } + if (mutation === "changed-episode") context.legacyDispositionEpisode.id = randomUUID(); + if (mutation === "exhausted-slot") context.legacyDispositionEpisode.attempt = 3; + await db.update(heartbeatRuns).set({ contextSnapshot: context }).where(eq(heartbeatRuns.id, second.id)); + expect((await heartbeatService(db).promoteDueScheduledRetries(new Date(second.scheduledRetryAt!.getTime() + 1))).runIds).not.toContain(second.id); + expect((await f.runs()).find(r => r.id === second.id)).toMatchObject({ status: "cancelled", errorCode: "issue_disposition_repair_superseded" }); + }); + it("persists the source identity while the second repair waits to dispatch", async () => { const f = await fixture(); await f.createRecovery().reconcileLegacyContinuation(f.runId); diff --git a/server/src/modules/run-dispatch/adapters/postgres.ts b/server/src/modules/run-dispatch/adapters/postgres.ts index 1213fa71953..c465c04daad 100644 --- a/server/src/modules/run-dispatch/adapters/postgres.ts +++ b/server/src/modules/run-dispatch/adapters/postgres.ts @@ -20,6 +20,7 @@ import { evaluateAgentInvokabilityFromDb } from "../../../services/agent-invokab import { budgetService } from "../../../services/budgets.js"; import { isHeartbeatWakeOnDemandEnabled } from "../../../services/heartbeat-policy.js"; import { collectDispositionRepairSourceState } from "../../../services/recovery/disposition-repair.js"; +import { legacyDispositionEpisode, legacyDispositionFingerprint } from "../../../services/recovery/legacy-continuation.js"; import { appendHeartbeatRunEvent } from "../../../services/heartbeat-run-events.js"; import { emitAgentTaskRun } from "../../../services/agent-task-run-telemetry.js"; import { issueService } from "../../../services/issues.js"; @@ -391,13 +392,31 @@ export function createPostgresRunDispatchAdapter( excludeRunId: input.runId, excludeWakeupRequestId: input.wakeupRequestId, }); + let currentFingerprint = sourceState.fingerprint; + let validSource = true; + if (readNonEmptyString(parseObject(input.contextSnapshot.legacyDispositionEpisode).id)) { + // Legacy repair reserves a slot in a persisted episode. Its fingerprint + // identifies that episode, not the older parked-summary state snapshot. + // Still recheck every active/wait/ownership gate before promotion. + const episode = legacyDispositionEpisode({ id: input.runId, contextSnapshot: input.contextSnapshot }); + const sourceId = readNonEmptyString(input.contextSnapshot.dispositionRepairSourceRunId) + ?? readNonEmptyString(input.contextSnapshot.retryOfRunId); + const source = sourceId ? await dbOrTx.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.id, sourceId), eq(heartbeatRuns.companyId, input.companyId), + )).limit(1).then(rows => rows[0]) : null; + validSource = Boolean(source && source.status === "succeeded" && source.agentId === input.agentId + && (source.contextSnapshot?.issueId ?? source.contextSnapshot?.taskId) === issueId + && legacyDispositionEpisode(source).id === episode.id + && episode.attempt >= 1 && episode.attempt <= episode.maxAttempts); + currentFingerprint = legacyDispositionFingerprint(input.companyId, issueId, input.agentId, episode.id); + } facts.dispositionRepair = { expectedFingerprintPresent: expectedFingerprint !== null, - fingerprintMatches: sourceState.fingerprint === expectedFingerprint, + fingerprintMatches: validSource && currentFingerprint === expectedFingerprint, hasActiveExecutionPath: sourceState.hasActiveExecutionPath, hasDurableWaitingPath: sourceState.hasDurableWaitingPath, expectedFingerprint, - currentFingerprint: sourceState.fingerprint, + currentFingerprint, durablePathReason: sourceState.durablePathReason, }; } diff --git a/server/src/services/execution-recovery-attempt.test.ts b/server/src/services/execution-recovery-attempt.test.ts index f18baa135a5..2673e2b33c3 100644 --- a/server/src/services/execution-recovery-attempt.test.ts +++ b/server/src/services/execution-recovery-attempt.test.ts @@ -20,8 +20,26 @@ describe("failure attempts across resource waits", () => { expect(executionFailureRetryCount({ scheduledRetryReason: "transient_failure", scheduledRetryAttempt: 2, contextSnapshot: { failureRetriesBeforeWorkspaceWait: 0 } })).toBe(2); }); - it("keeps ambiguous historical counts and starts a new incident after productive continuation", () => { + it("keeps ambiguous historical counts without treating a historical productive count as failures", () => { expect(executionFailureRetryCount({ scheduledRetryReason: "workspace_busy", scheduledRetryAttempt: 4 })).toBe(4); expect(executionFailureRetryCount({ scheduledRetryReason: "max_turns_continuation", scheduledRetryAttempt: 4 })).toBe(0); }); }); + +describe("persisted independent accounting", () => { + it.each(["max_turns_continuation", "issue_disposition_repair", "workspace_busy", "ai_connection_busy"])("%s cannot erase prior infrastructure debits or spend more", scheduledRetryReason => { + expect(executionFailureRetryCount({ scheduledRetryReason, scheduledRetryAttempt: 20, + contextSnapshot: { executionRetryAccounting: { version: 1, failureRetries: 2, maxTurnContinuations: 1 } }, + })).toBe(2); + }); + it("never lowers the current failure count from a partial or stale ledger", () => { + for (const executionRetryAccounting of [ + { version: 1, failureRetries: 0, maxTurnContinuations: 1 }, + { version: 1, failureRetries: -1, maxTurnContinuations: 1 }, + { version: 1, failureRetries: "0", maxTurnContinuations: 1 }, + { version: 2, failureRetries: 0, maxTurnContinuations: 1 }, + ]) expect(executionFailureRetryCount({ scheduledRetryReason: "transient_failure", scheduledRetryAttempt: 2, + contextSnapshot: { executionRetryAccounting }, + })).toBe(2); + }); +}); diff --git a/server/src/services/execution-recovery-attempt.ts b/server/src/services/execution-recovery-attempt.ts index 11449966a8e..689137d9d99 100644 --- a/server/src/services/execution-recovery-attempt.ts +++ b/server/src/services/execution-recovery-attempt.ts @@ -1,20 +1,72 @@ -/** Resource waits and productive continuations are not failed provider attempts. */ -export function executionFailureRetryCount(run: { +type RetryRun = { scheduledRetryAttempt?: number | null; scheduledRetryReason?: string | null; contextSnapshot?: Record | null; -}): number { - if (run.scheduledRetryReason === "max_turns_continuation") return 0; +}; + +/** Server-owned counts for one automatic continuation chain. Repair slots live + * in legacyDispositionEpisode and are never charged to either counter here. */ +export interface ExecutionRetryAccounting { + version: 1; + failureRetries: number; + maxTurnContinuations: number; +} + +function count(value: unknown): number | null { + return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null; +} + +function savedAccounting(run: RetryRun): ExecutionRetryAccounting | null { + const value = run.contextSnapshot?.executionRetryAccounting; + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const saved = value as Record; + const failureRetries = count(saved.failureRetries); + const maxTurnContinuations = count(saved.maxTurnContinuations); + if (saved.version !== 1 || failureRetries === null || maxTurnContinuations === null) return null; + return { version: 1, failureRetries, maxTurnContinuations }; +} + +function historicalFailureCount(run: RetryRun): number { + if (run.scheduledRetryReason === "max_turns_continuation" || run.scheduledRetryReason === "issue_disposition_repair") return 0; if (run.scheduledRetryReason === "ai_connection_busy") { - const count = run.contextSnapshot?.failureRetriesBeforeAiConnectionWait; - if (typeof count === "number" && Number.isInteger(count) && count >= 0) return count; + const saved = count(run.contextSnapshot?.failureRetriesBeforeAiConnectionWait); + if (saved !== null) return saved; } if (run.scheduledRetryReason === "workspace_busy") { - // Only a server-created workspace retry can consume this field. Its - // scheduler overwrites caller context with the predecessor's durable count. - const count = run.contextSnapshot?.failureRetriesBeforeWorkspaceWait; - if (typeof count === "number" && Number.isInteger(count) && count >= 0) return count; + const saved = count(run.contextSnapshot?.failureRetriesBeforeWorkspaceWait); + if (saved !== null) return saved; } // Historical ambiguous counters remain conservative rather than resetting. - return run.scheduledRetryAttempt ?? 0; + return count(run.scheduledRetryAttempt) ?? 0; +} + +export function executionRetryAccounting(run: RetryRun): ExecutionRetryAccounting { + const saved = savedAccounting(run); + const nonFailureLane = ["max_turns_continuation", "issue_disposition_repair", "workspace_busy", "ai_connection_busy"].includes(run.scheduledRetryReason ?? ""); + return { + version: 1, + failureRetries: Math.max(saved?.failureRetries ?? 0, saved && nonFailureLane ? 0 : historicalFailureCount(run)), + maxTurnContinuations: Math.max(saved?.maxTurnContinuations ?? 0, + run.scheduledRetryReason === "max_turns_continuation" ? count(run.scheduledRetryAttempt) ?? 0 : 0), + }; +} + +/** Resource waits, repairs and productive continuations do not spend failures. */ +export function executionFailureRetryCount(run: RetryRun): number { + return executionRetryAccounting(run).failureRetries; +} + +export function executionRetryAttemptCount(run: RetryRun, reason: string): number { + if (reason === "workspace_busy" || reason === "ai_connection_busy") { + return run.scheduledRetryReason === reason ? count(run.scheduledRetryAttempt) ?? 0 : 0; + } + const accounting = executionRetryAccounting(run); + return reason === "max_turns_continuation" ? accounting.maxTurnContinuations : accounting.failureRetries; +} + +export function accountingForScheduledRetry(run: RetryRun, reason: string, attempt: number): ExecutionRetryAccounting { + const accounting = executionRetryAccounting(run); + if (reason === "max_turns_continuation") accounting.maxTurnContinuations = attempt; + else if (reason !== "workspace_busy" && reason !== "ai_connection_busy") accounting.failureRetries = attempt; + return accounting; } diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 7693bafdd22..a991184b73c 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -28,7 +28,7 @@ import { registerAdapterExecutionControl, waitForAdapterStop, } from "./adapter-execution-control.js"; -import { executionFailureRetryCount } from "./execution-recovery-attempt.js"; +import { executionFailureRetryCount, executionRetryAttemptCount, accountingForScheduledRetry } from "./execution-recovery-attempt.js"; import { buildHeartbeatRunStatusLiveEventPayload } from "./heartbeat-run-status-payload.js"; export { buildHeartbeatRunStatusLiveEventPayload } from "./heartbeat-run-status-payload.js"; import { buildExecutionContinuation } from "./execution-continuation.js"; @@ -15192,12 +15192,8 @@ export function heartbeatService( opts?.maxAttempts ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS, ), ); - const nextAttempt = - (retryReason === WORKSPACE_BUSY_RETRY_REASON || - retryReason === AI_CONNECTION_BUSY_RETRY_REASON || - retryReason === MAX_TURN_CONTINUATION_RETRY_REASON - ? (run.scheduledRetryAttempt ?? 0) - : executionFailureRetryCount(run)) + 1; + const consumedAttempts = executionRetryAttemptCount(run, retryReason); + const nextAttempt = consumedAttempts + 1; const computedBaseSchedule = opts?.delayMs != null ? nextAttempt <= maxAttempts @@ -15237,14 +15233,14 @@ export function heartbeatService( if (!baseSchedule) { const exhaustion = { retryReason, - scheduledRetryAttempt: run.scheduledRetryAttempt ?? 0, + scheduledRetryAttempt: consumedAttempts, maxAttempts, }; await appendRunEvent(run, { eventType: "lifecycle", stream: "system", level: "warn", - message: `Bounded retry exhausted after ${run.scheduledRetryAttempt ?? 0} scheduled attempts; no further automatic retry will be queued`, + message: `Bounded retry exhausted after ${consumedAttempts} scheduled attempts; no further automatic retry will be queued`, payload: exhaustion, retryExhaustion: exhaustion, }); @@ -15253,7 +15249,7 @@ export function heartbeatService( run, issueId, attempt: Math.min( - run.scheduledRetryAttempt ?? maxAttempts, + consumedAttempts, maxAttempts, ), maxAttempts, @@ -15384,6 +15380,7 @@ export function heartbeatService( const retryContextSnapshot: Record = withRecoveryContext( { ...contextSnapshot, + executionRetryAccounting: accountingForScheduledRetry(run, retryReason, schedule.attempt), retryOfRunId: run.id, wakeReason, retryReason, diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 0c7b03d92d4..037446ab259 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -1,3 +1,4 @@ +import { executionRetryAccounting } from "../execution-recovery-attempt.js"; import { decideLegacyContinuation, legacyDispositionEpisode, legacyDispositionFingerprint, LEGACY_DISPOSITION_REPAIR_INSTRUCTION, type LegacyDispositionEpisode, @@ -3095,7 +3096,7 @@ export function recoveryService( if (!agentId) return null; // Preserve the initiating identity on the durable row, including while a // delayed repair is waiting to dispatch. Never substitute the issue owner. - const sourceRun = input.latestRun?.id ? await db.select({ responsibleUserId: heartbeatRuns.responsibleUserId }) + const sourceRun = input.latestRun?.id ? await db.select() .from(heartbeatRuns).where(and(eq(heartbeatRuns.id, input.latestRun.id), eq(heartbeatRuns.companyId, input.issue.companyId))) .limit(1).then(rows => rows[0]) : null; const timing = dispositionRepairDelayMs(input.attempt, input.fingerprint); @@ -3109,6 +3110,7 @@ export function recoveryService( wakeReason: ISSUE_DISPOSITION_REPAIR_RETRY_REASON, retryReason: ISSUE_DISPOSITION_REPAIR_RETRY_REASON, source: "issue.deliberate_wait_disposition_repair", + executionRetryAccounting: executionRetryAccounting(sourceRun ?? {}), retryOfRunId: input.latestRun?.id ?? null, recoveryActionId: input.action.id, dispositionRepairFingerprint: input.fingerprint, diff --git a/server/test-baselines/lifecycle-heartbeat.test.ts b/server/test-baselines/lifecycle-heartbeat.test.ts index 55c5a08c931..aecdbd6f05a 100644 --- a/server/test-baselines/lifecycle-heartbeat.test.ts +++ b/server/test-baselines/lifecycle-heartbeat.test.ts @@ -30,6 +30,9 @@ import { CONTROL_PLANE_CONFORMANCE_TERMINAL, } from "../src/vendor/paperclip-runner/testing.js"; import { observe } from "../../tests/lifecycle-baseline/observe.js"; +import { PaperclipRunnerToolAuthority } from "../src/services/native-runtime/paperclip-runner-tool-authority.js"; +import { issueThreadInteractionService } from "../src/services/issue-thread-interactions.js"; +import { questionResponseDeliveryService } from "../src/services/question-response-delivery.js"; const execute = vi.hoisted(() => vi.fn()); vi.mock("../src/adapters/index.js", async (importOriginal) => ({ ...(await importOriginal()), @@ -38,6 +41,7 @@ vi.mock("../src/adapters/index.js", async (importOriginal) => ({ vi.mock("../src/telemetry.js", () => ({ getTelemetryClient: () => ({ track: vi.fn(), + trackDynamic: vi.fn(), hashPrivateRef: (value: string) => value, }), })); @@ -133,7 +137,7 @@ describe("LCA full heartbeat observation", () => { input: NativeExecutionInput, ): NativeSessionBackend => { const capabilities = { - resume: false, + resume: true, typedEvents: true, steering: false, interruption: true, @@ -174,7 +178,7 @@ describe("LCA full heartbeat observation", () => { }, ]; result.continuation = { - kind: "same_agent", + kind: "response_wake", summary: "Perform the second fixture step", idempotencyKey: `step:${issueId}:${providerTurns + 1}`, }; @@ -185,6 +189,20 @@ describe("LCA full heartbeat observation", () => { async startTurn() { providerTurns++; operations++; + if (result.reportedWorkDisposition === "yielded") { + await new PaperclipRunnerToolAuthority(db, input.binding).execute({ + tool: "request_human_input", callId: `step-${providerTurns}`, + arguments: { + interactionKind: "questions", idempotencyKey: `step-${providerTurns}`, + title: `Input for step ${providerTurns + 1}`, prompt: "Choose the next step", + continuationPolicy: "wake_assignee", + payload: { version: 1, questions: [{ + id: "next", prompt: "Choose the next step", selectionMode: "single", required: true, + options: [{ id: "continue", label: "Continue" }, { id: "revise", label: "Revise" }], + }] }, + }, + }); + } start(); return { turnId }; }, @@ -243,6 +261,7 @@ describe("LCA full heartbeat observation", () => { }, }), openSession: async () => session, + recoverSession: async () => ({ recovered: true, session }), }; }; const heartbeat = heartbeatService(db, { @@ -294,6 +313,27 @@ describe("LCA full heartbeat observation", () => { }; }; const first = await collect(); + if (mode === "native" && !complete) { + for (let step = 1; step < requiredTurns; step++) { + const pending = (await db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.issueId, issueId))) + .filter(interaction => interaction.status === "pending"); + expect(pending).toHaveLength(1); + expect(providerTurns).toBe(step); + // A real persisted response, not summary text, owns the next run. + await heartbeat.dispatchPendingNativeStatusWakeups({ companyId }); + await heartbeat.drainActiveRunExecutions(); + expect(providerTurns).toBe(step); + await issueThreadInteractionService(db).answerQuestions({ id: issueId, companyId }, pending[0].id, + { answers: [{ questionId: "next", optionIds: ["continue"] }] }, { userId: "fixture-owner" }); + const delivery = questionResponseDeliveryService(db, { heartbeat }); + const delivered = await delivery.deliver(pending[0].id); + await heartbeat.drainActiveRunExecutions(); + // Replayed delivery cannot create a duplicate successor. + await delivery.deliver(pending[0].id); + await heartbeat.drainActiveRunExecutions(); + expect(providerTurns, JSON.stringify({ delivered, state: await collect() })).toBe(step + 1); + } + } await heartbeat.dispatchPendingNativeStatusWakeups({ companyId }); await heartbeat.resumeQueuedRuns(); await heartbeat.drainActiveRunExecutions(); @@ -334,7 +374,7 @@ describe("LCA full heartbeat observation", () => { expect(misleading.wakes).toEqual(neutral.wakes); }, ); - it("LCA-02 native explicit continuation survives misleading approval prose", async () => { + it("LCA-02 native answered-question continuation survives misleading approval prose", async () => { const neutral = await run("native", "First step recorded.", false); const misleading = await run( "native", @@ -386,7 +426,7 @@ describe("LCA full heartbeat observation", () => { .sort((a, b) => Number(a.repairAttempt) - Number(b.repairAttempt)); expect(effects(challenge)).toEqual(effects(neutral)); }); - it("LCA-02 native productive workflow continues beyond the failure retry allowance", async () => { + it("LCA-02 native answered-question workflow continues beyond the failure retry allowance", async () => { const steps = BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS.length + 2; const observed = await run( "native", diff --git a/tests/lifecycle-baseline/inventory.mjs b/tests/lifecycle-baseline/inventory.mjs index e8ed86280c3..f3adfb01225 100644 --- a/tests/lifecycle-baseline/inventory.mjs +++ b/tests/lifecycle-baseline/inventory.mjs @@ -6,6 +6,7 @@ export const lanes = { files: [ "tests/lifecycle-baseline/authority.test.ts", "tests/lifecycle-baseline/accounting.test.ts", + "server/src/services/execution-recovery-attempt.test.ts", "server/src/services/recovery/legacy-continuation.test.ts", `${server}run-liveness.test.ts`, `${server}heartbeat-context-summary.test.ts`, @@ -299,9 +300,10 @@ export const scenarios = [ ["ACCT-04", "Restart and replay accounting", "Consumed allowances and causal receipts survive restart and duplicates"], ].map(([id, name, expected]) => ({ id, name, expected, coverage: [ ref("unit", "tests/lifecycle-baseline/accounting.test.ts", id), + ...(id === "ACCT-01" ? [ref("unit", "server/src/services/execution-recovery-attempt.test.ts"), integ("legacy-continuation-authority", id)] : []), ...(id === "ACCT-01" || id === "ACCT-02" ? [integ("heartbeat-retry-scheduling", id)] : []), ...(id === "ACCT-02" || id === "ACCT-03" ? [integ("legacy-continuation-authority", id)] : []), - ...(id === "ACCT-04" ? [runner("ACCT-04"), integ("legacy-continuation-authority", "ACCT-04|replay|restart|fast repair"), integ("heartbeat-retry-scheduling", "concurrent|coalesces")] : []), + ...(id === "ACCT-04" ? [runner("ACCT-04"), integ("legacy-continuation-authority", "ACCT-04|replay|restart|fast repair"), integ("heartbeat-retry-scheduling", "concurrent|coalesces|after restart")] : []), ...(id === "ACCT-03" ? [ref("integration", "server/test-baselines/lifecycle-heartbeat.test.ts", "LCA-04 LCA-10"), integ("heartbeat-stale-queue-invalidation", "gate|cap|ownership")] : []), ref("grading", "tests/runner-e2e/accounting.test.ts"), ], live: ["continuation-accounting:accounting-productive-neutral", "continuation-accounting:accounting-productive-noisy", "continuation-accounting:accounting-exhaustion-neutral", "continuation-accounting:accounting-exhaustion-noisy", "continuation-accounting:accounting-repair-stop", "continuation-accounting:accounting-repair-approval"] })), diff --git a/tests/runner-e2e/playwright-support.config.ts b/tests/runner-e2e/playwright-support.config.ts index c9905f3ecd2..9e7b080d9bc 100644 --- a/tests/runner-e2e/playwright-support.config.ts +++ b/tests/runner-e2e/playwright-support.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from "@playwright/test"; /** Browser-only harness regressions: no Paperclip instance or provider credentials. */ export default defineConfig({ testDir: ".", - testMatch: "screenshot-readiness.spec.ts", + testMatch: ["screenshot-readiness.spec.ts", "service-worker-reload.spec.ts"], workers: 1, retries: 0, timeout: 10_000, diff --git a/tests/runner-e2e/service-worker-reload.spec.ts b/tests/runner-e2e/service-worker-reload.spec.ts new file mode 100644 index 00000000000..8c0a36ec6e1 --- /dev/null +++ b/tests/runner-e2e/service-worker-reload.spec.ts @@ -0,0 +1,50 @@ +import { test, expect } from "@playwright/test"; +import { createServer, type Server } from "node:http"; +import { readFile } from "node:fs/promises"; + +// Real browser/module-cache behavior without a Paperclip instance or LLM. +test("development module revalidation bypasses the offline worker across reloads", async ({ page }) => { + const worker = await readFile(new URL("../../ui/public/sw.js", import.meta.url), "utf8"); + let revalidations = 0; + const server: Server = createServer((request, response) => { + if (request.url === "/sw.js") { + response.writeHead(200, { "content-type": "text/javascript", "cache-control": "no-store" }); + response.end(worker); + } else if (request.url === "/fixture.js") { + if (request.headers["if-none-match"] === '"fixture-v1"') { + revalidations++; + response.writeHead(304); response.end(); + } else { + response.writeHead(200, { "content-type": "text/javascript", "cache-control": "no-cache", etag: '"fixture-v1"' }); + response.end('document.getElementById("root").textContent = "App mounted";'); + } + } else { + response.writeHead(200, { "content-type": "text/html", "cache-control": "no-store" }); + response.end('
'); + } + }); + await new Promise(resolve => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("Missing fixture port"); + const workerModuleRequests: string[] = []; + page.context().on("request", request => { + if (request.url().endsWith("/fixture.js") && request.serviceWorker()) workerModuleRequests.push(request.url()); + }); + try { + await page.goto(`http://127.0.0.1:${address.port}/`); + await page.evaluate(async () => { + await navigator.serviceWorker.register("/sw.js"); + await navigator.serviceWorker.ready; + }); + await expect.poll(() => page.evaluate(() => Boolean(navigator.serviceWorker.controller))).toBe(true); + for (let n = 0; n < 3; n++) { + await page.reload(); + await expect(page.locator("#root")).toHaveText("App mounted"); + } + expect(revalidations).toBeGreaterThan(0); + expect(workerModuleRequests).toEqual([]); + } finally { + server.closeAllConnections(); + await new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve())); + } +}); diff --git a/ui/public/sw.js b/ui/public/sw.js index c5f43d34ae0..4b527bfe2ac 100644 --- a/ui/public/sw.js +++ b/ui/public/sw.js @@ -31,6 +31,10 @@ self.addEventListener("activate", (event) => { }); self.addEventListener("fetch", (event) => { + // Vite owns development module revalidation and HMR. Passing that graph + // through an offline worker can forward bodyless 304 responses on reload. + // Only a stamped production build has an offline-cache contract. + if (BUILD_ID.startsWith("__")) return; const { request } = event; const url = new URL(request.url); // Only immutable Vite build assets have a public offline-cache contract. diff --git a/ui/src/lib/service-worker-cache-policy.test.ts b/ui/src/lib/service-worker-cache-policy.test.ts index 92116966484..c72b68725d4 100644 --- a/ui/src/lib/service-worker-cache-policy.test.ts +++ b/ui/src/lib/service-worker-cache-policy.test.ts @@ -3,13 +3,14 @@ import { readFileSync } from "node:fs"; import vm from "node:vm"; import { describe, expect, it, vi } from "vitest"; -function worker() { +function worker(development = false) { const handlers = new Map void>(); const put = vi.fn(); const match = vi.fn(); const remove = vi.fn().mockResolvedValue(true); const fetch = vi.fn().mockResolvedValue(new Response("public asset")); - vm.runInNewContext(readFileSync(new URL("../../public/sw.js", import.meta.url), "utf8"), { + const source = readFileSync(new URL("../../public/sw.js", import.meta.url), "utf8"); + vm.runInNewContext(development ? source : source.replace("__PAPERCLIP_BUILD_ID__", "fixture-production"), { self: { location: { origin: "https://example.test" }, addEventListener: (type: string, fn: (event: unknown) => void) => handlers.set(type, fn) }, URL, Response, fetch, caches: { keys: async () => ["paperclip-old", "paperclip-current"], open: async () => ({ put, delete: remove, match }), match }, }); @@ -22,6 +23,15 @@ function worker() { } describe("service worker privacy boundaries", () => { + it("leaves development navigation and module revalidation to the browser", () => { + const w = worker(true); + for (const path of ["/RUN/issues/RUN-1", "/src/main.tsx", "/@fs/vite-cache/deps/react.js?v=1"]) { + expect(w.request("default", path)).not.toHaveBeenCalled(); + } + expect(w.fetch).not.toHaveBeenCalled(); + expect(w.put).not.toHaveBeenCalled(); + expect(w.match).not.toHaveBeenCalled(); + }); it("bypasses both caching and offline fallback for a no-store request outside /api", () => { const w = worker(); expect(w.request("no-store")).not.toHaveBeenCalled();