mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-30 21:38:42 +01:00
Separate continuation allowances and preserve delayed repair authority
This commit is contained in:
1 parent
c0b5ee8b38
commit
cfbf03adee
14 files changed
+338
-33
No files matched your search
@@ -0,0 +1,41 @@
|
||||
# Continuation accounting fixes — September 23, 2026
|
||||
|
||||
Follow the [test-first accounting baseline](2026-09-22-continuation-accounting-baseline.md)
|
||||
and preserve its failed measurements. Repair the product against those assertions;
|
||||
do not change their intended outcomes.
|
||||
|
||||
## Implementation
|
||||
|
||||
1. Persist independent infrastructure-failure and max-turn continuation counts in
|
||||
server-owned run context. Repair slots remain in the durable disposition episode.
|
||||
Carry both counts across waits, repairs and controller restart. A change of retry
|
||||
reason cannot erase prior debt or charge another lane. Historical ambiguous
|
||||
counters remain conservative; known repair/productive counters are not failures.
|
||||
2. Promote delayed legacy repairs against their persisted episode fingerprint and
|
||||
successful source run, checking company, issue, agent and bounded repair slot.
|
||||
Retain the existing late Stop, wait/approval, ownership, pause and spending gates.
|
||||
3. Replace the two obsolete scripted native `same_agent` probes with the reachable
|
||||
public question/response contract: create a durable question, wait for its answer,
|
||||
deliver once, and reject duplicate delivery. Keep the misleading-prose pair and
|
||||
the sequence longer than the infrastructure retry allowance.
|
||||
4. Keep the development service worker out of Vite's module revalidation. The failed
|
||||
browser trace contains a bodyless module `304` through the worker before React
|
||||
mounts. A small real-browser regression checks repeated conditional reloads with
|
||||
the actual worker; production cache/privacy tests continue using a stamped build.
|
||||
The live rerun must establish whether the observed blank-page failure is resolved.
|
||||
|
||||
## Verification
|
||||
|
||||
- Existing failing unit, scheduler and promotion assertions must pass unchanged.
|
||||
- Exercise alternating failure/productive/resource-wait retries through the actual
|
||||
scheduler, recreate the controller each time, and exhaust each allowance separately.
|
||||
- Verify repairs preserve both debits, invalid source/episode/slot cannot promote,
|
||||
and late gates cancel delayed repairs without another debit.
|
||||
- Run all deterministic lifecycle layers, E2E support, the cheap browser suite,
|
||||
relevant typechecks, then the required repository checks.
|
||||
- Rerun all eight explicit `continuation-accounting` real-provider/browser cells in
|
||||
parallel GitHub Actions jobs. Retain original grades, source/definition fingerprints,
|
||||
cleanup, retries, usage/cost coverage and failure evidence. Save the new measurement
|
||||
separately from September 22; do not regrade the baseline.
|
||||
|
||||
Other retained findings remain in the [backlog](2026-09-22-legacy-continuation-authority.md).
|
||||
@@ -662,6 +662,42 @@ describeEmbeddedPostgres("heartbeat bounded retry scheduling", () => {
|
||||
expect(scheduled).toMatchObject({ outcome: "scheduled", attempt: 1 });
|
||||
});
|
||||
|
||||
it("ACCT-01 alternating failures, productive continuations and waits preserve both allowances after restart", async () => {
|
||||
const f = await seedMaxTurnFixture();
|
||||
let current = (await heartbeat.getRun(f.runId))!;
|
||||
for (const [reason, attempt, failures, continuations] of [
|
||||
["transient_failure", 1, 1, 0],
|
||||
[MAX_TURN_CONTINUATION_RETRY_REASON, 1, 1, 1],
|
||||
["workspace_busy", 1, 1, 1],
|
||||
["transient_failure", 2, 2, 1],
|
||||
[MAX_TURN_CONTINUATION_RETRY_REASON, 2, 2, 2],
|
||||
["ai_connection_busy", 1, 2, 2],
|
||||
] as const) {
|
||||
// Exercise the typed terminal outcome that requests each lane.
|
||||
const waiting = reason === "workspace_busy" || reason === "ai_connection_busy";
|
||||
await db.update(heartbeatRuns).set({
|
||||
status: waiting ? "cancelled" : "failed", errorCode: waiting ? reason : "adapter_failed",
|
||||
resultJson: { ...(reason === MAX_TURN_CONTINUATION_RETRY_REASON ? { stopReason: "max_turns_exhausted" } : {}),
|
||||
executionRecovery: { kind: waiting ? (reason === "workspace_busy" ? "workspace_wait" : "ai_connection_wait") : "bootstrap", providerWorkStarted: false } },
|
||||
}).where(eq(heartbeatRuns.id, current.id));
|
||||
const restarted = heartbeatService(db);
|
||||
const result = await restarted.scheduleBoundedRetry(current.id, { now: f.now, retryReason: reason, maxAttempts: 2, delayMs: 1 });
|
||||
expect(result, JSON.stringify({ reason, result })).toMatchObject({ outcome: "scheduled", attempt });
|
||||
if (result.outcome !== "scheduled") throw new Error("Missing successor");
|
||||
expect(result.run.contextSnapshot?.executionRetryAccounting).toEqual({ version: 1, failureRetries: failures, maxTurnContinuations: continuations });
|
||||
await db.update(heartbeatRuns).set({ status: "failed", finishedAt: f.now,
|
||||
resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } },
|
||||
}).where(eq(heartbeatRuns.id, result.run.id));
|
||||
await db.update(issues).set({ executionRunId: result.run.id }).where(eq(issues.id, f.issueId));
|
||||
current = (await restarted.getRun(result.run.id))!;
|
||||
}
|
||||
for (const retryReason of ["transient_failure", MAX_TURN_CONTINUATION_RETRY_REASON]) {
|
||||
expect(await heartbeatService(db).scheduleBoundedRetry(current.id, { now: f.now, retryReason, maxAttempts: 2, delayMs: 1 }))
|
||||
.toMatchObject({ outcome: "retry_exhausted", attempt: 3 });
|
||||
}
|
||||
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(7);
|
||||
});
|
||||
|
||||
it.each(["silent", "comments-and-tools"])("ACCT-02 exhausted infrastructure retries remain exhausted after %s and restart", async variant => {
|
||||
const f = await seedMaxTurnFixture({ scheduledRetryAttempt: BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS.length });
|
||||
await db.update(heartbeatRuns).set({ scheduledRetryReason: "transient_failure",
|
||||
|
||||
@@ -101,6 +101,8 @@ describe("legacy continuation persisted authority", () => {
|
||||
if (gate === "pause") await db.update(agents).set({ status: "paused" }).where(eq(agents.id, f.agentId));
|
||||
if (gate === "reassigned") await db.update(issues).set({ assigneeAgentId: null }).where(eq(issues.id, f.issueId));
|
||||
expect(await f.createRecovery().legacyRepairDispatchBlock(second.id)).not.toBeNull();
|
||||
expect((await heartbeatService(db).promoteDueScheduledRetries(new Date(second.scheduledRetryAt!.getTime() + 1))).runIds).not.toContain(second.id);
|
||||
expect((await f.runs()).find(r => r.id === second.id)?.status).toBe("cancelled");
|
||||
await f.createRecovery().reconcileLegacyContinuation(first.id);
|
||||
expect(await f.runs()).toHaveLength(3);
|
||||
expect((await f.actions())[0].attemptCount).toBe(2);
|
||||
@@ -121,6 +123,38 @@ describe("legacy continuation persisted authority", () => {
|
||||
expect((await f.actions())[0].attemptCount).toBe(2);
|
||||
});
|
||||
|
||||
it("ACCT-01 repairs preserve prior infrastructure and productive debits", async () => {
|
||||
const f = await fixture();
|
||||
const accounting = { version: 1, failureRetries: 2, maxTurnContinuations: 1 };
|
||||
await db.update(heartbeatRuns).set({ contextSnapshot: { issueId: f.issueId, executionRetryAccounting: accounting } }).where(eq(heartbeatRuns.id, f.runId));
|
||||
await f.createRecovery().reconcileLegacyContinuation(f.runId);
|
||||
const first = (await f.runs()).find(r => r.id !== f.runId)!;
|
||||
expect(first.contextSnapshot?.executionRetryAccounting).toEqual(accounting);
|
||||
await f.finish(first);
|
||||
await f.createRecovery().reconcileLegacyContinuation(first.id);
|
||||
expect((await f.runs()).find(r => r.status === "scheduled_retry")?.contextSnapshot?.executionRetryAccounting).toEqual(accounting);
|
||||
});
|
||||
|
||||
it.each(["foreign-source", "changed-episode", "exhausted-slot"])("ACCT-04 rejects a delayed repair with %s", async mutation => {
|
||||
const f = await fixture();
|
||||
await f.createRecovery().reconcileLegacyContinuation(f.runId);
|
||||
const first = (await f.runs()).find(r => r.id !== f.runId)!;
|
||||
await f.finish(first);
|
||||
await f.createRecovery().reconcileLegacyContinuation(first.id);
|
||||
const second = (await f.runs()).find(r => r.status === "scheduled_retry")!;
|
||||
const context = structuredClone(second.contextSnapshot!) as Record<string, any>;
|
||||
if (mutation === "foreign-source") {
|
||||
const foreign = await fixture();
|
||||
context.dispositionRepairSourceRunId = foreign.runId;
|
||||
await db.update(issues).set({ status: "done" }).where(eq(issues.id, foreign.issueId));
|
||||
}
|
||||
if (mutation === "changed-episode") context.legacyDispositionEpisode.id = randomUUID();
|
||||
if (mutation === "exhausted-slot") context.legacyDispositionEpisode.attempt = 3;
|
||||
await db.update(heartbeatRuns).set({ contextSnapshot: context }).where(eq(heartbeatRuns.id, second.id));
|
||||
expect((await heartbeatService(db).promoteDueScheduledRetries(new Date(second.scheduledRetryAt!.getTime() + 1))).runIds).not.toContain(second.id);
|
||||
expect((await f.runs()).find(r => r.id === second.id)).toMatchObject({ status: "cancelled", errorCode: "issue_disposition_repair_superseded" });
|
||||
});
|
||||
|
||||
it("persists the source identity while the second repair waits to dispatch", async () => {
|
||||
const f = await fixture();
|
||||
await f.createRecovery().reconcileLegacyContinuation(f.runId);
|
||||
|
||||
@@ -20,6 +20,7 @@ import { evaluateAgentInvokabilityFromDb } from "../../../services/agent-invokab
|
||||
import { budgetService } from "../../../services/budgets.js";
|
||||
import { isHeartbeatWakeOnDemandEnabled } from "../../../services/heartbeat-policy.js";
|
||||
import { collectDispositionRepairSourceState } from "../../../services/recovery/disposition-repair.js";
|
||||
import { legacyDispositionEpisode, legacyDispositionFingerprint } from "../../../services/recovery/legacy-continuation.js";
|
||||
import { appendHeartbeatRunEvent } from "../../../services/heartbeat-run-events.js";
|
||||
import { emitAgentTaskRun } from "../../../services/agent-task-run-telemetry.js";
|
||||
import { issueService } from "../../../services/issues.js";
|
||||
@@ -391,13 +392,31 @@ export function createPostgresRunDispatchAdapter(
|
||||
excludeRunId: input.runId,
|
||||
excludeWakeupRequestId: input.wakeupRequestId,
|
||||
});
|
||||
let currentFingerprint = sourceState.fingerprint;
|
||||
let validSource = true;
|
||||
if (readNonEmptyString(parseObject(input.contextSnapshot.legacyDispositionEpisode).id)) {
|
||||
// Legacy repair reserves a slot in a persisted episode. Its fingerprint
|
||||
// identifies that episode, not the older parked-summary state snapshot.
|
||||
// Still recheck every active/wait/ownership gate before promotion.
|
||||
const episode = legacyDispositionEpisode({ id: input.runId, contextSnapshot: input.contextSnapshot });
|
||||
const sourceId = readNonEmptyString(input.contextSnapshot.dispositionRepairSourceRunId)
|
||||
?? readNonEmptyString(input.contextSnapshot.retryOfRunId);
|
||||
const source = sourceId ? await dbOrTx.select().from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.id, sourceId), eq(heartbeatRuns.companyId, input.companyId),
|
||||
)).limit(1).then(rows => rows[0]) : null;
|
||||
validSource = Boolean(source && source.status === "succeeded" && source.agentId === input.agentId
|
||||
&& (source.contextSnapshot?.issueId ?? source.contextSnapshot?.taskId) === issueId
|
||||
&& legacyDispositionEpisode(source).id === episode.id
|
||||
&& episode.attempt >= 1 && episode.attempt <= episode.maxAttempts);
|
||||
currentFingerprint = legacyDispositionFingerprint(input.companyId, issueId, input.agentId, episode.id);
|
||||
}
|
||||
facts.dispositionRepair = {
|
||||
expectedFingerprintPresent: expectedFingerprint !== null,
|
||||
fingerprintMatches: sourceState.fingerprint === expectedFingerprint,
|
||||
fingerprintMatches: validSource && currentFingerprint === expectedFingerprint,
|
||||
hasActiveExecutionPath: sourceState.hasActiveExecutionPath,
|
||||
hasDurableWaitingPath: sourceState.hasDurableWaitingPath,
|
||||
expectedFingerprint,
|
||||
currentFingerprint: sourceState.fingerprint,
|
||||
currentFingerprint,
|
||||
durablePathReason: sourceState.durablePathReason,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -20,8 +20,26 @@ describe("failure attempts across resource waits", () => {
|
||||
expect(executionFailureRetryCount({ scheduledRetryReason: "transient_failure", scheduledRetryAttempt: 2,
|
||||
contextSnapshot: { failureRetriesBeforeWorkspaceWait: 0 } })).toBe(2);
|
||||
});
|
||||
it("keeps ambiguous historical counts and starts a new incident after productive continuation", () => {
|
||||
it("keeps ambiguous historical counts without treating a historical productive count as failures", () => {
|
||||
expect(executionFailureRetryCount({ scheduledRetryReason: "workspace_busy", scheduledRetryAttempt: 4 })).toBe(4);
|
||||
expect(executionFailureRetryCount({ scheduledRetryReason: "max_turns_continuation", scheduledRetryAttempt: 4 })).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("persisted independent accounting", () => {
|
||||
it.each(["max_turns_continuation", "issue_disposition_repair", "workspace_busy", "ai_connection_busy"])("%s cannot erase prior infrastructure debits or spend more", scheduledRetryReason => {
|
||||
expect(executionFailureRetryCount({ scheduledRetryReason, scheduledRetryAttempt: 20,
|
||||
contextSnapshot: { executionRetryAccounting: { version: 1, failureRetries: 2, maxTurnContinuations: 1 } },
|
||||
})).toBe(2);
|
||||
});
|
||||
it("never lowers the current failure count from a partial or stale ledger", () => {
|
||||
for (const executionRetryAccounting of [
|
||||
{ version: 1, failureRetries: 0, maxTurnContinuations: 1 },
|
||||
{ version: 1, failureRetries: -1, maxTurnContinuations: 1 },
|
||||
{ version: 1, failureRetries: "0", maxTurnContinuations: 1 },
|
||||
{ version: 2, failureRetries: 0, maxTurnContinuations: 1 },
|
||||
]) expect(executionFailureRetryCount({ scheduledRetryReason: "transient_failure", scheduledRetryAttempt: 2,
|
||||
contextSnapshot: { executionRetryAccounting },
|
||||
})).toBe(2);
|
||||
});
|
||||
});
|
||||
@@ -1,20 +1,72 @@
|
||||
/** Resource waits and productive continuations are not failed provider attempts. */
|
||||
export function executionFailureRetryCount(run: {
|
||||
type RetryRun = {
|
||||
scheduledRetryAttempt?: number | null;
|
||||
scheduledRetryReason?: string | null;
|
||||
contextSnapshot?: Record<string, unknown> | null;
|
||||
}): number {
|
||||
if (run.scheduledRetryReason === "max_turns_continuation") return 0;
|
||||
};
|
||||
|
||||
/** Server-owned counts for one automatic continuation chain. Repair slots live
|
||||
* in legacyDispositionEpisode and are never charged to either counter here. */
|
||||
export interface ExecutionRetryAccounting {
|
||||
version: 1;
|
||||
failureRetries: number;
|
||||
maxTurnContinuations: number;
|
||||
}
|
||||
|
||||
function count(value: unknown): number | null {
|
||||
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
|
||||
}
|
||||
|
||||
function savedAccounting(run: RetryRun): ExecutionRetryAccounting | null {
|
||||
const value = run.contextSnapshot?.executionRetryAccounting;
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
|
||||
const saved = value as Record<string, unknown>;
|
||||
const failureRetries = count(saved.failureRetries);
|
||||
const maxTurnContinuations = count(saved.maxTurnContinuations);
|
||||
if (saved.version !== 1 || failureRetries === null || maxTurnContinuations === null) return null;
|
||||
return { version: 1, failureRetries, maxTurnContinuations };
|
||||
}
|
||||
|
||||
function historicalFailureCount(run: RetryRun): number {
|
||||
if (run.scheduledRetryReason === "max_turns_continuation" || run.scheduledRetryReason === "issue_disposition_repair") return 0;
|
||||
if (run.scheduledRetryReason === "ai_connection_busy") {
|
||||
const count = run.contextSnapshot?.failureRetriesBeforeAiConnectionWait;
|
||||
if (typeof count === "number" && Number.isInteger(count) && count >= 0) return count;
|
||||
const saved = count(run.contextSnapshot?.failureRetriesBeforeAiConnectionWait);
|
||||
if (saved !== null) return saved;
|
||||
}
|
||||
if (run.scheduledRetryReason === "workspace_busy") {
|
||||
// Only a server-created workspace retry can consume this field. Its
|
||||
// scheduler overwrites caller context with the predecessor's durable count.
|
||||
const count = run.contextSnapshot?.failureRetriesBeforeWorkspaceWait;
|
||||
if (typeof count === "number" && Number.isInteger(count) && count >= 0) return count;
|
||||
const saved = count(run.contextSnapshot?.failureRetriesBeforeWorkspaceWait);
|
||||
if (saved !== null) return saved;
|
||||
}
|
||||
// Historical ambiguous counters remain conservative rather than resetting.
|
||||
return run.scheduledRetryAttempt ?? 0;
|
||||
return count(run.scheduledRetryAttempt) ?? 0;
|
||||
}
|
||||
|
||||
export function executionRetryAccounting(run: RetryRun): ExecutionRetryAccounting {
|
||||
const saved = savedAccounting(run);
|
||||
const nonFailureLane = ["max_turns_continuation", "issue_disposition_repair", "workspace_busy", "ai_connection_busy"].includes(run.scheduledRetryReason ?? "");
|
||||
return {
|
||||
version: 1,
|
||||
failureRetries: Math.max(saved?.failureRetries ?? 0, saved && nonFailureLane ? 0 : historicalFailureCount(run)),
|
||||
maxTurnContinuations: Math.max(saved?.maxTurnContinuations ?? 0,
|
||||
run.scheduledRetryReason === "max_turns_continuation" ? count(run.scheduledRetryAttempt) ?? 0 : 0),
|
||||
};
|
||||
}
|
||||
|
||||
/** Resource waits, repairs and productive continuations do not spend failures. */
|
||||
export function executionFailureRetryCount(run: RetryRun): number {
|
||||
return executionRetryAccounting(run).failureRetries;
|
||||
}
|
||||
|
||||
export function executionRetryAttemptCount(run: RetryRun, reason: string): number {
|
||||
if (reason === "workspace_busy" || reason === "ai_connection_busy") {
|
||||
return run.scheduledRetryReason === reason ? count(run.scheduledRetryAttempt) ?? 0 : 0;
|
||||
}
|
||||
const accounting = executionRetryAccounting(run);
|
||||
return reason === "max_turns_continuation" ? accounting.maxTurnContinuations : accounting.failureRetries;
|
||||
}
|
||||
|
||||
export function accountingForScheduledRetry(run: RetryRun, reason: string, attempt: number): ExecutionRetryAccounting {
|
||||
const accounting = executionRetryAccounting(run);
|
||||
if (reason === "max_turns_continuation") accounting.maxTurnContinuations = attempt;
|
||||
else if (reason !== "workspace_busy" && reason !== "ai_connection_busy") accounting.failureRetries = attempt;
|
||||
return accounting;
|
||||
}
|
||||
@@ -28,7 +28,7 @@ import {
|
||||
registerAdapterExecutionControl,
|
||||
waitForAdapterStop,
|
||||
} from "./adapter-execution-control.js";
|
||||
import { executionFailureRetryCount } from "./execution-recovery-attempt.js";
|
||||
import { executionFailureRetryCount, executionRetryAttemptCount, accountingForScheduledRetry } from "./execution-recovery-attempt.js";
|
||||
import { buildHeartbeatRunStatusLiveEventPayload } from "./heartbeat-run-status-payload.js";
|
||||
export { buildHeartbeatRunStatusLiveEventPayload } from "./heartbeat-run-status-payload.js";
|
||||
import { buildExecutionContinuation } from "./execution-continuation.js";
|
||||
@@ -15192,12 +15192,8 @@ export function heartbeatService(
|
||||
opts?.maxAttempts ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS,
|
||||
),
|
||||
);
|
||||
const nextAttempt =
|
||||
(retryReason === WORKSPACE_BUSY_RETRY_REASON ||
|
||||
retryReason === AI_CONNECTION_BUSY_RETRY_REASON ||
|
||||
retryReason === MAX_TURN_CONTINUATION_RETRY_REASON
|
||||
? (run.scheduledRetryAttempt ?? 0)
|
||||
: executionFailureRetryCount(run)) + 1;
|
||||
const consumedAttempts = executionRetryAttemptCount(run, retryReason);
|
||||
const nextAttempt = consumedAttempts + 1;
|
||||
const computedBaseSchedule =
|
||||
opts?.delayMs != null
|
||||
? nextAttempt <= maxAttempts
|
||||
@@ -15237,14 +15233,14 @@ export function heartbeatService(
|
||||
if (!baseSchedule) {
|
||||
const exhaustion = {
|
||||
retryReason,
|
||||
scheduledRetryAttempt: run.scheduledRetryAttempt ?? 0,
|
||||
scheduledRetryAttempt: consumedAttempts,
|
||||
maxAttempts,
|
||||
};
|
||||
await appendRunEvent(run, {
|
||||
eventType: "lifecycle",
|
||||
stream: "system",
|
||||
level: "warn",
|
||||
message: `Bounded retry exhausted after ${run.scheduledRetryAttempt ?? 0} scheduled attempts; no further automatic retry will be queued`,
|
||||
message: `Bounded retry exhausted after ${consumedAttempts} scheduled attempts; no further automatic retry will be queued`,
|
||||
payload: exhaustion,
|
||||
retryExhaustion: exhaustion,
|
||||
});
|
||||
@@ -15253,7 +15249,7 @@ export function heartbeatService(
|
||||
run,
|
||||
issueId,
|
||||
attempt: Math.min(
|
||||
run.scheduledRetryAttempt ?? maxAttempts,
|
||||
consumedAttempts,
|
||||
maxAttempts,
|
||||
),
|
||||
maxAttempts,
|
||||
@@ -15384,6 +15380,7 @@ export function heartbeatService(
|
||||
const retryContextSnapshot: Record<string, unknown> = withRecoveryContext(
|
||||
{
|
||||
...contextSnapshot,
|
||||
executionRetryAccounting: accountingForScheduledRetry(run, retryReason, schedule.attempt),
|
||||
retryOfRunId: run.id,
|
||||
wakeReason,
|
||||
retryReason,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { executionRetryAccounting } from "../execution-recovery-attempt.js";
|
||||
import {
|
||||
decideLegacyContinuation, legacyDispositionEpisode, legacyDispositionFingerprint,
|
||||
LEGACY_DISPOSITION_REPAIR_INSTRUCTION, type LegacyDispositionEpisode,
|
||||
@@ -3095,7 +3096,7 @@ export function recoveryService(
|
||||
if (!agentId) return null;
|
||||
// Preserve the initiating identity on the durable row, including while a
|
||||
// delayed repair is waiting to dispatch. Never substitute the issue owner.
|
||||
const sourceRun = input.latestRun?.id ? await db.select({ responsibleUserId: heartbeatRuns.responsibleUserId })
|
||||
const sourceRun = input.latestRun?.id ? await db.select()
|
||||
.from(heartbeatRuns).where(and(eq(heartbeatRuns.id, input.latestRun.id), eq(heartbeatRuns.companyId, input.issue.companyId)))
|
||||
.limit(1).then(rows => rows[0]) : null;
|
||||
const timing = dispositionRepairDelayMs(input.attempt, input.fingerprint);
|
||||
@@ -3109,6 +3110,7 @@ export function recoveryService(
|
||||
wakeReason: ISSUE_DISPOSITION_REPAIR_RETRY_REASON,
|
||||
retryReason: ISSUE_DISPOSITION_REPAIR_RETRY_REASON,
|
||||
source: "issue.deliberate_wait_disposition_repair",
|
||||
executionRetryAccounting: executionRetryAccounting(sourceRun ?? {}),
|
||||
retryOfRunId: input.latestRun?.id ?? null,
|
||||
recoveryActionId: input.action.id,
|
||||
dispositionRepairFingerprint: input.fingerprint,
|
||||
|
||||
@@ -30,6 +30,9 @@ import {
|
||||
CONTROL_PLANE_CONFORMANCE_TERMINAL,
|
||||
} from "../src/vendor/paperclip-runner/testing.js";
|
||||
import { observe } from "../../tests/lifecycle-baseline/observe.js";
|
||||
import { PaperclipRunnerToolAuthority } from "../src/services/native-runtime/paperclip-runner-tool-authority.js";
|
||||
import { issueThreadInteractionService } from "../src/services/issue-thread-interactions.js";
|
||||
import { questionResponseDeliveryService } from "../src/services/question-response-delivery.js";
|
||||
const execute = vi.hoisted(() => vi.fn());
|
||||
vi.mock("../src/adapters/index.js", async (importOriginal) => ({
|
||||
...(await importOriginal<object>()),
|
||||
@@ -38,6 +41,7 @@ vi.mock("../src/adapters/index.js", async (importOriginal) => ({
|
||||
vi.mock("../src/telemetry.js", () => ({
|
||||
getTelemetryClient: () => ({
|
||||
track: vi.fn(),
|
||||
trackDynamic: vi.fn(),
|
||||
hashPrivateRef: (value: string) => value,
|
||||
}),
|
||||
}));
|
||||
@@ -133,7 +137,7 @@ describe("LCA full heartbeat observation", () => {
|
||||
input: NativeExecutionInput,
|
||||
): NativeSessionBackend => {
|
||||
const capabilities = {
|
||||
resume: false,
|
||||
resume: true,
|
||||
typedEvents: true,
|
||||
steering: false,
|
||||
interruption: true,
|
||||
@@ -174,7 +178,7 @@ describe("LCA full heartbeat observation", () => {
|
||||
},
|
||||
];
|
||||
result.continuation = {
|
||||
kind: "same_agent",
|
||||
kind: "response_wake",
|
||||
summary: "Perform the second fixture step",
|
||||
idempotencyKey: `step:${issueId}:${providerTurns + 1}`,
|
||||
};
|
||||
@@ -185,6 +189,20 @@ describe("LCA full heartbeat observation", () => {
|
||||
async startTurn() {
|
||||
providerTurns++;
|
||||
operations++;
|
||||
if (result.reportedWorkDisposition === "yielded") {
|
||||
await new PaperclipRunnerToolAuthority(db, input.binding).execute({
|
||||
tool: "request_human_input", callId: `step-${providerTurns}`,
|
||||
arguments: {
|
||||
interactionKind: "questions", idempotencyKey: `step-${providerTurns}`,
|
||||
title: `Input for step ${providerTurns + 1}`, prompt: "Choose the next step",
|
||||
continuationPolicy: "wake_assignee",
|
||||
payload: { version: 1, questions: [{
|
||||
id: "next", prompt: "Choose the next step", selectionMode: "single", required: true,
|
||||
options: [{ id: "continue", label: "Continue" }, { id: "revise", label: "Revise" }],
|
||||
}] },
|
||||
},
|
||||
});
|
||||
}
|
||||
start();
|
||||
return { turnId };
|
||||
},
|
||||
@@ -243,6 +261,7 @@ describe("LCA full heartbeat observation", () => {
|
||||
},
|
||||
}),
|
||||
openSession: async () => session,
|
||||
recoverSession: async () => ({ recovered: true, session }),
|
||||
};
|
||||
};
|
||||
const heartbeat = heartbeatService(db, {
|
||||
@@ -294,6 +313,27 @@ describe("LCA full heartbeat observation", () => {
|
||||
};
|
||||
};
|
||||
const first = await collect();
|
||||
if (mode === "native" && !complete) {
|
||||
for (let step = 1; step < requiredTurns; step++) {
|
||||
const pending = (await db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.issueId, issueId)))
|
||||
.filter(interaction => interaction.status === "pending");
|
||||
expect(pending).toHaveLength(1);
|
||||
expect(providerTurns).toBe(step);
|
||||
// A real persisted response, not summary text, owns the next run.
|
||||
await heartbeat.dispatchPendingNativeStatusWakeups({ companyId });
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
expect(providerTurns).toBe(step);
|
||||
await issueThreadInteractionService(db).answerQuestions({ id: issueId, companyId }, pending[0].id,
|
||||
{ answers: [{ questionId: "next", optionIds: ["continue"] }] }, { userId: "fixture-owner" });
|
||||
const delivery = questionResponseDeliveryService(db, { heartbeat });
|
||||
const delivered = await delivery.deliver(pending[0].id);
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
// Replayed delivery cannot create a duplicate successor.
|
||||
await delivery.deliver(pending[0].id);
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
expect(providerTurns, JSON.stringify({ delivered, state: await collect() })).toBe(step + 1);
|
||||
}
|
||||
}
|
||||
await heartbeat.dispatchPendingNativeStatusWakeups({ companyId });
|
||||
await heartbeat.resumeQueuedRuns();
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
@@ -334,7 +374,7 @@ describe("LCA full heartbeat observation", () => {
|
||||
expect(misleading.wakes).toEqual(neutral.wakes);
|
||||
},
|
||||
);
|
||||
it("LCA-02 native explicit continuation survives misleading approval prose", async () => {
|
||||
it("LCA-02 native answered-question continuation survives misleading approval prose", async () => {
|
||||
const neutral = await run("native", "First step recorded.", false);
|
||||
const misleading = await run(
|
||||
"native",
|
||||
@@ -386,7 +426,7 @@ describe("LCA full heartbeat observation", () => {
|
||||
.sort((a, b) => Number(a.repairAttempt) - Number(b.repairAttempt));
|
||||
expect(effects(challenge)).toEqual(effects(neutral));
|
||||
});
|
||||
it("LCA-02 native productive workflow continues beyond the failure retry allowance", async () => {
|
||||
it("LCA-02 native answered-question workflow continues beyond the failure retry allowance", async () => {
|
||||
const steps = BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS.length + 2;
|
||||
const observed = await run(
|
||||
"native",
|
||||
|
||||
@@ -6,6 +6,7 @@ export const lanes = {
|
||||
files: [
|
||||
"tests/lifecycle-baseline/authority.test.ts",
|
||||
"tests/lifecycle-baseline/accounting.test.ts",
|
||||
"server/src/services/execution-recovery-attempt.test.ts",
|
||||
"server/src/services/recovery/legacy-continuation.test.ts",
|
||||
`${server}run-liveness.test.ts`,
|
||||
`${server}heartbeat-context-summary.test.ts`,
|
||||
@@ -299,9 +300,10 @@ export const scenarios = [
|
||||
["ACCT-04", "Restart and replay accounting", "Consumed allowances and causal receipts survive restart and duplicates"],
|
||||
].map(([id, name, expected]) => ({ id, name, expected, coverage: [
|
||||
ref("unit", "tests/lifecycle-baseline/accounting.test.ts", id),
|
||||
...(id === "ACCT-01" ? [ref("unit", "server/src/services/execution-recovery-attempt.test.ts"), integ("legacy-continuation-authority", id)] : []),
|
||||
...(id === "ACCT-01" || id === "ACCT-02" ? [integ("heartbeat-retry-scheduling", id)] : []),
|
||||
...(id === "ACCT-02" || id === "ACCT-03" ? [integ("legacy-continuation-authority", id)] : []),
|
||||
...(id === "ACCT-04" ? [runner("ACCT-04"), integ("legacy-continuation-authority", "ACCT-04|replay|restart|fast repair"), integ("heartbeat-retry-scheduling", "concurrent|coalesces")] : []),
|
||||
...(id === "ACCT-04" ? [runner("ACCT-04"), integ("legacy-continuation-authority", "ACCT-04|replay|restart|fast repair"), integ("heartbeat-retry-scheduling", "concurrent|coalesces|after restart")] : []),
|
||||
...(id === "ACCT-03" ? [ref("integration", "server/test-baselines/lifecycle-heartbeat.test.ts", "LCA-04 LCA-10"), integ("heartbeat-stale-queue-invalidation", "gate|cap|ownership")] : []),
|
||||
ref("grading", "tests/runner-e2e/accounting.test.ts"),
|
||||
], live: ["continuation-accounting:accounting-productive-neutral", "continuation-accounting:accounting-productive-noisy", "continuation-accounting:accounting-exhaustion-neutral", "continuation-accounting:accounting-exhaustion-noisy", "continuation-accounting:accounting-repair-stop", "continuation-accounting:accounting-repair-approval"] })),
|
||||
|
||||
@@ -3,7 +3,7 @@ import { defineConfig } from "@playwright/test";
|
||||
/** Browser-only harness regressions: no Paperclip instance or provider credentials. */
|
||||
export default defineConfig({
|
||||
testDir: ".",
|
||||
testMatch: "screenshot-readiness.spec.ts",
|
||||
testMatch: ["screenshot-readiness.spec.ts", "service-worker-reload.spec.ts"],
|
||||
workers: 1,
|
||||
retries: 0,
|
||||
timeout: 10_000,
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
import { test, expect } from "@playwright/test";
|
||||
import { createServer, type Server } from "node:http";
|
||||
import { readFile } from "node:fs/promises";
|
||||
|
||||
// Real browser/module-cache behavior without a Paperclip instance or LLM.
|
||||
test("development module revalidation bypasses the offline worker across reloads", async ({ page }) => {
|
||||
const worker = await readFile(new URL("../../ui/public/sw.js", import.meta.url), "utf8");
|
||||
let revalidations = 0;
|
||||
const server: Server = createServer((request, response) => {
|
||||
if (request.url === "/sw.js") {
|
||||
response.writeHead(200, { "content-type": "text/javascript", "cache-control": "no-store" });
|
||||
response.end(worker);
|
||||
} else if (request.url === "/fixture.js") {
|
||||
if (request.headers["if-none-match"] === '"fixture-v1"') {
|
||||
revalidations++;
|
||||
response.writeHead(304); response.end();
|
||||
} else {
|
||||
response.writeHead(200, { "content-type": "text/javascript", "cache-control": "no-cache", etag: '"fixture-v1"' });
|
||||
response.end('document.getElementById("root").textContent = "App mounted";');
|
||||
}
|
||||
} else {
|
||||
response.writeHead(200, { "content-type": "text/html", "cache-control": "no-store" });
|
||||
response.end('<div id="root"></div><script type="module" src="/fixture.js"></script>');
|
||||
}
|
||||
});
|
||||
await new Promise<void>(resolve => server.listen(0, "127.0.0.1", resolve));
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") throw new Error("Missing fixture port");
|
||||
const workerModuleRequests: string[] = [];
|
||||
page.context().on("request", request => {
|
||||
if (request.url().endsWith("/fixture.js") && request.serviceWorker()) workerModuleRequests.push(request.url());
|
||||
});
|
||||
try {
|
||||
await page.goto(`http://127.0.0.1:${address.port}/`);
|
||||
await page.evaluate(async () => {
|
||||
await navigator.serviceWorker.register("/sw.js");
|
||||
await navigator.serviceWorker.ready;
|
||||
});
|
||||
await expect.poll(() => page.evaluate(() => Boolean(navigator.serviceWorker.controller))).toBe(true);
|
||||
for (let n = 0; n < 3; n++) {
|
||||
await page.reload();
|
||||
await expect(page.locator("#root")).toHaveText("App mounted");
|
||||
}
|
||||
expect(revalidations).toBeGreaterThan(0);
|
||||
expect(workerModuleRequests).toEqual([]);
|
||||
} finally {
|
||||
server.closeAllConnections();
|
||||
await new Promise<void>((resolve, reject) => server.close(error => error ? reject(error) : resolve()));
|
||||
}
|
||||
});
|
||||
@@ -31,6 +31,10 @@ self.addEventListener("activate", (event) => {
|
||||
});
|
||||
|
||||
self.addEventListener("fetch", (event) => {
|
||||
// Vite owns development module revalidation and HMR. Passing that graph
|
||||
// through an offline worker can forward bodyless 304 responses on reload.
|
||||
// Only a stamped production build has an offline-cache contract.
|
||||
if (BUILD_ID.startsWith("__")) return;
|
||||
const { request } = event;
|
||||
const url = new URL(request.url);
|
||||
// Only immutable Vite build assets have a public offline-cache contract.
|
||||
|
||||
@@ -3,13 +3,14 @@ import { readFileSync } from "node:fs";
|
||||
import vm from "node:vm";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
function worker() {
|
||||
function worker(development = false) {
|
||||
const handlers = new Map<string, (event: unknown) => void>();
|
||||
const put = vi.fn();
|
||||
const match = vi.fn();
|
||||
const remove = vi.fn().mockResolvedValue(true);
|
||||
const fetch = vi.fn().mockResolvedValue(new Response("public asset"));
|
||||
vm.runInNewContext(readFileSync(new URL("../../public/sw.js", import.meta.url), "utf8"), {
|
||||
const source = readFileSync(new URL("../../public/sw.js", import.meta.url), "utf8");
|
||||
vm.runInNewContext(development ? source : source.replace("__PAPERCLIP_BUILD_ID__", "fixture-production"), {
|
||||
self: { location: { origin: "https://example.test" }, addEventListener: (type: string, fn: (event: unknown) => void) => handlers.set(type, fn) },
|
||||
URL, Response, fetch, caches: { keys: async () => ["paperclip-old", "paperclip-current"], open: async () => ({ put, delete: remove, match }), match },
|
||||
});
|
||||
@@ -22,6 +23,15 @@ function worker() {
|
||||
}
|
||||
|
||||
describe("service worker privacy boundaries", () => {
|
||||
it("leaves development navigation and module revalidation to the browser", () => {
|
||||
const w = worker(true);
|
||||
for (const path of ["/RUN/issues/RUN-1", "/src/main.tsx", "/@fs/vite-cache/deps/react.js?v=1"]) {
|
||||
expect(w.request("default", path)).not.toHaveBeenCalled();
|
||||
}
|
||||
expect(w.fetch).not.toHaveBeenCalled();
|
||||
expect(w.put).not.toHaveBeenCalled();
|
||||
expect(w.match).not.toHaveBeenCalled();
|
||||
});
|
||||
it("bypasses both caching and offline fallback for a no-store request outside /api", () => {
|
||||
const w = worker();
|
||||
expect(w.request("no-store")).not.toHaveBeenCalled();
|
||||
|
||||
Reference in new issue
Block a user