diff --git a/doc/plans/2026-10-07-native-connection-evidence-repair.md b/doc/plans/2026-10-07-native-connection-evidence-repair.md new file mode 100644 index 00000000000..7444aa2ef7b --- /dev/null +++ b/doc/plans/2026-10-07-native-connection-evidence-repair.md @@ -0,0 +1,47 @@ +# Native connection evidence and continuation repair + +## Finish line + +Fix the observed decline-grading races and qualify an explicit per-turn native +completion reminder. Keep the prompt reduction in #15489 draft. Do not merge. + +## Changes and evidence + +The evaluator waits for the final successful lead run's attributed saved reply +before grading a decline, with the original deadline unchanged. Wrong wording +still fails. Both explanation checks recognize the observed unavailable/rejected +and "wasn't able to pull" responses. The exact grading input is retained before +later refreshes. A recorded tool-action rejection may await its bound continuation +within the same deadline, just like an executed approval. + +Compact resumed inputs remind the provider to obtain an accepted completion or +block report for the current turn. Earlier reports and final prose are not a +replacement. The strict native result gate, original connection prompt, tool +catalog, permission enforcement and automatic retry policy remain in place. +This is an instruction repair hypothesis; local tests alone do not establish +that a model will follow it. + +## Bounded comparison + +Freeze a common evaluator baseline on master `fc6304dfe5f2e446e09bd052a7b45f51e930f250`. +Use identical evaluator, fixture, prompt, model, profile, permission and budget +inputs in both variants. The candidate alone adds the compact per-turn reminder. +Select the fifteen explicit `native-connection-guidance` local cells: five +connection stories across native Codex `gpt-5.6-sol`, ACPX Claude `claude-sonnet-5`, +and OpenCode `openrouter/deepseek/deepseek-v4-flash-0731`. One attempt per variant +and cell, no automatic retries, 720000 ms cell deadlines, 1000-cent company and +agent hard stops, and three concurrent cells per campaign. Count every actual +run, including failures; preserve source and grader hashes and incomplete billing. + +The old campaigns 37688433682 and 37688449963 remain original 11/15 and 12/15 +results with two new failures. This follow-up does not regrade or qualify those +trees. A passing workflow command is not a behavioral grade. Inspect saved output, +accepted reports, exact pairs, decisions, tool use and cleanup before readiness. + +## Validation so far + +- 25 focused evaluator tests pass, including delayed/missing/wrong-run replies, + incorrect output, and accepted/rejected continuation deadlines. +- 194 native contract, instruction-delivery and session tests pass. A correct + visible provider answer without a semantic completion still fails closed. +- Full validation and the fresh paired live comparison are pending. diff --git a/packages/paperclip-runner/README.md b/packages/paperclip-runner/README.md index ceb4c20c022..5d758d956bd 100644 --- a/packages/paperclip-runner/README.md +++ b/packages/paperclip-runner/README.md @@ -22,6 +22,13 @@ the provider conversation. Company, agent, task, workspace, model, instruction, and skill compatibility still gate recovery. An MCP-only assignment change can resume only when the selected harness explicitly supports refreshing tools. +Compact continuation messages include the active completion revision and criterion +IDs plus a reminder to obtain an accepted `paperclip_finish` or `paperclip_block` +result for this turn. Reports from earlier turns do not finish the new turn. +Provider final text alone remains insufficient; governed waits and strict native +completion validation keep their existing behavior. This reminder changes the +resumed model input, not the tool catalog or automatic retry policy. + When recovery needs a fresh conversation, the server supplies a deterministic handoff through a lazy history loader at the fresh attempt boundary. It includes the original request, recent messages, resolved interaction diff --git a/packages/paperclip-runner/src/backends/native-instruction-measurement.test.ts b/packages/paperclip-runner/src/backends/native-instruction-measurement.test.ts index ab56cdfe845..98507b24338 100644 --- a/packages/paperclip-runner/src/backends/native-instruction-measurement.test.ts +++ b/packages/paperclip-runner/src/backends/native-instruction-measurement.test.ts @@ -139,6 +139,8 @@ describe("native instruction payload measurement", () => { expect(JSON.stringify(turn.input)).toContain("Document saved"); } expect(JSON.stringify(turn.input)).toContain("objective"); + expect(JSON.stringify(turn.input)).toContain("Before ending this turn, obtain one accepted paperclip_finish or paperclip_block result."); + expect(JSON.stringify(turn.input)).toContain("Earlier reports belong to earlier turns"); } else { expect(JSON.stringify(turn.input)).toContain("Obtain one accepted result"); expect(JSON.stringify(turn.input)).toContain("Document saved"); diff --git a/packages/paperclip-runner/src/contracts/native-execution.test.ts b/packages/paperclip-runner/src/contracts/native-execution.test.ts index 05644579b8e..111e705b5db 100644 --- a/packages/paperclip-runner/src/contracts/native-execution.test.ts +++ b/packages/paperclip-runner/src/contracts/native-execution.test.ts @@ -104,7 +104,8 @@ describe("NativeExecutionInputV1", () => { expect(delta).toEqual({ schema: "paperclip.native-continuation.v1", events: '{"messages":[{"authorType":"user","body":"Just this new comment"}]}', - completion: { revision: "1", criterionIds: ["objective"] }, + completion: { revision: "1", criterionIds: ["objective"], + instruction: "Before ending this turn, obtain one accepted paperclip_finish or paperclip_block result. Earlier reports belong to earlier turns; a final message alone does not complete this turn." }, }); expect(JSON.stringify(delta)).not.toContain(input.task.title); expect(JSON.stringify(delta)).not.toContain(input.completionContract.contract.objective); diff --git a/packages/paperclip-runner/src/contracts/native-execution.ts b/packages/paperclip-runner/src/contracts/native-execution.ts index f0959a1713b..622aedb0f48 100644 --- a/packages/paperclip-runner/src/contracts/native-execution.ts +++ b/packages/paperclip-runner/src/contracts/native-execution.ts @@ -863,7 +863,7 @@ function parseCompletionSources(value: unknown): NativeCompletionSources { export interface NativeContinuationEnvelope { schema: "paperclip.native-continuation.v1"; events: string; - completion: { revision: string; criterionIds: string[] }; + completion: { revision: string; criterionIds: string[]; instruction: string }; } export function buildNativeModelEnvelope(input: NativeExecutionInput, options: { resumedSession: true }): NativeModelEnvelopeV1 | NativeModelEnvelopeV2 | NativeModelEnvelopeV3 | NativeContinuationEnvelope; @@ -876,6 +876,7 @@ export function buildNativeModelEnvelope(input: NativeExecutionInput, options?: completion: { revision: input.completionContract.contract.revision, criterionIds: input.completionContract.contract.criteria.map((criterion) => criterion.id), + instruction: "Before ending this turn, obtain one accepted paperclip_finish or paperclip_block result. Earlier reports belong to earlier turns; a final message alone does not complete this turn.", }, }; } diff --git a/packages/paperclip-runner/src/native-session-runtime.test.ts b/packages/paperclip-runner/src/native-session-runtime.test.ts index e65944682cd..610813ef9c6 100644 --- a/packages/paperclip-runner/src/native-session-runtime.test.ts +++ b/packages/paperclip-runner/src/native-session-runtime.test.ts @@ -508,7 +508,8 @@ describe("executeNativeSession recovery", () => { const submitted = startTurn.mock.calls[0]![0]!; if (snapshotBeforeUpdate) { expect(submitted.continuation).toBe(true); - expect(JSON.parse(submitted.message.text)).toMatchObject({ schema: "paperclip.native-continuation.v1", events: "Say bye" }); + expect(JSON.parse(submitted.message.text)).toMatchObject({ schema: "paperclip.native-continuation.v1", events: "Say bye", + completion: { instruction: expect.stringContaining("Earlier reports belong to earlier turns") } }); } else { expect(submitted).not.toHaveProperty("continuation"); expect(JSON.parse(submitted.message.text).task.prompt).toBe("Say bye"); @@ -7498,6 +7499,37 @@ describe("executeNativeSession recovery", () => { }); }); + it("rejects visible provider final text without an accepted current-turn completion report", async () => { + const capabilities = { resume: false, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + const events: PrpEvent[] = []; + const completeRun = vi.fn(async () => undefined); + const session: NativeSession = { + identity: () => identity, + async capabilities() { return capabilities; }, + async *events() { + yield runnerEvent(1, "item.completed", { kind: "agentMessage", channel: "final", text: "Contacts retrieved: Ada Fixture. Verification code: CONTACTS_fixture." }); + yield runnerEvent(2, "turn.completed"); + }, + async startTurn() { return { turnId: "turn-recovery" }; }, + async result() { return null; }, + async snapshot() { return { backendKind: "mock", sessionId: identity.sessionId, identity, + providerSessionId: "provider-recovery", cursor: "2", activeTurnId: "turn-recovery", pendingRuntimeRequests: [], lineage: [] }; }, + async close() {}, + }; + await expect(executeNativeSession({ input, + backend: { async descriptor() { return { kind: "mock", name: "missing-current-completion", version: "1", capabilities }; }, async openSession() { return session; } }, + controlPlane: { + async openRun() {}, async checkpointSession() {}, + async appendEvent(event) { events.push(structuredClone(event as PrpEvent)); return { cursor: event.sourceSeq, highestContiguousSourceSeq: event.sourceSeq, disposition: "committed" }; }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, completeRun, + }, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", timeoutMs: 1000, + resolveMissingResult: async () => null, + })).rejects.toThrow("native_finalization_missing"); + expect(completeRun).not.toHaveBeenCalled(); + expect(events.some(event => event.eventType === "item.completed")).toBe(true); + expect(events.some(event => event.eventType === "run.result.accepted")).toBe(false); + }); + it("accepts a control-plane governed wait when a completed turn omitted its semantic result", async () => { const terminalEvent: PrpEvent = { schema: "paperclip.prp.event.v1", diff --git a/tests/runner-e2e/EVERYDAY-WORKFLOWS.md b/tests/runner-e2e/EVERYDAY-WORKFLOWS.md index 675c4ed8db8..d2c9a2e899e 100644 --- a/tests/runner-e2e/EVERYDAY-WORKFLOWS.md +++ b/tests/runner-e2e/EVERYDAY-WORKFLOWS.md @@ -238,3 +238,25 @@ revision, harness digest, and existing usage/cost evidence accompany each attemp routing, incorrect ordering, early calls, duplicate questions, and fabricated reads. Run a single `everyday-workflows.runner-codex-mini.local.provider-decline` cell first; do not treat these fixtures as live provider compatibility tests. + +### Connection-guidance observation boundaries + +The explicit `native-connection-guidance` suite keeps one attempt per cell. +For decline cases, Done and a succeeded run are insufficient: settlement waits +within the existing cell deadline for a saved reply attributed by run ID to the +lead agent's final successful run on this task. Readiness checks storage and +identity, not favorable wording. A missing reply times out; an incorrect reply +settles and fails the independent explanation check. Both decline explanation +checks use the same bounded matcher, including unavailable/rejected access and +“wasn't able to pull” wording. This is textual evidence, not proof of cognition. + +An executed approval or a recorded tool-action rejection with `wake_assignee` +may precede its continuation run. Only a response bound to the same task, company, +agent, and completed source run can defer the stranded-blocker check, and only +within the original deadline. A failed execution, unrelated card, or already +consumed response cannot extend that wait. + +`connection-guidance-decline-grade.json` and the workflow's +`declineGradeEvidence` retain the exact cloned assertion input and original +checks. Later final-state or cleanup observations cannot overwrite that input. +Earlier campaign grades remain unchanged when the evaluator is corrected. diff --git a/tests/runner-e2e/connection-guidance-evidence.ts b/tests/runner-e2e/connection-guidance-evidence.ts index 48573317b23..5d2c651af05 100644 --- a/tests/runner-e2e/connection-guidance-evidence.ts +++ b/tests/runner-e2e/connection-guidance-evidence.ts @@ -9,7 +9,7 @@ type Run = { id: string; nativeIssueId?: string | null; agentId?: string | null; // Additional oracle for the new neutral-prompt suite only. Historical Everyday // grades remain intact. This proves attributed saved output, not cognition. -export function gradeConnectionGuidanceDecline(input: { +export type ConnectionGuidanceDeclineInput = { caseId: "service-decline" | "connection-decline" | "provider-decline"; decisionId: string; decisions: Decision[]; @@ -20,7 +20,9 @@ export function gradeConnectionGuidanceDecline(input: { calls?: number; marker: string; sameConnections: boolean; -}) { +}; + +function declineReplyEvidence(input: ConnectionGuidanceDeclineInput) { const decision = input.decisions.find(row => row.id === input.decisionId); const resolvedAt = Date.parse(decision?.resolvedAt ?? ""); const provider = input.caseId === "provider-decline"; @@ -36,17 +38,35 @@ export function gradeConnectionGuidanceDecline(input: { validDecision && reply.authorAgentId === input.leadAgentId && Number.isFinite(Date.parse(reply.createdAt ?? "")) && Date.parse(reply.createdAt!) >= resolvedAt); + const finalRunTime = Math.max(...input.runs.filter(run => + run.agentId === input.leadAgentId && run.nativeIssueId === input.issueId && run.status === "succeeded") + .map(run => Date.parse(run.finishedAt ?? "")).filter(Number.isFinite)); const attributed = afterDecision.filter(reply => input.runs.some(run => Boolean(reply.createdByRunId) && run.id === reply.createdByRunId && run.agentId === input.leadAgentId && run.nativeIssueId === input.issueId && - run.status === "succeeded" && Date.parse(run.finishedAt ?? "") >= resolvedAt)); - const explainsUnavailable = (text: string) => - /declin|not now|could(?:n.t| not)|cannot|can.t|unable|unavailable|not (?:connect|retriev)|without (?:access|connect)|\b(?:is|are)n['’]t\s+(?:connect|retriev|available)/i.test(text); + run.status === "succeeded" && Date.parse(run.finishedAt ?? "") >= resolvedAt && + Date.parse(run.finishedAt ?? "") === finalRunTime)); + return { validDecision, afterDecision, attributed }; +} + +/** Readiness waits for storage, not a passing explanation. Wrong wording must still fail grading. */ +export function hasConnectionGuidanceDeclineReply(input: ConnectionGuidanceDeclineInput): boolean { + return declineReplyEvidence(input).attributed.some(reply => typeof reply.body === "string" && reply.body.trim().length > 0); +} + +/** Bounded textual evidence, not a claim about model reasoning or native skill activation. */ +export function explainsConnectionUnavailable(text: string): boolean { + const normalized = text.replaceAll("’", "'"); + return /(? typeof reply.body === "string" && explainsUnavailable(reply.body)), - detail: "A saved explanation follows the decision and joins by run ID to the lead's successful execution on this task." }, + passed: attributed.some(reply => typeof reply.body === "string" && explainsConnectionUnavailable(reply.body)), + detail: "A saved explanation follows the decision and joins by run ID to the lead's final successful execution on this task." }, { id: "guidance-decline-no-use", passed: (input.caseId === "connection-decline" || input.calls === 0) && input.sameConnections && afterDecision.every(reply => (typeof reply.body !== "string" || !reply.body.includes(input.marker))), detail: "No connection changes or unread marker in replies. Installed-service/provider declines also require an observed zero fixture-call count; Notion setup does not execute a service." }, diff --git a/tests/runner-e2e/connection-guidance.test.ts b/tests/runner-e2e/connection-guidance.test.ts index 92469eb7f78..6609569cc0c 100644 --- a/tests/runner-e2e/connection-guidance.test.ts +++ b/tests/runner-e2e/connection-guidance.test.ts @@ -1,8 +1,9 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { runnerMatrix, runnerSuites } from "./catalog.js"; import { everydayTasks } from "./everyday-cases.js"; import { CONNECTION_GUIDANCE_SUITE, connectionGuidanceTasks, connectionGuidanceDefinitionDigest } from "./connection-guidance-cases.js"; -import { gradeConnectionGuidanceDecline } from "./connection-guidance-evidence.js"; +import { gradeConnectionGuidanceDecline, hasConnectionGuidanceDeclineReply } from "./connection-guidance-evidence.js"; +import { pollUntil } from "./api.js"; import { parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js"; describe("neutral connection guidance selection", () => { @@ -58,9 +59,11 @@ describe("neutral decline evidence", () => { for (const body of [ "HubSpot isn't connected because you chose None for now.", "HubSpot isn’t connected after the provider decision.", + "I wasn't able to pull your recent HubSpot contacts. You chose None for now, so no connection was made.", + "The page service approval was rejected, so the data was unavailable.", "The pages aren't retrievable after the connection decision.", ]) expect(passes({ ...valid, replies: [{ ...valid.replies[0], body }] })).toBe(true); - for (const body of ["HubSpot is connected.", "HubSpot isn't disconnected.", "HubSpot isn't slow.", "Contacts are available."]) { + for (const body of ["HubSpot is connected.", "HubSpot isn't disconnected.", "HubSpot isn't slow.", "Contacts are available.", "The request was not declined.", "The request was not rejected.", "Contacts are not unavailable.", "I wasn't able to finish my coffee."]) { expect(passes({ ...valid, replies: [{ ...valid.replies[0], body }] })).toBe(false); } }); @@ -82,6 +85,38 @@ describe("neutral decline evidence", () => { { ...valid, runs: [{ ...valid.runs[0], finishedAt: "2026-10-06T21:59:00Z" }] }, ]) expect(passes(input)).toBe(false); }); + it("requires output from the final successful task run, not a late comment from an earlier wait", () => { + const runs = [...valid.runs, { ...valid.runs[0], id: "final", finishedAt: "2026-10-06T22:00:06Z" }]; + expect(hasConnectionGuidanceDeclineReply({ ...valid, runs })).toBe(false); + expect(passes({ ...valid, runs })).toBe(false); + const input = { ...valid, runs, replies: [{ ...valid.replies[0], createdByRunId: "final", createdAt: "2026-10-06T22:00:07Z" }] }; + expect(hasConnectionGuidanceDeclineReply(input)).toBe(true); + expect(passes(input)).toBe(true); + }); + it.each(["delayed", "missing", "wrong-run", "wrong-content"])("waits for durable output within the original deadline: %s", async variant => { + vi.useFakeTimers(); + try { + const start = Date.now(); + const input = structuredClone({ ...valid, replies: [] as typeof valid.replies }); + const pending = pollUntil({ label: "final decline reply", deadlineAt: start + 30_000, intervalMs: 1000, + load: async () => { + if (variant !== "missing" && Date.now() >= start + 20_000) input.replies = [{ ...valid.replies[0], + createdByRunId: variant === "wrong-run" ? "other" : "run", + body: variant === "wrong-content" ? "Done." : valid.replies[0].body }]; + return input; + }, + accept: hasConnectionGuidanceDeclineReply, + }).catch((error: unknown) => error); + await vi.advanceTimersByTimeAsync(30_001); + const result = await pending; + if (variant === "missing" || variant === "wrong-run") expect(result).toBeInstanceOf(Error); + else { + expect(result).toEqual(input); + // Readiness must not wait for favorable wording or silently convert a bad answer into a pass. + expect(passes(input)).toBe(variant === "delayed"); + } + } finally { vi.useRealTimers(); } + }); it("rejects wrong or repeated decisions, early use, missing call evidence and connection changes", () => { for (const input of [ { ...valid, decisionId: "other" }, diff --git a/tests/runner-e2e/everyday-flow.ts b/tests/runner-e2e/everyday-flow.ts index fbf819b50d4..f3d20ad2407 100644 --- a/tests/runner-e2e/everyday-flow.ts +++ b/tests/runner-e2e/everyday-flow.ts @@ -1,6 +1,6 @@ import { gradeAgentmailSetup } from "./agentmail-setup-evidence.js"; import { CONNECTION_GUIDANCE_SUITE, CONNECTION_GUIDANCE_BUDGET_CENTS } from "./connection-guidance-cases.js"; -import { gradeConnectionGuidanceDecline } from "./connection-guidance-evidence.js"; +import { gradeConnectionGuidanceDecline, hasConnectionGuidanceDeclineReply, explainsConnectionUnavailable, type ConnectionGuidanceDeclineInput } from "./connection-guidance-evidence.js"; import { expect, type Page } from "@playwright/test"; import { runnerApiToolsEnabled } from "../../server/src/services/native-runtime/runner-api-rollout.js"; import { spawn } from "node:child_process"; @@ -68,6 +68,7 @@ export interface EverydayEvidence { connectionGuidanceBudgets?: { companyMonthlyCents: unknown; agentMonthlyCents: unknown }; }; documents?: Row[]; + declineGradeEvidence?: { capturedAt: string; input: ConnectionGuidanceDeclineInput; checks: StoryCheck[] }; checks: StoryCheck[]; timeline: Array<{ at: string; action: string; detail?: unknown }>; issues: Array; @@ -301,6 +302,18 @@ export async function runEverydayFlow(input: Input) { commentIds: added.map((c) => c.id), }); } + function declineInput(state: Pick): ConnectionGuidanceDeclineInput | undefined { + if (execution.suite.id !== CONNECTION_GUIDANCE_SUITE || + (caseId !== "service-decline" && caseId !== "connection-decline" && caseId !== "provider-decline")) return; + const issue = state.issues.find(i => i.id === parent?.id); + return { + caseId, decisionId: decisionId ?? "", decisions: issue?.interactions as any ?? [], + leadAgentId: fixtures.agent.id, issueId: parent?.id ?? "", replies: issue?.comments ?? [], + runs: state.runs, marker: (caseId === "provider-decline" ? "CONTACTS_" : "SERVICE_") + nonce, + // Readiness never grades these external observations. They are filled at the grading boundary below. + sameConnections: false, + }; + } async function settled(expectedAgentReply?: string) { const settledState = await pollUntil({ label: `everyday ${caseId} settled`, @@ -310,7 +323,7 @@ export async function runEverydayFlow(input: Input) { state.issues, parent?.id ?? "", fixtures.agent.id, state.runs, observableAgentIds(state), ) ?? (storyHasDurableServiceContinuation(state.issues, parent?.id ?? "", fixtures.agent.id, state.runs) - ? "task is Blocked without an active continuation after executed service approval" : undefined)), + ? "task is Blocked without an active continuation after resolved service approval" : undefined)), intervalMs: 1000, load: refresh, accept: (state) => @@ -324,6 +337,7 @@ export async function runEverydayFlow(input: Input) { state.runs.some( (r) => Date.parse(r.finishedAt ?? "") >= lastSubmissionAt, ) && + (!declineInput(state) || hasConnectionGuidanceDeclineReply(declineInput(state)!)) && (!expectedAgentReply || storyHasAgentReply( state.issues.find((issue) => issue.id === parent?.id), @@ -1175,15 +1189,17 @@ export async function runEverydayFlow(input: Input) { } if (execution.suite.id === CONNECTION_GUIDANCE_SUITE && (caseId === "service-decline" || caseId === "connection-decline" || caseId === "provider-decline")) { - const issue = ev.issues.find(i => i.id === parent!.id)!; const state = await api.get<{ connections: Row[] }>("/api/companies/" + fixtures.company.id + "/tools/connections"); - ev.checks.push(...gradeConnectionGuidanceDecline({ - caseId, decisionId: decisionId!, decisions: issue.interactions as any, - leadAgentId: fixtures.agent.id, issueId: parent!.id, replies: issue.comments ?? [], - runs: ev.runs, calls: review?.invocationCount() ?? aggregatorFixture?.invocationCount(), - marker: (caseId === "provider-decline" ? "CONTACTS_" : "SERVICE_") + nonce, + const gradeInput = structuredClone({ + ...declineInput(ev)!, + calls: review?.invocationCount() ?? aggregatorFixture?.invocationCount(), sameConnections: isDeepStrictEqual(state.connections.map(c => c.id).sort(), initialConnections.sort()), - })); + }); + const checks = gradeConnectionGuidanceDecline(gradeInput); + // Keep the actual assertion input independent of the later final/cleanup refreshes. + ev.declineGradeEvidence = { capturedAt: new Date().toISOString(), input: gradeInput, checks: structuredClone(checks) }; + await input.evidence("connection-guidance-decline-grade.json", ev.declineGradeEvidence); + ev.checks.push(...checks); } if (declining) { const issue = ev.issues.find((i) => i.id === parent!.id)!; @@ -1204,9 +1220,7 @@ export async function runEverydayFlow(input: Input) { check( "decline-visible-explanation", replies.length > 0 && - /declin|not now|could(?:n.t| not)|cannot|can.t|unable|not (?:connect|retriev)|without (?:access|connect)/i.test( - text, - ), + explainsConnectionUnavailable(text), "A new agent response explains the missing access after the saved decline.", ); check( diff --git a/tests/runner-e2e/everyday-observations.ts b/tests/runner-e2e/everyday-observations.ts index 6dad56a118c..15935a9c091 100644 --- a/tests/runner-e2e/everyday-observations.ts +++ b/tests/runner-e2e/everyday-observations.ts @@ -445,7 +445,7 @@ export function storyHasDurableAgentReviewContinuation( ); } -/** An executed approval may enqueue its wake just after run finalization. +/** An executed approval or recorded rejection may enqueue its wake just after run finalization. * Wait within the original deadline only for this task/run's recorded response. * Once a follow-up run has consumed it, a new blocker is a real failure. */ @@ -456,16 +456,18 @@ export function storyHasDurableServiceContinuation(issues: StoryIssue[], parentI const source = runs.find(run => run.id === interaction.sourceRunId); const action = interaction.payload?.toolAction as Record | undefined; const result = interaction.result?.toolAction as Record | undefined; + const resolved = (interaction.status === "accepted" && interaction.result?.outcome === "accepted" && result?.status === "executed") || + (interaction.status === "rejected" && interaction.result?.outcome === "rejected" && result === undefined); if (interaction.issueId !== parentId || interaction.createdByAgentId !== agentId || interaction.kind !== "request_confirmation" || interaction.continuationPolicy !== "wake_assignee" || - interaction.status !== "accepted" || interaction.result?.outcome !== "accepted" || - action?.version !== 1 || typeof action.actionRequestId !== "string" || result?.status !== "executed" || - source?.agentId !== agentId || source.companyId !== issue.companyId || source.status !== "succeeded" || - !(Date.parse(source.finishedAt ?? "") >= Date.parse(interaction.resolvedAt ?? ""))) return false; + !resolved || + action?.version !== 1 || typeof action.actionRequestId !== "string" || !action.actionRequestId.trim() || + source?.agentId !== agentId || source.companyId !== issue.companyId || source.nativeIssueId !== parentId || source.status !== "succeeded" || + (!Number.isFinite(Date.parse(source.finishedAt ?? "")) || !Number.isFinite(Date.parse(interaction.resolvedAt ?? "")))) return false; return !runs.some(run => { const input = run.runnerProfileJson?.nativeExecutionInput as { interactionResponses?: Array<{ interactionId?: string }> } | undefined; - return run.id !== source.id && run.agentId === agentId && + return run.id !== source.id && run.agentId === agentId && run.companyId === issue.companyId && input?.interactionResponses?.some(response => response.interactionId === interaction.id); }); }) ?? false; diff --git a/tests/runner-e2e/workflow-timeout.test.ts b/tests/runner-e2e/workflow-timeout.test.ts index aa75104aa38..d15c6cf1440 100644 --- a/tests/runner-e2e/workflow-timeout.test.ts +++ b/tests/runner-e2e/workflow-timeout.test.ts @@ -146,16 +146,16 @@ describe("review continuation deadline", () => { }); }); -describe("approved service continuation deadline", () => { +describe.each([["accepted", false], ["accepted", true], ["rejected", false], ["rejected", true]] as const)("%s service continuation deadline (decision after run: %s)", (status, decisionAfterRun) => { it.each([true, false])("keeps the original deadline for a delayed dispatch (arrives: %s)", async arrives => { vi.useFakeTimers(); try { const start = Date.now(); const issues: StoryIssue[] = [{ id: "task", companyId: "company", title: "Briefing", status: "blocked", assigneeAgentId: "agent", interactions: [{ - id: "approval", issueId: "task", sourceRunId: "first", createdByAgentId: "agent", kind: "request_confirmation", status: "accepted", continuationPolicy: "wake_assignee", - resolvedAt: new Date(start - 1000).toISOString(), payload: { toolAction: { version: 1, actionRequestId: "action" } }, result: { outcome: "accepted", toolAction: { status: "executed" } }, + id: "approval", issueId: "task", sourceRunId: "first", createdByAgentId: "agent", kind: "request_confirmation", status, continuationPolicy: "wake_assignee", + resolvedAt: new Date(start - 1000).toISOString(), payload: { toolAction: { version: 1, actionRequestId: "action" } }, result: status === "accepted" ? { outcome: "accepted", toolAction: { status: "executed" } } : { outcome: "rejected" }, }] }]; - const runs = [{ id: "first", companyId: "company", agentId: "agent", status: "succeeded", finishedAt: new Date(start).toISOString() }]; + const runs = [{ id: "first", companyId: "company", nativeIssueId: "task", agentId: "agent", status: "succeeded", finishedAt: new Date(start - (decisionAfterRun ? 2000 : 0)).toISOString() }]; const caught = pollUntil({ label: "service continuation", deadlineAt: start + 30_000, intervalMs: 1000, load: async () => { if (arrives && Date.now() >= start + 20_000) issues[0]!.status = "done"; return { issues, runs }; }, accept: state => state.issues[0]!.status === "done", @@ -170,12 +170,27 @@ describe("approved service continuation deadline", () => { }); it("waits for dispatch only until the same response has been consumed", () => { const issues: StoryIssue[] = [{ id: "task", companyId: "company", title: "Briefing", status: "blocked", assigneeAgentId: "agent", interactions: [{ - id: "approval", issueId: "task", sourceRunId: "first", createdByAgentId: "agent", kind: "request_confirmation", status: "accepted", continuationPolicy: "wake_assignee", - resolvedAt: "2026-10-07T05:37:19Z", payload: { toolAction: { version: 1, actionRequestId: "action" } }, result: { outcome: "accepted", toolAction: { status: "executed" } }, + id: "approval", issueId: "task", sourceRunId: "first", createdByAgentId: "agent", kind: "request_confirmation", status, continuationPolicy: "wake_assignee", + resolvedAt: "2026-10-07T05:37:19Z", payload: { toolAction: { version: 1, actionRequestId: "action" } }, result: status === "accepted" ? { outcome: "accepted", toolAction: { status: "executed" } } : { outcome: "rejected" }, }] }]; - const runs = [{ id: "first", companyId: "company", agentId: "agent", status: "succeeded", finishedAt: "2026-10-07T05:37:28Z" }]; + const runs = [{ id: "first", companyId: "company", nativeIssueId: "task", agentId: "agent", status: "succeeded", finishedAt: decisionAfterRun ? "2026-10-07T05:37:18Z" : "2026-10-07T05:37:28Z" }]; expect(storyHasDurableServiceContinuation(issues, "task", "agent", runs)).toBe(true); expect(storyHasDurableServiceContinuation(issues, "task", "other", runs)).toBe(false); + for (const mutate of [ + (row: StoryIssue) => { row.interactions![0]!.sourceRunId = "other-run"; }, + (row: StoryIssue) => { row.interactions![0]!.issueId = "other-task"; }, + (row: StoryIssue) => { row.companyId = "other-company"; }, + (row: StoryIssue) => { row.interactions![0]!.resolvedAt = "invalid"; }, + (row: StoryIssue) => { row.interactions![0]!.continuationPolicy = "none"; }, + (row: StoryIssue) => { row.interactions![0]!.payload = { toolAction: { version: 1, actionRequestId: "" } }; }, + (row: StoryIssue) => { row.interactions![0]!.result = { outcome: "accepted", toolAction: { status: "failed" } }; }, + ]) { + const changed = structuredClone(issues); + mutate(changed[0]!); + expect(storyHasDurableServiceContinuation(changed, "task", "agent", runs)).toBe(false); + } + expect(storyHasDurableServiceContinuation(issues, "task", "agent", [{ ...runs[0]!, nativeIssueId: "other-task" }])).toBe(false); + expect(storyHasDurableServiceContinuation(issues, "task", "agent", [{ ...runs[0]!, finishedAt: "invalid" }])).toBe(false); const continued = [...runs, { id: "next", companyId: "company", agentId: "agent", status: "succeeded", runnerProfileJson: { nativeExecutionInput: { interactionResponses: [{ interactionId: "approval" }] } } }]; expect(storyHasDurableServiceContinuation(issues, "task", "agent", continued)).toBe(false); issues[0]!.interactions![0]!.result = { outcome: "accepted", toolAction: { status: "failed" } };