fix(evals): await attributed decline replies and resolved continuations

Preserve exact grading inputs and keep original attempt deadlines and grades.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/EVERYDAY-WORKFLOWS.md

@@ -238,3 +238,25 @@ revision, harness digest, and existing usage/cost evidence accompany each attemp
 routing, incorrect ordering, early calls, duplicate questions, and fabricated reads.
 Run a single `everyday-workflows.runner-codex-mini.local.provider-decline` cell
 first; do not treat these fixtures as live provider compatibility tests.
+
+### Connection-guidance observation boundaries
+
+The explicit `native-connection-guidance` suite keeps one attempt per cell.
+For decline cases, Done and a succeeded run are insufficient: settlement waits
+within the existing cell deadline for a saved reply attributed by run ID to the
+lead agent's final successful run on this task. Readiness checks storage and
+identity, not favorable wording. A missing reply times out; an incorrect reply
+settles and fails the independent explanation check. Both decline explanation
+checks use the same bounded matcher, including unavailable/rejected access and
+“wasn't able to pull” wording. This is textual evidence, not proof of cognition.
+
+An executed approval or a recorded tool-action rejection with `wake_assignee`
+may precede its continuation run. Only a response bound to the same task, company,
+agent, and completed source run can defer the stranded-blocker check, and only
+within the original deadline. A failed execution, unrelated card, or already
+consumed response cannot extend that wait.
+
+`connection-guidance-decline-grade.json` and the workflow's
+`declineGradeEvidence` retain the exact cloned assertion input and original
+checks. Later final-state or cleanup observations cannot overwrite that input.
+Earlier campaign grades remain unchanged when the evaluator is corrected.

tests/runner-e2e/connection-guidance-evidence.ts

@@ -9,7 +9,7 @@ type Run = { id: string; nativeIssueId?: string | null; agentId?: string | null;
 
 // Additional oracle for the new neutral-prompt suite only. Historical Everyday
 // grades remain intact. This proves attributed saved output, not cognition.
-export function gradeConnectionGuidanceDecline(input: {
+export type ConnectionGuidanceDeclineInput = {
   caseId: "service-decline" | "connection-decline" | "provider-decline";
   decisionId: string;
   decisions: Decision[];
@@ -20,7 +20,9 @@ export function gradeConnectionGuidanceDecline(input: {
   calls?: number;
   marker: string;
   sameConnections: boolean;
-}) {
+};
+
+function declineReplyEvidence(input: ConnectionGuidanceDeclineInput) {
   const decision = input.decisions.find(row => row.id === input.decisionId);
   const resolvedAt = Date.parse(decision?.resolvedAt ?? "");
   const provider = input.caseId === "provider-decline";
@@ -36,17 +38,35 @@ export function gradeConnectionGuidanceDecline(input: {
     validDecision && reply.authorAgentId === input.leadAgentId &&
     Number.isFinite(Date.parse(reply.createdAt ?? "")) &&
     Date.parse(reply.createdAt!) >= resolvedAt);
+  const finalRunTime = Math.max(...input.runs.filter(run =>
+    run.agentId === input.leadAgentId && run.nativeIssueId === input.issueId && run.status === "succeeded")
+    .map(run => Date.parse(run.finishedAt ?? "")).filter(Number.isFinite));
   const attributed = afterDecision.filter(reply => input.runs.some(run =>
     Boolean(reply.createdByRunId) && run.id === reply.createdByRunId && run.agentId === input.leadAgentId && run.nativeIssueId === input.issueId &&
-    run.status === "succeeded" && Date.parse(run.finishedAt ?? "") >= resolvedAt));
-  const explainsUnavailable = (text: string) =>
-    /declin|not now|could(?:n.t| not)|cannot|can.t|unable|unavailable|not (?:connect|retriev)|without (?:access|connect)|\b(?:is|are)n['’]t\s+(?:connect|retriev|available)/i.test(text);
+    run.status === "succeeded" && Date.parse(run.finishedAt ?? "") >= resolvedAt &&
+    Date.parse(run.finishedAt ?? "") === finalRunTime));
+  return { validDecision, afterDecision, attributed };
+}
+
+/** Readiness waits for storage, not a passing explanation. Wrong wording must still fail grading. */
+export function hasConnectionGuidanceDeclineReply(input: ConnectionGuidanceDeclineInput): boolean {
+  return declineReplyEvidence(input).attributed.some(reply => typeof reply.body === "string" && reply.body.trim().length > 0);
+}
+
+/** Bounded textual evidence, not a claim about model reasoning or native skill activation. */
+export function explainsConnectionUnavailable(text: string): boolean {
+  const normalized = text.replaceAll("’", "'");
+  return /(?<!not )\b(?:declin(?:e|ed|ing)|rejected|unable|unavailable)\b|\bnot now\b|\bcould(?:n't| not)\b|\bcannot\b|\bcan't\b|\bnot (?:connect|retriev)|\bwithout (?:access|connect)|\b(?:is|are)n't\s+(?:connect|retriev|available)|\b(?:was|were)n't able to (?:pull|access|read|retriev|connect)|\bno connection was made\b/i.test(normalized);
+}
+
+export function gradeConnectionGuidanceDecline(input: ConnectionGuidanceDeclineInput) {
+  const { validDecision, afterDecision, attributed } = declineReplyEvidence(input);
   return [
     { id: "guidance-decline-decision", passed: validDecision && input.decisions.length === 1,
       detail: "Exactly one correctly typed, resolved decline belongs to the selected decision." },
     { id: "guidance-decline-attributed-explanation",
-      passed: attributed.some(reply => typeof reply.body === "string" && explainsUnavailable(reply.body)),
-      detail: "A saved explanation follows the decision and joins by run ID to the lead's successful execution on this task." },
+      passed: attributed.some(reply => typeof reply.body === "string" && explainsConnectionUnavailable(reply.body)),
+      detail: "A saved explanation follows the decision and joins by run ID to the lead's final successful execution on this task." },
     { id: "guidance-decline-no-use", passed: (input.caseId === "connection-decline" || input.calls === 0) && input.sameConnections &&
         afterDecision.every(reply => (typeof reply.body !== "string" || !reply.body.includes(input.marker))),
       detail: "No connection changes or unread marker in replies. Installed-service/provider declines also require an observed zero fixture-call count; Notion setup does not execute a service." },

tests/runner-e2e/connection-guidance.test.ts

@@ -1,8 +1,9 @@
-import { describe, expect, it } from "vitest";
+import { describe, expect, it, vi } from "vitest";
 import { runnerMatrix, runnerSuites } from "./catalog.js";
 import { everydayTasks } from "./everyday-cases.js";
 import { CONNECTION_GUIDANCE_SUITE, connectionGuidanceTasks, connectionGuidanceDefinitionDigest } from "./connection-guidance-cases.js";
-import { gradeConnectionGuidanceDecline } from "./connection-guidance-evidence.js";
+import { gradeConnectionGuidanceDecline, hasConnectionGuidanceDeclineReply } from "./connection-guidance-evidence.js";
+import { pollUntil } from "./api.js";
 import { parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js";
 
 describe("neutral connection guidance selection", () => {
@@ -58,9 +59,11 @@ describe("neutral decline evidence", () => {
     for (const body of [
       "HubSpot isn't connected because you chose None for now.",
       "HubSpot isn’t connected after the provider decision.",
+      "I wasn't able to pull your recent HubSpot contacts. You chose None for now, so no connection was made.",
+      "The page service approval was rejected, so the data was unavailable.",
       "The pages aren't retrievable after the connection decision.",
     ]) expect(passes({ ...valid, replies: [{ ...valid.replies[0], body }] })).toBe(true);
-    for (const body of ["HubSpot is connected.", "HubSpot isn't disconnected.", "HubSpot isn't slow.", "Contacts are available."]) {
+    for (const body of ["HubSpot is connected.", "HubSpot isn't disconnected.", "HubSpot isn't slow.", "Contacts are available.", "The request was not declined.", "The request was not rejected.", "Contacts are not unavailable.", "I wasn't able to finish my coffee."]) {
       expect(passes({ ...valid, replies: [{ ...valid.replies[0], body }] })).toBe(false);
     }
   });
@@ -82,6 +85,38 @@ describe("neutral decline evidence", () => {
       { ...valid, runs: [{ ...valid.runs[0], finishedAt: "2026-10-06T21:59:00Z" }] },
     ]) expect(passes(input)).toBe(false);
   });
+  it("requires output from the final successful task run, not a late comment from an earlier wait", () => {
+    const runs = [...valid.runs, { ...valid.runs[0], id: "final", finishedAt: "2026-10-06T22:00:06Z" }];
+    expect(hasConnectionGuidanceDeclineReply({ ...valid, runs })).toBe(false);
+    expect(passes({ ...valid, runs })).toBe(false);
+    const input = { ...valid, runs, replies: [{ ...valid.replies[0], createdByRunId: "final", createdAt: "2026-10-06T22:00:07Z" }] };
+    expect(hasConnectionGuidanceDeclineReply(input)).toBe(true);
+    expect(passes(input)).toBe(true);
+  });
+  it.each(["delayed", "missing", "wrong-run", "wrong-content"])("waits for durable output within the original deadline: %s", async variant => {
+    vi.useFakeTimers();
+    try {
+      const start = Date.now();
+      const input = structuredClone({ ...valid, replies: [] as typeof valid.replies });
+      const pending = pollUntil({ label: "final decline reply", deadlineAt: start + 30_000, intervalMs: 1000,
+        load: async () => {
+          if (variant !== "missing" && Date.now() >= start + 20_000) input.replies = [{ ...valid.replies[0],
+            createdByRunId: variant === "wrong-run" ? "other" : "run",
+            body: variant === "wrong-content" ? "Done." : valid.replies[0].body }];
+          return input;
+        },
+        accept: hasConnectionGuidanceDeclineReply,
+      }).catch((error: unknown) => error);
+      await vi.advanceTimersByTimeAsync(30_001);
+      const result = await pending;
+      if (variant === "missing" || variant === "wrong-run") expect(result).toBeInstanceOf(Error);
+      else {
+        expect(result).toEqual(input);
+        // Readiness must not wait for favorable wording or silently convert a bad answer into a pass.
+        expect(passes(input)).toBe(variant === "delayed");
+      }
+    } finally { vi.useRealTimers(); }
+  });
   it("rejects wrong or repeated decisions, early use, missing call evidence and connection changes", () => {
     for (const input of [
       { ...valid, decisionId: "other" },

tests/runner-e2e/everyday-flow.ts

@@ -1,6 +1,6 @@
 import { gradeAgentmailSetup } from "./agentmail-setup-evidence.js";
 import { CONNECTION_GUIDANCE_SUITE, CONNECTION_GUIDANCE_BUDGET_CENTS } from "./connection-guidance-cases.js";
-import { gradeConnectionGuidanceDecline } from "./connection-guidance-evidence.js";
+import { gradeConnectionGuidanceDecline, hasConnectionGuidanceDeclineReply, explainsConnectionUnavailable, type ConnectionGuidanceDeclineInput } from "./connection-guidance-evidence.js";
 import { expect, type Page } from "@playwright/test";
 import { runnerApiToolsEnabled } from "../../server/src/services/native-runtime/runner-api-rollout.js";
 import { spawn } from "node:child_process";
@@ -68,6 +68,7 @@ export interface EverydayEvidence {
     connectionGuidanceBudgets?: { companyMonthlyCents: unknown; agentMonthlyCents: unknown };
   };
   documents?: Row[];
+  declineGradeEvidence?: { capturedAt: string; input: ConnectionGuidanceDeclineInput; checks: StoryCheck[] };
   checks: StoryCheck[];
   timeline: Array<{ at: string; action: string; detail?: unknown }>;
   issues: Array<StoryIssue & Row>;
@@ -301,6 +302,18 @@ export async function runEverydayFlow(input: Input) {
       commentIds: added.map((c) => c.id),
     });
   }
+  function declineInput(state: Pick<EverydayEvidence, "issues" | "runs">): ConnectionGuidanceDeclineInput | undefined {
+    if (execution.suite.id !== CONNECTION_GUIDANCE_SUITE ||
+      (caseId !== "service-decline" && caseId !== "connection-decline" && caseId !== "provider-decline")) return;
+    const issue = state.issues.find(i => i.id === parent?.id);
+    return {
+      caseId, decisionId: decisionId ?? "", decisions: issue?.interactions as any ?? [],
+      leadAgentId: fixtures.agent.id, issueId: parent?.id ?? "", replies: issue?.comments ?? [],
+      runs: state.runs, marker: (caseId === "provider-decline" ? "CONTACTS_" : "SERVICE_") + nonce,
+      // Readiness never grades these external observations. They are filled at the grading boundary below.
+      sameConnections: false,
+    };
+  }
   async function settled(expectedAgentReply?: string) {
     const settledState = await pollUntil({
       label: `everyday ${caseId} settled`,
@@ -310,7 +323,7 @@ export async function runEverydayFlow(input: Input) {
           state.issues, parent?.id ?? "", fixtures.agent.id, state.runs,
           observableAgentIds(state),
         ) ?? (storyHasDurableServiceContinuation(state.issues, parent?.id ?? "", fixtures.agent.id, state.runs)
-          ? "task is Blocked without an active continuation after executed service approval" : undefined)),
+          ? "task is Blocked without an active continuation after resolved service approval" : undefined)),
       intervalMs: 1000,
       load: refresh,
       accept: (state) =>
@@ -324,6 +337,7 @@ export async function runEverydayFlow(input: Input) {
         state.runs.some(
           (r) => Date.parse(r.finishedAt ?? "") >= lastSubmissionAt,
         ) &&
+        (!declineInput(state) || hasConnectionGuidanceDeclineReply(declineInput(state)!)) &&
         (!expectedAgentReply ||
           storyHasAgentReply(
             state.issues.find((issue) => issue.id === parent?.id),
@@ -1175,15 +1189,17 @@ export async function runEverydayFlow(input: Input) {
     }
     if (execution.suite.id === CONNECTION_GUIDANCE_SUITE &&
       (caseId === "service-decline" || caseId === "connection-decline" || caseId === "provider-decline")) {
-      const issue = ev.issues.find(i => i.id === parent!.id)!;
       const state = await api.get<{ connections: Row[] }>("/api/companies/" + fixtures.company.id + "/tools/connections");
-      ev.checks.push(...gradeConnectionGuidanceDecline({
-        caseId, decisionId: decisionId!, decisions: issue.interactions as any,
-        leadAgentId: fixtures.agent.id, issueId: parent!.id, replies: issue.comments ?? [],
-        runs: ev.runs, calls: review?.invocationCount() ?? aggregatorFixture?.invocationCount(),
-        marker: (caseId === "provider-decline" ? "CONTACTS_" : "SERVICE_") + nonce,
+      const gradeInput = structuredClone({
+        ...declineInput(ev)!,
+        calls: review?.invocationCount() ?? aggregatorFixture?.invocationCount(),
         sameConnections: isDeepStrictEqual(state.connections.map(c => c.id).sort(), initialConnections.sort()),
-      }));
+      });
+      const checks = gradeConnectionGuidanceDecline(gradeInput);
+      // Keep the actual assertion input independent of the later final/cleanup refreshes.
+      ev.declineGradeEvidence = { capturedAt: new Date().toISOString(), input: gradeInput, checks: structuredClone(checks) };
+      await input.evidence("connection-guidance-decline-grade.json", ev.declineGradeEvidence);
+      ev.checks.push(...checks);
     }
     if (declining) {
       const issue = ev.issues.find((i) => i.id === parent!.id)!;
@@ -1204,9 +1220,7 @@ export async function runEverydayFlow(input: Input) {
       check(
         "decline-visible-explanation",
         replies.length > 0 &&
-          /declin|not now|could(?:n.t| not)|cannot|can.t|unable|not (?:connect|retriev)|without (?:access|connect)/i.test(
-            text,
-          ),
+          explainsConnectionUnavailable(text),
         "A new agent response explains the missing access after the saved decline.",
       );
       check(

tests/runner-e2e/everyday-observations.ts

@@ -445,7 +445,7 @@ export function storyHasDurableAgentReviewContinuation(
   );
 }
 
-/** An executed approval may enqueue its wake just after run finalization.
+/** An executed approval or recorded rejection may enqueue its wake just after run finalization.
  * Wait within the original deadline only for this task/run's recorded response.
  * Once a follow-up run has consumed it, a new blocker is a real failure.
  */
@@ -456,16 +456,18 @@ export function storyHasDurableServiceContinuation(issues: StoryIssue[], parentI
     const source = runs.find(run => run.id === interaction.sourceRunId);
     const action = interaction.payload?.toolAction as Record<string, unknown> | undefined;
     const result = interaction.result?.toolAction as Record<string, unknown> | undefined;
+    const resolved = (interaction.status === "accepted" && interaction.result?.outcome === "accepted" && result?.status === "executed") ||
+      (interaction.status === "rejected" && interaction.result?.outcome === "rejected" && result === undefined);
     if (interaction.issueId !== parentId || interaction.createdByAgentId !== agentId ||
         interaction.kind !== "request_confirmation" || interaction.continuationPolicy !== "wake_assignee" ||
-        interaction.status !== "accepted" || interaction.result?.outcome !== "accepted" ||
-        action?.version !== 1 || typeof action.actionRequestId !== "string" || result?.status !== "executed" ||
-        source?.agentId !== agentId || source.companyId !== issue.companyId || source.status !== "succeeded" ||
+        !resolved ||
+        action?.version !== 1 || typeof action.actionRequestId !== "string" || !action.actionRequestId.trim() ||
+        source?.agentId !== agentId || source.companyId !== issue.companyId || source.nativeIssueId !== parentId || source.status !== "succeeded" ||
         !(Date.parse(source.finishedAt ?? "") >= Date.parse(interaction.resolvedAt ?? ""))) return false;
     return !runs.some(run => {
       const input = run.runnerProfileJson?.nativeExecutionInput as
         { interactionResponses?: Array<{ interactionId?: string }> } | undefined;
-      return run.id !== source.id && run.agentId === agentId &&
+      return run.id !== source.id && run.agentId === agentId && run.companyId === issue.companyId &&
         input?.interactionResponses?.some(response => response.interactionId === interaction.id);
     });
   }) ?? false;

tests/runner-e2e/workflow-timeout.test.ts

@@ -146,16 +146,16 @@ describe("review continuation deadline", () => {
   });
 });
 
-describe("approved service continuation deadline", () => {
+describe.each(["accepted", "rejected"] as const)("%s service continuation deadline", status => {
   it.each([true, false])("keeps the original deadline for a delayed dispatch (arrives: %s)", async arrives => {
     vi.useFakeTimers();
     try {
       const start = Date.now();
       const issues: StoryIssue[] = [{ id: "task", companyId: "company", title: "Briefing", status: "blocked", assigneeAgentId: "agent", interactions: [{
-        id: "approval", issueId: "task", sourceRunId: "first", createdByAgentId: "agent", kind: "request_confirmation", status: "accepted", continuationPolicy: "wake_assignee",
-        resolvedAt: new Date(start - 1000).toISOString(), payload: { toolAction: { version: 1, actionRequestId: "action" } }, result: { outcome: "accepted", toolAction: { status: "executed" } },
+        id: "approval", issueId: "task", sourceRunId: "first", createdByAgentId: "agent", kind: "request_confirmation", status, continuationPolicy: "wake_assignee",
+        resolvedAt: new Date(start - 1000).toISOString(), payload: { toolAction: { version: 1, actionRequestId: "action" } }, result: status === "accepted" ? { outcome: "accepted", toolAction: { status: "executed" } } : { outcome: "rejected" },
       }] }];
-      const runs = [{ id: "first", companyId: "company", agentId: "agent", status: "succeeded", finishedAt: new Date(start).toISOString() }];
+      const runs = [{ id: "first", companyId: "company", nativeIssueId: "task", agentId: "agent", status: "succeeded", finishedAt: new Date(start).toISOString() }];
       const caught = pollUntil({ label: "service continuation", deadlineAt: start + 30_000, intervalMs: 1000,
         load: async () => { if (arrives && Date.now() >= start + 20_000) issues[0]!.status = "done"; return { issues, runs }; },
         accept: state => state.issues[0]!.status === "done",
@@ -170,12 +170,25 @@ describe("approved service continuation deadline", () => {
   });
   it("waits for dispatch only until the same response has been consumed", () => {
     const issues: StoryIssue[] = [{ id: "task", companyId: "company", title: "Briefing", status: "blocked", assigneeAgentId: "agent", interactions: [{
-      id: "approval", issueId: "task", sourceRunId: "first", createdByAgentId: "agent", kind: "request_confirmation", status: "accepted", continuationPolicy: "wake_assignee",
-      resolvedAt: "2026-10-07T05:37:19Z", payload: { toolAction: { version: 1, actionRequestId: "action" } }, result: { outcome: "accepted", toolAction: { status: "executed" } },
+      id: "approval", issueId: "task", sourceRunId: "first", createdByAgentId: "agent", kind: "request_confirmation", status, continuationPolicy: "wake_assignee",
+      resolvedAt: "2026-10-07T05:37:19Z", payload: { toolAction: { version: 1, actionRequestId: "action" } }, result: status === "accepted" ? { outcome: "accepted", toolAction: { status: "executed" } } : { outcome: "rejected" },
     }] }];
-    const runs = [{ id: "first", companyId: "company", agentId: "agent", status: "succeeded", finishedAt: "2026-10-07T05:37:28Z" }];
+    const runs = [{ id: "first", companyId: "company", nativeIssueId: "task", agentId: "agent", status: "succeeded", finishedAt: "2026-10-07T05:37:28Z" }];
     expect(storyHasDurableServiceContinuation(issues, "task", "agent", runs)).toBe(true);
     expect(storyHasDurableServiceContinuation(issues, "task", "other", runs)).toBe(false);
+    for (const mutate of [
+      (row: StoryIssue) => { row.interactions![0]!.sourceRunId = "other-run"; },
+      (row: StoryIssue) => { row.interactions![0]!.issueId = "other-task"; },
+      (row: StoryIssue) => { row.companyId = "other-company"; },
+      (row: StoryIssue) => { row.interactions![0]!.resolvedAt = "invalid"; },
+      (row: StoryIssue) => { row.interactions![0]!.continuationPolicy = "none"; },
+      (row: StoryIssue) => { row.interactions![0]!.payload = { toolAction: { version: 1, actionRequestId: "" } }; },
+      (row: StoryIssue) => { row.interactions![0]!.result = { outcome: "accepted", toolAction: { status: "failed" } }; },
+    ]) {
+      const changed = structuredClone(issues);
+      mutate(changed[0]!);
+      expect(storyHasDurableServiceContinuation(changed, "task", "agent", runs)).toBe(false);
+    }
     const continued = [...runs, { id: "next", companyId: "company", agentId: "agent", status: "succeeded", runnerProfileJson: { nativeExecutionInput: { interactionResponses: [{ interactionId: "approval" }] } } }];
     expect(storyHasDurableServiceContinuation(issues, "task", "agent", continued)).toBe(false);
     issues[0]!.interactions![0]!.result = { outcome: "accepted", toolAction: { status: "failed" } };