diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 6c796464629..c1aff01ccca 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -1258,7 +1258,7 @@ "chat-catalog-domains/api-read-receipts.ts", "run-observations.ts" ], - "oracleVersion": "paperclip.chat-catalog.spend-breakdown.structural.v4", + "oracleVersion": "paperclip.chat-catalog.spend-breakdown.structural.v5", "expectedRunCount": 2, "turnTimeoutMs": 120000, "attemptTimeoutMs": 600000, @@ -5079,7 +5079,7 @@ "../../ui/src/pages/secrets/SetMyUserSecretDialog.tsx", "chat-scenarios/secrets.yaml" ], - "oracleVersion": "paperclip.chat-catalog.personal-secret.v19", + "oracleVersion": "paperclip.chat-catalog.personal-secret.v20", "expectedRunCount": 3, "expectedHumanMessageCount": 2, "minimumRunCount": 3, @@ -5275,7 +5275,7 @@ "chat-catalog-domains/internal-support.ts", "chat-scenarios/skills.yaml" ], - "oracleVersion": "paperclip.chat-catalog.integrator.v1", + "oracleVersion": "paperclip.chat-catalog.integrator.skill-fork.v2", "expectedRunCount": 1, "expectedHumanMessageCount": 1, "turnTimeoutMs": 120000, @@ -5394,7 +5394,7 @@ "run-observations.ts", "chat-scenarios/status-cards.yaml" ], - "oracleVersion": "paperclip.chat-catalog.status-card.explain.v5", + "oracleVersion": "paperclip.chat-catalog.status-card.explain.v6", "expectedRunCount": 5, "minimumRunCount": 3, "expectedHumanMessageCount": 2, @@ -5493,7 +5493,7 @@ "chat-catalog-domains/identity-support.ts", "chat-scenarios/steering.yaml" ], - "oracleVersion": "paperclip.chat-catalog.runtime-discard.v4", + "oracleVersion": "paperclip.chat-catalog.runtime-discard.v6", "coverageGaps": [], "expectedRunCount": 5, "expectedHumanMessageCount": 2, diff --git a/tests/runner-e2e/chat-catalog-domains/external-personal-secret-receipt.test.ts b/tests/runner-e2e/chat-catalog-domains/external-personal-secret-receipt.test.ts index 02ba333dad6..70a6226023e 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-personal-secret-receipt.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-personal-secret-receipt.test.ts @@ -86,3 +86,14 @@ it("does not enable the personal receipt option for other credential cases", asy const ctx = await context(); ctx.scenario.id = "secrets.least-scope-delivery"; await expect(secretTestEndpoint(ctx, "morgan", "synthetic-personal-value", "PUBLISHING_API_KEY", undefined, { personalWorkerReceipt: true })).rejects.toThrow("dedicated personal-secret endpoint"); }); +it("reads the receipt object out of output that carries other text, still requiring the endpoint's receipt", () => { + const receipt = JSON.stringify({ ok: true, status: 200, check: "publishing endpoint", receiptId: "a".repeat(48) }); + const withOutput = (output: string) => { const f = delivery(); f.evidence.events[0]!.payload.prpEvent.payload.output = output; return personalSecretWorkerReceipt(f.run, f.evidence, f.expected); }; + expect(withOutput(`${receipt}EXIT_CODE=0\n`)).toMatchObject({ receiptId: "a".repeat(48) }); + expect(withOutput(`checking publishing endpoint...\n${JSON.stringify(JSON.parse(receipt), null, 2)}\nexit: 0 {done}\n`)).toMatchObject({ receiptId: "a".repeat(48) }); + expect(withOutput(`${receipt.replace("a".repeat(48), "b".repeat(48))}EXIT_CODE=0\n`)).toBeUndefined(); + expect(withOutput(`${JSON.stringify({ ok: false, status: 403, check: "publishing endpoint", receiptId: "a".repeat(48) })}EXIT_CODE=1\n`)).toBeUndefined(); + expect(withOutput(`${receipt}\n${receipt.replace("a".repeat(48), "b".repeat(48))}`)).toBeUndefined(); + expect(withOutput("EXIT_CODE=0\n")).toBeUndefined(); + expect(withOutput(`receiptId=${"a".repeat(48)} ok=true status=200`)).toBeUndefined(); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-personal-secret.test.ts b/tests/runner-e2e/chat-catalog-domains/external-personal-secret.test.ts index cd1db18e3e1..47d9469c950 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-personal-secret.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-personal-secret.test.ts @@ -338,7 +338,7 @@ it("keeps a confirmation dependent on actual worker delivery",()=>{ }); it("declares the prospective personal acknowledgment calibration without changing its bounds", () => { - expect(personalSecretExternalCases[0]).toMatchObject({ oracleVersion: "paperclip.chat-catalog.personal-secret.v19", + expect(personalSecretExternalCases[0]).toMatchObject({ oracleVersion: "paperclip.chat-catalog.personal-secret.v20", expectedRunCount: 3, minimumRunCount: 3, expectedHumanMessageCount: 2, attemptTimeoutMs: 660000 }); }); // Full cohort 3: the recorded guidance linked My secrets and excluded chat; a variant that also asks diff --git a/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts b/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts index ee37909a501..56351ccbbe7 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts @@ -194,6 +194,22 @@ export interface PersonalSecretWorkerEvidence { calls: Awaited>["calls"]; events: PersonalSecretWorkerEvent[]; } +export function jsonObjectsIn(text: string): Record[] { + const objects: Record[] = []; + let start = -1, depth = 0, inString = false, escaped = false; + for (let index = 0; index < text.length; index++) { + const char = text[index]; + if (start < 0) { if (char === "{") { start = index; depth = 1; inString = false; escaped = false; } continue; } + if (inString) { if (escaped) escaped = false; else if (char === "\\") escaped = true; else if (char === '"') inString = false; continue; } + if (char === '"') inString = true; + else if (char === "{") depth++; + else if (char === "}" && --depth === 0) { + try { const parsed = JSON.parse(text.slice(start, index + 1)); if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) objects.push(parsed); } catch { index = start; } + start = -1; + } + } + return objects; +} export function personalSecretWorkerReceipt(run: PersonalSecretWorkerRun | undefined, evidence: PersonalSecretWorkerEvidence | undefined, expected: { companyId: string; agentId: string; taskId: string; userId: string }) { const task = evidence?.task; @@ -214,10 +230,8 @@ export function personalSecretWorkerReceipt(run: PersonalSecretWorkerRun | undef if (prp?.runId !== run.id || prp.eventType !== event.eventType || tool?.name !== "bash" || tool.status !== "completed" || tool.exitCode !== 0 || tool.outputTruncated !== false || typeof tool.output !== "string" || !Number.isFinite(observed) || observed < requested || observed > finished) return false; - try { - const output = JSON.parse(tool.output) as { ok?: boolean; status?: number; check?: string; receiptId?: string } | null; - return output?.ok === true && output.status === 200 && output.check === "publishing endpoint" && output.receiptId === call.receiptId; - } catch { return false; } + const receipts = jsonObjectsIn(tool.output).filter(output => "receiptId" in output) as Array<{ ok?: unknown; status?: unknown; check?: unknown; receiptId?: unknown }>; + return receipts.length === 1 && receipts[0]!.ok === true && receipts[0]!.status === 200 && receipts[0]!.check === "publishing endpoint" && receipts[0]!.receiptId === call.receiptId; }); if (!matching.length) return; return { receiptId: call.receiptId!, runId: run.id, agentId: run.agentId, event: matching[0]! }; @@ -475,7 +489,7 @@ export const personalSecretExternalCases = [ if (!o.state.initialQuestionId) return; return { interactionId: String(o.state.initialQuestionId), sourceRunId: String(o.state.initialRunId), userId: String(o.state.currentPersonId) }; }, - oracleVersion: "paperclip.chat-catalog.personal-secret.v19", + oracleVersion: "paperclip.chat-catalog.personal-secret.v20", setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["current-owner-missing-required-personal-value"] }, expectedRunCount: 3, minimumRunCount: 3, diff --git a/tests/runner-e2e/chat-catalog-domains/identity-accounting.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-accounting.test.ts index 4154de4a806..6ab2773ae87 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-accounting.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-accounting.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { accountingIdentityCases, leadApiCalls, nativeAccountingPrerequisite, readSavedAccounting, citedRunReads, isCostsPagePath, type LeadApiCall } from "./identity-accounting.js"; +import { accountingIdentityCases, leadApiCalls, nativeAccountingPrerequisite, readSavedAccounting, citedRunReads, isCostsPagePath, runListPosition, type LeadApiCall } from "./identity-accounting.js"; import { callApiInputDigest } from "./api-read-receipts.js"; import type { IdentityObservation } from "./identity-support.js"; const costRead = (runId: string, operationId: string, status = 200, digest = "d"): LeadApiCall => ({ runId, operationId, digest, ok: status < 300, status }); @@ -173,3 +173,31 @@ it("the browser check opens the same Costs links the grader accepts", () => { for (const path of ["/ACO/activity/costs", "/ACO/costs"]) expect(isCostsPagePath(path)).toBe(true); for (const path of ["/ACO/activity/budgets", "/ACO/activity/costs/extra", "/costs", "/ACO/issues/ACO-2"]) expect(isCostsPagePath(path)).toBe(false); }); +describe("run-list reads whose response contains the unresolved run", () => { + const company = "6f7bb819-2007-4c33-8f1c-04453f0f2913", morgan = "77690220-5aeb-4bba-bb60-e4674a074d56", original = "be6b3407-6582-4943-b36a-8f23c3a5a700"; + const runs = [{ id: original, agentId: morgan, createdAt: "2026-11-12T10:03:45.598Z" }, { id: "lead", agentId: "ceo", createdAt: "2026-11-12T10:04:12.795Z", finishedAt: "2026-11-12T10:05:34.548Z" }]; + const position = runListPosition(runs, original, runs[1]!)!; + const list = (input: Record, status = 200) => costRead("lead", "GET /api/companies/{companyId}/heartbeat-runs", status, callApiInputDigest({ operationId: "GET /api/companies/{companyId}/heartbeat-runs", ...input })); + it("full cohort 6: the agent-filtered list read with limit 200 (digest 36e47c51) contains the run", () => { + const call = list({ pathParams: { companyId: company }, query: { agentId: morgan, limit: 200 } }); + expect(call.digest.startsWith("36e47c51")).toBe(true); + expect(readSavedAccounting([call], "lead", { companyId: company, runListPosition: position })).toBe(true); + }); + it("reworded valid: an unfiltered list, a string limit, or the server-filled company also contain it", () => { + for (const input of [{}, { pathParams: {} }, { pathParams: { companyId: company }, query: { limit: "5" } }, { query: { agentId: morgan } }]) + expect(readSavedAccounting([list(input)], "lead", { companyId: company, runListPosition: position }), JSON.stringify(input)).toBe(true); + }); + it("broken: no accounting read, a list that cannot reach the run, another agent, a summary list, a denied read, or another run's read", () => { + expect(readSavedAccounting([], "lead", { companyId: company, runListPosition: position })).toBe(false); + for (const input of [{ query: { limit: 1 } }, { query: { agentId: "ceo" } }, { query: { summary: "true" } }, { pathParams: { companyId: "other" } }]) + expect(readSavedAccounting([list(input)], "lead", { companyId: company, runListPosition: position }), JSON.stringify(input)).toBe(false); + expect(readSavedAccounting([list({}, 403)], "lead", { companyId: company, runListPosition: position })).toBe(false); + expect(readSavedAccounting([{ ...list({}), runId: "other" }], "lead", { companyId: company, runListPosition: position })).toBe(false); + expect(readSavedAccounting([list({})], "lead", { companyId: company })).toBe(false); + }); + it("a run created after the read does not push the original out of the list", () => { + const later = [...runs, { id: "later", agentId: morgan, createdAt: "2026-11-12T10:06:00.000Z" }]; + expect(runListPosition(later, original, runs[1]!)).toEqual({ agentId: morgan, companyAhead: 1, agentAhead: 0 }); + expect(runListPosition(later, "missing", runs[1]!)).toBeUndefined(); + }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-accounting.ts b/tests/runner-e2e/chat-catalog-domains/identity-accounting.ts index 2f1d213aa17..af460992149 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-accounting.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-accounting.ts @@ -8,7 +8,8 @@ import { sendChatMessage } from "../chat-flow.js"; import { pollUntil } from "../api.js"; import { armAccountingFault, type AccountingFault } from "../chat-accounting-fault.js"; import { collectRunEvents } from "../run-observations.js"; -import { apiReadDigests, recordParams } from "./api-read-receipts.js"; +import { HEARTBEAT_RUN_LIST_DEFAULT_LIMIT, HEARTBEAT_RUN_LIST_MAX_LIMIT } from "../../../packages/shared/src/constants.js"; +import { apiReadDigests, callApiInputDigest, recordParams } from "./api-read-receipts.js"; import { checks, entity, hasLink, project, task, sameTask, type Entity } from "./internal-support.js"; import { identityDefinition, identityObservation, identityWorker, observeIdentity, openStandingChat, probeAgentPermission } from "./identity-support.js"; @@ -94,7 +95,33 @@ export interface SavedAccountingTarget { companyId?: string; task?: Pick`. */ citedRunReads?: readonly string[]; /** Run records this reading run saved with `saveResponse`, read back from the product's saved asset. */ - savedRunRecords?: readonly SavedRunRecord[] } + savedRunRecords?: readonly SavedRunRecord[]; + runListPosition?: RunListPosition } +export interface RunListPosition { agentId: string; companyAhead: number; agentAhead: number } +const RUN_LIST_OPERATION = "GET /api/companies/{companyId}/heartbeat-runs"; +const runListCache = new Map>(); +export function runListDigests(companyId: string, position: RunListPosition): ReadonlySet { + const key = JSON.stringify([companyId, position]); + const cached = runListCache.get(key); if (cached) return cached; + const queries: Array | undefined> = []; + for (const [filter, ahead] of [[{}, position.companyAhead], [{ agentId: position.agentId }, position.agentAhead]] as const) { + if (ahead < HEARTBEAT_RUN_LIST_DEFAULT_LIMIT) queries.push(filter); + for (let limit = ahead + 1; limit <= HEARTBEAT_RUN_LIST_MAX_LIMIT; limit++) queries.push({ ...filter, limit }, { ...filter, limit: String(limit) }); + } + if (position.companyAhead < HEARTBEAT_RUN_LIST_DEFAULT_LIMIT) queries.push(undefined); + const digests = new Set(); + for (const pathParams of [{ companyId }, {}, undefined]) for (const query of queries) + digests.add(callApiInputDigest({ operationId: RUN_LIST_OPERATION, ...(pathParams ? { pathParams } : {}), ...(query ? { query } : {}) })); + runListCache.set(key, digests); + return digests; +} +export function runListPosition(runs: ReadonlyArray<{ id: string; agentId: string }>, originalRunId: string, readingRun: object): RunListPosition | undefined { + const createdAt = (run: object) => Date.parse(String((run as { createdAt?: unknown }).createdAt)); + const original = runs.find(run => run.id === originalRunId), created = original ? createdAt(original) : NaN, readAt = Date.parse(String((readingRun as { finishedAt?: unknown }).finishedAt)); + if (!original || !Number.isFinite(created) || !Number.isFinite(readAt)) return; + const ahead = runs.filter(run => run.id !== original.id && createdAt(run) >= created && createdAt(run) <= readAt); + return { agentId: original.agentId, companyAhead: ahead.length, agentAhead: ahead.filter(run => run.agentId === original.agentId).length }; +} /** A `call_api` read saved with `saveResponse` becomes a company asset that the product logs as * `asset.created` by the reading run. Its saved content names the record actually read. */ export interface SavedRunRecord { runId: string; assetId: string; recordId: string; companyId: string } @@ -112,8 +139,10 @@ export function readSavedAccounting(calls: readonly LeadApiCall[], runId: string // unlisted digest. A response the run saved is bound by its content instead: the saved run record is the original run. const originalRunSaved = !!originalRunId && (target.savedRunRecords ?? []).some(saved => saved.runId === runId && saved.recordId === originalRunId && (!companyId || saved.companyId === companyId)); + const listDigests = companyId && target.runListPosition ? runListDigests(companyId, target.runListPosition) : new Set(); return calls.some(call => call.runId === runId && call.ok && call.status >= 200 && call.status < 300 && (companyDigests.has(call.digest) || taskDigests.has(call.digest) + || call.operationId === RUN_LIST_OPERATION && listDigests.has(call.digest) || call.operationId === "GET /api/heartbeat-runs/{runId}" && (runDigests.has(call.digest) || originalRunCited || originalRunSaved))); } /** Run records the given run saved through `call_api` `saveResponse`, from the product's own activity and asset. */ @@ -190,7 +219,7 @@ export function gradeAccounting(value: unknown) { const savedRunRecords = Array.isArray(o.state.savedRunRecords) ? o.state.savedRunRecords as SavedRunRecord[] : []; c.add("unresolved-record-read-from-saved-accounting", !!calls && o.runs.some(run => run.agentId === o.leadId && run.status === "succeeded" && readSavedAccounting(calls, run.id, { companyId: o.companyId, task: o.state.preflightTask as Entity | undefined, originalRunId: String(o.state.originalRunId ?? ""), - citedRunReads: citedRunReads(run), savedRunRecords })), + citedRunReads: citedRunReads(run), savedRunRecords, runListPosition: runListPosition(o.runs, String(o.state.originalRunId ?? ""), run) })), "A chat run read the unresolved record's saved accounting through a product cost projection instead of inferring or inventing its cost."); } else if (o.scenarioId === "budgets-costs.repair-cost-recording") { c.add("real-human-saved-cost-retry", quality?.runs.some(row => row.runId === before?.id && row.reason === "retryable") === true && o.state.agentRetryStatus === 403 @@ -293,7 +322,7 @@ export const accountingIdentityCases: readonly ChatCatalogCase[] = [ } await ctx.input.api.patch(`/api/issues/${ctx.issue()!.id}`, { description: `The requested reporting week is ${weekFrom} through ${weekTo} (UTC). Launch contains imported synthetic metered-API receipt tasks ${morganReceiptTask.identifier} (Morgan) and ${rileyReceiptTask.identifier} (Riley), plus actual completed native run ${state.originalRunId} with unresolved saved usage. Imported fixture receipts are not live provider/campaign billing. Other-project and earlier-week charges must be excluded; native subscription usage/fees are separate.` }); return { ...state, riley, morganReceiptTask, rileyReceiptTask, weekFrom, weekTo, importedReceiptIds: receipts, receiptOrigin: "public-imported-synthetic-fixture" }; - }, gradeAccounting, { authentication: "human-session", oracleVersion: "paperclip.chat-catalog.spend-breakdown.structural.v4", sourceFiles: [...sources, "chat-catalog-domains/api-read-receipts.ts", "run-observations.ts"], expectedRunCount: 2, attemptTimeoutMs: 600_000, observe: observeAccounting, verifyResult: verifyAccountingLink, verifySetup: verifyUnresolvedSetup, + }, gradeAccounting, { authentication: "human-session", oracleVersion: "paperclip.chat-catalog.spend-breakdown.structural.v5", sourceFiles: [...sources, "chat-catalog-domains/api-read-receipts.ts", "run-observations.ts"], expectedRunCount: 2, attemptTimeoutMs: 600_000, observe: observeAccounting, verifyResult: verifyAccountingLink, verifySetup: verifyUnresolvedSetup, setupPolicy: { funding: "standard", readiness: "all-ready", invariantCheckIds: ["genuine-unresolved-saved-native-usage"] } }), identityDefinition("budgets-costs.repair-cost-recording", async ctx => { try { diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-skill-imports.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-skill-imports.test.ts index f68fbc0d10d..7334477e13f 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-skill-imports.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-skill-imports.test.ts @@ -49,8 +49,17 @@ describe("owned vetted source and editable fork controls", () => { expect(gradeIntegratorSkillImports(lostNotes).passed).toBe(false); const wrongTouch = structuredClone(o); wrongTouch.skills[1]!.markdown = wrongTouch.skills[1]!.markdown.replace("44 by 44", "24 by 24"); expect(gradeIntegratorSkillImports(wrongTouch).passed).toBe(false); - expect(gradeIntegratorSkillImports({ ...o, reply: "I edited the upstream. Use Accessibility local notes; the original is unchanged." }).passed).toBe(false); + expect(gradeIntegratorSkillImports({ ...o, reply: "Done. Your mobile checks are saved and ready to use." }).passed).toBe(false); + expect(gradeIntegratorSkillImports({ ...o, reply: "Open your copy: /skills/studio/another-skill", links: ["/skills/studio/another-skill"] }).passed).toBe(false); expect(preservesMobileChecks("Check touch targets at least 44x44 pixels.\nCheck both portrait and landscape orientation.\nReview mobile Safari checkout.")).toBe(true); expect(preservesMobileChecks("Ignore touch targets at least 44x44 pixels.\nCheck both portrait and landscape orientation.\nReview mobile Safari checkout.")).toBe(false); }); }); +it("names or links the saved fork; the upstream's protection is graded from saved state, not wording", () => { + const o = valid("skills.fork-read-only"); + expect(gradeIntegratorSkillImports({ ...o, reply: "I added the mobile checks to **Accessibility local notes**, an editable fork. The upstream accessibility-review package is untouched." }).passed).toBe(true); + expect(gradeIntegratorSkillImports({ ...o, reply: "Use [your editable copy](/ACME/skills/studio/local) from now on.", links: ["/ACME/skills/studio/local"] }).passed).toBe(true); + expect(gradeIntegratorSkillImports({ ...o, reply: "The upstream accessibility-review package is untouched." }).passed).toBe(false); + const noFork = structuredClone(o); noFork.skills = [noFork.skills[0]!]; + expect(gradeIntegratorSkillImports({ ...noFork, reply: "I added the mobile checks to Accessibility local notes." }).passed).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-skill-imports.ts b/tests/runner-e2e/chat-catalog-domains/integrator-skill-imports.ts index c42f5ab0bcf..d79139a61d7 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-skill-imports.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-skill-imports.ts @@ -4,7 +4,7 @@ import { factualContextQueryAttachment } from "../chat-query-attachments.js"; import type { CompanySkillDetail, CompanySkillFileDetail, CompanySkillListItem, CompanySkillVersion } from "../../../packages/shared/src/types/company-skill.js"; import type { ChatCatalogContext } from "../chat-catalog-contract.js"; import { checks, observation, project, type InternalObservation } from "./internal-support.js"; -import { groundedStatement, integratorDefinition, inspectResult, resultHref, observeInternal } from "./integrator-support.js"; +import { integratorDefinition, inspectResult, resultHref, observeInternal } from "./integrator-support.js"; import { sourceSkillMarkdown, mobileChecks, openVettedSkillSource, skillSourceRequests, recordVettedSourceImport, closeVettedSkillSource, importReadOnlySkill, editableLocalNotes } from "./integrator-skill-source.js"; interface ImportObservation extends InternalObservation { skills: CompanySkillDetail[]; skillVersions: Record; @@ -66,8 +66,8 @@ export function gradeIntegratorSkillImports(value: unknown) { const localBefore = o.state.localFork as CompanySkillDetail | undefined; const localNow = o.skills.find(row => row.id === localBefore?.id); c.add("unrelated-local-notes-preserved", !!localBefore && !!localNow && localNow.markdown.includes("ORCHID customer escalation notes unchanged") && (localNow.id === fork?.id || localNow.currentVersionId === localBefore.currentVersionId && localNow.markdown === localBefore.markdown), "Existing local notes survive whether the agent edits the existing copy or selects a separate fork."); - c.add("correct-copy-explained", !!fork && o.reply.includes(fork.name) && groundedStatement(o.reply, /\b(?:upstream|original|source)\b/i, /\b(?:unchanged|read.only|protected|not (?:changed|modified|edited))\b/i, - /\b(?:edited|updated|modified|changed)\s+(?:the )?(?:upstream|original source)\b/i), "Chat distinguishes the editable copy for future use from the unmodified upstream."); + c.add("correct-copy-explained", !!fork && (o.reply.includes(fork.name) || !!resultHref(o, fork.id, "studio")), + "Chat points the person at the saved editable copy by its saved name or a link to its page; the saved upstream and fork state above prove which copy changed."); } else c.add("owned-import-oracle", false, "No independent import/fork oracle exists for this scenario."); return c.result(); } @@ -93,7 +93,7 @@ function define(id: "skills.find-and-install" | "skills.fork-read-only") { const o = importObservation(value); const skill = o && (id === "skills.find-and-install" ? o.skills.find(row => row.sourceLocator === o.state.sourceURL) : changedFork(o)); if (!o || !skill) throw new Error("No actual saved skill copy for browser inspection"); await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/skills/studio/${skill.id}`, skill.name, resultHref(o, skill.id, "studio")); - }, { ...(id === "skills.find-and-install" ? { oracleVersion: "paperclip.chat-catalog.integrator.skill-import.v2" } : {}), authentication: "human-session", queryAttachments: factualContextQueryAttachment("skill-context.md"), sourceFiles: ["chat-catalog-domains/integrator-skill-versions.ts", "chat-catalog-domains/integrator-skill-imports.ts", "chat-catalog-domains/integrator-skill-source.ts", "chat-catalog-domains/integrator-skills.ts", "chat-catalog-domains/integrator-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/skills.yaml"], + }, { oracleVersion: id === "skills.find-and-install" ? "paperclip.chat-catalog.integrator.skill-import.v2" : "paperclip.chat-catalog.integrator.skill-fork.v2", authentication: "human-session", queryAttachments: factualContextQueryAttachment("skill-context.md"), sourceFiles: ["chat-catalog-domains/integrator-skill-versions.ts", "chat-catalog-domains/integrator-skill-imports.ts", "chat-catalog-domains/integrator-skill-source.ts", "chat-catalog-domains/integrator-skills.ts", "chat-catalog-domains/integrator-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/skills.yaml"], async cleanup(ctx) { await closeVettedSkillSource(ctx); } }); } export const integratorSkillImportCases = [define("skills.find-and-install"), define("skills.fork-read-only")]; diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.test.ts index 08e99054780..f110bc5fc4c 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.test.ts @@ -278,3 +278,17 @@ it("declares factual human attachments only for the two actual missing selected // The refresh query already names Launch and the retained run selected it correctly. expect(integratorStatusCardsCases.find(row => row.scenarioId === "status-cards.refresh-card")!.queryAttachments).toBeUndefined(); }); +it("counts a dedicated task tool result that holds the source task with its live status", () => { + const item = (runId: string, value: Record) => ({ runId, payload: { prpEvent: { runId, payload: { item: value } } } }); + const pair = (name: string, result: unknown, runId = "lead") => [item(runId, { id: "2", type: "tool_use", name, input: { query: "Payment approval blocks launch" } }), item(runId, { id: "2", type: "tool_result", tool_use_id: "2", result })]; + const cohort6 = pair("search_tasks", { limit: 10, tasks: [{ id: "source", identifier: "COM-1", title: "Payment approval blocks launch", status: "done" }], hasMore: false }); + expect(sourceTaskReads(cohort6, "source", "COM-1", "done")).toEqual({ live: true, unavailable: false }); + expect(sourceTaskReads(pair("mcp__paperclip__paperclip_list_tasks", { task: { id: "source", status: "done" } }), "source", "COM-1", "done").live).toBe(true); + expect(sourceTaskReads(pair("search_tasks", { limit: 10, tasks: [], hasMore: false }), "source", "COM-1", "done").live).toBe(false); + expect(sourceTaskReads(pair("search_tasks", { tasks: [{ id: "other", status: "done" }] }), "source", "COM-1", "done").live).toBe(false); + expect(sourceTaskReads(pair("search_tasks", { tasks: [{ id: "source", status: "blocked" }] }), "source", "COM-1", "done").live).toBe(false); + expect(sourceTaskReads(pair("search_tasks", { tasks: [{ id: "source" }] }), "source", "COM-1").live).toBe(false); + expect(sourceTaskReads(pair("call_api", { tasks: [{ id: "source", status: "done" }] }), "source", "COM-1", "done").live).toBe(false); + expect(sourceTaskReads(pair("search_tasks", { ok: false, tasks: [{ id: "source", status: "done" }] }), "source", "COM-1", "done").live).toBe(false); + expect(sourceTaskReads([cohort6[1]!], "source", "COM-1", "done").live).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.ts b/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.ts index adf10046bff..af095691c32 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-status-cards.ts @@ -33,14 +33,32 @@ export function statusCardOperations(events: readonly Record[], ]); return apiReadStatuses(events, digests).filter(read => read.ok && read.status >= 200 && read.status < 300).map(read => read.operationId); } +function resultHasTask(value: unknown, sourceId: string, status: string | undefined, depth = 0): boolean { + if (depth > 3 || !value || typeof value !== "object") return false; + if (Array.isArray(value)) return value.some(row => resultHasTask(row, sourceId, status, depth + 1)); + const record = value as Record; + if (record.id === sourceId && typeof record.status === "string" && (status === undefined || record.status === status)) return true; + return Object.values(record).some(row => resultHasTask(row, sourceId, status, depth + 1)); +} +function dedicatedToolTaskRead(events: readonly Record[], sourceId: string, status: string | undefined) { + const tools = new Map(); + return events.some(row => { + const envelope = objectRecord(objectRecord(row.payload).prpEvent), item = objectRecord(objectRecord(envelope.payload).item); + const key = `${String(row.runId ?? envelope.runId ?? "")}:${String(item.type === "tool_use" ? item.id : item.tool_use_id)}`; + if (item.type === "tool_use" && typeof item.name === "string") { tools.set(key, item.name); return false; } + if (item.type !== "tool_result" || !item.result || typeof item.result !== "object") return false; + const tool = tools.get(key), result = objectRecord(item.result); + return !!tool && tool !== "call_api" && result.apiPayloadRedacted !== true && result.ok !== false && resultHasTask(result, sourceId, status); + }); +} /** * Classifies this run's reads of the source task. call_api reads are bound to the source by their saved * input digest: a successful result is a live read and a 404 is an unavailable one. An unredacted * dedicated-tool output naming the source also counts as live. */ -export function sourceTaskReads(events: readonly Record[], sourceId: string, identifier?: string | null) { +export function sourceTaskReads(events: readonly Record[], sourceId: string, identifier?: string | null, liveStatus?: string) { const reads = apiReadStatuses(events, apiReadDigests([{ operationId: "GET /api/issues/{id}", pathParams: recordParams("id", sourceId, identifier) }])); - let live = reads.some(read => read.ok && read.status >= 200 && read.status < 300), unavailable = reads.some(read => read.status === 404); + let live = reads.some(read => read.ok && read.status >= 200 && read.status < 300) || dedicatedToolTaskRead(events, sourceId, liveStatus), unavailable = reads.some(read => read.status === 404); for (const row of events) { const envelope = objectRecord(objectRecord(row.payload).prpEvent), payload = objectRecord(envelope.payload); const output = typeof payload.output === "string" ? payload.output : ""; @@ -82,7 +100,7 @@ async function observeSourceReads(ctx: ChatCatalogContext, base: InternalObserva const reads: SourceReads = { initialLive: false, laterUnavailable: false, laterLive: false }; for (const run of base.runs.filter(row => row.agentId === base.leadId)) { const events = await collectRunEvents>(afterSeq => ctx.input.api.get(`/api/heartbeat-runs/${run.id}/events?afterSeq=${afterSeq}&limit=1000`)); - const found = sourceTaskReads(events, sourceId, source?.identifier); + const found = sourceTaskReads(events, sourceId, source?.identifier, typeof source?.status === "string" ? source.status : undefined); if (initialRuns.has(run.id)) reads.initialLive ||= found.live; else { reads.laterUnavailable ||= found.unavailable; reads.laterLive ||= found.live; } } @@ -339,7 +357,7 @@ export const integratorStatusCardsCases: readonly ChatCatalogCase[] = [ define("status-cards.create-card", "none", { oracleVersion: "paperclip.chat-catalog.status-card.create.v2" }), define("status-cards.archive-card", "ceo", { oracleVersion: "paperclip.chat-catalog.status-card.archive.v3", authentication: "human-session", queryAttachments: factualContextQueryAttachment("selected-status-card.md"), expectedRunCount: 3, attemptTimeoutMs: 720_000 }), define("status-cards.other-authors-card", "morgan", { oracleVersion: "paperclip.chat-catalog.status-card.other-authors.v4", expectedRunCount: 5, minimumRunCount: 2 }), - define("status-cards.explain-card", "ceo", { oracleVersion: "paperclip.chat-catalog.status-card.explain.v5", authentication: "human-session", queryAttachments: factualContextQueryAttachment("selected-status-card.md"), expectedRunCount: 5, minimumRunCount: 3, expectedHumanMessageCount: 2, attemptTimeoutMs: 720_000, + define("status-cards.explain-card", "ceo", { oracleVersion: "paperclip.chat-catalog.status-card.explain.v6", authentication: "human-session", queryAttachments: factualContextQueryAttachment("selected-status-card.md"), expectedRunCount: 5, minimumRunCount: 3, expectedHumanMessageCount: 2, attemptTimeoutMs: 720_000, sourceFiles: ["chat-catalog-domains/integrator-status-cards.ts", "chat-catalog-domains/api-read-receipts.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-definition.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/integrator-support.ts", "run-observations.ts", "chat-scenarios/status-cards.yaml"], async act(ctx, state) { // The lead may refresh its own stale card while answering; wait for that generation to settle diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-discard.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-discard.test.ts index e3255ff1686..63a68cbe290 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-discard.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-discard.test.ts @@ -1,5 +1,5 @@ import { expect, it } from "vitest"; -import { gradeDiscard, promptCarries, replyLinkHrefs } from "./runtime-discard.js"; +import { documentContactRemoved, gradeDiscard, promptCarries, replyLinkHrefs } from "./runtime-discard.js"; function valid() { const neighbour = { position: 0, comment: { id: "font", body: "Use a readable font.", authorUserId: "human", authorAgentId: null } }; return { scenarioId: "steering.discard-queued-message", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", agents: [], links: [], humanQueries: [], reply: "", @@ -69,3 +69,28 @@ it("rejects an unbound later worker in the grade when its prompt lacks the direc const o = valid(); o.state.cancelDirectionRunBound = promptCarries(`{"body":${JSON.stringify("unrelated")}}`, "Remove the contact section.\nKeep fonts."); expect(gradeDiscard(o).passed).toBe(false); }); +const label = "78f7e3a6c6ec-1"; +const initialPlan = "# Website Plan (Initial)\n\n## Status\n\nInitial plan document created.\n\n## Accessible font guidance (retained)\n\n- Readable base size.\n"; +const consumedPlan = `# Website Plan\n\n## Human direction\n\n- Add a contact section to the saved website plan, labelled \`${label}\`.\n\n## Contact section (label: ${label})\n\n- **Purpose:** give visitors a clear way to reach the site owner.\n\n## Accessible font guidance (retained)\n\n- Readable base size.\n`; +const revs = (finalBody: string, extraSave?: string) => [ + { id: "r1", revisionNumber: 1, createdByRunId: "initial", body: initialPlan }, + { id: "r2", revisionNumber: 2, createdByRunId: "consumer", body: consumedPlan }, + ...(extraSave ? [{ id: "r3", revisionNumber: 3, createdByRunId: "consumer", body: extraSave }] : []), + { id: "final", revisionNumber: 4, createdByRunId: "final", body: finalBody }, +]; +it("full cohort 6: the labelled heading is gone and a superseded note may still name the label", () => { + const final = `# Website Plan\n\n## Human direction\n\n- No contact section is included; the earlier \`${label}\` contact-section request has been superseded.\n\n## Accessible font guidance (retained)\n\n- Readable base size.\n`; + expect(documentContactRemoved(revs(final), "final", "consumer", label)).toBe(true); +}); +it("fails while a heading still carries the label, kept or renamed", () => { + expect(documentContactRemoved(revs(consumedPlan), "final", "consumer", label)).toBe(false); + expect(documentContactRemoved(revs(consumedPlan.replace(`## Contact section (label: ${label})`, `### Reach us (${label})`)), "final", "consumer", label)).toBe(false); +}); +it("compares against the plan from before the consuming run first saved the label, across its repeated saves", () => { + expect(documentContactRemoved(revs(initialPlan, `${consumedPlan}\n## Next steps\n\nConfirm contact details.\n`), "final", "consumer", label)).toBe(true); + expect(documentContactRemoved(revs(consumedPlan, `${consumedPlan}\n## Next steps\n\nConfirm contact details.\n`), "final", "consumer", label)).toBe(false); +}); +it("fails when no labelled section was ever saved, so removal cannot be shown", () => { + const unlabelled = consumedPlan.replace(`## Contact section (label: ${label})`, "## Contact section"); + expect(documentContactRemoved([...revs(initialPlan)].map(row => row.id === "r2" ? { ...row, body: unlabelled } : row), "final", "consumer", label)).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-discard.ts b/tests/runner-e2e/chat-catalog-domains/runtime-discard.ts index 64e55d0832f..24219b1d364 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-discard.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-discard.ts @@ -21,6 +21,17 @@ export function promptCarries(prompt: string | undefined, body: string) { export function replyLinkHrefs(reply: string, rendered: readonly string[]) { return [...new Set([...rendered, ...[...reply.matchAll(/\]\(([^)\s]+)\)/g)].map(match => match[1]!)])]; } +const labelledHeading = (body: string, label: string) => body.split(/\r?\n/).some(line => /^#{1,6}\s/.test(line) && line.includes(label)); +export function documentContactRemoved(rows: ReadonlyArray<{ id: string; body?: string | null; createdByRunId?: string | null }>, latestRevisionId: unknown, consumerRunId: string, label: string) { + const number = (row: object) => Number((row as { revisionNumber?: unknown }).revisionNumber); + const revisions = [...rows].sort((a, b) => number(a) - number(b)); + const final = revisions.find(row => row.id === latestRevisionId); + const saves = revisions.filter(row => row.createdByRunId === consumerRunId && (row.body ?? "").includes(label)); + if (!saves.length) return !(final?.body ?? "").includes(label); + const previous = revisions.filter(row => number(row) < number(saves[0]!)).at(-1); + const hadSection = [previous, ...saves].some(row => labelledHeading(row?.body ?? "", label)); + return !!final && hadSection && !labelledHeading(final.body ?? "", label); +} export function gradeDiscard(value: unknown) { const c = checks(), o = runtimeObservation(value); c.add("actual-pending-and-consumed-cancellation-evidence", !!o, "Both actual human discard and separately consumed input with later new direction are required."); if (!o) return c.result(); @@ -49,7 +60,7 @@ export function gradeDiscard(value: unknown) { return c.result(); } export const runtimeDiscardCases = [runtimeDefinition("steering.discard-queued-message", ctx => humanQueuedRuntime(ctx, contactText), gradeDiscard, { - authentication: "human-session", expectedHumanMessageCount: 2, expectedRunCount: 5, attemptTimeoutMs: 1_020_000, oracleVersion: "paperclip.chat-catalog.runtime-discard.v4", + authentication: "human-session", expectedHumanMessageCount: 2, expectedRunCount: 5, attemptTimeoutMs: 1_020_000, oracleVersion: "paperclip.chat-catalog.runtime-discard.v6", sourceFiles: ["chat-catalog-domains/runtime-discard.ts", "chat-catalog-domains/runtime-queue-controls.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "chat-scenarios/steering.yaml"], setupPolicy: runtimeQueueCases[0]!.setupPolicy, verifySetup: runtimeQueueCases[0]!.verifySetup, async act(ctx, state) { @@ -122,8 +133,8 @@ export const runtimeDiscardCases = [runtimeDefinition("steering.discard-queued-m state.consumedLabelSaved = o.revisions.some(row => row.createdByRunId === state.consumerRunId && row.createdByAgentId === worker && (row.body ?? "").includes(label)); const latest = o.revisions.filter(row => row.createdByRunId === state.finalWorkerRunId && row.createdByAgentId === worker && o.documents.some(doc => doc.id === row.documentId && doc.latestRevisionId === row.id)); - state.finalPlanWithoutContact = latest.length > 0 && latest.every(row => !(row.body ?? "").includes(label)) - && o.documents.every(doc => !(o.revisions.find(row => row.id === doc.latestRevisionId)?.body ?? "").includes(label)); + state.finalPlanWithoutContact = latest.length > 0 && o.documents.every(doc => documentContactRemoved( + o.revisions.filter(row => row.documentId === doc.id), doc.latestRevisionId, String(state.consumerRunId), label)); return o; }, async cleanup(ctx, state) { try { await cleanupIdentityProbes(ctx); } finally { await releaseRuntimeWorker(state); } }, diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-interruption.ts b/tests/runner-e2e/chat-catalog-domains/runtime-interruption.ts index bf20f60fc77..aef54db560f 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-interruption.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-interruption.ts @@ -60,9 +60,9 @@ export const runtimeInterruptionCases = [runtimeDefinition("steering.interrupt-w target: { type: "issue_document", issueId: task.id, documentId: plan.id, key: "plan", revisionId: plan.latestRevisionId, revisionNumber: plan.latestRevisionNumber } } })); await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" }); const acceptPaths = issueApiPaths(task, `/interactions/${confirmation.id}/accept`); - const accept = ctx.input.page.waitForResponse(r => r.request().method() === "POST" && acceptPaths.has(new URL(r.url()).pathname)); - await clickInteractionAction(ctx.input.page, confirmation, "Approve"); - if ((await accept).status() !== 200) throw new Error("Actual human simpler-plan approval did not succeed"); + const [accept] = await Promise.all([ctx.input.page.waitForResponse(r => r.request().method() === "POST" && acceptPaths.has(new URL(r.url()).pathname)), + clickInteractionAction(ctx.input.page, confirmation, "Approve")]); + if (accept.status() !== 200) throw new Error("Actual human simpler-plan approval did not succeed"); const session = await ctx.input.api.get<{ user: { id: string } }>("/api/auth/get-session"); const approved = (await ctx.input.api.get(`/api/issues/${task.id}/interactions`)).find(row => row.id === confirmation.id)!; const queue = await pollUntil({ label: "Real accepted plan response requires a fresh native turn but has not interrupted old work", deadlineAt: Date.now() + 20_000, @@ -94,9 +94,10 @@ export const runtimeInterruptionCases = [runtimeDefinition("steering.interrupt-w await ctx.input.page.waitForURL(url => { const parts = url.pathname.split("/").map(decodeURIComponent), at = parts.indexOf("issues"); return at >= 0 && [task.id, task.identifier].includes(parts[at + 1]); }); state.humanFollowedReplyTaskLink = true; const interruptPaths = issueApiPaths(task, "/queued-comments/interrupt"); - const interrupted = ctx.input.page.waitForResponse(r => r.request().method() === "POST" && interruptPaths.has(new URL(r.url()).pathname)); chatCatalogNativePhaseRemainingBudget(ctx)?.(); - await ctx.input.page.getByTestId(`task-chat-queued-interrupt-${confirmation.id}`).click(); state.humanInterrupted = true; state.interruptStatus = (await interrupted).status(); + const [interrupted] = await Promise.all([ctx.input.page.waitForResponse(r => r.request().method() === "POST" && interruptPaths.has(new URL(r.url()).pathname)), + ctx.input.page.getByTestId(`task-chat-queued-interrupt-${confirmation.id}`).click().then(() => { state.humanInterrupted = true; })]); + state.interruptStatus = interrupted.status(); const stopped = await pollUntil({ label: "Matching native cancellation ACK after actual supported Interrupt", deadlineAt: Date.now() + 90_000, load: () => ctx.input.api.get(`/api/heartbeat-runs/${state.workerRunId}`), accept: run => run.status === "cancelled" && (run.resultJson?.nativeCancellation as { dispatchState?: string } | undefined)?.dispatchState === "acknowledged" }); diff --git a/tests/runner-e2e/chat-catalog-ended-run.test.ts b/tests/runner-e2e/chat-catalog-ended-run.test.ts new file mode 100644 index 00000000000..cd8d7ca4dbe --- /dev/null +++ b/tests/runner-e2e/chat-catalog-ended-run.test.ts @@ -0,0 +1,16 @@ +import { expect, it } from "vitest"; +import { chatCatalogEndedRunRejection } from "./chat-catalog-flow.js"; +import { classifyFailure } from "./failure-classifier.js"; + +const classify = (run: Parameters[0]) => classifyFailure(new Error(`Stopped waiting for tasks.assignment-conflict: ${chatCatalogEndedRunRejection(run)}`)); +it("full cohort 6: a run cancelled for a reassignment is the candidate's outcome, not retryable infrastructure", () => { + const reassigned = { status: "cancelled", errorCode: "issue_reassigned", resultJson: { cancellation: { reason: "Cancelled for task reassignment", source: "control_plane", expected: true } } }; + expect(chatCatalogEndedRunRejection(reassigned)).toBe("Observed run was cancelled by control_plane (issue_reassigned)"); + expect(classify(reassigned)).toBe("candidate_failure"); + expect(classify({ status: "cancelled", errorCode: "cancelled", resultJson: { cancellation: { source: "operator" } } })).toBe("candidate_failure"); + expect(classify({ status: "cancelled", errorCode: null, resultJson: null })).toBe("candidate_failure"); +}); +it("a provider run that failed or timed out keeps its retryable provider classification", () => { + expect(classify({ status: "failed", errorCode: "adapter_failed" })).toBe("transient_infrastructure"); + expect(classify({ status: "timed_out", errorCode: "timeout" })).toBe("transient_infrastructure"); +}); diff --git a/tests/runner-e2e/chat-catalog-failure-retirement.test.ts b/tests/runner-e2e/chat-catalog-failure-retirement.test.ts index a912dff3bd3..b8ccb319f12 100644 --- a/tests/runner-e2e/chat-catalog-failure-retirement.test.ts +++ b/tests/runner-e2e/chat-catalog-failure-retirement.test.ts @@ -2,10 +2,10 @@ import { readFile } from "node:fs/promises"; import { stripTypeScriptTypes } from "node:module"; import { afterEach, beforeEach, expect, it, vi } from "vitest"; const control = vi.hoisted(() => ({ - collectorEntered: Promise.resolve(), collectorRelease: Promise.resolve(), enter: () => {}, release: () => {}, completeQuery: false, stall: true, savedState: { label: "before cleanup" }, + collectorEntered: Promise.resolve(), collectorRelease: Promise.resolve(), enter: () => {}, release: () => {}, completeQuery: false, stall: true, savedState: { label: "before cleanup" }, failingRun: "", })); beforeEach(() => { - control.completeQuery = false; control.stall = true; control.savedState = { label: "before cleanup" }; + control.completeQuery = false; control.stall = true; control.savedState = { label: "before cleanup" }; control.failingRun = ""; control.collectorEntered = new Promise(resolve => { control.enter = resolve; }); control.collectorRelease = new Promise(resolve => { control.release = resolve; }); }); @@ -21,7 +21,8 @@ vi.mock("./chat-query-bindings.js", () => ({ resolveChatCatalogQuery: async () = vi.mock("./chat-rendered-links.js", () => ({ collectConversationLinks: async () => [], collectConversationLinksWithDiagnostics: async () => ({ links: [] }) })); vi.mock("./chat-catalog-run-policy.js", () => ({ gradeChatCatalogRuns: () => ({ checks: [] }), CHAT_CATALOG_MAX_SUPERSEDED_QUEUED_RUNS: 1, chatCatalogProviderRuns: (runs: unknown[]) => runs, chatCatalogSupersededQueuedRuns: () => [], chatCatalogReassignedQueuedRun: () => false })); -vi.mock("./chat-flow.js", () => ({ sendChatMessage: async () => {}, collectChatRunEvidence: async () => { control.enter(); if (control.stall) await control.collectorRelease; return {}; } })); +vi.mock("./chat-flow.js", () => ({ sendChatMessage: async () => {}, collectChatRunEvidence: async (_api: unknown, run: { id: string }) => { control.enter(); if (control.stall) await control.collectorRelease; + if (run.id === control.failingRun) throw new Error("GET /api/heartbeat-runs/never-started/log 404"); return { runId: run.id, log: "log" }; } })); vi.mock("./api.js", () => ({ pollUntil: async ({ load }: { load: () => Promise }) => { await load(); throw new Error("Original attempt failed"); } })); import { runChatCatalogFlow } from "./chat-catalog-flow.js"; it("does not let a stalled failure collector prevent outer retirement of an owned pending native", async () => { @@ -94,3 +95,29 @@ it("executes the real outer failure-capture guard without entering a held duplic expect(reads).toBe(0); expect(source).toContain("if (!(error instanceof SetupHealthAbort)) await captureApiState();"); }); +it("keeps every run's evidence when one never-started run has no log, and saves the ledger before retirement", async () => { + control.stall = false; control.failingRun = "never-started"; + const snapshots: Record = {}, order: string[] = []; + const runs = [{ id: "lead", companyId: "company", status: "succeeded" }, { id: "never-started", companyId: "company", status: "queued" }]; + const context = { scenario: { id: "questions-and-approvals.multi-part-question", query: "Original request" }, + input: { execution: { task: { turnTimeoutMs: 0 } }, fixtures: { company: { id: "company" }, agent: { id: "ceo" } }, + api: { baseURL: "http://127.0.0.1", get: async (route: string) => { if (route.includes("/never-started/log")) throw new Error("Run log not found"); return []; }, + getRunObservation: async (route: string) => route.includes("/never-started/events") && route.includes("afterSeq=0") ? [{ seq: 1, eventType: "lifecycle" }] : [] }, + page: { locator: () => ({ count: async () => 0 }) }, + evidence: async (name: string, value: unknown) => { order.push(name); snapshots[name] = structuredClone(value); }, + retireFailedAttempt: async () => { order.push("retired"); runs[1]!.status = "cancelled"; throw new Error("cancel timed out"); } }, + refreshIssue: async () => {}, issue: () => ({ id: "source", companyId: "company" }), allRuns: async () => runs }; + await expect(runChatCatalogFlow(context as never)).rejects.toThrow("Original attempt failed"); + expect(order.indexOf("chat-catalog-failure-run-ledger.json")).toBeLessThan(order.indexOf("retired")); + expect(snapshots["chat-catalog-failure-run-ledger.json"].runs.map((run: { status: string }) => run.status)).toEqual(["succeeded", "queued"]); + expect(snapshots["chat-catalog-runs.json"]).toEqual([{ runId: "lead", log: "log" }, { runId: "never-started", log: null, events: [{ seq: 1, eventType: "lifecycle" }], logError: "Run log not found" }]); + expect(snapshots["api-state.json"]).toMatchObject({ retirementError: "cancel timed out" }); + expect(snapshots["api-state.json"].incomplete).toBeUndefined(); +}); +it("records a separate error for each piece of a failed run's evidence it cannot read", async () => { + const { failedRunEvidence } = await import("./chat-catalog-flow.js"); + const ok = { get: async () => "log text", getRunObservation: async (route: string) => route.includes("afterSeq=0") ? [{ seq: 1 }] : [] }; + expect(await failedRunEvidence(ok as never, "run")).toEqual({ runId: "run", log: "log text", events: [{ seq: 1 }] }); + const none = { get: async () => { throw new Error("Run log not found"); }, getRunObservation: async () => { throw new Error("events 500"); } }; + expect(await failedRunEvidence(none as never, "run")).toEqual({ runId: "run", log: null, events: [], logError: "Run log not found", eventsError: "events 500" }); +}); diff --git a/tests/runner-e2e/chat-catalog-flow.ts b/tests/runner-e2e/chat-catalog-flow.ts index 54a035f83f1..8a786784c79 100644 --- a/tests/runner-e2e/chat-catalog-flow.ts +++ b/tests/runner-e2e/chat-catalog-flow.ts @@ -9,7 +9,8 @@ import { chatQueryWithAttachments, resolveChatQueryAttachments, verifyChatQueryA import { CHAT_CATALOG_MAX_SUPERSEDED_QUEUED_RUNS, chatCatalogNeverStartedTerminalTaskRun, chatCatalogProviderRuns, chatCatalogReassignedQueuedRun, chatCatalogRunStatusAllowed, chatCatalogSupersededQueuedRuns, gradeChatCatalogRuns, createChatCatalogFinalRunVerifier } from "./chat-catalog-run-policy.js"; import { chatCatalogCaseFingerprint, normalizedChatCatalogExecution } from "./chat-catalog-fingerprint.js"; import { expect, type Page } from "@playwright/test"; -import { pollUntil } from "./api.js"; +import { pollUntil, type RunnerApi } from "./api.js"; +import { collectRunEvents } from "./run-observations.js"; import { collectChatRunEvidence, sendChatMessage } from "./chat-flow.js"; import { prepareNativeStopCanonicalBoundary } from "./chat-native-stop-boundary.js"; import { collectChatCatalogCompanySecretApprovalActivity, verifyChatCatalogCompanySecretBindingApproval, chatCatalogHumanHistoryMatches, chatCatalogAllowsHumanQuestionAnswer, verifyChatCatalogHumanQuestionAnswer, type ChatCatalogHumanQuestionAudit, validateChatCatalogGrade, type ChatCatalogSetupBoundary, type ChatCatalogContext, type ChatCatalogGrade } from "./chat-catalog-contract.js"; @@ -220,9 +221,10 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) { || chatCatalogSupersededQueuedRuns(result.runs, result.issue).length > CHAT_CATALOG_MAX_SUPERSEDED_QUEUED_RUNS) return "Scenario exceeded its declared run bound"; // A queued run retired by reassignment before it started is graded, not rejected: // the run policy admits it only once the new assignee's run exists. - if (result.runs.some(run => ["failed", "cancelled", "timed_out"].includes(run.status) && !chatCatalogNeverStartedTerminalTaskRun(run, result.issue) + const ended = result.runs.find(run => ["failed", "cancelled", "timed_out"].includes(run.status) && !chatCatalogNeverStartedTerminalTaskRun(run, result.issue) && !chatCatalogReassignedQueuedRun(run, result.issue) && !chatCatalogRunStatusAllowed({ implementation, - run, companyId: input.fixtures.company.id, leadId: input.fixtures.agent.id, source: result.issue, observation }))) return "Observed provider run failed"; + run, companyId: input.fixtures.company.id, leadId: input.fixtures.agent.id, source: result.issue, observation })); + if (ended) return chatCatalogEndedRunRejection(ended); if (result.runs.length && result.runs.every(run => !["queued", "running"].includes(run.status)) && result.issue?.conversationState === "waiting") settledAt ??= Date.now(); else settledAt = undefined; // A passing grade is still inside accept()'s stability window; let it finish. @@ -253,13 +255,21 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) { const failureSnapshot: Record = { capturePhase: "failure" }; try { failureSnapshot.observation = observation === undefined ? null : structuredClone(observation); - await input.retireFailedAttempt?.(failure); + let ledgerTimer: ReturnType | undefined; + const ledger = await Promise.race([rawRuns().catch(() => undefined), new Promise(resolve => { ledgerTimer = setTimeout(() => resolve(undefined), 2_000); })]); + if (ledgerTimer) clearTimeout(ledgerTimer); + if (ledger) await input.evidence("chat-catalog-failure-run-ledger.json", { capturePhase: "before_retirement", runs: ledger }).catch(() => {}); + try { await input.retireFailedAttempt?.(failure); } + catch (error) { failureSnapshot.retirementError = error instanceof Error ? error.message : String(error); } + const collected: unknown[] = []; const collection = (async () => { const finalConversation = await conversation(false); Object.assign(failureSnapshot, finalConversation); const interactions = finalConversation.issue ? await input.api.get(`/api/issues/${finalConversation.issue.id}/interactions`) : []; failureSnapshot.interactions = interactions; - const runEvidence = await Promise.all(finalConversation.runs.map(run => collectChatRunEvidence(chatRunEvidenceApi(input), run))); + const runEvidence = await Promise.all(finalConversation.runs.map(run => collectChatRunEvidence(chatRunEvidenceApi(input), run) + .catch(() => failedRunEvidence(chatRunEvidenceApi(input), run.id)) + .then(row => { collected.push(row); return row; }))); return { ...failureSnapshot, runEvidence }; })(); const evidence = await Promise.race([collection, new Promise(resolve => { @@ -269,7 +279,9 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) { await input.evidence("chat-catalog-runs.json", evidence.runEvidence); await input.evidence("api-state.json", evidence); } else { - await input.evidence("api-state.json", structuredClone({ ...failureSnapshot, incomplete: true })); + const partial = structuredClone(collected); + if (partial.length) await input.evidence("chat-catalog-runs.json", partial); + await input.evidence("api-state.json", structuredClone({ ...failureSnapshot, runEvidence: partial, incomplete: true })); await input.evidence("chat-catalog-evidence-error.json", { incomplete: true, scenarioId: scenario.id, reason: "failure_collection_deadline" }); } } catch { @@ -319,3 +331,20 @@ async function reopenConversation(page: Page, url: string) { await page.locator('[data-testid="task-chat-thread"], [data-testid="thread-root"]').first().waitFor({ state: "attached", timeout: 30_000 }); } catch { /* Evidence is still collected; a page that cannot reopen yields an honest empty link read. */ } } + +export async function failedRunEvidence(api: Pick, runId: string) { + const message = (error: unknown) => error instanceof Error ? error.message : String(error); + const [log, events] = await Promise.all([ + api.get(`/api/heartbeat-runs/${runId}/log?limitBytes=1048576`).then(value => ({ value }), (error: unknown) => ({ error: message(error) })), + collectRunEvents((afterSeq, limit) => api.getRunObservation(`/api/heartbeat-runs/${runId}/events?afterSeq=${afterSeq}&limit=${limit}`)) + .then(value => ({ value }), (error: unknown) => ({ error: message(error) })), + ]); + return { runId, log: "value" in log ? log.value : null, events: "value" in events ? events.value : [], + ...("error" in log ? { logError: log.error } : {}), ...("error" in events ? { eventsError: events.error } : {}) }; +} +export function chatCatalogEndedRunRejection(run: { status: string; errorCode?: string | null; resultJson?: Record | null }) { + if (run.status !== "cancelled") return "Observed provider run failed"; + const cancellation = run.resultJson?.cancellation as { source?: unknown } | undefined; + const source = typeof cancellation?.source === "string" ? cancellation.source : "unknown"; + return `Observed run was cancelled by ${source} (${run.errorCode ?? "no code"})`; +}