diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 924891de680..461ba165a13 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -527,7 +527,7 @@ "chat-catalog-domains/runtime-agents.ts", "chat-scenarios/agents.yaml" ], - "oracleVersion": "paperclip.chat-catalog.runtime.edit-instructions.v3", + "oracleVersion": "paperclip.chat-catalog.runtime.edit-instructions.v4", "coverageGaps": [], "expectedRunCount": 1, "turnTimeoutMs": 120000, @@ -2052,7 +2052,7 @@ "chat-catalog-domains/internal-support.ts", "chat-scenarios/companies.yaml" ], - "oracleVersion": "paperclip.chat-catalog.identity.v1", + "oracleVersion": "paperclip.chat-catalog.identity.configure-decision-model.v2", "expectedRunCount": 1, "turnTimeoutMs": 120000, "attemptTimeoutMs": 420000, @@ -2834,7 +2834,7 @@ "chat-catalog-domains/internal-website-delivery.ts", "registered-artifact.ts" ], - "oracleVersion": "paperclip.chat-catalog.internal.website-download.v2", + "oracleVersion": "paperclip.chat-catalog.internal.website-download.v3", "coverageGaps": [], "expectedRunCount": 2, "minimumRunCount": 2, @@ -2859,7 +2859,7 @@ "chat-scenarios/documents-artifacts.yaml", "../../packages/shared/src/document-anchors.ts" ], - "oracleVersion": "paperclip.chat-catalog.internal.v1", + "oracleVersion": "paperclip.chat-catalog.internal.resolve-feedback.v2", "coverageGaps": [], "expectedRunCount": 1, "turnTimeoutMs": 120000, @@ -4433,7 +4433,7 @@ "../../server/src/vendor/paperclip-runner/index.ts", "chat-scenarios/recovery.yaml" ], - "oracleVersion": "paperclip.chat-catalog.runtime.infrastructure-recovery.v6", + "oracleVersion": "paperclip.chat-catalog.runtime.infrastructure-recovery.v7", "coverageGaps": [], "expectedRunCount": 3, "turnTimeoutMs": 120000, @@ -4549,7 +4549,7 @@ "../../server/src/vendor/paperclip-runner/index.ts", "chat-scenarios/recovery.yaml" ], - "oracleVersion": "paperclip.chat-catalog.runtime.infrastructure-recovery.v6", + "oracleVersion": "paperclip.chat-catalog.runtime.infrastructure-recovery.v7", "coverageGaps": [], "expectedRunCount": 3, "turnTimeoutMs": 120000, @@ -4661,7 +4661,7 @@ "chat-catalog-domains/integrator-routines.ts", "chat-scenarios/routines.yaml" ], - "oracleVersion": "paperclip.chat-catalog.integrator.v1", + "oracleVersion": "paperclip.chat-catalog.integrator.create-daily-brief.v2", "expectedRunCount": 1, "expectedHumanMessageCount": 1, "turnTimeoutMs": 120000, @@ -4774,7 +4774,7 @@ "chat-catalog-domains/internal-support.ts", "chat-scenarios/routines.yaml" ], - "oracleVersion": "paperclip.chat-catalog.integrator.routine-invocation.v4", + "oracleVersion": "paperclip.chat-catalog.integrator.routine-invocation.v5", "expectedRunCount": 3, "expectedHumanMessageCount": 2, "turnTimeoutMs": 120000, @@ -5524,7 +5524,7 @@ "chat-rendered-links.ts", "chat-scenarios/steering.yaml" ], - "oracleVersion": "paperclip.chat-catalog.edit-queued-message.structural.v5", + "oracleVersion": "paperclip.chat-catalog.edit-queued-message.structural.v6", "coverageGaps": [], "expectedRunCount": 3, "minimumRunCount": 2, diff --git a/tests/runner-e2e/chat-catalog-domains/identity-company-models.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-company-models.test.ts index a1ff480aa50..dcf80f58f00 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-company-models.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-company-models.test.ts @@ -14,7 +14,10 @@ function valid(scenarioId: string): IdentityObservation { state: { connection: { id: "shared", name: "Shared QA", grantId: "grant", companyId: "acorn", ownership: "shared", provider: "openrouter", method: "api_key", status: "connected" }, humanId: "alex", saved: { companyId: "acorn", enabled: true, connectionId: "shared", grantId: "grant", model }, modelBeforeTest: model, beforeEnabled: false, afterReplyEnabled: false, agentWriteStatus: 403, agentTestStatus: 403, persistedAfterReload: true, testHttpStatus: 200, visibleTestReceipt: true, history: [history], testCostEvents: 2 } }; - if (scenarioId.endsWith("configure-decision-model")) o.state.testResult = { status: "succeeded", invocationId: "test", answers: {}, usage }; + if (scenarioId.endsWith("configure-decision-model")) { + o.state.testResult = { status: "succeeded", invocationId: "test", answers: {}, usage }; + o.links = ["/acorn/company/settings/connections"]; o.state.replyLinkedControlHref = "/acorn/company/settings/connections"; + } else if (scenarioId.endsWith("fast-responses")) { o.state.testResult = { status: "succeeded", invocationId: "test", text: "I will check that for you.", durationMs: 140, usage }; o.links = ["/acorn/company/settings/connections"]; o.state.replyLinkedControlHref = "/acorn/company/settings/connections"; @@ -101,8 +104,23 @@ describe("company model service configuration and actual tests", () => { const notRendered = valid(d.scenarioId); notRendered.links = ["/acorn/company/settings"]; expect(d.grade(notRendered).passed).toBe(false); }); - it("fast response act follows only the reply's links and stops when none shows the control", async () => { - const definition = companyModelIdentityCases.find(row => row.scenarioId === "companies.fast-responses")!; + // Full cohort 9 (master b1ef1f3022): the reply said choosing and testing the decision model is board-only and linked + // Connections, and also "I tested it at the provider level" about a real usage read. A wording regex for "I ... tested" + // failed it. The service stayed off after the reply and the person reached the control from the reply's link. + it("grades the decision model on saved state and the reply link whose page showed the control, not reply wording", () => { + const d = companyModelIdentityCases.find(row => row.scenarioId === "companies.configure-decision-model")!; + const retained = valid(d.scenarioId); retained.links = ["/acorn/agents/ceo/overview", "/acorn/company/settings/connections"]; + retained.reply = "I tested it at the provider level just now with a live usage probe. The company Decision model is currently not configured; choosing this account and running the decision test requires a board user. Open [connection settings](/acorn/company/settings/connections)."; + expect(d.grade(retained).passed).toBe(true); + const reworded = valid(d.scenarioId); reworded.reply = "Only a board member can turn this on: [Connections](/acorn/company/settings/connections), Decision model, then Run test."; + expect(d.grade(reworded).passed).toBe(true); + const enabledByAgent = valid(d.scenarioId); enabledByAgent.state.afterReplyEnabled = true; expect(d.grade(enabledByAgent).passed).toBe(false); + const unlinked = valid(d.scenarioId); unlinked.state.replyLinkedControlHref = null; + expect(d.grade(unlinked).checks.find(check => check.id === "reply-link-opens-decision-model-control")?.passed).toBe(false); + const notRendered = valid(d.scenarioId); notRendered.links = ["/acorn/company/settings"]; expect(d.grade(notRendered).passed).toBe(false); + }); + it.each(["companies.fast-responses", "companies.configure-decision-model"])("%s act follows only the reply's links and stops when none shows the control", async id => { + const definition = companyModelIdentityCases.find(row => row.scenarioId === id)!; const support = await import("./identity-support.js"); const observe = vi.spyOn(support, "observeIdentity").mockResolvedValue({ links: ["/RUN/company/settings", "https://elsewhere.test/RUN/company/settings/connections"], origin: "https://fixture.test" } as never); const fill = vi.fn(async () => {}), save = vi.fn(async () => {}), goto = vi.fn(async () => {}); @@ -110,8 +128,8 @@ describe("company model service configuration and actual tests", () => { const evidence = vi.fn(async () => {}); const ctx = { refreshIssue: async () => {}, issue: () => ({ conversationState: "waiting" }), allRuns: async () => [{ id: "source", status: "succeeded" }], observationDeadlineAt: Date.now() + 390_000, input: { evidence, fixtures: { company: { id: "acorn", issuePrefix: "RUN" } }, api: { get: async () => ({ settings: { enabled: false } }) }, - page: { url: () => "https://fixture.test/RUN/issues/RUN-1", goto, getByRole: () => section } } }; - const state = valid("companies.fast-responses").state; delete state.replyLinkedControlHref; + page: { url: () => "https://fixture.test/RUN/issues/RUN-1", goto, getByRole: () => section, locator: () => section } } }; + const state = valid(id).state; delete state.replyLinkedControlHref; await definition.act!(ctx as never, state); expect(state.replyLinkedControlHref).toBeNull(); expect(fill).not.toHaveBeenCalled(); expect(save).not.toHaveBeenCalled(); expect(goto.mock.calls.map(call => (call as unknown[])[0])).not.toContain("/RUN/company/settings/connections"); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-company-models.ts b/tests/runner-e2e/chat-catalog-domains/identity-company-models.ts index 3c3721b3e68..c6924cd41d8 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-company-models.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-company-models.ts @@ -77,6 +77,10 @@ export function gradeCompanyModel(value: unknown) { const href = o.state.replyLinkedControlHref; c.add("reply-link-opens-fast-response-control", typeof href === "string" && o.links.includes(href), "The person clicked a link rendered in the reply and that page showed the Fast response control; a settings page without it does not qualify."); + } else if (o.scenarioId === "companies.configure-decision-model") { + const href = o.state.replyLinkedControlHref; + c.add("reply-link-opens-decision-model-control", typeof href === "string" && o.links.includes(href), + "The service was still off after the reply, and the person reached the board-only Decision model control by clicking a link rendered in the reply; the reply's wording is not graded."); } else c.add("honest-human-settings-guidance", /settings|connections/i.test(o.reply) && !/\bI(?:'ve| have)?\s+(?:enabled|configured|turned on|tested)\b/i.test(o.reply), "The chat names the human settings surface and does not claim a board-only change or test it could not perform."); if (["companies.prompt-injection-guard", "companies.fast-responses"].includes(o.scenarioId)) { c.add("requested-service-model-persisted-before-test", o.state.modelBeforeTest === requestedAuxiliaryServiceModel && saved?.model === requestedAuxiliaryServiceModel, @@ -127,7 +131,8 @@ export const companyModelIdentityCases: readonly ChatCatalogCase[] = specs.map(s }, gradeCompanyModel, { sourceFiles: sources, authentication: "human-session", requiredCredentials: ["OPENROUTER_API_KEY"], attemptTimeoutMs: 420_000, ...(spec.id === "companies.fast-responses" ? { oracleVersion: "paperclip.chat-catalog.identity.fast-responses.v4" } - : spec.id === "companies.prompt-injection-guard" ? { oracleVersion: "paperclip.chat-catalog.identity.prompt-injection-guard.v2" } : {}), + : spec.id === "companies.prompt-injection-guard" ? { oracleVersion: "paperclip.chat-catalog.identity.prompt-injection-guard.v2" } + : { oracleVersion: "paperclip.chat-catalog.identity.configure-decision-model.v2" }), async act(ctx, state) { await waitForCompanySettingsTurn(ctx); const root = `/api/companies/${ctx.input.fixtures.company.id}`; const source = ctx.input.page.url(); state.afterReplyEnabled = (await ctx.input.api.get<{ settings: Settings }>(`${root}/${spec.path}`)).settings.enabled; @@ -135,7 +140,7 @@ export const companyModelIdentityCases: readonly ChatCatalogCase[] = specs.map(s ...(spec.path === "decision-model" ? { allowBackground: false } : spec.path === "fast-response" ? { model: requestedAuxiliaryServiceModel, allowSponsored: false } : { model: requestedAuxiliaryServiceModel }) }); state.agentTestStatus = await probeAgentPermission(ctx, `${root}/${spec.path}/test`, "POST", {}); const section = spec.path === "decision-model" ? ctx.input.page.locator('section[aria-labelledby="decision-model-heading"]') : ctx.input.page.getByRole("region", { name: spec.section, exact: true }); - if (spec.path === "fast-response") { + if (spec.path !== "prompt-injection-prevention") { // The run is recorded finished before its reply is saved and rendered; read the reply's links only once it is on the page. await waitForTurnReplyLinks(ctx); const o = await observeIdentity(ctx, state); diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-routine-invocation.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-routine-invocation.test.ts index a1085273a6b..010ce6aff52 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-routine-invocation.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-routine-invocation.test.ts @@ -16,7 +16,7 @@ function valid() { { runId: "follow-up-run", name: "paperclip_get_task", at: "2026-01-01T00:03:00.000Z", output: '{"data":{"id":"task","status":"done"}}' }, ] }; const first = structuredClone(base); first.invocations[0]!.status = "issue_created"; first.tasks[0]!.status = "todo"; - return { ...base, runs: [...base.runs, followUpRun], state: { ...base.state, replayRunId: "invocation", replayIssueId: "task", firstInvocation: first }, + return { ...base, runs: [...base.runs, followUpRun], state: { ...base.state, replayProbe: "replayed", replayRunId: "invocation", replayIssueId: "task", firstInvocation: first } as Record & { replayIssueId?: string; firstInvocation: typeof first }, reply: `${first.reply}\nThe summary completed: Launch is still waiting on payment approval. Here is its result task.` }; } type Observation = ReturnType; @@ -46,6 +46,21 @@ describe("real manual invocation and delivery distinction", () => { const unread = valid(); unread.toolReads = unread.toolReads.filter(row => row.runId !== "output-run"); expect(gradeIntegratorRoutineInvocation(unread).passed).toBe(false); }); + // Full cohort 9 (master b1ef1f3022): after six 400s the CEO started the routine (202) without an idempotency key. + // The harness threw before grading because its replay probe needs the agent's key. + it("records the replay probe as unverified when the agent supplied no idempotency key, and still rejects a duplicate", () => { + const keyless = () => { const o = valid(); delete (o.state.firstInvocation.invocations[0] as { idempotencyKey?: string }).idempotencyKey; + delete (o.invocations[0] as { idempotencyKey?: string }).idempotencyKey; o.state = { ...o.state, replayProbe: "skipped-no-idempotency-key" }; + delete o.state.replayRunId; delete o.state.replayIssueId; return o; }; + const o = keyless(); const result = gradeIntegratorRoutineInvocation(o); + expect(result.passed).toBe(true); + expect(result.checks.find(row => row.id === "same-delivery-is-not-new-intent")?.detail).toMatch(/^Unverified/); + const duplicate = keyless(); duplicate.invocations.push({ ...duplicate.invocations[0]!, id: "another" }); + expect(gradeIntegratorRoutineInvocation(duplicate).passed).toBe(false); + const unrecorded = keyless(); delete unrecorded.state.replayProbe; expect(gradeIntegratorRoutineInvocation(unrecorded).passed).toBe(false); + const keyedButSkipped = valid(); keyedButSkipped.state = { ...keyedButSkipped.state, replayProbe: "skipped-no-idempotency-key" }; + expect(gradeIntegratorRoutineInvocation(keyedButSkipped).passed).toBe(false); + }); it("rejects false early completion and a later answer not grounded on the completed result", () => { const early = valid(); early.state.firstInvocation.reply = "The Launch summary completed successfully."; early.reply = `${early.state.firstInvocation.reply}\nThe summary completed; payment approval remains pending for Launch.`; diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-routine-invocation.ts b/tests/runner-e2e/chat-catalog-domains/integrator-routine-invocation.ts index f1ddbead9ba..71d9d50f8e9 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-routine-invocation.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-routine-invocation.ts @@ -74,9 +74,11 @@ export function gradeIntegratorRoutineInvocation(value: unknown) { && invocation.triggerPayload?.variables != null && (invocation.triggerPayload.variables as Record).projectName === "Launch" && !!task && task.projectId === projectId && task.assigneeAgentId === o.leadId, "Exactly one manual invocation retains the supplied project-name input and creates the intended owned Launch task."); - c.add("same-delivery-is-not-new-intent", !!firstInvocation?.idempotencyKey && o.state.replayRunId === invocation?.id - && o.state.replayIssueId === task?.id && firstInvocation.id === invocation?.id && o.invocations.length === 1, - "Replaying the committed public invocation with its original key retains the same invocation and task IDs."); + const replaySkipped = !firstInvocation?.idempotencyKey && o.state.replayProbe === "skipped-no-idempotency-key"; + c.add("same-delivery-is-not-new-intent", !!firstInvocation && firstInvocation.id === invocation?.id && o.invocations.length === 1 && (replaySkipped + || !!firstInvocation.idempotencyKey && o.state.replayProbe === "replayed" && o.state.replayRunId === invocation?.id && o.state.replayIssueId === task?.id), + replaySkipped ? "Unverified: the agent started the routine without an idempotency key, so no replay probe ran; exactly one invocation exists." + : "Replaying the committed public invocation with its original key retains the same invocation and task IDs."); c.add("accepted-versus-completed-at-first-reply", !!first && first.invocations.length === 1 && first.reply.trim().length > 0 && (!prematureCompletionClaim(first.reply) || firstInvocation?.status === "completed" && first.tasks.some(row => row.id === firstInvocation.linkedIssueId && row.status === "done")), "The first answer cannot claim a completed summary before the saved invocation and task actually complete."); @@ -110,7 +112,7 @@ export const integratorRoutineInvocationCases = [integratorDefinition("routines" if (!o || !task) throw new Error("Missing actual routine result task"); await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, task.title!, resultHref(o, task.identifier ?? task.id, "issues") ?? resultHref(o, task.id, "issues")); }, { - oracleVersion: "paperclip.chat-catalog.integrator.routine-invocation.v4", + oracleVersion: "paperclip.chat-catalog.integrator.routine-invocation.v5", sourceFiles: ["chat-catalog-domains/integrator-routine-invocation.ts", "chat-catalog-domains/api-read-receipts.ts", "run-observations.ts", "chat-catalog-domains/integrator-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/routines.yaml"], expectedRunCount: 3, expectedHumanMessageCount: 2, attemptTimeoutMs: 720_000, async act(ctx, state) { @@ -119,14 +121,16 @@ export const integratorRoutineInvocationCases = [integratorDefinition("routines" reject: o => o.invocations.length > 1 ? "Initial request created duplicate routine invocations" : undefined }); state.firstInvocation = { ...first, state: structuredClone(state) }; const invocation = first.invocations[0]!; - if (!invocation.idempotencyKey || !invocation.linkedIssueId) throw new Error("Committed routine invocation has no replayable public identity"); - // Independent transport-delivery probe of the existing committed request; - // it must not supply another user intent or admit another execution. - const replay = await ctx.input.api.post(`/api/routines/${invocation.routineId}/run`, { - source: invocation.source, triggerId: invocation.triggerId, idempotencyKey: invocation.idempotencyKey, - payload: invocation.triggerPayload, variables: invocation.triggerPayload?.variables, - }); - state.replayRunId = replay.id; state.replayIssueId = replay.linkedIssueId; + if (!invocation.linkedIssueId) throw new Error("Committed routine invocation created no task"); + if (invocation.idempotencyKey) { + // Independent transport-delivery probe of the existing committed request; + // it must not supply another user intent or admit another execution. + const replay = await ctx.input.api.post(`/api/routines/${invocation.routineId}/run`, { + source: invocation.source, triggerId: invocation.triggerId, idempotencyKey: invocation.idempotencyKey, + payload: invocation.triggerPayload, variables: invocation.triggerPayload?.variables, + }); + state.replayProbe = "replayed"; state.replayRunId = replay.id; state.replayIssueId = replay.linkedIssueId; + } else state.replayProbe = "skipped-no-idempotency-key"; await ctx.input.evidence("chat-routine-invocation-first-response.json", state.firstInvocation); await pollUntil({ label: "real routine execution completed", deadlineAt: Date.now() + 240_000, intervalMs: 1000, load: () => observeInvocation(ctx, state), accept: o => o.invocations.length === 1 && o.invocations[0]?.status === "completed" && o.runs.length === 2 && o.runs.every(run => run.status === "succeeded"), diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-routines.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-routines.test.ts index 5204e1be300..a087ff93d96 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-routines.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-routines.test.ts @@ -6,7 +6,7 @@ function valid(id: string) { triggers: [{ id: "schedule", kind: "schedule", enabled: true, cronExpression: "0 9 * * 1-5", timezone: "America/Chicago" }] }; const revision = { id: "revision", createdByAgentId: "ceo", createdByRunId: "run" }; return { scenarioId: id, origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: { project: { id: "project" }, worker: { id: "morgan" }, routine: structuredClone(routine), originalRevisions: [structuredClone(revision)], setupActivityIds: ["setup-created"] }, - tasks: [], projects: [], goals: [], comments: [], documents: [], revisions: [], humanQueries: [], activity: (managed ? [{ id: "setup-created", action: "routine.created", entityType: "routine", entityId: "routine", actorType: "user", actorId: "local-board", agentId: null }] : []) as Array>, links: managed ? ["/ACORN/routines/routine"] : [] as string[], runs: [{ id: "run", agentId: "ceo", status: "succeeded" }] as Array>, + tasks: [], projects: [], goals: [], comments: [], documents: [], revisions: [], humanQueries: [], activity: (managed ? [{ id: "setup-created", action: "routine.created", entityType: "routine", entityId: "routine", actorType: "user", actorId: "local-board", agentId: null }] : []) as Array>, links: managed ? ["/ACORN/routines/routine"] : ["/ACORN/routines/routine"] as string[], runs: [{ id: "run", agentId: "ceo", status: "succeeded" }] as Array>, reply: managed ? "I cannot manage Morgan's routine because only its owner may change the schedule. Morgan or a human operator can make this change." : "Saved one weekday routine for 9 in the morning America/Chicago. Each occurrence creates a task in Launch; future summaries have not run.", routines: [routine], histories: { routine: [] }, routineRevisions: { routine: [revision] }, browserTimezone: "America/Chicago", sourceId: "source" }; } @@ -24,6 +24,21 @@ describe("routine definition and management independent controls", () => { expect(gradeIntegratorRoutines({ ...o, routineRevisions: { routine: [] } }).passed).toBe(false); expect(gradeIntegratorRoutines({ ...o, reply: "Saved weekday schedule for 9 UTC. The task is in Launch, America/Chicago." }).passed).toBe(false); }); + // Full cohort 9 (gha, master b1ef1f3022): the saved routine was weekday 09:00 America/Chicago and the reply said + // "9:00 AM Central" and linked the Launch project, but a timezone keyword list rejected bare "Central". + it("grades the saved timezone and the reply's link, not how the reply names the timezone", () => { + const o = valid("routines.create-daily-brief"); + const retained = { ...o, links: ["/ACORN/agents/ceo/overview", "/ACORN/projects/project"], + reply: "Done. The Launch summary routine is set for Monday–Friday at 9:00 AM Central, with the first run this morning. Each run posts a summary task in the [Launch project](/ACORN/projects/project) assigned to me." }; + expect(gradeIntegratorRoutines(retained).passed).toBe(true); + expect(gradeIntegratorRoutines({ ...o, reply: "Set: weekdays at 9 a.m. CT. Summaries land in [the routine](/ACORN/routines/routine)." }).passed).toBe(true); + expect(gradeIntegratorRoutines({ ...o, reply: "Every weekday at 9:00 AM your time, see [Launch summary](/ACORN/routines/routine)." }).passed).toBe(true); + expect(gradeIntegratorRoutines({ ...retained, links: ["/ACORN/agents/ceo/overview"] }).passed).toBe(false); + expect(gradeIntegratorRoutines({ ...retained, links: ["/ACORN/projects/other"] }).passed).toBe(false); + expect(gradeIntegratorRoutines({ ...retained, reply: "Saved for weekdays at 9:00 AM UTC in [Launch](/ACORN/projects/project)." }).passed).toBe(false); + const utc = structuredClone(retained); utc.routines[0]!.triggers[0]!.timezone = "UTC"; expect(gradeIntegratorRoutines(utc).passed).toBe(false); + const hour = structuredClone(retained); hour.routines[0]!.triggers[0]!.cronExpression = "0 21 * * 1-5"; expect(gradeIntegratorRoutines(hour).passed).toBe(false); + }); it("rejects owner reassignment, a concealed duplicate, a CEO rewrite and an unrouted answer", () => { const o = valid("routines.manage-other-owner"); const stolen = structuredClone(o); stolen.routines[0]!.assigneeAgentId = "ceo"; expect(gradeIntegratorRoutines(stolen).passed).toBe(false); diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-routines.ts b/tests/runner-e2e/chat-catalog-domains/integrator-routines.ts index 59e4d6bde31..688d54d6d9e 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-routines.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-routines.ts @@ -4,7 +4,7 @@ import { checks, hasLink, project, observation, type Entity, type InternalObserv import { chatCompletionFollowUp, runIssueId } from "./run-roles.js"; import { parseCron } from "../../../server/src/services/cron.js"; import { fixtureWorker } from "./internal-definition.js"; -import { integratorDefinition, groundedStatement, inspectResult, observeInternal } from "./integrator-support.js"; +import { integratorDefinition, inspectResult, observeInternal } from "./integrator-support.js"; interface RoutinesObservation extends InternalObservation { routines: RoutineDetail[]; histories: Record; routineRevisions: Record; browserTimezone: string; sourceId?: string; @@ -59,9 +59,10 @@ export function gradeIntegratorRoutines(value: unknown) { && /^0\s+9\s+\*\s+\*\s+(?:1-5|MON-FRI|1,2,3,4,5)$/i.test(schedule[0]?.cronExpression ?? "") && schedule[0]?.timezone === "America/Chicago", "One enabled 09:00 weekday schedule uses the genuine browser person's Chicago timezone, never UTC."); c.add("saved-agent-revision", !!routine && !!o.routineRevisions[routine.id]?.some(row => row.id === routine.latestRevisionId && row.createdByAgentId === o.leadId && row.createdByRunId != null && o.runs.some(run => run.id === row.createdByRunId)), "The current saved revision is independently attributed to a measured agent run."); - c.add("schedule-and-work-location-explained", groundedStatement(o.reply, /\b(?:weekday|monday|mon.fri)\b/i, /\b(?:9|nine|09:00)\b/i, /\b(?:9|nine|09:00)\s*(?:a\.?m\.?\s*)?UTC\b/i) - && /(?:America\/Chicago|Chicago|Central (?:time|timezone))/i.test(o.reply) && /\b(?:task|routine|work)\b/i.test(o.reply) - && !/\b(?:summary (?:is )?(?:complete|ready)|already summarized|all future summaries (?:are )?(?:complete|ready))\b/i.test(o.reply), "The response describes the real saved local schedule and resulting work without claiming future summaries already ran."); + c.add("schedule-and-work-location-explained", !!routine && o.links.some(href => linksTo(o, href, "routines", [routine.id]) || linksTo(o, href, "projects", [p?.id, (p as Entity | undefined)?.urlKey])) + && !/\b(?:0?9(?::00)?|nine)\s*(?:a\.?m\.?\s*)?UTC\b/i.test(o.reply) + && !/\b(?:summary (?:is )?(?:complete|ready)|already summarized|all future summaries (?:are )?(?:complete|ready))\b/i.test(o.reply), + "The saved schedule is graded above; the reply links the saved routine or the Launch project where its work lands, never states nine as UTC, and does not claim future summaries already ran."); } else if (o.scenarioId === "routines.manage-other-owner") { const before = o.state.routine as RoutineDetail; const current = o.routines.find(row => row.id === before?.id); const worker = o.state.worker as Entity; const route = ownerRoute(o, worker); @@ -102,7 +103,7 @@ async function verifyRoutine(ctx: ChatCatalogContext, _state: Record = {}) => integratorDefinition("routines", id, setup, observeRoutines, gradeIntegratorRoutines, verifyRoutine, extra); export const integratorRoutinesCases: readonly ChatCatalogCase[] = [ - define("routines.create-daily-brief", async ctx => ({ project: await project(ctx, "Launch") }), { browserTimezone: "America/Chicago" }), + define("routines.create-daily-brief", async ctx => ({ project: await project(ctx, "Launch") }), { browserTimezone: "America/Chicago", oracleVersion: "paperclip.chat-catalog.integrator.create-daily-brief.v2" }), define("routines.manage-other-owner", async ctx => { const p = await project(ctx, "Launch"); const worker = await fixtureWorker(ctx); const created = await ctx.input.api.post(`${root(ctx)}/routines`, { title: "Morgan's launch summary", description: "Summarize Launch without changing project tasks.", projectId: p.id, assigneeAgentId: worker.id }); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-documents-artifacts.ts b/tests/runner-e2e/chat-catalog-domains/internal-documents-artifacts.ts index c36c847daf8..f8b4734b380 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-documents-artifacts.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-documents-artifacts.ts @@ -37,7 +37,8 @@ export function grade_documents_artifacts(value: unknown) { c.add("correct-thread-resolved", annotation?.status === "resolved" && annotation.resolvedByAgentId === o.leadId && unrelated?.status === "open" && annotation.documentId === plan?.id && annotation.originalRevisionId === o.state.originalRevisionId, "Only the anchored pricing annotation is resolved by the CEO, preserving its original document and revision relationship."); - const edited = o.activity.find(row => row.action === "issue.document_updated" && row.agentId === o.leadId); + const edited = o.activity.filter(row => row.action === "issue.document_updated" && row.agentId === o.leadId) + .sort((a, b) => Date.parse(a.createdAt ?? "") - Date.parse(b.createdAt ?? ""))[0]; const resolved = o.activity.find(row => row.action === "issue.document_annotation_thread_resolved" && row.agentId === o.leadId && row.details?.threadId === target.id); c.add("edit-before-resolution", !!edited && !!resolved && Date.parse(edited.createdAt ?? "") <= Date.parse(resolved.createdAt ?? "") && !!plan && o.revisions.some(row => row.id === plan.latestRevisionId && row.createdByAgentId === o.leadId && row.body === plan.body), @@ -113,7 +114,7 @@ function resolveFeedback(): ChatCatalogCase { document: { id: doc.id, key: "plan", latestRevisionId: doc.latestRevisionId! }, annotationId: pricingThread.id }); return { project: p, task: t, pricingThread, siblingThread, supportText, originalRevisionId: doc.latestRevisionId, selectedSource, factualContextAttachment: selectedSource.factualContextAttachment }; }); - return { ...base, sourceFiles: [...base.sourceFiles, "../../packages/shared/src/document-anchors.ts"], async observe(ctx, state) { + return { ...base, oracleVersion: "paperclip.chat-catalog.internal.resolve-feedback.v2", sourceFiles: [...base.sourceFiles, "../../packages/shared/src/document-anchors.ts"], async observe(ctx, state) { const o = await observeInternal(ctx, state); return { ...o, annotations: await ctx.input.api.get(`/api/issues/${entity(state, "task").id}/documents/plan/annotations?status=all&includeComments=true`) }; } }; diff --git a/tests/runner-e2e/chat-catalog-domains/internal-pricing-oracle.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-pricing-oracle.test.ts index d300327f6ee..1f3ed11a85a 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-pricing-oracle.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-pricing-oracle.test.ts @@ -41,3 +41,22 @@ it("preserves unrelated content, exact annotation relationships, saved revision expect(grade_documents_artifacts({ ...o, annotations: o.annotations.map(row => ({ ...row, status: "resolved" })) }).passed).toBe(false); expect(grade_documents_artifacts({ ...o, activity: [o.activity[1], { ...o.activity[0], createdAt: "2026-10-27T10:00:02Z" }] }).passed).toBe(false); }); + +// Full cohort 9 (master b1ef1f3022): the CEO saved the edit at 00:41:59, resolved at 00:42:03 and saved an identical body +// again at 00:42:06. Activity is listed newest first, so the check read the later save and reported no edit before resolution. +it("establishes edit-before-resolution from the earliest CEO edit, whatever order activity is listed in", () => { + const task = { id: "task", companyId: "company", title: "Website task", status: "backlog", projectId: "project" }; + const supportText = "Support is provided by the original launch team fixture-42."; + const body = `$19 monthly total for the entire team, not per user.\n\n${supportText}`; + const edit = (id: string, at: string) => ({ id, action: "issue.document_updated", agentId: "ceo", createdAt: at }); + const resolve = { id: "resolve", action: "issue.document_annotation_thread_resolved", agentId: "ceo", details: { threadId: "pricing" }, createdAt: "2026-11-12T00:42:03Z" }; + const o = (activity: unknown[]) => ({ scenarioId: "documents-artifacts.resolve-feedback", companyId: "company", origin: "https://fixture.test", leadId: "ceo", + state: { task, pricingThread: { id: "pricing" }, siblingThread: { id: "sibling" }, supportText, originalRevisionId: "original" }, tasks: [task], projects: [], goals: [], comments: [], + documents: [{ id: "document", key: "plan", latestRevisionId: "revision", body }], revisions: [{ id: "revision", body, createdByAgentId: "ceo" }], + annotations: [{ id: "pricing", status: "resolved", resolvedByAgentId: "ceo", documentId: "document", originalRevisionId: "original" }, { id: "sibling", status: "open" }], + activity, reply: "Fixed.", links: ["/RUN/issues/task"], humanQueries: [], runs: [] }); + expect(grade_documents_artifacts(o([edit("resave", "2026-11-12T00:42:06Z"), resolve, edit("edit", "2026-11-12T00:41:59Z")])).passed).toBe(true); + expect(grade_documents_artifacts(o([edit("edit", "2026-11-12T00:41:59Z"), resolve, edit("resave", "2026-11-12T00:42:06Z")])).passed).toBe(true); + expect(grade_documents_artifacts(o([edit("late", "2026-11-12T00:42:06Z"), resolve])).passed).toBe(false); + expect(grade_documents_artifacts(o([resolve])).passed).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-support.ts b/tests/runner-e2e/chat-catalog-domains/internal-support.ts index 7c109012f88..ac3c2a3aa3e 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-support.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-support.ts @@ -17,7 +17,7 @@ export interface Entity { export interface InternalObservation { scenarioId: string; origin: string; companyId: string; leadId: string; state: Record; tasks: Entity[]; projects: Entity[]; goals: Entity[]; comments: Entity[]; - renderedLinkEvidence?: RenderedConversationLink[]; linkDiagnostics?: ConversationLinkDiagnostics; activity: Entity[]; annotations?: Entity[]; agents?: Entity[]; documents: Entity[]; revisions: Entity[]; reply: string; links: string[]; humanQueries: string[]; runs: ChatRun[]; + renderedLinkEvidence?: RenderedConversationLink[]; unservedLinkEvidence?: Array; linkDiagnostics?: ConversationLinkDiagnostics; activity: Entity[]; annotations?: Entity[]; agents?: Entity[]; documents: Entity[]; revisions: Entity[]; reply: string; links: string[]; humanQueries: string[]; runs: ChatRun[]; } export function entity(state: Record, key: string): Entity { const value = state[key]; @@ -72,7 +72,7 @@ export async function observeInternal(ctx: ChatCatalogContext, state: Record row.id !== source?.id), projects, goals, runs, comments, activity, documents, revisions, - renderedLinkEvidence, ...(collected.diagnostics ? { linkDiagnostics: collected.diagnostics } : {}), humanQueries: chat.filter(row => !row.authorAgentId).map(row => row.body ?? ""), reply: replies.map(row => row.body ?? "").join("\n"), links }; + renderedLinkEvidence, ...("rejected" in collected && collected.rejected ? { unservedLinkEvidence: collected.rejected } : {}), ...(collected.diagnostics ? { linkDiagnostics: collected.diagnostics } : {}), humanQueries: chat.filter(row => !row.authorAgentId).map(row => row.body ?? ""), reply: replies.map(row => row.body ?? "").join("\n"), links }; } export function hasLink(o: InternalObservation, kind: "issues" | "projects" | "goals", row: Entity): boolean { return o.links.some(href => { diff --git a/tests/runner-e2e/chat-catalog-domains/internal-website-delivery.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-website-delivery.test.ts index 3acad2c8250..d04d5f77d8d 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-website-delivery.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-website-delivery.test.ts @@ -34,6 +34,11 @@ describe("registered website delivery calibration", () => { const cancelled = structuredClone(o); cancelled.tasks[0]!.status = "cancelled"; expect(gradeWebsiteDelivery(cancelled).passed).toBe(false); const moved = structuredClone(o); (moved.tasks[0] as { projectId?: string }).projectId = "elsewhere"; expect(gradeWebsiteDelivery(moved).passed).toBe(false); }); + // Full cohort 9: the answer linked /COM/api/attachments//content?download=1. The server serves its API only at /api. + it("accepts the served attachment routes and rejects the company-prefixed API path the server does not serve", () => { + const opened = valid(); opened.links = ["/RUN/issues/task", "/api/attachments/new/content"]; expect(gradeWebsiteDelivery(opened).passed).toBe(true); + const prefixed = valid(); prefixed.links = ["/RUN/issues/task", "/RUN/api/attachments/new/content?download=1"]; expect(gradeWebsiteDelivery(prefixed).passed).toBe(false); + }); it("rejects a local path without registered download", () => { const o = valid(); o.links = ["/RUN/issues/task", "/tmp/website.html"]; expect(gradeWebsiteDelivery(o).passed).toBe(false); }); it("rejects stale or mismatched downloaded bytes", () => { const o = valid(); o.downloaded[0]!.body = "stale"; expect(gradeWebsiteDelivery(o).passed).toBe(false); }); it("rejects wrong immutable origin or work-product source", () => { const o = valid(); o.attachments[0]!.originatingRunId = "build"; expect(gradeWebsiteDelivery(o).passed).toBe(false); const source = valid(true); source.products[0]!.issueId = "another-task"; expect(gradeWebsiteDelivery(source).passed).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-website-delivery.ts b/tests/runner-e2e/chat-catalog-domains/internal-website-delivery.ts index 6e020bf9c36..d54b518a2c8 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-website-delivery.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-website-delivery.ts @@ -104,7 +104,7 @@ function websiteDelivery(revising: boolean): ChatCatalogCase { const state = { project: p, task: handedOff, worker, priorRunId, filename, producedBody: actual, producedHash: hash(actual), marker, bodyFact, signupHref, originalAttachment, originalProductId, selectedSource, factualContextAttachment: selectedSource.factualContextAttachment }; await ctx.input.evidence("chat-catalog-actual-prior-website.json", state); return state; }, gradeWebsiteDelivery); - return { ...base, oracleVersion: revising ? "paperclip.chat-catalog.internal.website-revision.v3" : "paperclip.chat-catalog.internal.website-download.v2", authentication: "human-session", queryAttachments: factualContextQueryAttachment("selected-source.md"), expectedRunCount: 2, minimumRunCount: 2, attemptTimeoutMs: 600_000, + return { ...base, oracleVersion: revising ? "paperclip.chat-catalog.internal.website-revision.v3" : "paperclip.chat-catalog.internal.website-download.v3", authentication: "human-session", queryAttachments: factualContextQueryAttachment("selected-source.md"), expectedRunCount: 2, minimumRunCount: 2, attemptTimeoutMs: 600_000, sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-website-delivery.ts", "registered-artifact.ts"], async observe(ctx, state): Promise { const o = await observeInternal(ctx, state); const ids = [entity(state, "task").id, ...(ctx.issue() ? [ctx.issue()!.id] : [])]; diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-agents.ts b/tests/runner-e2e/chat-catalog-domains/runtime-agents.ts index 7590690ca1d..5e3531d6f79 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-agents.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-agents.ts @@ -98,7 +98,7 @@ export const runtimeAgentsCases = [ const originalInstruction = await ctx.input.api.get(`/api/agents/${worker.id}/instructions-bundle/file?path=AGENTS.md`); return { worker, instructionAgentId: worker.id, originalInstruction, sideContent, concurrentMarker: `Concurrent human guidance ${ctx.input.nonce}: preserve keyboard support.` }; }, gradeRuntimeAgents, { - oracleVersion: "paperclip.chat-catalog.runtime.edit-instructions.v3", + oracleVersion: "paperclip.chat-catalog.runtime.edit-instructions.v4", async act(ctx, state) { const active = await pollUntil({ label: "Active instructions update before concurrent human revision", deadlineAt: Date.now() + 60_000, load: () => ctx.allRuns(), accept: runs => runs.length === 1 && runs[0]?.status === "running", diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-busy-worker.ts b/tests/runner-e2e/chat-catalog-domains/runtime-busy-worker.ts index 7e1d75e657e..1ccffc72bd0 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-busy-worker.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-busy-worker.ts @@ -70,12 +70,12 @@ export async function prepareRuntimeWorkerGate(ctx: ChatCatalogContext, options: await ctx.input.api.post(`${company}/budgets/policies`, { ...scope, amount: 500, hardStopEnabled: true, isActive: true, unpricedUsagePolicy: "allow" }); return { worker, gate, previousRuns, company }; } -export async function busyRuntimeWorker(ctx: ChatCatalogContext, options: { worker?: RuntimeAgent; observationDeadlineAt?: number } = {}) { +export async function busyRuntimeWorker(ctx: ChatCatalogContext, options: { worker?: RuntimeAgent; observationDeadlineAt?: number; briefSource?: string } = {}) { const { worker, gate, previousRuns, company } = await prepareRuntimeWorkerGate(ctx, { ...options, taskBound: true }); // Each busy worker owns its own task. Without allowDuplicate the product's recent-open-title // deduplication returns the first worker's identically titled "Website task" (HTTP 200). const task = await ctx.input.api.post(`${company}/issues`, { title: "Website task", status: "todo", assigneeAgentId: worker.id, allowDuplicate: true, - description: `The website brief is being supplied by the human. Run ${gate.command} once and wait for its real output. Do not invent a brief, replace the script, or mark work complete before it returns. Once the brief arrives, save it as a task document and finish normally.` }); + description: `${options.briefSource ?? "The website brief is being supplied by the human."} Run ${gate.command} once and wait for its real output. Do not invent a brief, replace the script, or mark work complete before it returns. Once the brief arrives, save it as a task document and finish normally.` }); if (task.assigneeAgentId !== worker.id || previousRuns.some(run => run.contextSnapshot?.issueId === task.id)) throw new Error("Busy worker fixture did not create its own task for this worker"); const running = await pollUntil({ label: "Real Morgan process waiting for its human brief", deadlineAt: options.observationDeadlineAt ?? Date.now() + 150_000, diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-infrastructure.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-infrastructure.test.ts index cb9a057d02f..f7e5cf66711 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-infrastructure.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-infrastructure.test.ts @@ -1,7 +1,8 @@ import { expect, it, vi } from "vitest"; -import { freshContinuationRecord, gradeInfrastructureRecovery, nativeInfrastructureLoss, readSourceRecoveryRecord } from "./runtime-infrastructure.js"; +import { freshContinuationRecord, gradeInfrastructureRecovery, INFRASTRUCTURE_BRIEF_SOURCE, nativeInfrastructureLoss, readSourceRecoveryRecord } from "./runtime-infrastructure.js"; +import { busyRuntimeWorker } from "./runtime-busy-worker.js"; vi.mock("./runtime-busy-worker.js", async importOriginal => ({ ...await importOriginal(), - busyRuntimeWorker: async () => ({ worker: { id: "morgan" }, task: { id: "website" }, gate: {}, workerRunId: "run" }) })); + busyRuntimeWorker: vi.fn(async () => ({ worker: { id: "morgan" }, task: { id: "website" }, gate: {}, workerRunId: "run" })) })); vi.mock("./runtime-native-fault.js", async importOriginal => ({ ...await importOriginal(), faultOwnedNativeProcess: async () => ({ id: "run", processPid: 7, errorCode: "native_runner_process_exited", status: "running", nativePhase: "running" }) })); import { savedCall } from "./api-read-receipts.test-fixtures.js"; @@ -116,3 +117,14 @@ it("rejects a run after the last Runner loss that does not continue the interrup await expect(nativeInfrastructureLoss(lossContext(3, { retryOfRunId: "other" }) as never)).rejects.toThrow(/undeclared replacement/); await expect(nativeInfrastructureLoss(lossContext(3, { agentId: "riley" }) as never)).rejects.toThrow(/undeclared replacement/); }); + +// Full cohort 9 (recovery.source-action, master b1ef1f3022): the recovery mechanics passed, but the gated Website task said +// "The website brief is being supplied by the human", so the CEO asked the chat person for the brief instead of explaining +// that the system was already continuing the task. The brief comes from the gated export command itself. +it("gives the interrupted Website task a brief source that does not ask the chat person to supply anything", async () => { + vi.mocked(busyRuntimeWorker).mockClear(); + await nativeInfrastructureLoss(lossContext(3) as never); + expect(vi.mocked(busyRuntimeWorker).mock.calls[0]?.[1]).toEqual({ briefSource: INFRASTRUCTURE_BRIEF_SOURCE }); + expect(INFRASTRUCTURE_BRIEF_SOURCE).not.toMatch(/\b(?:human|person|you|user|supplied)\b/i); + expect(INFRASTRUCTURE_BRIEF_SOURCE).toMatch(/command below/); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-infrastructure.ts b/tests/runner-e2e/chat-catalog-domains/runtime-infrastructure.ts index a76cdc28fea..6bf2e65348b 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-infrastructure.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-infrastructure.ts @@ -11,7 +11,7 @@ import { checks, hasLink, type Entity } from "./internal-support.js"; import { busyRuntimeWorker, releasedRetryListener, releaseRuntimeWorker, type RuntimeBriefGate } from "./runtime-busy-worker.js"; import { faultOwnedNativeProcess, NATIVE_INFRASTRUCTURE_FAULT_CODES, type NativeFaultRun } from "./runtime-native-fault.js"; import { runtimeDefinition, runtimeObservation, observeRuntime, type RuntimeAgent, type RuntimeObservation } from "./runtime-support.js"; -export const INFRASTRUCTURE_RECOVERY_ORACLE = "paperclip.chat-catalog.runtime.infrastructure-recovery.v6"; +export const INFRASTRUCTURE_RECOVERY_ORACLE = "paperclip.chat-catalog.runtime.infrastructure-recovery.v7"; const objectRecord = (value: unknown): Record => value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; /** * The chat run read the source task's saved recovery record through a product @@ -69,8 +69,9 @@ export function continuesFailedRun(run: Pick = []; let previousPid: number | null = null; for (let attempt = 1; attempt <= 3; attempt++) { diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-queue-controls.ts b/tests/runner-e2e/chat-catalog-domains/runtime-queue-controls.ts index 5402ae5a871..4b73110be47 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-queue-controls.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-queue-controls.ts @@ -85,7 +85,7 @@ export function gradeQueueEdit(value: unknown) { export const runtimeQueueControlCases = [runtimeDefinition("steering.edit-queued-message", ctx => humanQueuedRuntime(ctx), gradeQueueEdit, { sourceFiles: ["chat-catalog-domains/runtime-queue-controls.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-rendered-links.ts", "chat-scenarios/steering.yaml"], - oracleVersion: "paperclip.chat-catalog.edit-queued-message.structural.v5", + oracleVersion: "paperclip.chat-catalog.edit-queued-message.structural.v6", authentication: "human-session", expectedRunCount: 3, minimumRunCount: 2, attemptTimeoutMs: 720_000, runPolicy: { kind: "terminal-counts", terminalStatusCounts: { succeeded: 2 }, maximumTerminalStatusCounts: { succeeded: 3 } }, setupPolicy: runtimeQueueCases[0]!.setupPolicy, verifySetup: runtimeQueueCases[0]!.verifySetup, @@ -128,9 +128,18 @@ export const runtimeQueueControlCases = [runtimeDefinition("steering.edit-queued } finally { release(); await ctx.input.page.unroute(pattern); await concurrent.close(); } await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); await edit(ctx.input.page, targetId, `${blueText} ${state.concurrentMarker}`); + const saved = ctx.input.page.waitForResponse(response => response.request().method() === "PATCH" && new URL(response.url()).pathname === mutation, { timeout: 15_000 }); await ctx.input.page.getByRole("button", { name: "Save queued message", exact: true }).click(); + state.finalEditStatus = (await saved).status(); + if (state.finalEditStatus !== 200) throw new Error(`The person's final queued edit was not saved (${state.finalEditStatus})`); + state.queueAfterEdit = await pollUntil({ label: "The saved edit is the queue's current revision", deadlineAt: Date.now() + 15_000, load: () => queue(ctx, state), + accept: q => q.entries.some(row => row.comment.id === targetId && row.comment.body === `${blueText} ${state.concurrentMarker}` && row.position === 1) }); const handle = ctx.input.page.getByRole("button", { name: `Reorder queued message: ${blueText} ${state.concurrentMarker}`, exact: true }); - state.reorderRequest = await keyboardMoveQueuedMessageUp(ctx.input.page, handle); + await expect(handle).toBeVisible(); + state.reorderRequest = await keyboardMoveQueuedMessageUp(ctx.input.page, handle, 15_000, 100, { attempts: 2, beforeRetry: async () => { + await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); await expect(handle).toBeVisible(); + } }); + if ((state.reorderRequest as { status: number }).status !== 200) throw new Error(`The person's reorder was refused (${(state.reorderRequest as { status: number }).status})`); state.queueAfter = await pollUntil({ label: "Actual full blue text and first queue position persisted", deadlineAt: Date.now() + 15_000, load: () => queue(ctx, state), accept: q => q.entries[0]?.comment.id === targetId && q.entries[0]?.comment.body === `${blueText} ${state.concurrentMarker}` }); await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-queue.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-queue.test.ts index bb20ee85147..59fc6835c3a 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-queue.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-queue.test.ts @@ -88,6 +88,21 @@ it("fails when the handle never enables or the gesture sends no order request", await expect(keyboardMoveQueuedMessageUp(silent.page, handle as never, 200, 20)).rejects.toThrow(/sent no queue order request/); }); +// Full cohort 9 (master b1ef1f3022, steering.edit-queued-message): the person's edit saved (PATCH 200), the handle was +// enabled, but the keyboard gesture sent no PUT /queued-comments/order within 15 s. +it("refetches and presses the gesture again when the first gesture sends no order request, and still fails when none does", async () => { + const fake = reorderPage(0); let reloads = 0, sends = false; + const handle = { ...(fake.handle as object), press: async (key: string) => { if (sends) await (fake.handle as { press: (k: string) => Promise }).press(key); } }; + const result = await keyboardMoveQueuedMessageUp(fake.page, handle as never, 200, 20, { attempts: 2, beforeRetry: async () => { reloads++; sends = true; } }); + expect(result).toMatchObject({ status: 200, gestures: 2 }); expect(reloads).toBe(1); + const first = reorderPage(0); const once = await keyboardMoveQueuedMessageUp(first.page, first.handle, 3_000, 20, { attempts: 2, beforeRetry: async () => { reloads++; } }); + expect(once.gestures).toBe(1); expect(reloads).toBe(1); + const silent = reorderPage(0); let retried = 0; + await expect(keyboardMoveQueuedMessageUp(silent.page, { ...(silent.handle as object), press: async () => undefined } as never, 200, 20, + { attempts: 2, beforeRetry: async () => { retried++; } })).rejects.toThrow(/sent no queue order request after 2 gesture/); + expect(retried).toBe(1); +}); + describe("task history reads", () => { const event = (comments: unknown, eventType = "task_history.read") => ({ eventType, payload: { comments } }); it("keeps the newest revision each comment was read at, from history-read events only", () => { diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-queue.ts b/tests/runner-e2e/chat-catalog-domains/runtime-queue.ts index a5c0da82243..3ec0a8b7559 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-queue.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-queue.ts @@ -14,22 +14,28 @@ import type { IssueWorkProduct } from "../../../packages/shared/src/types/work-p /** Move a queued message up one place with the drag handle's keyboard gesture, as a keyboard user does. * Saving an edit disables every handle until that queue change settles, and a gesture pressed on a disabled * handle does nothing, so wait for this handle to be enabled first. The gesture must send the queue order - * request; its status is returned so the caller can tell an accepted move from a refused one. */ + * request; its status is returned so the caller can tell an accepted move from a refused one. When a gesture + * sends nothing, `beforeRetry` (for example a reload that refetches the queue) runs and the gesture is pressed + * again, up to `attempts` gestures in all; every gesture is reported. */ export async function keyboardMoveQueuedMessageUp(page: Pick, handle: Pick, - timeoutMs = 15_000, intervalMs = 100): Promise<{ status: number; waitedForEnabledMs: number }> { - const started = Date.now(); - while (!await handle.isEnabled()) { - if (Date.now() - started >= timeoutMs) throw new Error("The queued message's reorder handle stayed disabled"); - await new Promise(resolve => setTimeout(resolve, intervalMs)); + timeoutMs = 15_000, intervalMs = 100, options: { attempts?: number; beforeRetry?: () => Promise } = {}): Promise<{ status: number; waitedForEnabledMs: number; gestures: number }> { + const attempts = Math.max(1, options.attempts ?? 1); + for (let gesture = 1; ; gesture++) { + const started = Date.now(); + while (!await handle.isEnabled()) { + if (Date.now() - started >= timeoutMs) throw new Error("The queued message's reorder handle stayed disabled"); + await new Promise(resolve => setTimeout(resolve, intervalMs)); + } + const waitedForEnabledMs = Date.now() - started; + const sent = page.waitForResponse((response: Response) => response.request().method() === "PUT" + && new URL(response.url()).pathname.endsWith("/queued-comments/order"), { timeout: timeoutMs }); + sent.catch(() => undefined); + await handle.focus(); await handle.press("Space"); await handle.press("ArrowUp"); await handle.press("Space"); + const response = await sent.catch(() => null); + if (response) return { status: response.status(), waitedForEnabledMs, gestures: gesture }; + if (gesture >= attempts) throw new Error(`The keyboard reorder gesture sent no queue order request after ${gesture} gesture(s)`); + await options.beforeRetry?.(); } - const waitedForEnabledMs = Date.now() - started; - const sent = page.waitForResponse((response: Response) => response.request().method() === "PUT" - && new URL(response.url()).pathname.endsWith("/queued-comments/order"), { timeout: timeoutMs }); - sent.catch(() => undefined); - await handle.focus(); await handle.press("Space"); await handle.press("ArrowUp"); await handle.press("Space"); - const response = await sent.catch(() => null); - if (!response) throw new Error("The keyboard reorder gesture sent no queue order request"); - return { status: response.status(), waitedForEnabledMs }; } const MAX_DELIVERABLE_BYTES = 262_144; /** Text of each stored or deflated entry in a zip deliverable, read from its central directory; [] when it is not a readable zip. */ diff --git a/tests/runner-e2e/chat-rendered-links.test.ts b/tests/runner-e2e/chat-rendered-links.test.ts index 6f7ce902ae6..9b6ec4a1560 100644 --- a/tests/runner-e2e/chat-rendered-links.test.ts +++ b/tests/runner-e2e/chat-rendered-links.test.ts @@ -1,7 +1,7 @@ // @vitest-environment jsdom import { readFileSync } from "node:fs"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { collectConversationLinks, collectConversationLinksWithDiagnostics, conversationLinkBindings, mislabeledIssueLinks, readConversationDomLinks, readLatestConversationReplyRender, waitForLatestConversationReplyRender, waitForSavedTurnReplyRender, waitForSavedTurnReplyRenderReloadingOnce, type ConversationLinkInput } from "./chat-rendered-links.js"; +import { APP_ROUTE_ROOTS, collectConversationLinks, collectConversationLinksWithDiagnostics, conversationLinkBindings, unservedLinkReason, mislabeledIssueLinks, readConversationDomLinks, readLatestConversationReplyRender, waitForLatestConversationReplyRender, waitForSavedTurnReplyRender, waitForSavedTurnReplyRenderReloadingOnce, type ConversationLinkInput } from "./chat-rendered-links.js"; const card = '
'; const thread = (content = card) => `
Create a project called Launch for the new website.
${content}
Created the Launch project. No repositories are connected yet.
`; @@ -218,3 +218,42 @@ describe("records thread and anchor diagnostics when a saved reply yields no ren expect(result.links.map(row => row.href)).toEqual(["/RUN/issues/task"]); expect(result.diagnostics).toBeUndefined(); }); }); + +// Full cohort 9 (documents-artifacts.produce-download, agents.edit-instructions): a reply's download link was written as +// /COM/api/attachments/.../content?download=1, which the server does not serve (its API is only at /api), and a reply's +// Morgan link was saved as //COM/agents/..., which the board repairs but notifications and chat channels deliver as a +// link to a host named COM. Both counted as valid links. +describe("only links that open a served app page or API route count", () => { + const origin = "https://fixture.test"; + it("accepts company pages, unprefixed board pages, global pages and served API downloads", () => { + for (const href of ["/RUN/issues/RUN-1", "/run/agents/morgan/overview", "/RUN/company/settings/connections", "/RUN/routines/routine", "/RUN", + "/issues/RUN-1", "/instance/settings", "/api/attachments/file/content?download=1", "/api/attachments/file/content", "https://github.com/owner/repository"]) + expect(unservedLinkReason(href, origin), href).toBeUndefined(); + }); + it("rejects a company-prefixed API path, a doubled prefix and other paths the app does not route", () => { + for (const href of ["/RUN/api/attachments/file/content?download=1", "/RUN/RUN/issues/RUN-1", "/RUN/attachments/file/content", "/attachments/file/content"]) + expect(unservedLinkReason(href, origin), href).toBe("not-a-served-route"); + }); + it("rejects a protocol-relative company link, rendered raw or repaired by the board, but not the same page saved correctly", () => { + expect(unservedLinkReason("//RUN/agents/morgan", origin)).toBe("protocol-relative"); + expect(unservedLinkReason("/RUN/agents/morgan", origin, ["//RUN/agents/morgan"])).toBe("protocol-relative"); + expect(unservedLinkReason("/RUN/agents/morgan", origin, ["/RUN/agents/morgan"])).toBeUndefined(); + expect(unservedLinkReason("/RUN/agents/morgan", origin, ["//RUN/agents/morgan", "/RUN/agents/morgan"])).toBeUndefined(); + }); + it("lists exactly the first path segments that the board router declares", () => { + const app = readFileSync(`${import.meta.dirname}/../../ui/src/App.tsx`, "utf8"); + const roots = new Set([...app.matchAll(/path="\/?([^"/]+)/g)].map(match => match[1]!).filter(root => !root.startsWith(":") && root !== "*")); + expect([...APP_ROUTE_ROOTS].sort()).toEqual([...roots].sort()); + expect(APP_ROUTE_ROOTS).not.toContain("api"); + }); + it("drops unserved reply links from collection, keeps them as rejected evidence and keeps served ones", async () => { + const page = { evaluate: async (read: (arg: A) => T, arg: A) => read(arg) } as unknown as Parameters[0]; + vi.spyOn(console, "info").mockImplementation(() => undefined); + document.querySelector('#comment-reply')!.innerHTML = 'website.htmlDownloadMorgan'; + const value = input(); value.projects = []; value.activity = []; + value.comments[1]!.body = "Prepared [website.html](/api/attachments/file/content). [Download](/RUN/api/attachments/file/content?download=1) from [Morgan](//RUN/agents/morgan)."; + const result = await collectConversationLinksWithDiagnostics(page, value); + expect(result.links.map(row => row.href)).toEqual(["/api/attachments/file/content"]); + expect(result.rejected?.map(row => [row.href, row.reason])).toEqual([["/RUN/api/attachments/file/content?download=1", "not-a-served-route"], ["/RUN/agents/morgan", "protocol-relative"]]); + }); +}); diff --git a/tests/runner-e2e/chat-rendered-links.ts b/tests/runner-e2e/chat-rendered-links.ts index 7d17eb4c3e2..0423cfa688c 100644 --- a/tests/runner-e2e/chat-rendered-links.ts +++ b/tests/runner-e2e/chat-rendered-links.ts @@ -1,7 +1,7 @@ import type { Locator, Page } from "@playwright/test"; import type { ChatRun } from "./chat-flow.js"; -interface Row { id: string; companyId?: string; issueId?: string; authorAgentId?: string | null; createdByRunId?: string | null; name?: string; action?: string; entityType?: string; entityId?: string; agentId?: string | null; actorType?: string; actorId?: string; runId?: string | null } +interface Row { id: string; companyId?: string; issueId?: string; body?: string | null; authorAgentId?: string | null; createdByRunId?: string | null; name?: string; action?: string; entityType?: string; entityId?: string; agentId?: string | null; actorType?: string; actorId?: string; runId?: string | null } export interface RenderedConversationLink { threadTestId: "task-chat-thread" | "thread-root"; href: string; source: "reply" | "project-created-card"; conversationId: string; anchorCommentId: string; commentId?: string; resourceId?: string; cardLabel?: string; createdByRunId?: string; @@ -42,6 +42,36 @@ export function unresolvedLinkTarget(href: string): boolean { try { decoded = decodeURIComponent(href); } catch { /* keep the raw target */ } return /[<>{}]/.test(decoded); } +/** First path segments the board app routes, as declared in ui/src/App.tsx. A company-prefixed page is + * ///...; the server serves its API only at /api. */ +export const APP_ROUTE_ROOTS: readonly string[] = ["activity", "agents", "approvals", "apps", "artifacts", "assistant-connections", "audit", "auth", + "board-chat", "board-claim", "budgets", "cases", "chat-identity", "chats", "cli-auth", "companies", "company", "costs", "dashboard", "decisions", + "design-guide", "dev", "dot-connect", "execution-workspaces", "goals", "inbox", "instance", "invite", "issues", "mcp-connect", "mcp-device", + "oauth-handoff", "onboarding", "org", "plugins", "projects", "routines", "runs", "search", "settings", "skills", "status", "status-cards", "tasks", + "tests", "timeline", "tools", "u", "ux-lab", "workspaces"]; +export type UnservedLinkReason = "protocol-relative" | "not-a-served-route"; +/** Why a rendered same-origin link opens nothing the product serves, or undefined when it does. `savedTargets` are the + * Markdown targets of the saved reply that rendered it: the board repairs `//PREFIX/...` to `/PREFIX/...`, but the saved + * reply is also what notifications, chat channels and the API deliver, where a protocol-relative target leaves the app. */ +export function unservedLinkReason(href: string, origin: string, savedTargets: readonly string[] = []): UnservedLinkReason | undefined { + if (href.trim().startsWith("//") || !savedTargets.includes(href) && savedTargets.some(target => target.startsWith("//") && target.slice(1) === href)) return "protocol-relative"; + let url: URL; + try { url = new URL(href, origin); } catch { return "not-a-served-route"; } + if (url.origin !== new URL(origin).origin) return undefined; + const parts = url.pathname.split("/").filter(Boolean).map(part => part.toLowerCase()); + if (!parts.length || parts[0] === "api" || APP_ROUTE_ROOTS.includes(parts[0]!)) return undefined; + return parts.length === 1 || APP_ROUTE_ROOTS.includes(parts[1]!) ? undefined : "not-a-served-route"; +} +/** Split rendered links into those that open a served app page or API route and those that do not. */ +export function partitionServedConversationLinks(rows: readonly RenderedConversationLink[], input: Pick) { + const links: RenderedConversationLink[] = [], rejected: Array = []; + for (const row of rows) { + const body = row.commentId ? input.comments.find(comment => comment.id === row.commentId)?.body : undefined; + const reason = unservedLinkReason(row.href, input.origin, body ? markdownLinkTargets(body) : []); + if (reason) rejected.push({ ...row, reason }); else links.push(row); + } + return { links, rejected }; +} export function conversationLinkBindings(input: ConversationLinkInput) { const comments = input.comments.filter(row => row.companyId === input.companyId && row.issueId === input.sourceId); const replies = comments.filter(row => row.authorAgentId === input.leadId && input.runs.some(run => @@ -227,17 +257,20 @@ export function readConversationLinkDiagnostics(input: Pick [id, root.querySelector(`[id="comment-${id}"]`)?.querySelectorAll("a[href]").length ?? 0])) }; }) }; } -export async function collectConversationLinksWithDiagnostics(page: Page, input: ConversationLinkInput): Promise<{ links: RenderedConversationLink[]; diagnostics?: ConversationLinkDiagnostics }> { +export async function collectConversationLinksWithDiagnostics(page: Page, input: ConversationLinkInput): Promise<{ links: RenderedConversationLink[]; + rejected?: Array; diagnostics?: ConversationLinkDiagnostics }> { const bindings = conversationLinkBindings(input); const render = await waitForLatestConversationReplyRender(page, bindings); - const links = await page.evaluate(readConversationDomLinks, bindings); - if (links.length || !bindings.replies.length) return { links }; + const { links, rejected } = partitionServedConversationLinks(await page.evaluate(readConversationDomLinks, bindings), input); + if (rejected.length) console.info(`[chat-rendered-links] links that open no served route: ${JSON.stringify(rejected.map(row => ({ href: row.href, reason: row.reason })))}`); + const kept = rejected.length ? { rejected } : {}; + if (links.length || !bindings.replies.length) return { links, ...kept }; let diagnostics: ConversationLinkDiagnostics; try { diagnostics = { render, ...await page.evaluate(readConversationLinkDiagnostics, { anchors: bindings.anchors, replies: bindings.replies }) }; } catch { diagnostics = { render, expectedReplyIds: bindings.replies, savedCommentCount: bindings.anchors.length, threads: [] }; } // Logged so an empty capture says whether the thread was closed, another conversation, or the reply had no anchors yet. console.info(`[chat-rendered-links] no rendered reply links: ${JSON.stringify(diagnostics)}`); - return { links, diagnostics }; + return { links, ...kept, diagnostics }; } export function conversationLinkLocator(page: Page, link: RenderedConversationLink): Locator { const thread = page.getByTestId(link.threadTestId).filter({ has: page.locator(`[id=${JSON.stringify(`comment-${link.anchorCommentId}`)}]`) });