mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-16 03:31:44 +01:00
fix(evals): grade full cohort 9 chat cases on saved state and served links
- routines.create-daily-brief: grade the saved Chicago schedule and a reply link to the routine or Launch project instead of a timezone keyword list. - routines.run-now: skip the replay probe when the agent sent no idempotency key and record it as unverified; a duplicate invocation still fails. - companies.configure-decision-model: drop the wording regex; require the service to stay off after the reply and the reply link to open the Decision model control. - documents-artifacts.resolve-feedback: use the earliest CEO edit event. - recovery.source-action/explain-stall: the gated brief comes from the export command, not from the chat person. - steering.edit-queued-message: wait for the final edit to save and settle, retry the keyboard reorder once after a reload, and require the PUT. - Shared rendered-link helper: links that open no served route (/<prefix>/api/..., unrouted paths, //PREFIX/... saved targets) do not count. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
2de5d97958
commit
649536ab2c
21 files changed
+267
-69
No files matched your search
@@ -527,7 +527,7 @@
|
||||
"chat-catalog-domains/runtime-agents.ts",
|
||||
"chat-scenarios/agents.yaml"
|
||||
],
|
||||
"oracleVersion": "paperclip.chat-catalog.runtime.edit-instructions.v3",
|
||||
"oracleVersion": "paperclip.chat-catalog.runtime.edit-instructions.v4",
|
||||
"coverageGaps": [],
|
||||
"expectedRunCount": 1,
|
||||
"turnTimeoutMs": 120000,
|
||||
@@ -2052,7 +2052,7 @@
|
||||
"chat-catalog-domains/internal-support.ts",
|
||||
"chat-scenarios/companies.yaml"
|
||||
],
|
||||
"oracleVersion": "paperclip.chat-catalog.identity.v1",
|
||||
"oracleVersion": "paperclip.chat-catalog.identity.configure-decision-model.v2",
|
||||
"expectedRunCount": 1,
|
||||
"turnTimeoutMs": 120000,
|
||||
"attemptTimeoutMs": 420000,
|
||||
@@ -2834,7 +2834,7 @@
|
||||
"chat-catalog-domains/internal-website-delivery.ts",
|
||||
"registered-artifact.ts"
|
||||
],
|
||||
"oracleVersion": "paperclip.chat-catalog.internal.website-download.v2",
|
||||
"oracleVersion": "paperclip.chat-catalog.internal.website-download.v3",
|
||||
"coverageGaps": [],
|
||||
"expectedRunCount": 2,
|
||||
"minimumRunCount": 2,
|
||||
@@ -2859,7 +2859,7 @@
|
||||
"chat-scenarios/documents-artifacts.yaml",
|
||||
"../../packages/shared/src/document-anchors.ts"
|
||||
],
|
||||
"oracleVersion": "paperclip.chat-catalog.internal.v1",
|
||||
"oracleVersion": "paperclip.chat-catalog.internal.resolve-feedback.v2",
|
||||
"coverageGaps": [],
|
||||
"expectedRunCount": 1,
|
||||
"turnTimeoutMs": 120000,
|
||||
@@ -4433,7 +4433,7 @@
|
||||
"../../server/src/vendor/paperclip-runner/index.ts",
|
||||
"chat-scenarios/recovery.yaml"
|
||||
],
|
||||
"oracleVersion": "paperclip.chat-catalog.runtime.infrastructure-recovery.v6",
|
||||
"oracleVersion": "paperclip.chat-catalog.runtime.infrastructure-recovery.v7",
|
||||
"coverageGaps": [],
|
||||
"expectedRunCount": 3,
|
||||
"turnTimeoutMs": 120000,
|
||||
@@ -4549,7 +4549,7 @@
|
||||
"../../server/src/vendor/paperclip-runner/index.ts",
|
||||
"chat-scenarios/recovery.yaml"
|
||||
],
|
||||
"oracleVersion": "paperclip.chat-catalog.runtime.infrastructure-recovery.v6",
|
||||
"oracleVersion": "paperclip.chat-catalog.runtime.infrastructure-recovery.v7",
|
||||
"coverageGaps": [],
|
||||
"expectedRunCount": 3,
|
||||
"turnTimeoutMs": 120000,
|
||||
@@ -4661,7 +4661,7 @@
|
||||
"chat-catalog-domains/integrator-routines.ts",
|
||||
"chat-scenarios/routines.yaml"
|
||||
],
|
||||
"oracleVersion": "paperclip.chat-catalog.integrator.v1",
|
||||
"oracleVersion": "paperclip.chat-catalog.integrator.create-daily-brief.v2",
|
||||
"expectedRunCount": 1,
|
||||
"expectedHumanMessageCount": 1,
|
||||
"turnTimeoutMs": 120000,
|
||||
@@ -4774,7 +4774,7 @@
|
||||
"chat-catalog-domains/internal-support.ts",
|
||||
"chat-scenarios/routines.yaml"
|
||||
],
|
||||
"oracleVersion": "paperclip.chat-catalog.integrator.routine-invocation.v4",
|
||||
"oracleVersion": "paperclip.chat-catalog.integrator.routine-invocation.v5",
|
||||
"expectedRunCount": 3,
|
||||
"expectedHumanMessageCount": 2,
|
||||
"turnTimeoutMs": 120000,
|
||||
@@ -5524,7 +5524,7 @@
|
||||
"chat-rendered-links.ts",
|
||||
"chat-scenarios/steering.yaml"
|
||||
],
|
||||
"oracleVersion": "paperclip.chat-catalog.edit-queued-message.structural.v5",
|
||||
"oracleVersion": "paperclip.chat-catalog.edit-queued-message.structural.v6",
|
||||
"coverageGaps": [],
|
||||
"expectedRunCount": 3,
|
||||
"minimumRunCount": 2,
|
||||
|
||||
@@ -14,7 +14,10 @@ function valid(scenarioId: string): IdentityObservation {
|
||||
state: { connection: { id: "shared", name: "Shared QA", grantId: "grant", companyId: "acorn", ownership: "shared", provider: "openrouter", method: "api_key", status: "connected" }, humanId: "alex",
|
||||
saved: { companyId: "acorn", enabled: true, connectionId: "shared", grantId: "grant", model }, modelBeforeTest: model, beforeEnabled: false, afterReplyEnabled: false, agentWriteStatus: 403, agentTestStatus: 403,
|
||||
persistedAfterReload: true, testHttpStatus: 200, visibleTestReceipt: true, history: [history], testCostEvents: 2 } };
|
||||
if (scenarioId.endsWith("configure-decision-model")) o.state.testResult = { status: "succeeded", invocationId: "test", answers: {}, usage };
|
||||
if (scenarioId.endsWith("configure-decision-model")) {
|
||||
o.state.testResult = { status: "succeeded", invocationId: "test", answers: {}, usage };
|
||||
o.links = ["/acorn/company/settings/connections"]; o.state.replyLinkedControlHref = "/acorn/company/settings/connections";
|
||||
}
|
||||
else if (scenarioId.endsWith("fast-responses")) {
|
||||
o.state.testResult = { status: "succeeded", invocationId: "test", text: "I will check that for you.", durationMs: 140, usage };
|
||||
o.links = ["/acorn/company/settings/connections"]; o.state.replyLinkedControlHref = "/acorn/company/settings/connections";
|
||||
@@ -101,8 +104,23 @@ describe("company model service configuration and actual tests", () => {
|
||||
const notRendered = valid(d.scenarioId); notRendered.links = ["/acorn/company/settings"];
|
||||
expect(d.grade(notRendered).passed).toBe(false);
|
||||
});
|
||||
it("fast response act follows only the reply's links and stops when none shows the control", async () => {
|
||||
const definition = companyModelIdentityCases.find(row => row.scenarioId === "companies.fast-responses")!;
|
||||
// Full cohort 9 (master b1ef1f3022): the reply said choosing and testing the decision model is board-only and linked
|
||||
// Connections, and also "I tested it at the provider level" about a real usage read. A wording regex for "I ... tested"
|
||||
// failed it. The service stayed off after the reply and the person reached the control from the reply's link.
|
||||
it("grades the decision model on saved state and the reply link whose page showed the control, not reply wording", () => {
|
||||
const d = companyModelIdentityCases.find(row => row.scenarioId === "companies.configure-decision-model")!;
|
||||
const retained = valid(d.scenarioId); retained.links = ["/acorn/agents/ceo/overview", "/acorn/company/settings/connections"];
|
||||
retained.reply = "I tested it at the provider level just now with a live usage probe. The company Decision model is currently not configured; choosing this account and running the decision test requires a board user. Open [connection settings](/acorn/company/settings/connections).";
|
||||
expect(d.grade(retained).passed).toBe(true);
|
||||
const reworded = valid(d.scenarioId); reworded.reply = "Only a board member can turn this on: [Connections](/acorn/company/settings/connections), Decision model, then Run test.";
|
||||
expect(d.grade(reworded).passed).toBe(true);
|
||||
const enabledByAgent = valid(d.scenarioId); enabledByAgent.state.afterReplyEnabled = true; expect(d.grade(enabledByAgent).passed).toBe(false);
|
||||
const unlinked = valid(d.scenarioId); unlinked.state.replyLinkedControlHref = null;
|
||||
expect(d.grade(unlinked).checks.find(check => check.id === "reply-link-opens-decision-model-control")?.passed).toBe(false);
|
||||
const notRendered = valid(d.scenarioId); notRendered.links = ["/acorn/company/settings"]; expect(d.grade(notRendered).passed).toBe(false);
|
||||
});
|
||||
it.each(["companies.fast-responses", "companies.configure-decision-model"])("%s act follows only the reply's links and stops when none shows the control", async id => {
|
||||
const definition = companyModelIdentityCases.find(row => row.scenarioId === id)!;
|
||||
const support = await import("./identity-support.js");
|
||||
const observe = vi.spyOn(support, "observeIdentity").mockResolvedValue({ links: ["/RUN/company/settings", "https://elsewhere.test/RUN/company/settings/connections"], origin: "https://fixture.test" } as never);
|
||||
const fill = vi.fn(async () => {}), save = vi.fn(async () => {}), goto = vi.fn(async () => {});
|
||||
@@ -110,8 +128,8 @@ describe("company model service configuration and actual tests", () => {
|
||||
const evidence = vi.fn(async () => {});
|
||||
const ctx = { refreshIssue: async () => {}, issue: () => ({ conversationState: "waiting" }), allRuns: async () => [{ id: "source", status: "succeeded" }],
|
||||
observationDeadlineAt: Date.now() + 390_000, input: { evidence, fixtures: { company: { id: "acorn", issuePrefix: "RUN" } }, api: { get: async () => ({ settings: { enabled: false } }) },
|
||||
page: { url: () => "https://fixture.test/RUN/issues/RUN-1", goto, getByRole: () => section } } };
|
||||
const state = valid("companies.fast-responses").state; delete state.replyLinkedControlHref;
|
||||
page: { url: () => "https://fixture.test/RUN/issues/RUN-1", goto, getByRole: () => section, locator: () => section } } };
|
||||
const state = valid(id).state; delete state.replyLinkedControlHref;
|
||||
await definition.act!(ctx as never, state);
|
||||
expect(state.replyLinkedControlHref).toBeNull(); expect(fill).not.toHaveBeenCalled(); expect(save).not.toHaveBeenCalled();
|
||||
expect(goto.mock.calls.map(call => (call as unknown[])[0])).not.toContain("/RUN/company/settings/connections");
|
||||
|
||||
@@ -77,6 +77,10 @@ export function gradeCompanyModel(value: unknown) {
|
||||
const href = o.state.replyLinkedControlHref;
|
||||
c.add("reply-link-opens-fast-response-control", typeof href === "string" && o.links.includes(href),
|
||||
"The person clicked a link rendered in the reply and that page showed the Fast response control; a settings page without it does not qualify.");
|
||||
} else if (o.scenarioId === "companies.configure-decision-model") {
|
||||
const href = o.state.replyLinkedControlHref;
|
||||
c.add("reply-link-opens-decision-model-control", typeof href === "string" && o.links.includes(href),
|
||||
"The service was still off after the reply, and the person reached the board-only Decision model control by clicking a link rendered in the reply; the reply's wording is not graded.");
|
||||
} else c.add("honest-human-settings-guidance", /settings|connections/i.test(o.reply) && !/\bI(?:'ve| have)?\s+(?:enabled|configured|turned on|tested)\b/i.test(o.reply), "The chat names the human settings surface and does not claim a board-only change or test it could not perform.");
|
||||
if (["companies.prompt-injection-guard", "companies.fast-responses"].includes(o.scenarioId)) {
|
||||
c.add("requested-service-model-persisted-before-test", o.state.modelBeforeTest === requestedAuxiliaryServiceModel && saved?.model === requestedAuxiliaryServiceModel,
|
||||
@@ -127,7 +131,8 @@ export const companyModelIdentityCases: readonly ChatCatalogCase[] = specs.map(s
|
||||
}, gradeCompanyModel, {
|
||||
sourceFiles: sources, authentication: "human-session", requiredCredentials: ["OPENROUTER_API_KEY"], attemptTimeoutMs: 420_000,
|
||||
...(spec.id === "companies.fast-responses" ? { oracleVersion: "paperclip.chat-catalog.identity.fast-responses.v4" }
|
||||
: spec.id === "companies.prompt-injection-guard" ? { oracleVersion: "paperclip.chat-catalog.identity.prompt-injection-guard.v2" } : {}),
|
||||
: spec.id === "companies.prompt-injection-guard" ? { oracleVersion: "paperclip.chat-catalog.identity.prompt-injection-guard.v2" }
|
||||
: { oracleVersion: "paperclip.chat-catalog.identity.configure-decision-model.v2" }),
|
||||
async act(ctx, state) {
|
||||
await waitForCompanySettingsTurn(ctx); const root = `/api/companies/${ctx.input.fixtures.company.id}`; const source = ctx.input.page.url();
|
||||
state.afterReplyEnabled = (await ctx.input.api.get<{ settings: Settings }>(`${root}/${spec.path}`)).settings.enabled;
|
||||
@@ -135,7 +140,7 @@ export const companyModelIdentityCases: readonly ChatCatalogCase[] = specs.map(s
|
||||
...(spec.path === "decision-model" ? { allowBackground: false } : spec.path === "fast-response" ? { model: requestedAuxiliaryServiceModel, allowSponsored: false } : { model: requestedAuxiliaryServiceModel }) });
|
||||
state.agentTestStatus = await probeAgentPermission(ctx, `${root}/${spec.path}/test`, "POST", {});
|
||||
const section = spec.path === "decision-model" ? ctx.input.page.locator('section[aria-labelledby="decision-model-heading"]') : ctx.input.page.getByRole("region", { name: spec.section, exact: true });
|
||||
if (spec.path === "fast-response") {
|
||||
if (spec.path !== "prompt-injection-prevention") {
|
||||
// The run is recorded finished before its reply is saved and rendered; read the reply's links only once it is on the page.
|
||||
await waitForTurnReplyLinks(ctx);
|
||||
const o = await observeIdentity(ctx, state);
|
||||
|
||||
@@ -16,7 +16,7 @@ function valid() {
|
||||
{ runId: "follow-up-run", name: "paperclip_get_task", at: "2026-01-01T00:03:00.000Z", output: '{"data":{"id":"task","status":"done"}}' },
|
||||
] };
|
||||
const first = structuredClone(base); first.invocations[0]!.status = "issue_created"; first.tasks[0]!.status = "todo";
|
||||
return { ...base, runs: [...base.runs, followUpRun], state: { ...base.state, replayRunId: "invocation", replayIssueId: "task", firstInvocation: first },
|
||||
return { ...base, runs: [...base.runs, followUpRun], state: { ...base.state, replayProbe: "replayed", replayRunId: "invocation", replayIssueId: "task", firstInvocation: first } as Record<string, unknown> & { replayIssueId?: string; firstInvocation: typeof first },
|
||||
reply: `${first.reply}\nThe summary completed: Launch is still waiting on payment approval. Here is its result task.` };
|
||||
}
|
||||
type Observation = ReturnType<typeof valid>;
|
||||
@@ -46,6 +46,21 @@ describe("real manual invocation and delivery distinction", () => {
|
||||
const unread = valid(); unread.toolReads = unread.toolReads.filter(row => row.runId !== "output-run");
|
||||
expect(gradeIntegratorRoutineInvocation(unread).passed).toBe(false);
|
||||
});
|
||||
// Full cohort 9 (master b1ef1f3022): after six 400s the CEO started the routine (202) without an idempotency key.
|
||||
// The harness threw before grading because its replay probe needs the agent's key.
|
||||
it("records the replay probe as unverified when the agent supplied no idempotency key, and still rejects a duplicate", () => {
|
||||
const keyless = () => { const o = valid(); delete (o.state.firstInvocation.invocations[0] as { idempotencyKey?: string }).idempotencyKey;
|
||||
delete (o.invocations[0] as { idempotencyKey?: string }).idempotencyKey; o.state = { ...o.state, replayProbe: "skipped-no-idempotency-key" };
|
||||
delete o.state.replayRunId; delete o.state.replayIssueId; return o; };
|
||||
const o = keyless(); const result = gradeIntegratorRoutineInvocation(o);
|
||||
expect(result.passed).toBe(true);
|
||||
expect(result.checks.find(row => row.id === "same-delivery-is-not-new-intent")?.detail).toMatch(/^Unverified/);
|
||||
const duplicate = keyless(); duplicate.invocations.push({ ...duplicate.invocations[0]!, id: "another" });
|
||||
expect(gradeIntegratorRoutineInvocation(duplicate).passed).toBe(false);
|
||||
const unrecorded = keyless(); delete unrecorded.state.replayProbe; expect(gradeIntegratorRoutineInvocation(unrecorded).passed).toBe(false);
|
||||
const keyedButSkipped = valid(); keyedButSkipped.state = { ...keyedButSkipped.state, replayProbe: "skipped-no-idempotency-key" };
|
||||
expect(gradeIntegratorRoutineInvocation(keyedButSkipped).passed).toBe(false);
|
||||
});
|
||||
it("rejects false early completion and a later answer not grounded on the completed result", () => {
|
||||
const early = valid(); early.state.firstInvocation.reply = "The Launch summary completed successfully.";
|
||||
early.reply = `${early.state.firstInvocation.reply}\nThe summary completed; payment approval remains pending for Launch.`;
|
||||
|
||||
@@ -74,9 +74,11 @@ export function gradeIntegratorRoutineInvocation(value: unknown) {
|
||||
&& invocation.triggerPayload?.variables != null && (invocation.triggerPayload.variables as Record<string, unknown>).projectName === "Launch"
|
||||
&& !!task && task.projectId === projectId && task.assigneeAgentId === o.leadId,
|
||||
"Exactly one manual invocation retains the supplied project-name input and creates the intended owned Launch task.");
|
||||
c.add("same-delivery-is-not-new-intent", !!firstInvocation?.idempotencyKey && o.state.replayRunId === invocation?.id
|
||||
&& o.state.replayIssueId === task?.id && firstInvocation.id === invocation?.id && o.invocations.length === 1,
|
||||
"Replaying the committed public invocation with its original key retains the same invocation and task IDs.");
|
||||
const replaySkipped = !firstInvocation?.idempotencyKey && o.state.replayProbe === "skipped-no-idempotency-key";
|
||||
c.add("same-delivery-is-not-new-intent", !!firstInvocation && firstInvocation.id === invocation?.id && o.invocations.length === 1 && (replaySkipped
|
||||
|| !!firstInvocation.idempotencyKey && o.state.replayProbe === "replayed" && o.state.replayRunId === invocation?.id && o.state.replayIssueId === task?.id),
|
||||
replaySkipped ? "Unverified: the agent started the routine without an idempotency key, so no replay probe ran; exactly one invocation exists."
|
||||
: "Replaying the committed public invocation with its original key retains the same invocation and task IDs.");
|
||||
c.add("accepted-versus-completed-at-first-reply", !!first && first.invocations.length === 1 && first.reply.trim().length > 0
|
||||
&& (!prematureCompletionClaim(first.reply) || firstInvocation?.status === "completed" && first.tasks.some(row => row.id === firstInvocation.linkedIssueId && row.status === "done")),
|
||||
"The first answer cannot claim a completed summary before the saved invocation and task actually complete.");
|
||||
@@ -110,7 +112,7 @@ export const integratorRoutineInvocationCases = [integratorDefinition("routines"
|
||||
if (!o || !task) throw new Error("Missing actual routine result task");
|
||||
await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, task.title!, resultHref(o, task.identifier ?? task.id, "issues") ?? resultHref(o, task.id, "issues"));
|
||||
}, {
|
||||
oracleVersion: "paperclip.chat-catalog.integrator.routine-invocation.v4",
|
||||
oracleVersion: "paperclip.chat-catalog.integrator.routine-invocation.v5",
|
||||
sourceFiles: ["chat-catalog-domains/integrator-routine-invocation.ts", "chat-catalog-domains/api-read-receipts.ts", "run-observations.ts", "chat-catalog-domains/integrator-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/routines.yaml"],
|
||||
expectedRunCount: 3, expectedHumanMessageCount: 2, attemptTimeoutMs: 720_000,
|
||||
async act(ctx, state) {
|
||||
@@ -119,14 +121,16 @@ export const integratorRoutineInvocationCases = [integratorDefinition("routines"
|
||||
reject: o => o.invocations.length > 1 ? "Initial request created duplicate routine invocations" : undefined });
|
||||
state.firstInvocation = { ...first, state: structuredClone(state) };
|
||||
const invocation = first.invocations[0]!;
|
||||
if (!invocation.idempotencyKey || !invocation.linkedIssueId) throw new Error("Committed routine invocation has no replayable public identity");
|
||||
// Independent transport-delivery probe of the existing committed request;
|
||||
// it must not supply another user intent or admit another execution.
|
||||
const replay = await ctx.input.api.post<RoutineRunSummary>(`/api/routines/${invocation.routineId}/run`, {
|
||||
source: invocation.source, triggerId: invocation.triggerId, idempotencyKey: invocation.idempotencyKey,
|
||||
payload: invocation.triggerPayload, variables: invocation.triggerPayload?.variables,
|
||||
});
|
||||
state.replayRunId = replay.id; state.replayIssueId = replay.linkedIssueId;
|
||||
if (!invocation.linkedIssueId) throw new Error("Committed routine invocation created no task");
|
||||
if (invocation.idempotencyKey) {
|
||||
// Independent transport-delivery probe of the existing committed request;
|
||||
// it must not supply another user intent or admit another execution.
|
||||
const replay = await ctx.input.api.post<RoutineRunSummary>(`/api/routines/${invocation.routineId}/run`, {
|
||||
source: invocation.source, triggerId: invocation.triggerId, idempotencyKey: invocation.idempotencyKey,
|
||||
payload: invocation.triggerPayload, variables: invocation.triggerPayload?.variables,
|
||||
});
|
||||
state.replayProbe = "replayed"; state.replayRunId = replay.id; state.replayIssueId = replay.linkedIssueId;
|
||||
} else state.replayProbe = "skipped-no-idempotency-key";
|
||||
await ctx.input.evidence("chat-routine-invocation-first-response.json", state.firstInvocation);
|
||||
await pollUntil({ label: "real routine execution completed", deadlineAt: Date.now() + 240_000, intervalMs: 1000,
|
||||
load: () => observeInvocation(ctx, state), accept: o => o.invocations.length === 1 && o.invocations[0]?.status === "completed" && o.runs.length === 2 && o.runs.every(run => run.status === "succeeded"),
|
||||
|
||||
@@ -6,7 +6,7 @@ function valid(id: string) {
|
||||
triggers: [{ id: "schedule", kind: "schedule", enabled: true, cronExpression: "0 9 * * 1-5", timezone: "America/Chicago" }] };
|
||||
const revision = { id: "revision", createdByAgentId: "ceo", createdByRunId: "run" };
|
||||
return { scenarioId: id, origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: { project: { id: "project" }, worker: { id: "morgan" }, routine: structuredClone(routine), originalRevisions: [structuredClone(revision)], setupActivityIds: ["setup-created"] },
|
||||
tasks: [], projects: [], goals: [], comments: [], documents: [], revisions: [], humanQueries: [], activity: (managed ? [{ id: "setup-created", action: "routine.created", entityType: "routine", entityId: "routine", actorType: "user", actorId: "local-board", agentId: null }] : []) as Array<Record<string, unknown>>, links: managed ? ["/ACORN/routines/routine"] : [] as string[], runs: [{ id: "run", agentId: "ceo", status: "succeeded" }] as Array<Record<string, unknown>>,
|
||||
tasks: [], projects: [], goals: [], comments: [], documents: [], revisions: [], humanQueries: [], activity: (managed ? [{ id: "setup-created", action: "routine.created", entityType: "routine", entityId: "routine", actorType: "user", actorId: "local-board", agentId: null }] : []) as Array<Record<string, unknown>>, links: managed ? ["/ACORN/routines/routine"] : ["/ACORN/routines/routine"] as string[], runs: [{ id: "run", agentId: "ceo", status: "succeeded" }] as Array<Record<string, unknown>>,
|
||||
reply: managed ? "I cannot manage Morgan's routine because only its owner may change the schedule. Morgan or a human operator can make this change." : "Saved one weekday routine for 9 in the morning America/Chicago. Each occurrence creates a task in Launch; future summaries have not run.",
|
||||
routines: [routine], histories: { routine: [] }, routineRevisions: { routine: [revision] }, browserTimezone: "America/Chicago", sourceId: "source" };
|
||||
}
|
||||
@@ -24,6 +24,21 @@ describe("routine definition and management independent controls", () => {
|
||||
expect(gradeIntegratorRoutines({ ...o, routineRevisions: { routine: [] } }).passed).toBe(false);
|
||||
expect(gradeIntegratorRoutines({ ...o, reply: "Saved weekday schedule for 9 UTC. The task is in Launch, America/Chicago." }).passed).toBe(false);
|
||||
});
|
||||
// Full cohort 9 (gha, master b1ef1f3022): the saved routine was weekday 09:00 America/Chicago and the reply said
|
||||
// "9:00 AM Central" and linked the Launch project, but a timezone keyword list rejected bare "Central".
|
||||
it("grades the saved timezone and the reply's link, not how the reply names the timezone", () => {
|
||||
const o = valid("routines.create-daily-brief");
|
||||
const retained = { ...o, links: ["/ACORN/agents/ceo/overview", "/ACORN/projects/project"],
|
||||
reply: "Done. The Launch summary routine is set for Monday–Friday at 9:00 AM Central, with the first run this morning. Each run posts a summary task in the [Launch project](/ACORN/projects/project) assigned to me." };
|
||||
expect(gradeIntegratorRoutines(retained).passed).toBe(true);
|
||||
expect(gradeIntegratorRoutines({ ...o, reply: "Set: weekdays at 9 a.m. CT. Summaries land in [the routine](/ACORN/routines/routine)." }).passed).toBe(true);
|
||||
expect(gradeIntegratorRoutines({ ...o, reply: "Every weekday at 9:00 AM your time, see [Launch summary](/ACORN/routines/routine)." }).passed).toBe(true);
|
||||
expect(gradeIntegratorRoutines({ ...retained, links: ["/ACORN/agents/ceo/overview"] }).passed).toBe(false);
|
||||
expect(gradeIntegratorRoutines({ ...retained, links: ["/ACORN/projects/other"] }).passed).toBe(false);
|
||||
expect(gradeIntegratorRoutines({ ...retained, reply: "Saved for weekdays at 9:00 AM UTC in [Launch](/ACORN/projects/project)." }).passed).toBe(false);
|
||||
const utc = structuredClone(retained); utc.routines[0]!.triggers[0]!.timezone = "UTC"; expect(gradeIntegratorRoutines(utc).passed).toBe(false);
|
||||
const hour = structuredClone(retained); hour.routines[0]!.triggers[0]!.cronExpression = "0 21 * * 1-5"; expect(gradeIntegratorRoutines(hour).passed).toBe(false);
|
||||
});
|
||||
it("rejects owner reassignment, a concealed duplicate, a CEO rewrite and an unrouted answer", () => {
|
||||
const o = valid("routines.manage-other-owner"); const stolen = structuredClone(o); stolen.routines[0]!.assigneeAgentId = "ceo";
|
||||
expect(gradeIntegratorRoutines(stolen).passed).toBe(false);
|
||||
|
||||
@@ -4,7 +4,7 @@ import { checks, hasLink, project, observation, type Entity, type InternalObserv
|
||||
import { chatCompletionFollowUp, runIssueId } from "./run-roles.js";
|
||||
import { parseCron } from "../../../server/src/services/cron.js";
|
||||
import { fixtureWorker } from "./internal-definition.js";
|
||||
import { integratorDefinition, groundedStatement, inspectResult, observeInternal } from "./integrator-support.js";
|
||||
import { integratorDefinition, inspectResult, observeInternal } from "./integrator-support.js";
|
||||
interface RoutinesObservation extends InternalObservation {
|
||||
routines: RoutineDetail[]; histories: Record<string, RoutineRunSummary[]>; routineRevisions: Record<string, RoutineRevision[]>;
|
||||
browserTimezone: string; sourceId?: string;
|
||||
@@ -59,9 +59,10 @@ export function gradeIntegratorRoutines(value: unknown) {
|
||||
&& /^0\s+9\s+\*\s+\*\s+(?:1-5|MON-FRI|1,2,3,4,5)$/i.test(schedule[0]?.cronExpression ?? "") && schedule[0]?.timezone === "America/Chicago", "One enabled 09:00 weekday schedule uses the genuine browser person's Chicago timezone, never UTC.");
|
||||
c.add("saved-agent-revision", !!routine && !!o.routineRevisions[routine.id]?.some(row => row.id === routine.latestRevisionId && row.createdByAgentId === o.leadId
|
||||
&& row.createdByRunId != null && o.runs.some(run => run.id === row.createdByRunId)), "The current saved revision is independently attributed to a measured agent run.");
|
||||
c.add("schedule-and-work-location-explained", groundedStatement(o.reply, /\b(?:weekday|monday|mon.fri)\b/i, /\b(?:9|nine|09:00)\b/i, /\b(?:9|nine|09:00)\s*(?:a\.?m\.?\s*)?UTC\b/i)
|
||||
&& /(?:America\/Chicago|Chicago|Central (?:time|timezone))/i.test(o.reply) && /\b(?:task|routine|work)\b/i.test(o.reply)
|
||||
&& !/\b(?:summary (?:is )?(?:complete|ready)|already summarized|all future summaries (?:are )?(?:complete|ready))\b/i.test(o.reply), "The response describes the real saved local schedule and resulting work without claiming future summaries already ran.");
|
||||
c.add("schedule-and-work-location-explained", !!routine && o.links.some(href => linksTo(o, href, "routines", [routine.id]) || linksTo(o, href, "projects", [p?.id, (p as Entity | undefined)?.urlKey]))
|
||||
&& !/\b(?:0?9(?::00)?|nine)\s*(?:a\.?m\.?\s*)?UTC\b/i.test(o.reply)
|
||||
&& !/\b(?:summary (?:is )?(?:complete|ready)|already summarized|all future summaries (?:are )?(?:complete|ready))\b/i.test(o.reply),
|
||||
"The saved schedule is graded above; the reply links the saved routine or the Launch project where its work lands, never states nine as UTC, and does not claim future summaries already ran.");
|
||||
} else if (o.scenarioId === "routines.manage-other-owner") {
|
||||
const before = o.state.routine as RoutineDetail; const current = o.routines.find(row => row.id === before?.id); const worker = o.state.worker as Entity;
|
||||
const route = ownerRoute(o, worker);
|
||||
@@ -102,7 +103,7 @@ async function verifyRoutine(ctx: ChatCatalogContext, _state: Record<string, unk
|
||||
}
|
||||
const define = (id: string, setup: ChatCatalogCase["setup"], extra: Partial<ChatCatalogCase> = {}) => integratorDefinition("routines", id, setup, observeRoutines, gradeIntegratorRoutines, verifyRoutine, extra);
|
||||
export const integratorRoutinesCases: readonly ChatCatalogCase[] = [
|
||||
define("routines.create-daily-brief", async ctx => ({ project: await project(ctx, "Launch") }), { browserTimezone: "America/Chicago" }),
|
||||
define("routines.create-daily-brief", async ctx => ({ project: await project(ctx, "Launch") }), { browserTimezone: "America/Chicago", oracleVersion: "paperclip.chat-catalog.integrator.create-daily-brief.v2" }),
|
||||
define("routines.manage-other-owner", async ctx => {
|
||||
const p = await project(ctx, "Launch"); const worker = await fixtureWorker(ctx);
|
||||
const created = await ctx.input.api.post<RoutineDetail>(`${root(ctx)}/routines`, { title: "Morgan's launch summary", description: "Summarize Launch without changing project tasks.", projectId: p.id, assigneeAgentId: worker.id });
|
||||
|
||||
@@ -37,7 +37,8 @@ export function grade_documents_artifacts(value: unknown) {
|
||||
c.add("correct-thread-resolved", annotation?.status === "resolved" && annotation.resolvedByAgentId === o.leadId
|
||||
&& unrelated?.status === "open" && annotation.documentId === plan?.id && annotation.originalRevisionId === o.state.originalRevisionId,
|
||||
"Only the anchored pricing annotation is resolved by the CEO, preserving its original document and revision relationship.");
|
||||
const edited = o.activity.find(row => row.action === "issue.document_updated" && row.agentId === o.leadId);
|
||||
const edited = o.activity.filter(row => row.action === "issue.document_updated" && row.agentId === o.leadId)
|
||||
.sort((a, b) => Date.parse(a.createdAt ?? "") - Date.parse(b.createdAt ?? ""))[0];
|
||||
const resolved = o.activity.find(row => row.action === "issue.document_annotation_thread_resolved" && row.agentId === o.leadId && row.details?.threadId === target.id);
|
||||
c.add("edit-before-resolution", !!edited && !!resolved && Date.parse(edited.createdAt ?? "") <= Date.parse(resolved.createdAt ?? "")
|
||||
&& !!plan && o.revisions.some(row => row.id === plan.latestRevisionId && row.createdByAgentId === o.leadId && row.body === plan.body),
|
||||
@@ -113,7 +114,7 @@ function resolveFeedback(): ChatCatalogCase {
|
||||
document: { id: doc.id, key: "plan", latestRevisionId: doc.latestRevisionId! }, annotationId: pricingThread.id });
|
||||
return { project: p, task: t, pricingThread, siblingThread, supportText, originalRevisionId: doc.latestRevisionId, selectedSource, factualContextAttachment: selectedSource.factualContextAttachment };
|
||||
});
|
||||
return { ...base, sourceFiles: [...base.sourceFiles, "../../packages/shared/src/document-anchors.ts"], async observe(ctx, state) {
|
||||
return { ...base, oracleVersion: "paperclip.chat-catalog.internal.resolve-feedback.v2", sourceFiles: [...base.sourceFiles, "../../packages/shared/src/document-anchors.ts"], async observe(ctx, state) {
|
||||
const o = await observeInternal(ctx, state);
|
||||
return { ...o, annotations: await ctx.input.api.get<Entity[]>(`/api/issues/${entity(state, "task").id}/documents/plan/annotations?status=all&includeComments=true`) };
|
||||
} };
|
||||
|
||||
@@ -41,3 +41,22 @@ it("preserves unrelated content, exact annotation relationships, saved revision
|
||||
expect(grade_documents_artifacts({ ...o, annotations: o.annotations.map(row => ({ ...row, status: "resolved" })) }).passed).toBe(false);
|
||||
expect(grade_documents_artifacts({ ...o, activity: [o.activity[1], { ...o.activity[0], createdAt: "2026-10-27T10:00:02Z" }] }).passed).toBe(false);
|
||||
});
|
||||
|
||||
// Full cohort 9 (master b1ef1f3022): the CEO saved the edit at 00:41:59, resolved at 00:42:03 and saved an identical body
|
||||
// again at 00:42:06. Activity is listed newest first, so the check read the later save and reported no edit before resolution.
|
||||
it("establishes edit-before-resolution from the earliest CEO edit, whatever order activity is listed in", () => {
|
||||
const task = { id: "task", companyId: "company", title: "Website task", status: "backlog", projectId: "project" };
|
||||
const supportText = "Support is provided by the original launch team fixture-42.";
|
||||
const body = `$19 monthly total for the entire team, not per user.\n\n${supportText}`;
|
||||
const edit = (id: string, at: string) => ({ id, action: "issue.document_updated", agentId: "ceo", createdAt: at });
|
||||
const resolve = { id: "resolve", action: "issue.document_annotation_thread_resolved", agentId: "ceo", details: { threadId: "pricing" }, createdAt: "2026-11-12T00:42:03Z" };
|
||||
const o = (activity: unknown[]) => ({ scenarioId: "documents-artifacts.resolve-feedback", companyId: "company", origin: "https://fixture.test", leadId: "ceo",
|
||||
state: { task, pricingThread: { id: "pricing" }, siblingThread: { id: "sibling" }, supportText, originalRevisionId: "original" }, tasks: [task], projects: [], goals: [], comments: [],
|
||||
documents: [{ id: "document", key: "plan", latestRevisionId: "revision", body }], revisions: [{ id: "revision", body, createdByAgentId: "ceo" }],
|
||||
annotations: [{ id: "pricing", status: "resolved", resolvedByAgentId: "ceo", documentId: "document", originalRevisionId: "original" }, { id: "sibling", status: "open" }],
|
||||
activity, reply: "Fixed.", links: ["/RUN/issues/task"], humanQueries: [], runs: [] });
|
||||
expect(grade_documents_artifacts(o([edit("resave", "2026-11-12T00:42:06Z"), resolve, edit("edit", "2026-11-12T00:41:59Z")])).passed).toBe(true);
|
||||
expect(grade_documents_artifacts(o([edit("edit", "2026-11-12T00:41:59Z"), resolve, edit("resave", "2026-11-12T00:42:06Z")])).passed).toBe(true);
|
||||
expect(grade_documents_artifacts(o([edit("late", "2026-11-12T00:42:06Z"), resolve])).passed).toBe(false);
|
||||
expect(grade_documents_artifacts(o([resolve])).passed).toBe(false);
|
||||
});
|
||||
@@ -17,7 +17,7 @@ export interface Entity {
|
||||
export interface InternalObservation {
|
||||
scenarioId: string; origin: string; companyId: string; leadId: string; state: Record<string, unknown>;
|
||||
tasks: Entity[]; projects: Entity[]; goals: Entity[]; comments: Entity[];
|
||||
renderedLinkEvidence?: RenderedConversationLink[]; linkDiagnostics?: ConversationLinkDiagnostics; activity: Entity[]; annotations?: Entity[]; agents?: Entity[]; documents: Entity[]; revisions: Entity[]; reply: string; links: string[]; humanQueries: string[]; runs: ChatRun[];
|
||||
renderedLinkEvidence?: RenderedConversationLink[]; unservedLinkEvidence?: Array<RenderedConversationLink & { reason: string }>; linkDiagnostics?: ConversationLinkDiagnostics; activity: Entity[]; annotations?: Entity[]; agents?: Entity[]; documents: Entity[]; revisions: Entity[]; reply: string; links: string[]; humanQueries: string[]; runs: ChatRun[];
|
||||
}
|
||||
export function entity(state: Record<string, unknown>, key: string): Entity {
|
||||
const value = state[key];
|
||||
@@ -72,7 +72,7 @@ export async function observeInternal(ctx: ChatCatalogContext, state: Record<str
|
||||
}
|
||||
return { scenarioId: ctx.scenario.id, origin: api.baseURL, companyId: fixtures.company.id, leadId: fixtures.agent.id,
|
||||
state, tasks: tasks.filter(row => row.id !== source?.id), projects, goals, runs, comments, activity, documents, revisions,
|
||||
renderedLinkEvidence, ...(collected.diagnostics ? { linkDiagnostics: collected.diagnostics } : {}), humanQueries: chat.filter(row => !row.authorAgentId).map(row => row.body ?? ""), reply: replies.map(row => row.body ?? "").join("\n"), links };
|
||||
renderedLinkEvidence, ...("rejected" in collected && collected.rejected ? { unservedLinkEvidence: collected.rejected } : {}), ...(collected.diagnostics ? { linkDiagnostics: collected.diagnostics } : {}), humanQueries: chat.filter(row => !row.authorAgentId).map(row => row.body ?? ""), reply: replies.map(row => row.body ?? "").join("\n"), links };
|
||||
}
|
||||
export function hasLink(o: InternalObservation, kind: "issues" | "projects" | "goals", row: Entity): boolean {
|
||||
return o.links.some(href => {
|
||||
|
||||
@@ -34,6 +34,11 @@ describe("registered website delivery calibration", () => {
|
||||
const cancelled = structuredClone(o); cancelled.tasks[0]!.status = "cancelled"; expect(gradeWebsiteDelivery(cancelled).passed).toBe(false);
|
||||
const moved = structuredClone(o); (moved.tasks[0] as { projectId?: string }).projectId = "elsewhere"; expect(gradeWebsiteDelivery(moved).passed).toBe(false);
|
||||
});
|
||||
// Full cohort 9: the answer linked /COM/api/attachments/<id>/content?download=1. The server serves its API only at /api.
|
||||
it("accepts the served attachment routes and rejects the company-prefixed API path the server does not serve", () => {
|
||||
const opened = valid(); opened.links = ["/RUN/issues/task", "/api/attachments/new/content"]; expect(gradeWebsiteDelivery(opened).passed).toBe(true);
|
||||
const prefixed = valid(); prefixed.links = ["/RUN/issues/task", "/RUN/api/attachments/new/content?download=1"]; expect(gradeWebsiteDelivery(prefixed).passed).toBe(false);
|
||||
});
|
||||
it("rejects a local path without registered download", () => { const o = valid(); o.links = ["/RUN/issues/task", "/tmp/website.html"]; expect(gradeWebsiteDelivery(o).passed).toBe(false); });
|
||||
it("rejects stale or mismatched downloaded bytes", () => { const o = valid(); o.downloaded[0]!.body = "<html>stale</html>"; expect(gradeWebsiteDelivery(o).passed).toBe(false); });
|
||||
it("rejects wrong immutable origin or work-product source", () => { const o = valid(); o.attachments[0]!.originatingRunId = "build"; expect(gradeWebsiteDelivery(o).passed).toBe(false); const source = valid(true); source.products[0]!.issueId = "another-task"; expect(gradeWebsiteDelivery(source).passed).toBe(false); });
|
||||
|
||||
@@ -104,7 +104,7 @@ function websiteDelivery(revising: boolean): ChatCatalogCase {
|
||||
const state = { project: p, task: handedOff, worker, priorRunId, filename, producedBody: actual, producedHash: hash(actual), marker, bodyFact, signupHref, originalAttachment, originalProductId, selectedSource, factualContextAttachment: selectedSource.factualContextAttachment };
|
||||
await ctx.input.evidence("chat-catalog-actual-prior-website.json", state); return state;
|
||||
}, gradeWebsiteDelivery);
|
||||
return { ...base, oracleVersion: revising ? "paperclip.chat-catalog.internal.website-revision.v3" : "paperclip.chat-catalog.internal.website-download.v2", authentication: "human-session", queryAttachments: factualContextQueryAttachment("selected-source.md"), expectedRunCount: 2, minimumRunCount: 2, attemptTimeoutMs: 600_000,
|
||||
return { ...base, oracleVersion: revising ? "paperclip.chat-catalog.internal.website-revision.v3" : "paperclip.chat-catalog.internal.website-download.v3", authentication: "human-session", queryAttachments: factualContextQueryAttachment("selected-source.md"), expectedRunCount: 2, minimumRunCount: 2, attemptTimeoutMs: 600_000,
|
||||
sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-website-delivery.ts", "registered-artifact.ts"],
|
||||
async observe(ctx, state): Promise<DeliveryObservation> {
|
||||
const o = await observeInternal(ctx, state); const ids = [entity(state, "task").id, ...(ctx.issue() ? [ctx.issue()!.id] : [])];
|
||||
|
||||
@@ -98,7 +98,7 @@ export const runtimeAgentsCases = [
|
||||
const originalInstruction = await ctx.input.api.get<InstructionFile>(`/api/agents/${worker.id}/instructions-bundle/file?path=AGENTS.md`);
|
||||
return { worker, instructionAgentId: worker.id, originalInstruction, sideContent, concurrentMarker: `Concurrent human guidance ${ctx.input.nonce}: preserve keyboard support.` };
|
||||
}, gradeRuntimeAgents, {
|
||||
oracleVersion: "paperclip.chat-catalog.runtime.edit-instructions.v3",
|
||||
oracleVersion: "paperclip.chat-catalog.runtime.edit-instructions.v4",
|
||||
async act(ctx, state) {
|
||||
const active = await pollUntil({ label: "Active instructions update before concurrent human revision", deadlineAt: Date.now() + 60_000,
|
||||
load: () => ctx.allRuns(), accept: runs => runs.length === 1 && runs[0]?.status === "running",
|
||||
|
||||
@@ -70,12 +70,12 @@ export async function prepareRuntimeWorkerGate(ctx: ChatCatalogContext, options:
|
||||
await ctx.input.api.post(`${company}/budgets/policies`, { ...scope, amount: 500, hardStopEnabled: true, isActive: true, unpricedUsagePolicy: "allow" });
|
||||
return { worker, gate, previousRuns, company };
|
||||
}
|
||||
export async function busyRuntimeWorker(ctx: ChatCatalogContext, options: { worker?: RuntimeAgent; observationDeadlineAt?: number } = {}) {
|
||||
export async function busyRuntimeWorker(ctx: ChatCatalogContext, options: { worker?: RuntimeAgent; observationDeadlineAt?: number; briefSource?: string } = {}) {
|
||||
const { worker, gate, previousRuns, company } = await prepareRuntimeWorkerGate(ctx, { ...options, taskBound: true });
|
||||
// Each busy worker owns its own task. Without allowDuplicate the product's recent-open-title
|
||||
// deduplication returns the first worker's identically titled "Website task" (HTTP 200).
|
||||
const task = await ctx.input.api.post<Entity>(`${company}/issues`, { title: "Website task", status: "todo", assigneeAgentId: worker.id, allowDuplicate: true,
|
||||
description: `The website brief is being supplied by the human. Run ${gate.command} once and wait for its real output. Do not invent a brief, replace the script, or mark work complete before it returns. Once the brief arrives, save it as a task document and finish normally.` });
|
||||
description: `${options.briefSource ?? "The website brief is being supplied by the human."} Run ${gate.command} once and wait for its real output. Do not invent a brief, replace the script, or mark work complete before it returns. Once the brief arrives, save it as a task document and finish normally.` });
|
||||
if (task.assigneeAgentId !== worker.id || previousRuns.some(run => run.contextSnapshot?.issueId === task.id))
|
||||
throw new Error("Busy worker fixture did not create its own task for this worker");
|
||||
const running = await pollUntil({ label: "Real Morgan process waiting for its human brief", deadlineAt: options.observationDeadlineAt ?? Date.now() + 150_000,
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import { expect, it, vi } from "vitest";
|
||||
import { freshContinuationRecord, gradeInfrastructureRecovery, nativeInfrastructureLoss, readSourceRecoveryRecord } from "./runtime-infrastructure.js";
|
||||
import { freshContinuationRecord, gradeInfrastructureRecovery, INFRASTRUCTURE_BRIEF_SOURCE, nativeInfrastructureLoss, readSourceRecoveryRecord } from "./runtime-infrastructure.js";
|
||||
import { busyRuntimeWorker } from "./runtime-busy-worker.js";
|
||||
vi.mock("./runtime-busy-worker.js", async importOriginal => ({ ...await importOriginal<typeof import("./runtime-busy-worker.js")>(),
|
||||
busyRuntimeWorker: async () => ({ worker: { id: "morgan" }, task: { id: "website" }, gate: {}, workerRunId: "run" }) }));
|
||||
busyRuntimeWorker: vi.fn(async () => ({ worker: { id: "morgan" }, task: { id: "website" }, gate: {}, workerRunId: "run" })) }));
|
||||
vi.mock("./runtime-native-fault.js", async importOriginal => ({ ...await importOriginal<typeof import("./runtime-native-fault.js")>(),
|
||||
faultOwnedNativeProcess: async () => ({ id: "run", processPid: 7, errorCode: "native_runner_process_exited", status: "running", nativePhase: "running" }) }));
|
||||
import { savedCall } from "./api-read-receipts.test-fixtures.js";
|
||||
@@ -116,3 +117,14 @@ it("rejects a run after the last Runner loss that does not continue the interrup
|
||||
await expect(nativeInfrastructureLoss(lossContext(3, { retryOfRunId: "other" }) as never)).rejects.toThrow(/undeclared replacement/);
|
||||
await expect(nativeInfrastructureLoss(lossContext(3, { agentId: "riley" }) as never)).rejects.toThrow(/undeclared replacement/);
|
||||
});
|
||||
|
||||
// Full cohort 9 (recovery.source-action, master b1ef1f3022): the recovery mechanics passed, but the gated Website task said
|
||||
// "The website brief is being supplied by the human", so the CEO asked the chat person for the brief instead of explaining
|
||||
// that the system was already continuing the task. The brief comes from the gated export command itself.
|
||||
it("gives the interrupted Website task a brief source that does not ask the chat person to supply anything", async () => {
|
||||
vi.mocked(busyRuntimeWorker).mockClear();
|
||||
await nativeInfrastructureLoss(lossContext(3) as never);
|
||||
expect(vi.mocked(busyRuntimeWorker).mock.calls[0]?.[1]).toEqual({ briefSource: INFRASTRUCTURE_BRIEF_SOURCE });
|
||||
expect(INFRASTRUCTURE_BRIEF_SOURCE).not.toMatch(/\b(?:human|person|you|user|supplied)\b/i);
|
||||
expect(INFRASTRUCTURE_BRIEF_SOURCE).toMatch(/command below/);
|
||||
});
|
||||
@@ -11,7 +11,7 @@ import { checks, hasLink, type Entity } from "./internal-support.js";
|
||||
import { busyRuntimeWorker, releasedRetryListener, releaseRuntimeWorker, type RuntimeBriefGate } from "./runtime-busy-worker.js";
|
||||
import { faultOwnedNativeProcess, NATIVE_INFRASTRUCTURE_FAULT_CODES, type NativeFaultRun } from "./runtime-native-fault.js";
|
||||
import { runtimeDefinition, runtimeObservation, observeRuntime, type RuntimeAgent, type RuntimeObservation } from "./runtime-support.js";
|
||||
export const INFRASTRUCTURE_RECOVERY_ORACLE = "paperclip.chat-catalog.runtime.infrastructure-recovery.v6";
|
||||
export const INFRASTRUCTURE_RECOVERY_ORACLE = "paperclip.chat-catalog.runtime.infrastructure-recovery.v7";
|
||||
const objectRecord = (value: unknown): Record<string, unknown> => value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
||||
/**
|
||||
* The chat run read the source task's saved recovery record through a product
|
||||
@@ -69,8 +69,9 @@ export function continuesFailedRun(run: Pick<HeartbeatRun, "id" | "companyId" |
|
||||
return !!run && run.id !== failedRunId && run.companyId === companyId && run.agentId === agentId && run.retryOfRunId === failedRunId
|
||||
&& [run.contextSnapshot?.issueId, run.contextSnapshot?.taskId].includes(issueId);
|
||||
}
|
||||
export const INFRASTRUCTURE_BRIEF_SOURCE = "The website brief comes from the brief export command below, which can take several minutes to return.";
|
||||
export async function nativeInfrastructureLoss(ctx: ChatCatalogContext) {
|
||||
const state = await busyRuntimeWorker(ctx), task = state.task as Entity;
|
||||
const state = await busyRuntimeWorker(ctx, { briefSource: INFRASTRUCTURE_BRIEF_SOURCE }), task = state.task as Entity;
|
||||
const faults: Array<{ runId: string; pid: number | null; errorCode: string | null }> = [];
|
||||
let previousPid: number | null = null;
|
||||
for (let attempt = 1; attempt <= 3; attempt++) {
|
||||
|
||||
@@ -85,7 +85,7 @@ export function gradeQueueEdit(value: unknown) {
|
||||
|
||||
export const runtimeQueueControlCases = [runtimeDefinition("steering.edit-queued-message", ctx => humanQueuedRuntime(ctx), gradeQueueEdit, {
|
||||
sourceFiles: ["chat-catalog-domains/runtime-queue-controls.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-rendered-links.ts", "chat-scenarios/steering.yaml"],
|
||||
oracleVersion: "paperclip.chat-catalog.edit-queued-message.structural.v5",
|
||||
oracleVersion: "paperclip.chat-catalog.edit-queued-message.structural.v6",
|
||||
authentication: "human-session", expectedRunCount: 3, minimumRunCount: 2, attemptTimeoutMs: 720_000,
|
||||
runPolicy: { kind: "terminal-counts", terminalStatusCounts: { succeeded: 2 }, maximumTerminalStatusCounts: { succeeded: 3 } },
|
||||
setupPolicy: runtimeQueueCases[0]!.setupPolicy, verifySetup: runtimeQueueCases[0]!.verifySetup,
|
||||
@@ -128,9 +128,18 @@ export const runtimeQueueControlCases = [runtimeDefinition("steering.edit-queued
|
||||
} finally { release(); await ctx.input.page.unroute(pattern); await concurrent.close(); }
|
||||
await ctx.input.page.reload({ waitUntil: "domcontentloaded" });
|
||||
await edit(ctx.input.page, targetId, `${blueText} ${state.concurrentMarker}`);
|
||||
const saved = ctx.input.page.waitForResponse(response => response.request().method() === "PATCH" && new URL(response.url()).pathname === mutation, { timeout: 15_000 });
|
||||
await ctx.input.page.getByRole("button", { name: "Save queued message", exact: true }).click();
|
||||
state.finalEditStatus = (await saved).status();
|
||||
if (state.finalEditStatus !== 200) throw new Error(`The person's final queued edit was not saved (${state.finalEditStatus})`);
|
||||
state.queueAfterEdit = await pollUntil({ label: "The saved edit is the queue's current revision", deadlineAt: Date.now() + 15_000, load: () => queue(ctx, state),
|
||||
accept: q => q.entries.some(row => row.comment.id === targetId && row.comment.body === `${blueText} ${state.concurrentMarker}` && row.position === 1) });
|
||||
const handle = ctx.input.page.getByRole("button", { name: `Reorder queued message: ${blueText} ${state.concurrentMarker}`, exact: true });
|
||||
state.reorderRequest = await keyboardMoveQueuedMessageUp(ctx.input.page, handle);
|
||||
await expect(handle).toBeVisible();
|
||||
state.reorderRequest = await keyboardMoveQueuedMessageUp(ctx.input.page, handle, 15_000, 100, { attempts: 2, beforeRetry: async () => {
|
||||
await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); await expect(handle).toBeVisible();
|
||||
} });
|
||||
if ((state.reorderRequest as { status: number }).status !== 200) throw new Error(`The person's reorder was refused (${(state.reorderRequest as { status: number }).status})`);
|
||||
state.queueAfter = await pollUntil({ label: "Actual full blue text and first queue position persisted", deadlineAt: Date.now() + 15_000,
|
||||
load: () => queue(ctx, state), accept: q => q.entries[0]?.comment.id === targetId && q.entries[0]?.comment.body === `${blueText} ${state.concurrentMarker}` });
|
||||
await ctx.input.page.reload({ waitUntil: "domcontentloaded" });
|
||||
|
||||
@@ -88,6 +88,21 @@ it("fails when the handle never enables or the gesture sends no order request",
|
||||
await expect(keyboardMoveQueuedMessageUp(silent.page, handle as never, 200, 20)).rejects.toThrow(/sent no queue order request/);
|
||||
});
|
||||
|
||||
// Full cohort 9 (master b1ef1f3022, steering.edit-queued-message): the person's edit saved (PATCH 200), the handle was
|
||||
// enabled, but the keyboard gesture sent no PUT /queued-comments/order within 15 s.
|
||||
it("refetches and presses the gesture again when the first gesture sends no order request, and still fails when none does", async () => {
|
||||
const fake = reorderPage(0); let reloads = 0, sends = false;
|
||||
const handle = { ...(fake.handle as object), press: async (key: string) => { if (sends) await (fake.handle as { press: (k: string) => Promise<void> }).press(key); } };
|
||||
const result = await keyboardMoveQueuedMessageUp(fake.page, handle as never, 200, 20, { attempts: 2, beforeRetry: async () => { reloads++; sends = true; } });
|
||||
expect(result).toMatchObject({ status: 200, gestures: 2 }); expect(reloads).toBe(1);
|
||||
const first = reorderPage(0); const once = await keyboardMoveQueuedMessageUp(first.page, first.handle, 3_000, 20, { attempts: 2, beforeRetry: async () => { reloads++; } });
|
||||
expect(once.gestures).toBe(1); expect(reloads).toBe(1);
|
||||
const silent = reorderPage(0); let retried = 0;
|
||||
await expect(keyboardMoveQueuedMessageUp(silent.page, { ...(silent.handle as object), press: async () => undefined } as never, 200, 20,
|
||||
{ attempts: 2, beforeRetry: async () => { retried++; } })).rejects.toThrow(/sent no queue order request after 2 gesture/);
|
||||
expect(retried).toBe(1);
|
||||
});
|
||||
|
||||
describe("task history reads", () => {
|
||||
const event = (comments: unknown, eventType = "task_history.read") => ({ eventType, payload: { comments } });
|
||||
it("keeps the newest revision each comment was read at, from history-read events only", () => {
|
||||
|
||||
@@ -14,22 +14,28 @@ import type { IssueWorkProduct } from "../../../packages/shared/src/types/work-p
|
||||
/** Move a queued message up one place with the drag handle's keyboard gesture, as a keyboard user does.
|
||||
* Saving an edit disables every handle until that queue change settles, and a gesture pressed on a disabled
|
||||
* handle does nothing, so wait for this handle to be enabled first. The gesture must send the queue order
|
||||
* request; its status is returned so the caller can tell an accepted move from a refused one. */
|
||||
* request; its status is returned so the caller can tell an accepted move from a refused one. When a gesture
|
||||
* sends nothing, `beforeRetry` (for example a reload that refetches the queue) runs and the gesture is pressed
|
||||
* again, up to `attempts` gestures in all; every gesture is reported. */
|
||||
export async function keyboardMoveQueuedMessageUp(page: Pick<Page, "waitForResponse">, handle: Pick<Locator, "isEnabled" | "focus" | "press">,
|
||||
timeoutMs = 15_000, intervalMs = 100): Promise<{ status: number; waitedForEnabledMs: number }> {
|
||||
const started = Date.now();
|
||||
while (!await handle.isEnabled()) {
|
||||
if (Date.now() - started >= timeoutMs) throw new Error("The queued message's reorder handle stayed disabled");
|
||||
await new Promise(resolve => setTimeout(resolve, intervalMs));
|
||||
timeoutMs = 15_000, intervalMs = 100, options: { attempts?: number; beforeRetry?: () => Promise<void> } = {}): Promise<{ status: number; waitedForEnabledMs: number; gestures: number }> {
|
||||
const attempts = Math.max(1, options.attempts ?? 1);
|
||||
for (let gesture = 1; ; gesture++) {
|
||||
const started = Date.now();
|
||||
while (!await handle.isEnabled()) {
|
||||
if (Date.now() - started >= timeoutMs) throw new Error("The queued message's reorder handle stayed disabled");
|
||||
await new Promise(resolve => setTimeout(resolve, intervalMs));
|
||||
}
|
||||
const waitedForEnabledMs = Date.now() - started;
|
||||
const sent = page.waitForResponse((response: Response) => response.request().method() === "PUT"
|
||||
&& new URL(response.url()).pathname.endsWith("/queued-comments/order"), { timeout: timeoutMs });
|
||||
sent.catch(() => undefined);
|
||||
await handle.focus(); await handle.press("Space"); await handle.press("ArrowUp"); await handle.press("Space");
|
||||
const response = await sent.catch(() => null);
|
||||
if (response) return { status: response.status(), waitedForEnabledMs, gestures: gesture };
|
||||
if (gesture >= attempts) throw new Error(`The keyboard reorder gesture sent no queue order request after ${gesture} gesture(s)`);
|
||||
await options.beforeRetry?.();
|
||||
}
|
||||
const waitedForEnabledMs = Date.now() - started;
|
||||
const sent = page.waitForResponse((response: Response) => response.request().method() === "PUT"
|
||||
&& new URL(response.url()).pathname.endsWith("/queued-comments/order"), { timeout: timeoutMs });
|
||||
sent.catch(() => undefined);
|
||||
await handle.focus(); await handle.press("Space"); await handle.press("ArrowUp"); await handle.press("Space");
|
||||
const response = await sent.catch(() => null);
|
||||
if (!response) throw new Error("The keyboard reorder gesture sent no queue order request");
|
||||
return { status: response.status(), waitedForEnabledMs };
|
||||
}
|
||||
const MAX_DELIVERABLE_BYTES = 262_144;
|
||||
/** Text of each stored or deflated entry in a zip deliverable, read from its central directory; [] when it is not a readable zip. */
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// @vitest-environment jsdom
|
||||
import { readFileSync } from "node:fs";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { collectConversationLinks, collectConversationLinksWithDiagnostics, conversationLinkBindings, mislabeledIssueLinks, readConversationDomLinks, readLatestConversationReplyRender, waitForLatestConversationReplyRender, waitForSavedTurnReplyRender, waitForSavedTurnReplyRenderReloadingOnce, type ConversationLinkInput } from "./chat-rendered-links.js";
|
||||
import { APP_ROUTE_ROOTS, collectConversationLinks, collectConversationLinksWithDiagnostics, conversationLinkBindings, unservedLinkReason, mislabeledIssueLinks, readConversationDomLinks, readLatestConversationReplyRender, waitForLatestConversationReplyRender, waitForSavedTurnReplyRender, waitForSavedTurnReplyRenderReloadingOnce, type ConversationLinkInput } from "./chat-rendered-links.js";
|
||||
|
||||
const card = '<div data-thread-item-kind="project_created" data-thread-anchor="project-created:project"><article aria-label="Project created: Launch"><div><p>Project created</p><a href="/RUN/projects/project">Launch</a></div></article></div>';
|
||||
const thread = (content = card) => `<div data-testid="task-chat-thread"><div data-testid="task-chat-history-content"><div data-testid="task-chat-scroller"><div id="comment-human" data-thread-item-kind="human">Create a project called Launch for the new website.</div>${content}<div id="comment-reply" data-thread-item-kind="agent">Created the Launch project. No repositories are connected yet.</div></div></div></div>`;
|
||||
@@ -218,3 +218,42 @@ describe("records thread and anchor diagnostics when a saved reply yields no ren
|
||||
expect(result.links.map(row => row.href)).toEqual(["/RUN/issues/task"]); expect(result.diagnostics).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// Full cohort 9 (documents-artifacts.produce-download, agents.edit-instructions): a reply's download link was written as
|
||||
// /COM/api/attachments/.../content?download=1, which the server does not serve (its API is only at /api), and a reply's
|
||||
// Morgan link was saved as //COM/agents/..., which the board repairs but notifications and chat channels deliver as a
|
||||
// link to a host named COM. Both counted as valid links.
|
||||
describe("only links that open a served app page or API route count", () => {
|
||||
const origin = "https://fixture.test";
|
||||
it("accepts company pages, unprefixed board pages, global pages and served API downloads", () => {
|
||||
for (const href of ["/RUN/issues/RUN-1", "/run/agents/morgan/overview", "/RUN/company/settings/connections", "/RUN/routines/routine", "/RUN",
|
||||
"/issues/RUN-1", "/instance/settings", "/api/attachments/file/content?download=1", "/api/attachments/file/content", "https://github.com/owner/repository"])
|
||||
expect(unservedLinkReason(href, origin), href).toBeUndefined();
|
||||
});
|
||||
it("rejects a company-prefixed API path, a doubled prefix and other paths the app does not route", () => {
|
||||
for (const href of ["/RUN/api/attachments/file/content?download=1", "/RUN/RUN/issues/RUN-1", "/RUN/attachments/file/content", "/attachments/file/content"])
|
||||
expect(unservedLinkReason(href, origin), href).toBe("not-a-served-route");
|
||||
});
|
||||
it("rejects a protocol-relative company link, rendered raw or repaired by the board, but not the same page saved correctly", () => {
|
||||
expect(unservedLinkReason("//RUN/agents/morgan", origin)).toBe("protocol-relative");
|
||||
expect(unservedLinkReason("/RUN/agents/morgan", origin, ["//RUN/agents/morgan"])).toBe("protocol-relative");
|
||||
expect(unservedLinkReason("/RUN/agents/morgan", origin, ["/RUN/agents/morgan"])).toBeUndefined();
|
||||
expect(unservedLinkReason("/RUN/agents/morgan", origin, ["//RUN/agents/morgan", "/RUN/agents/morgan"])).toBeUndefined();
|
||||
});
|
||||
it("lists exactly the first path segments that the board router declares", () => {
|
||||
const app = readFileSync(`${import.meta.dirname}/../../ui/src/App.tsx`, "utf8");
|
||||
const roots = new Set([...app.matchAll(/path="\/?([^"/]+)/g)].map(match => match[1]!).filter(root => !root.startsWith(":") && root !== "*"));
|
||||
expect([...APP_ROUTE_ROOTS].sort()).toEqual([...roots].sort());
|
||||
expect(APP_ROUTE_ROOTS).not.toContain("api");
|
||||
});
|
||||
it("drops unserved reply links from collection, keeps them as rejected evidence and keeps served ones", async () => {
|
||||
const page = { evaluate: async <T, A>(read: (arg: A) => T, arg: A) => read(arg) } as unknown as Parameters<typeof collectConversationLinks>[0];
|
||||
vi.spyOn(console, "info").mockImplementation(() => undefined);
|
||||
document.querySelector('#comment-reply')!.innerHTML = '<a href="/api/attachments/file/content">website.html</a><a href="/RUN/api/attachments/file/content?download=1">Download</a><a href="/RUN/agents/morgan">Morgan</a>';
|
||||
const value = input(); value.projects = []; value.activity = [];
|
||||
value.comments[1]!.body = "Prepared [website.html](/api/attachments/file/content). [Download](/RUN/api/attachments/file/content?download=1) from [Morgan](//RUN/agents/morgan).";
|
||||
const result = await collectConversationLinksWithDiagnostics(page, value);
|
||||
expect(result.links.map(row => row.href)).toEqual(["/api/attachments/file/content"]);
|
||||
expect(result.rejected?.map(row => [row.href, row.reason])).toEqual([["/RUN/api/attachments/file/content?download=1", "not-a-served-route"], ["/RUN/agents/morgan", "protocol-relative"]]);
|
||||
});
|
||||
});
|
||||
@@ -1,7 +1,7 @@
|
||||
import type { Locator, Page } from "@playwright/test";
|
||||
import type { ChatRun } from "./chat-flow.js";
|
||||
|
||||
interface Row { id: string; companyId?: string; issueId?: string; authorAgentId?: string | null; createdByRunId?: string | null; name?: string; action?: string; entityType?: string; entityId?: string; agentId?: string | null; actorType?: string; actorId?: string; runId?: string | null }
|
||||
interface Row { id: string; companyId?: string; issueId?: string; body?: string | null; authorAgentId?: string | null; createdByRunId?: string | null; name?: string; action?: string; entityType?: string; entityId?: string; agentId?: string | null; actorType?: string; actorId?: string; runId?: string | null }
|
||||
export interface RenderedConversationLink {
|
||||
threadTestId: "task-chat-thread" | "thread-root"; href: string; source: "reply" | "project-created-card"; conversationId: string; anchorCommentId: string;
|
||||
commentId?: string; resourceId?: string; cardLabel?: string; createdByRunId?: string;
|
||||
@@ -42,6 +42,36 @@ export function unresolvedLinkTarget(href: string): boolean {
|
||||
try { decoded = decodeURIComponent(href); } catch { /* keep the raw target */ }
|
||||
return /[<>{}]/.test(decoded);
|
||||
}
|
||||
/** First path segments the board app routes, as declared in ui/src/App.tsx. A company-prefixed page is
|
||||
* /<prefix>/<root>/...; the server serves its API only at /api. */
|
||||
export const APP_ROUTE_ROOTS: readonly string[] = ["activity", "agents", "approvals", "apps", "artifacts", "assistant-connections", "audit", "auth",
|
||||
"board-chat", "board-claim", "budgets", "cases", "chat-identity", "chats", "cli-auth", "companies", "company", "costs", "dashboard", "decisions",
|
||||
"design-guide", "dev", "dot-connect", "execution-workspaces", "goals", "inbox", "instance", "invite", "issues", "mcp-connect", "mcp-device",
|
||||
"oauth-handoff", "onboarding", "org", "plugins", "projects", "routines", "runs", "search", "settings", "skills", "status", "status-cards", "tasks",
|
||||
"tests", "timeline", "tools", "u", "ux-lab", "workspaces"];
|
||||
export type UnservedLinkReason = "protocol-relative" | "not-a-served-route";
|
||||
/** Why a rendered same-origin link opens nothing the product serves, or undefined when it does. `savedTargets` are the
|
||||
* Markdown targets of the saved reply that rendered it: the board repairs `//PREFIX/...` to `/PREFIX/...`, but the saved
|
||||
* reply is also what notifications, chat channels and the API deliver, where a protocol-relative target leaves the app. */
|
||||
export function unservedLinkReason(href: string, origin: string, savedTargets: readonly string[] = []): UnservedLinkReason | undefined {
|
||||
if (href.trim().startsWith("//") || !savedTargets.includes(href) && savedTargets.some(target => target.startsWith("//") && target.slice(1) === href)) return "protocol-relative";
|
||||
let url: URL;
|
||||
try { url = new URL(href, origin); } catch { return "not-a-served-route"; }
|
||||
if (url.origin !== new URL(origin).origin) return undefined;
|
||||
const parts = url.pathname.split("/").filter(Boolean).map(part => part.toLowerCase());
|
||||
if (!parts.length || parts[0] === "api" || APP_ROUTE_ROOTS.includes(parts[0]!)) return undefined;
|
||||
return parts.length === 1 || APP_ROUTE_ROOTS.includes(parts[1]!) ? undefined : "not-a-served-route";
|
||||
}
|
||||
/** Split rendered links into those that open a served app page or API route and those that do not. */
|
||||
export function partitionServedConversationLinks(rows: readonly RenderedConversationLink[], input: Pick<ConversationLinkInput, "origin" | "comments">) {
|
||||
const links: RenderedConversationLink[] = [], rejected: Array<RenderedConversationLink & { reason: UnservedLinkReason }> = [];
|
||||
for (const row of rows) {
|
||||
const body = row.commentId ? input.comments.find(comment => comment.id === row.commentId)?.body : undefined;
|
||||
const reason = unservedLinkReason(row.href, input.origin, body ? markdownLinkTargets(body) : []);
|
||||
if (reason) rejected.push({ ...row, reason }); else links.push(row);
|
||||
}
|
||||
return { links, rejected };
|
||||
}
|
||||
export function conversationLinkBindings(input: ConversationLinkInput) {
|
||||
const comments = input.comments.filter(row => row.companyId === input.companyId && row.issueId === input.sourceId);
|
||||
const replies = comments.filter(row => row.authorAgentId === input.leadId && input.runs.some(run =>
|
||||
@@ -227,17 +257,20 @@ export function readConversationLinkDiagnostics(input: Pick<ReturnType<typeof co
|
||||
replyAnchorCounts: Object.fromEntries(input.replies.map(id => [id, root.querySelector(`[id="comment-${id}"]`)?.querySelectorAll("a[href]").length ?? 0])) };
|
||||
}) };
|
||||
}
|
||||
export async function collectConversationLinksWithDiagnostics(page: Page, input: ConversationLinkInput): Promise<{ links: RenderedConversationLink[]; diagnostics?: ConversationLinkDiagnostics }> {
|
||||
export async function collectConversationLinksWithDiagnostics(page: Page, input: ConversationLinkInput): Promise<{ links: RenderedConversationLink[];
|
||||
rejected?: Array<RenderedConversationLink & { reason: UnservedLinkReason }>; diagnostics?: ConversationLinkDiagnostics }> {
|
||||
const bindings = conversationLinkBindings(input);
|
||||
const render = await waitForLatestConversationReplyRender(page, bindings);
|
||||
const links = await page.evaluate(readConversationDomLinks, bindings);
|
||||
if (links.length || !bindings.replies.length) return { links };
|
||||
const { links, rejected } = partitionServedConversationLinks(await page.evaluate(readConversationDomLinks, bindings), input);
|
||||
if (rejected.length) console.info(`[chat-rendered-links] links that open no served route: ${JSON.stringify(rejected.map(row => ({ href: row.href, reason: row.reason })))}`);
|
||||
const kept = rejected.length ? { rejected } : {};
|
||||
if (links.length || !bindings.replies.length) return { links, ...kept };
|
||||
let diagnostics: ConversationLinkDiagnostics;
|
||||
try { diagnostics = { render, ...await page.evaluate(readConversationLinkDiagnostics, { anchors: bindings.anchors, replies: bindings.replies }) }; }
|
||||
catch { diagnostics = { render, expectedReplyIds: bindings.replies, savedCommentCount: bindings.anchors.length, threads: [] }; }
|
||||
// Logged so an empty capture says whether the thread was closed, another conversation, or the reply had no anchors yet.
|
||||
console.info(`[chat-rendered-links] no rendered reply links: ${JSON.stringify(diagnostics)}`);
|
||||
return { links, diagnostics };
|
||||
return { links, ...kept, diagnostics };
|
||||
}
|
||||
export function conversationLinkLocator(page: Page, link: RenderedConversationLink): Locator {
|
||||
const thread = page.getByTestId(link.threadTestId).filter({ has: page.locator(`[id=${JSON.stringify(`comment-${link.anchorCommentId}`)}]`) });
|
||||
|
||||
Reference in new issue
Block a user