PaperClipAI/tests/runner-e2e/chat-catalog-domains/runtime-dependency.ts

import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely, chatCatalogNativePhaseRemainingBudget } from "../chat-catalog-contract.js";
import { expect, type Page } from "@playwright/test";
import type { Issue } from "../../../packages/shared/src/types/issue.js";
import type { HeartbeatRun } from "../../../packages/shared/src/types/heartbeat.js";
import { pollUntil } from "../api.js";
import { collectRunEvents } from "../run-observations.js";
import { readRecord, recordParams } from "./api-read-receipts.js";
import { sendChatMessage } from "../chat-flow.js";
import { checks, type Entity } from "./internal-support.js";
import { runtimeDefinition, runtimeWorker, runtimeObservation, observeRuntime } from "./runtime-support.js";
const record = (value: unknown): Record<string, unknown> => value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
/**
 * Whether a run re-read Website's saved state (its own retained events, never reply text): a successful
 * call_api read of the task, its documents or the company task list, bound by its saved input digest
 * (bodies are redacted before saving), or a completed dedicated task tool whose output names the task.
 */
export function readTaskState(events: readonly Record<string, unknown>[], task: { id?: string; identifier?: string | null; companyId?: string | null }) {
  const params = recordParams("id", task.id, task.identifier);
  if (task.id && readRecord(events, [
    ...["GET /api/issues/{id}", "GET /api/issues/{id}/documents", "GET /api/issues/{id}/work-products"].map(operationId => ({ operationId, pathParams: params })),
    { operationId: "GET /api/companies/{companyId}/issues", pathParams: [...recordParams("companyId", task.companyId), undefined], paged: true },
  ])) return true;
  return events.some(row => {
    const envelope = record(record(row.payload).prpEvent), payload = record(envelope.payload), output = typeof payload.output === "string" ? payload.output : "";
    return row.eventType === "tool.execution.completed" && envelope.sourceKind === "runner" && payload.status === "completed"
      && ["paperclip_get_task", "paperclip_search_tasks"].includes(String(payload.name))
      && [task.id, task.identifier].some(id => !!id && output.includes(id));
  });
}
async function chooseStatus(page: Page, status: "Todo" | "Done") {
  await page.getByRole("button", { name: /^Change status \(current:/ }).first().click();
  await page.getByRole("button", { name: new RegExp(`^${status}(?:\\s*${status})?$`) }).filter({ visible: true }).first().click();
}
type RunLike = { id: string; agentId?: string | null; status: string; contextSnapshot?: Record<string, unknown> | null };
export function isBlockedInteractionRun(run: RunLike, task: { id: string }, design: { id: string }, workerId: string) {
  const context = run.contextSnapshot ?? {};
  return run.agentId === workerId && run.status === "succeeded" && context.dependencyBlockedInteraction === true && context.issueId === task.id
    && Array.isArray(context.unresolvedBlockerIssueIds) && context.unresolvedBlockerIssueIds.includes(design.id);
}
export function gradeDesignDependency(value: unknown) {
  const c = checks(), o = runtimeObservation(value);
  c.add("actual-dependency-and-native-admission-evidence", !!o, "The saved dependency, real unfinished prerequisite and later eligible native result must be independently read."); if (!o) return c.result();
  const s = o.state, task = s.task as Entity, design = s.design as Entity, waiting = s.waitingTask as Issue, finished = o.tasks.find(row => row.id === task.id);
  c.add("actual-first-class-blocker-saved", waiting?.id === task.id && waiting.companyId === o.companyId && waiting.assigneeAgentId === (s.worker as Entity).id
    && waiting.blockedBy?.length === 1 && waiting.blockedBy[0]?.id === design.id && s.designUnfinishedAtWait === true
    && o.activity.some(row => row.action === "issue.blockers_updated" && row.entityId === task.id && row.agentId === o.leadId
      && (row.details?.addedBlockedByIssueIds as string[] | undefined)?.includes(design.id)),
  "The CEO actually installs the Design edge on the intended Website task; a comment alone is insufficient.");
  const interactionIds = Array.isArray(s.preApprovalInteractionRunIds) ? s.preApprovalInteractionRunIds as string[] : [];
  const interactions = o.runs.filter(row => interactionIds.includes(row.id)), work = o.runs.filter(row => !interactionIds.includes(row.id));
  c.add("blocked-ordinary-admission-and-usable-ui", s.blockerVisible === true && s.noWorkerBeforeApproval === true && s.attemptedBeforeApproval === true
    && [200, 422, 409].includes(Number(s.beforeApprovalStatus)) && s.noWebsiteResultBeforeApproval === true
    && interactions.length === interactionIds.length && interactions.every(row => isBlockedInteractionRun(row, task, design, (s.worker as Entity).id)),
  "Ordinary human start is attempted while Design is unfinished, but no Website work is admitted (a server-marked blocked-interaction reply to a comment is allowed) and nothing is saved on Website; the task UI exposes the exact blocker.");
  c.add("one-eligible-worker-after-real-human-approval", s.humanApprovedDesign === true && s.workerAdmissionAfterApproval === true && finished?.status === "done"
    && finished.assigneeAgentId === (s.worker as Entity).id && o.tasks.length === 2 && s.actualWebsiteResult === true
    && work.length === 3 && o.runs.every(row => row.status === "succeeded") && work.filter(row => row.agentId === (s.worker as Entity).id).length === 1,
  "After actual human Design completion clears the prerequisite, ordinary admission runs Morgan once and produces the genuine saved Website result.");
  // The saved edge, admission and document prove the outcome; a person reads the replies for meaning.
  c.add("later-own-phase-reports-actual-result", s.finalReplyAttributed === true && String(s.finalReply ?? "").trim().length > 0 && s.finalReadTaskState === true,
  "A declared factual human follow-up receives its own native reply, answered from a fresh product read of saved task state rather than a pre-approval response.");
  return c.result();
}
export const runtimeDependencyCases = [runtimeDefinition("task-coordination.block-on-design", async ctx => {
  const worker = await runtimeWorker(ctx), root = `/api/companies/${ctx.input.fixtures.company.id}`, marker = `Approved website ${ctx.input.nonce}`;
  for (const scope of [{ scopeType: "company", scopeId: ctx.input.fixtures.company.id }, { scopeType: "agent", scopeId: worker.id }])
    await ctx.input.api.post(`${root}/budgets/policies`, { ...scope, amount: 500, hardStopEnabled: true, isActive: true, unpricedUsagePolicy: "allow" });
  const design = await ctx.input.api.post<Entity>(`${root}/issues`, { title: "Design", status: "backlog", description: `A semantic HTML website with accessible labels. Await the actual human approval; reference ${marker}.` });
  const task = await ctx.input.api.post<Entity>(`${root}/issues`, { title: "Website", status: "backlog", assigneeAgentId: worker.id,
    description: `Implement the approved Design as a small website plan. Once eligible, read the actual Design task and save a task document with the exact reference ${marker}, accessible labels and a clear primary action, then finish normally. Do not deploy, create other tasks or invent approval.` });
  return { worker, task, design, marker };
}, gradeDesignDependency, {
  expectedHumanMessageCount: 2, expectedRunCount: 4, minimumRunCount: 3, attemptTimeoutMs: 720_000, oracleVersion: "paperclip.chat-catalog.runtime.block-on-design.v8",
  sourceFiles: ["chat-catalog-domains/runtime-dependency.ts", "chat-catalog-domains/api-read-receipts.ts", "chat-catalog-domains/runtime-support.ts", "run-observations.ts", "chat-scenarios/task-coordination.yaml"],
  async act(ctx, state) {
    const workerId = (state.worker as Entity | undefined)?.id ?? "", websiteTask = state.task as Entity, gate = state.design as Entity;
    const settled = await pollUntil({ label: "Saved dependency and canonical reply before human approval", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000),
      load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; },
      accept: v => chatCatalogNativePhaseIsTimely(ctx) && v.source?.conversationState === "waiting" && v.runs.filter(row => row.agentId !== workerId).length === 1
        && v.runs.every(row => row.status === "succeeded") && v.runs.filter(row => row.agentId === workerId).every(row => isBlockedInteractionRun(row, websiteTask, gate, workerId)),
      reject: v => v.runs.filter(row => row.agentId !== workerId).length > 1 || v.runs.some(row => ["failed", "cancelled", "timed_out"].includes(row.status))
        || v.runs.some(row => row.agentId === workerId && !isBlockedInteractionRun(row, websiteTask, gate, workerId) && row.status !== "queued" && row.status !== "running")
        ? "Work was admitted on Website, or a pre-approval run failed or duplicated, before Design approval" : undefined });
    state.preApprovalInteractionRunIds = settled.runs.filter(row => row.agentId === workerId).map(row => row.id);
    state.queryReply = (await observeRuntime(ctx, state)).reply;
    const source = ctx.input.page.url(), task = state.task as Entity, design = state.design as Entity, websiteRoot = `/api/issues/${task.id}`;
    state.waitingTask = await ctx.input.api.get<Issue>(websiteRoot);
    state.designUnfinishedAtWait = (await ctx.input.api.get<Issue>(`/api/issues/${design.id}`)).status !== "done";
    await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" });
    // A backlog task shows its blockers in the properties panel; the thread's blocker banner appears only once it is Blocked.
    const blockedBy = ctx.input.page.locator('[data-property-row]:not([inert] *):not([aria-hidden="true"] *)')
      .filter({ has: ctx.input.page.locator('[data-property-label="Blocked by"]') }).filter({ visible: true });
    // Open the panel through the control the page renders ("Show properties"); a fresh panel shows its tab
    // launcher first, from which a person chooses Properties.
    const panel = ctx.input.page.getByRole("button", { name: /^(Show properties|Properties|Toggle side panel)$/ }).filter({ visible: true }).first();
    const propertiesTab = ctx.input.page.getByRole("option", { name: /^Properties/ }).filter({ visible: true }).first();
    await expect.poll(async () => await blockedBy.isVisible() || await propertiesTab.isVisible() || await panel.isVisible()).toBe(true);
    if (!await blockedBy.isVisible() && !await propertiesTab.isVisible()) await panel.click();
    await expect.poll(async () => await blockedBy.isVisible() || await propertiesTab.isVisible()).toBe(true);
    if (!await blockedBy.isVisible()) await propertiesTab.click();
    await expect(blockedBy.getByText(design.identifier ?? "Design").first()).toBeVisible(); state.blockerVisible = true;
    const websitePaths = [websiteRoot, ...(task.identifier ? [`/api/issues/${task.identifier}`] : [])];
    const pending = ctx.input.page.waitForResponse(row => row.request().method() === "PATCH" && websitePaths.includes(new URL(row.url()).pathname));
    await chooseStatus(ctx.input.page, "Todo"); state.beforeApprovalStatus = (await pending).status(); state.attemptedBeforeApproval = true;
    // Reload and independent reads observe the real admission gate, rather
    // than setting a run's blocked status or pretending the worker executed.
    await ctx.input.page.reload({ waitUntil: "domcontentloaded" });
    state.noWorkerBeforeApproval = (await ctx.allRuns()).every(row => row.agentId !== workerId || (state.preApprovalInteractionRunIds as string[]).includes(row.id));
    state.noWebsiteResultBeforeApproval = (await ctx.input.api.get<Entity[]>(`${websiteRoot}/documents`)).length === 0
      && !["in_progress", "in_review", "done"].includes((await ctx.input.api.get<Issue>(websiteRoot)).status);
    await ctx.input.capture("actual-design-dependency-wait", "Website retains its actual Design blocker before human approval", "runtime-design-blocker.png");
    await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${design.identifier ?? design.id}`, { waitUntil: "domcontentloaded" });
    await chooseStatus(ctx.input.page, "Done");
    const approved = await pollUntil({ label: "Real human Design completion", deadlineAt: Date.now() + 20_000,
      load: () => ctx.input.api.get<Issue>(`/api/issues/${design.id}`), accept: row => row.status === "done" });
    state.humanApprovedDesign = approved.status === "done"; state.designApprovedAt = Date.now();
    const website = await ctx.input.api.get<Issue>(websiteRoot);
    if (website.status === "backlog") {
      await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" });
      await chooseStatus(ctx.input.page, "Todo");
    }
    const interactionCount = (state.preApprovalInteractionRunIds as string[]).length;
    const result = await pollUntil({ label: "One ordinary Website admission after Design approval", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 240_000),
      load: async () => ({ runs: await ctx.allRuns(), task: await ctx.input.api.get<Issue>(websiteRoot) }),
      accept: v => chatCatalogNativePhaseIsTimely(ctx) && v.runs.length === 2 + interactionCount && v.runs.every(row => row.status === "succeeded") && v.task.status === "done",
      reject: v => v.runs.length > 2 + interactionCount || v.runs.some(row => ["failed", "cancelled", "timed_out"].includes(row.status)) ? "Eligible Website admission failed or duplicated native work" : undefined });
    const worker = result.runs.find(row => row.agentId === workerId && !(state.preApprovalInteractionRunIds as string[]).includes(row.id))!;
    const designActivity = await ctx.input.api.get<Entity[]>(`/api/issues/${design.id}/activity`);
    const approval = designActivity.find(row => row.action === "issue.updated" && row.actorType === "user" && row.details?.status === "done");
    const workerDetail = await ctx.input.api.get<HeartbeatRun>(`/api/heartbeat-runs/${worker.id}`);
    state.workerAdmissionAfterApproval = !!approval && Date.parse(String(workerDetail.createdAt)) >= Date.parse(approval.createdAt ?? ""); state.websiteRunId = worker.id;
    await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
    const followup = `I approved Design ${design.identifier ?? design.id}. What is the actual result of Website ${task.identifier ?? task.id} now?`;
    await sendChatMessage(ctx.input.page, followup, chatCatalogNativePhaseRemainingBudget(ctx));
    const final = await pollUntil({ label: "Own native reply reporting actual post-approval Website outcome", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000),
      load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; },
      accept: v => chatCatalogNativePhaseIsTimely(ctx) && v.source?.conversationState === "waiting" && v.runs.length === 3 + interactionCount && v.runs.every(row => row.status === "succeeded") });
    const run = final.runs.find(row => !result.runs.some(prior => prior.id === row.id))!, comments = await ctx.input.api.get<Entity[]>(`/api/issues/${ctx.issue()!.id}/comments?order=asc`);
    const human = comments.filter(row => !row.authorAgentId && row.body === followup), replies = comments.filter(row => row.authorAgentId === ctx.input.fixtures.agent.id && row.createdByRunId === run.id);
    const detail = await ctx.input.api.get<{ runnerProfileJson?: { nativeExecutionInput?: { task?: { prompt?: string } } } }>(`/api/heartbeat-runs/${run.id}`);
    state.finalReplyAttributed = human.length === 1 && replies.length > 0 && detail.runnerProfileJson?.nativeExecutionInput?.task?.prompt?.includes(human[0]!.id) === true;
    state.finalReply = replies.map(row => row.body).join("\n");
    state.finalReadTaskState = readTaskState(await collectRunEvents<Record<string, unknown>>(afterSeq => ctx.input.api.get(`/api/heartbeat-runs/${run.id}/events?afterSeq=${afterSeq}&limit=1000`)), task);
  },
  async observe(ctx, state) {
    const o = await observeRuntime(ctx, state);
    state.actualWebsiteResult = o.revisions.some(row => row.createdByRunId === state.websiteRunId && row.createdByAgentId === (state.worker as Entity).id
      && (row.body ?? "").includes(String(state.marker)) && o.documents.some(doc => doc.id === row.documentId && doc.latestRevisionId === row.id));
    return o;
  },
  async verifyResult(ctx, state) {
    const source = ctx.input.page.url(), task = state.task as Entity;
    await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" });
    await ctx.input.page.reload({ waitUntil: "domcontentloaded" });
    await expect(ctx.input.page.getByRole("heading", { name: "Website", exact: true }).first()).toBeVisible();
    await expect(ctx.input.page.getByRole("button", { name: "Change status (current: Done)", exact: true }).first()).toBeVisible();
    await ctx.input.capture("actual-approved-website-result", "The same Website task completed after its real Design prerequisite", "runtime-approved-website.png");
    await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
  },
})];