From e0b618b02a818f0d1db1d070c3a19936892c2be2 Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 12 Nov 2026 22:53:41 -0600 Subject: [PATCH] test(evals): fix cohort-10 chat-eval defects in secrets entry, steering graders, private-task settle and queue brief - secrets.configure-company-secret / secrets.personal-secret: the private browser worker now drives the redesigned New secret dialog (radiogroup, More options, optional Stored in) and binds the personal entry form by its rendered heading. The saved company secret must be Paperclip-encrypted. - steering.steer-active / steering.discard-queued-message: accept the #21243 delivery contract where the running turn reads a queued comment at its current revision and cites it in its accepted result, with one fewer run. Uncited reads, cited-but-unapplied comments, replays and extra runs fail. The discard gate no longer steers the worker toward either path. - tasks.private-task: settle the delegated work and its completion follow-up before final run custody. - steering.queue-follow-up: the worker's gate now carries a complete website brief that does not pre-complete the queued contact requirement. Co-Authored-By: Claude Opus 5.5 --- .../runner-e2e/chat-catalog-descriptors.json | 15 +++--- ...ernal-company-secret-create-dialog.test.ts | 13 +++-- .../external-company-secret.ts | 2 +- ...ernal-personal-secret-entry-dialog.test.ts | 17 +++++++ .../external-personal-secret.test.ts | 2 +- .../external-personal-secret.ts | 2 +- .../external-private-browser-worker.ts | 15 +++--- .../internal-private-delegation.test.ts | 24 ++++++++++ .../internal-private-delegation.ts | 28 ++++++++++- .../runtime-active-steering.test.ts | 31 +++++++++++- .../runtime-active-steering.ts | 40 ++++++++++------ .../runtime-cited-delivery.test.ts | 28 +++++++++++ .../runtime-cited-delivery.ts | 23 +++++++++ .../runtime-discard.test.ts | 31 +++++++++++- .../chat-catalog-domains/runtime-discard.ts | 48 +++++++++++++------ .../runtime-queue.test.ts | 11 ++++- .../chat-catalog-domains/runtime-queue.ts | 16 ++++++- 17 files changed, 287 insertions(+), 59 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/internal-private-delegation.test.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-cited-delivery.test.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-cited-delivery.ts diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 53434a786bf..3b5b747c1e8 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -4994,7 +4994,7 @@ "chat-flow.ts", "chat-scenarios/secrets.yaml" ], - "oracleVersion": "paperclip.chat-catalog.company-secret-controller-baseline.v21", + "oracleVersion": "paperclip.chat-catalog.company-secret-controller-baseline.v22", "expectedRunCount": 3, "minimumRunCount": 2, "expectedHumanMessageCount": 2, @@ -5080,7 +5080,7 @@ "../../ui/src/pages/secrets/SetMyUserSecretDialog.tsx", "chat-scenarios/secrets.yaml" ], - "oracleVersion": "paperclip.chat-catalog.personal-secret.v20", + "oracleVersion": "paperclip.chat-catalog.personal-secret.v21", "expectedRunCount": 3, "expectedHumanMessageCount": 2, "minimumRunCount": 3, @@ -5489,14 +5489,16 @@ "chat-catalog-domains/runtime-discard.ts", "chat-catalog-domains/runtime-queue-controls.ts", "chat-catalog-domains/runtime-queue.ts", + "chat-catalog-domains/runtime-cited-delivery.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "chat-scenarios/steering.yaml" ], - "oracleVersion": "paperclip.chat-catalog.runtime-discard.v6", + "oracleVersion": "paperclip.chat-catalog.runtime-discard.v7", "coverageGaps": [], "expectedRunCount": 5, + "minimumRunCount": 4, "expectedHumanMessageCount": 2, "turnTimeoutMs": 120000, "attemptTimeoutMs": 1020000, @@ -5622,7 +5624,7 @@ "chat-catalog-domains/runtime-support.ts", "chat-scenarios/steering.yaml" ], - "oracleVersion": "paperclip.chat-catalog.runtime-queue.v3", + "oracleVersion": "paperclip.chat-catalog.runtime-queue.v4", "coverageGaps": [], "expectedRunCount": 3, "turnTimeoutMs": 120000, @@ -5644,6 +5646,7 @@ "chat-catalog-domains/runtime-active-steering.ts", "chat-catalog-domains/runtime-rendered-artifact.ts", "chat-catalog-domains/runtime-queue.ts", + "chat-catalog-domains/runtime-cited-delivery.ts", "chat-catalog-domains/runtime-interruption.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", @@ -5651,7 +5654,7 @@ "registered-artifact.ts", "chat-scenarios/steering.yaml" ], - "oracleVersion": "paperclip.chat-catalog.runtime.steer-active.v8", + "oracleVersion": "paperclip.chat-catalog.runtime.steer-active.v9", "coverageGaps": [], "expectedRunCount": 3, "minimumRunCount": 2, @@ -6002,7 +6005,7 @@ "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/identity-support.ts" ], - "oracleVersion": "paperclip.chat-catalog.internal.private-task.v2", + "oracleVersion": "paperclip.chat-catalog.internal.private-task.v3", "coverageGaps": [], "expectedRunCount": 3, "minimumRunCount": 2, diff --git a/tests/runner-e2e/chat-catalog-domains/external-company-secret-create-dialog.test.ts b/tests/runner-e2e/chat-catalog-domains/external-company-secret-create-dialog.test.ts index e8c3060bd7c..eb1a5fd2c0c 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-company-secret-create-dialog.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-company-secret-create-dialog.test.ts @@ -12,21 +12,24 @@ it.skipIf(process.env.PAPERCLIP_CHAT_EVAL_SECRET_ROUTE_CANARY !== "1")("waits fo const code = stripTypeScriptTypes(`async function operation(input:any,page:any){${source.slice(start, end)}}`); const operation = new Function("validCompanySecretMetadata","canonicalCompanySecretKey",`${code};return operation`)(validCompanySecretMetadata,canonicalCompanySecretKey) as (input: unknown, page: Page) => Promise<{ secretId: string; storageClean: boolean }>; const ui = await readFile(new URL("../../../ui/src/pages/Secrets.tsx", import.meta.url), "utf8"); - expect(ui).toContain('{editingDefinition ? "Edit user-provided secret" : "Create secret"}'); + for (const anchor of ['ariaLabel="Who provides the value"', 'title: "Organization"', 'htmlFor="new-secret-name">Name', 'htmlFor="new-secret-value">Value', 'htmlFor="new-secret-vault">Stored in', 'aria-expanded={createMoreOpen}', 'More options', ': "Create secret"}']) expect(ui).toContain(anchor); + expect(ui).toContain('{secretValueProvider === "user" ? "Environment variable" : "Key"}'); expect(ui).toContain("setCreateOpen(false)"); expect(ui).toContain("setDetailSelection(result.item.id)"); const browser = await chromium.launch({ headless: true, ...(process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE ? { executablePath: process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE } : {}) }); const companyId = "11111111-1111-4111-8111-111111111111", secretId = "22222222-2222-4222-8222-222222222222"; try { - for (const scenario of ["created", "requested metadata", "wrong stored metadata", "failed POST", "missing form"]) { + for (const scenario of ["created", "requested metadata", "several vaults", "wrong stored metadata", "wrong storage", "failed POST", "missing form", "retired form"]) { const page = await browser.newPage(); page.setDefaultTimeout(700); const companyMetadata=scenario==="requested metadata"||scenario==="wrong stored metadata"?{name:"Deployment API Key",key:"DEPLOYMENT_API_KEY"}:undefined,expectedName=companyMetadata?.name??"Deployment API key"; await page.route("**/*", async route => { - if (route.request().method() === "POST") {const posted=route.request().postDataJSON();return route.fulfill({ status: scenario === "failed POST" ? 400 : 201, contentType: "application/json", body: JSON.stringify({ id:secretId,companyId,name:posted.name,key:scenario==="wrong stored metadata"?"other-key":canonicalCompanySecretKey(posted.key),scope:"company",status:"active" }) });} - await route.fulfill({ contentType: "text/html; charset=utf-8", body: `` }); + if (route.request().method() === "POST") {const posted=route.request().postDataJSON();return route.fulfill({ status: scenario === "failed POST" ? 400 : 201, contentType: "application/json", body: JSON.stringify({ id:secretId,companyId,name:posted.name,key:scenario==="wrong stored metadata"?"other-key":canonicalCompanySecretKey(posted.key),scope:"company",status:"active",provider:scenario==="wrong storage"?"aws_secrets_manager":"local_encrypted" }) });} + const vaults=scenario==="several vaults"?``:""; + const form=scenario==="retired form"?``:``; + await route.fulfill({ contentType: "text/html; charset=utf-8", body: `${form}` }); }); const result = operation({ mode: "company-secret-create", companyId, setupURL: "http://127.0.0.1:3300/RUN/company/settings/secrets", credential: "synthetic-private-value",companyMetadata }, page); - if (scenario === "created"||scenario==="requested metadata") { + if (scenario === "created"||scenario==="requested metadata"||scenario==="several vaults") { expect(await result).toMatchObject({ secretId, storageClean: true,...(companyMetadata?{requestedCompanyName:"Deployment API Key",requestedCompanyKey:"DEPLOYMENT_API_KEY",savedCompanyName:"Deployment API Key",savedCompanyKey:"deployment_api_key"}:{}) }); expect(await page.getByRole("dialog", { name: expectedName, exact: true }).isVisible()).toBe(true); } else await expect(result).rejects.toThrow(); diff --git a/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts b/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts index 167d5259e30..d7a9291079a 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts @@ -215,7 +215,7 @@ export const companySecretExternalCases=[definition("secrets.configure-company-s }, optionalCompanySecretBindingApproval:true, expectedCompanySecretBindingApproval(value){if(!gradeCompanySecret(value).passed)return;return companySecretApprovalProof(value);}, - oracleVersion:"paperclip.chat-catalog.company-secret-controller-baseline.v21",expectedRunCount:3,minimumRunCount:2,expectedHumanMessageCount:2,attemptTimeoutMs:480000, + oracleVersion:"paperclip.chat-catalog.company-secret-controller-baseline.v22",expectedRunCount:3,minimumRunCount:2,expectedHumanMessageCount:2,attemptTimeoutMs:480000, sourceFiles:["../../server/src/routes/secrets.ts","../../packages/paperclip-runner/src/cli/opencode-app-server-proxy.ts","../../packages/paperclip-runner/src/cli/opencode-proxy-finish-diagnostics.ts","../../packages/paperclip-runner/src/protocol-actions/request-human-input.ts","chat-catalog-domains/external-company-secret.ts","chat-catalog-domains/external-company-secret-confirmation.ts","chat-catalog-domains/external-secret-proposal.ts","chat-catalog-domains/external-secret-question.ts","chat-catalog-domains/external-secret-route.ts","chat-catalog-domains/external-company-secret-menu.ts","chat-catalog-contract.ts","chat-catalog-flow.ts","chat-catalog-domains/external-private-browser.ts","chat-catalog-domains/external-private-browser-worker.ts","chat-catalog-domains/external-credential-language.ts","chat-catalog-domains/external-secret-scan.ts","chat-catalog-domains/external-support.ts","chat-catalog-domains/internal-support.ts","chat-catalog-domains/identity-support.ts","live-fixtures.ts","../../server/src/services/secrets.ts","../../server/src/services/native-runtime/runner-api-catalog.ts","../../server/src/routes/issues.ts","run-observations.ts","chat-flow.ts","chat-scenarios/secrets.yaml"], async act(ctx,state){ const deadline=chatCatalogOriginalPhaseDeadline(ctx),remaining=()=>remainingIdentityObservationBudget(ctx); diff --git a/tests/runner-e2e/chat-catalog-domains/external-personal-secret-entry-dialog.test.ts b/tests/runner-e2e/chat-catalog-domains/external-personal-secret-entry-dialog.test.ts index accfbeb7555..29e27167a91 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-personal-secret-entry-dialog.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-personal-secret-entry-dialog.test.ts @@ -29,3 +29,20 @@ it.skipIf(process.env.PAPERCLIP_CHAT_EVAL_SECRET_ROUTE_CANARY !== "1")("observes expect(persisted).toBe(false); } finally { await browser.close(); } }, 10000); +it.skipIf(process.env.PAPERCLIP_CHAT_EVAL_SECRET_ROUTE_CANARY !== "1")("binds the private entry form to the fixture definition by the dialog heading the product renders", async () => { + const { readFile } = await import("node:fs/promises"); + const ui = await readFile(new URL("../../../ui/src/pages/secrets/SetMyUserSecretDialog.tsx", import.meta.url), "utf8"); + const worker = await readFile(new URL("./external-private-browser-worker.ts", import.meta.url), "utf8"); + expect(ui).toContain('{existingSecret ? "Replace your value" : "Set your value"}'); + expect(ui).toContain('{definition ? ` for ${definition.name}` : ""}'); + const check = worker.match(/dialog\.getByRole\('heading',\{name:'([^']+)',exact:true\}\)\.isVisible\(\)\)throw new Error\('Wrong personal credential form'\)/); + expect(check?.[1]).toBe("Set your value for Personal publishing credential"); + const browser = await chromium.launch({ headless: true, ...process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE ? { executablePath: process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE } : {} }), page = await browser.newPage(); + try { + for (const [heading, bound] of [["Set your value for Personal publishing credential", true], ["Set your value for Shared publishing credential", false], ["Replace your value for Personal publishing credential", false]] as const) { + await page.setContent(`

${heading}

`); + const dialog = personalSecretEntryDialog(page); + expect(await dialog.getByRole("heading", { name: check![1], exact: true }).isVisible()).toBe(bound); + } + } finally { await browser.close(); } +}, 10000); diff --git a/tests/runner-e2e/chat-catalog-domains/external-personal-secret.test.ts b/tests/runner-e2e/chat-catalog-domains/external-personal-secret.test.ts index 47d9469c950..49b00e2b6aa 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-personal-secret.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-personal-secret.test.ts @@ -338,7 +338,7 @@ it("keeps a confirmation dependent on actual worker delivery",()=>{ }); it("declares the prospective personal acknowledgment calibration without changing its bounds", () => { - expect(personalSecretExternalCases[0]).toMatchObject({ oracleVersion: "paperclip.chat-catalog.personal-secret.v20", + expect(personalSecretExternalCases[0]).toMatchObject({ oracleVersion: "paperclip.chat-catalog.personal-secret.v21", expectedRunCount: 3, minimumRunCount: 3, expectedHumanMessageCount: 2, attemptTimeoutMs: 660000 }); }); // Full cohort 3: the recorded guidance linked My secrets and excluded chat; a variant that also asks diff --git a/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts b/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts index 56351ccbbe7..034c1ac2f95 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts @@ -489,7 +489,7 @@ export const personalSecretExternalCases = [ if (!o.state.initialQuestionId) return; return { interactionId: String(o.state.initialQuestionId), sourceRunId: String(o.state.initialRunId), userId: String(o.state.currentPersonId) }; }, - oracleVersion: "paperclip.chat-catalog.personal-secret.v20", + oracleVersion: "paperclip.chat-catalog.personal-secret.v21", setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["current-owner-missing-required-personal-value"] }, expectedRunCount: 3, minimumRunCount: 3, diff --git a/tests/runner-e2e/chat-catalog-domains/external-private-browser-worker.ts b/tests/runner-e2e/chat-catalog-domains/external-private-browser-worker.ts index 50157010d80..82fa7e02182 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-private-browser-worker.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-private-browser-worker.ts @@ -225,7 +225,7 @@ async function main() { await page.getByRole('listitem').filter({hasText:entry.definition.key}).getByRole('button',{name:'Set value',exact:true}).click(); } else await page.getByRole('button',{name:'Set my value',exact:true}).click(); const dialog=personalSecretEntryDialog(page);await dialog.getByRole('heading',{name:/^Set your value/}).waitFor(); - if(!await dialog.getByText('Personal publishing credential',{exact:true}).isVisible())throw new Error('Wrong personal credential form'); + if(!await dialog.getByRole('heading',{name:'Set your value for Personal publishing credential',exact:true}).isVisible())throw new Error('Wrong personal credential form'); await dialog.getByRole('textbox').fill(input.credential); const created=page.waitForResponse(response=>new URL(response.url()).pathname===`/api/companies/${input.companyId}/me/user-secrets`&&response.request().method()==='POST'&&response.status()===201); await dialog.getByRole('button',{name:'Save value',exact:true}).click();const secret=await(await created).json() as {id:string;ownerUserId:string;userSecretDefinitionId:string}; @@ -237,16 +237,17 @@ async function main() { if(!validCompanySecretMetadata(input.companyMetadata))throw new Error("Invalid requested company secret metadata"); const requestedName=input.companyMetadata?.name??"Deployment API key",requestedKey=input.companyMetadata?.key; if(!input.companyId||!/^[a-f0-9-]{36}$/i.test(input.companyId)||!new URL(input.setupURL).pathname.endsWith("/company/settings/secrets"))throw new Error("Invalid finite company secret operation"); - await page.goto(input.setupURL,{waitUntil:"domcontentloaded"});await page.getByRole("button",{name:"New secret",exact:true}).click(); - const dialog=page.getByRole("dialog",{name:"Create secret",exact:true});await dialog.getByLabel("Name",{exact:true}).fill(requestedName); - if(requestedKey){await dialog.getByRole("button",{name:"Edit",exact:true}).click();await dialog.getByLabel("Key",{exact:true}).fill(requestedKey);} - await dialog.getByLabel("Provider",{exact:true}).selectOption("local_encrypted"); + await page.goto(input.setupURL,{waitUntil:"domcontentloaded"});await page.getByRole("button",{name:"New secret",exact:true}).first().click(); + const dialog=page.getByRole("dialog").filter({has:page.getByRole("radiogroup",{name:"Who provides the value",exact:true})});await dialog.getByLabel("Name",{exact:true}).fill(requestedName); + await dialog.getByRole("radiogroup",{name:"Who provides the value",exact:true}).getByRole("radio",{name:/^Organization\b/}).click(); + if(requestedKey){const more=dialog.getByRole("button",{name:"More options",exact:true});if(await more.getAttribute("aria-expanded")!=="true")await more.click();await dialog.getByLabel("Key",{exact:true}).fill(requestedKey);} + const storedIn=dialog.getByLabel("Stored in",{exact:true});if(await storedIn.count()){const local=await storedIn.locator("option").evaluateAll(options=>options.map(option=>(option as HTMLOptionElement).value).filter(value=>value.startsWith("local_encrypted:")));if(local.length!==1)throw new Error("Paperclip-encrypted storage is not a single choice");await storedIn.selectOption(local[0]!);} await dialog.getByLabel("Value",{exact:true}).fill(input.credential); const created=page.waitForResponse(response=>new URL(response.url()).pathname===`/api/companies/${input.companyId}/secrets`&&response.request().method()==="POST"&&response.status()===201); await dialog.getByRole("button",{name:"Create secret",exact:true}).click();const response=await created,posted=response.request().postDataJSON() as {name?:string;key?:string}; if(posted.name!==requestedName||requestedKey&&posted.key!==requestedKey)throw new Error("Outgoing company secret metadata differs from the actual request"); - const submittedKey=requestedKey??posted.key??requestedName,secret=await response.json() as {id:string;companyId:string;name:string;key:string;scope:string;status:string}; - if(secret.companyId!==input.companyId||secret.scope!=="company"||secret.status!=="active"||secret.name!==requestedName||secret.key!==canonicalCompanySecretKey(submittedKey))throw new Error("Saved company secret metadata differs from the actual request"); + const submittedKey=requestedKey??posted.key??requestedName,secret=await response.json() as {id:string;companyId:string;name:string;key:string;scope:string;status:string;provider:string}; + if(secret.companyId!==input.companyId||secret.provider!=="local_encrypted"||secret.scope!=="company"||secret.status!=="active"||secret.name!==requestedName||secret.key!==canonicalCompanySecretKey(submittedKey))throw new Error("Saved company secret metadata differs from the actual request"); if(!/^[a-f0-9-]{36}$/i.test(secret.id))throw new Error("Invalid created secret identity");await dialog.waitFor({state:"hidden"}); await page.getByText(requestedName,{exact:true}).first().waitFor(); const storageClean=await page.evaluate(secret=>!JSON.stringify([Object.entries(localStorage),Object.entries(sessionStorage),document.body.innerText,location.href]).includes(secret),input.credential); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-private-delegation.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-private-delegation.test.ts new file mode 100644 index 00000000000..3dd6036dbef --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/internal-private-delegation.test.ts @@ -0,0 +1,24 @@ +import { expect, it } from "vitest"; +import { privateDelegationSettled } from "./internal-private-delegation.js"; + +const waiting = { conversationState: "waiting" }; +const lead = "ceo", worker = "morgan"; +const retained = () => [{ agentId: lead, status: "succeeded" }, { agentId: worker, status: "succeeded" }, { agentId: lead, status: "running" }]; + +it("keeps waiting while the completion follow-up from the retained cohort-10 attempt is still running", () => + expect(privateDelegationSettled({ source: waiting, runs: retained(), leadId: lead, workerId: worker })).toBe("unsettled")); +it("is settled once the completion follow-up has reported back", () => { + const runs = retained(); runs[2]!.status = "succeeded"; + expect(privateDelegationSettled({ source: waiting, runs, leadId: lead, workerId: worker })).toBe("reported"); +}); +it("waits out a quiet window when the worker finished and no follow-up has appeared yet", () => + expect(privateDelegationSettled({ source: waiting, runs: retained().slice(0, 2), leadId: lead, workerId: worker })).toBe("quiet")); +it.each([ + ["the worker is still running", (runs: ReturnType) => { runs[1]!.status = "running"; runs.pop(); }], + ["the worker never ran", (runs: ReturnType) => { runs.splice(1, 1); }], + ["the follow-up was cancelled", (runs: ReturnType) => { runs[2]!.status = "cancelled"; }], +])("is not settled when %s", (_label, mutate) => { const runs = retained(); mutate(runs); expect(privateDelegationSettled({ source: waiting, runs, leadId: lead, workerId: worker })).toBe("unsettled"); }); +it("is not settled while the conversation is still working", () => { + const runs = retained(); runs[2]!.status = "succeeded"; + expect(privateDelegationSettled({ source: { conversationState: "working" }, runs, leadId: lead, workerId: worker })).toBe("unsettled"); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-private-delegation.ts b/tests/runner-e2e/chat-catalog-domains/internal-private-delegation.ts index 9f2bc73c9a1..14112c0f6ef 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-private-delegation.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-private-delegation.ts @@ -1,5 +1,6 @@ import { expect } from "@playwright/test"; -import type { ChatCatalogCase } from "../chat-catalog-contract.js"; +import { pollUntil } from "../api.js"; +import { chatCatalogOriginalPhaseDeadline, type ChatCatalogCase, type ChatCatalogContext } from "../chat-catalog-contract.js"; import { fixtureWorker, makeDefinition } from "./internal-definition.js"; import { openStandingChat } from "./identity-support.js"; import { agentStatus, cleanupPrivacyProbes, mainPerson, observerPerson, personRead } from "./internal-privacy.js"; @@ -35,6 +36,28 @@ export function gradePrivateDelegation(value: unknown) { "Actual worker execution belongs to the private work task instead of the standing conversation."); return c.result(); } +const followUpFailure = "Private delegated work or its completion follow-up failed"; +export function privateDelegationSettled(input: { source?: { conversationState?: string | null }; runs: Array<{ agentId: string; status: string }>; leadId: string; workerId: string }) { + if (input.source?.conversationState !== "waiting" || !input.runs.every(run => run.status === "succeeded") || !input.runs.some(run => run.agentId === input.workerId)) return "unsettled"; + return input.runs.filter(run => run.agentId === input.leadId).length >= 2 ? "reported" : "quiet"; +} +async function settleDelegatedCompletion(ctx: ChatCatalogContext, workerId: string) { + const original = chatCatalogOriginalPhaseDeadline(ctx), deadlineAt = Math.min(original === undefined ? Number.POSITIVE_INFINITY : original - 15_000, Date.now() + 120_000); + let quietSince: number | undefined; + try { + await pollUntil({ label: "Private delegated work and its completion follow-up settled", deadlineAt, intervalMs: 1000, + load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; }, + accept: ({ source, runs }) => { + const state = privateDelegationSettled({ source, runs, leadId: ctx.input.fixtures.agent.id, workerId }); + if (state === "unsettled") { quietSince = undefined; return false; } + if (state === "reported") return true; + quietSince ??= Date.now(); return Date.now() - quietSince >= 30_000; + }, + reject: ({ runs }) => runs.some(run => ["failed", "cancelled", "timed_out"].includes(run.status)) ? followUpFailure : undefined }); + } catch (error) { + if (error instanceof Error && error.message.includes(followUpFailure)) throw error; + } +} function privateDelegation(): ChatCatalogCase { let observer: Awaited> | undefined; const base = makeDefinition("tasks.private-task", async ctx => { @@ -49,7 +72,7 @@ function privateDelegation(): ChatCatalogCase { await putDocument(ctx, source.id, `# Confidential launch brief\n${confidentialMarker}. Keep the announcement inside Private Launch and its authorized team.\n`); return { project: p, worker, ownerId, observerId: observer.id, sourceId: source.id, confidentialMarker }; }, gradePrivateDelegation); - return { ...base, oracleVersion: "paperclip.chat-catalog.internal.private-task.v2", authentication: "human-session", expectedRunCount: 3, minimumRunCount: 2, + return { ...base, oracleVersion: "paperclip.chat-catalog.internal.private-task.v3", authentication: "human-session", expectedRunCount: 3, minimumRunCount: 2, runPolicy: { kind: "source-settled-with-active-work", maximumActiveWorkerRuns: 1 }, sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-private-delegation.ts", "chat-catalog-domains/internal-privacy.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/identity-support.ts"], async observe(ctx, state): Promise { @@ -71,6 +94,7 @@ function privateDelegation(): ChatCatalogCase { observerSearchResults: JSON.stringify(search.results ?? []) }; }, async verifyResult(ctx, state, value) { + await settleDelegatedCompletion(ctx, entity(state, "worker").id); await base.verifyResult(ctx, state, value); if (!observer) throw new Error("Missing authenticated observer browser"); const o = value as PrivateDelegationObservation; const saved = o.tasks[0]!; const page = await observer.context.newPage(); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.test.ts index 5e88c65f23e..60f3e491a97 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.test.ts @@ -10,7 +10,9 @@ function valid() { queryReply: "Use Steer beside your saved message to send this direction to Morgan's active turn; waiting in the queue is separate from delivery.", steerObservedBranch: "same-run-acknowledged", steerStatus: 200, acknowledgedSameRun: true, steerAuditBound: true, idempotentRetryStatus: 200, ordinaryConsumerCount: 0, actualBlueArtifact: true, artifactRegisteredAfterDirection: true, deliveryRevisionBound: true, workerTaskDone: true, queueEmptyAtEnd: true, - steerFailureCode: "" as string | undefined, providerFailureVisible: false, failureRetainedMessage: false, supportedContinuationVisible: false, continuationInputBound: false } }; + steerFailureCode: "" as string | undefined, providerFailureVisible: false, failureRetainedMessage: false, supportedContinuationVisible: false, continuationInputBound: false, + leadRelayCount: 0, queuedComments: [{ id: "blue", updatedAt: "2026-11-13T04:01:38.950Z", body: "Switch to a blue design while you are working." }] as Array<{ id: string; updatedAt: string; body: string }>, + workerCommentReceipts: { reads: {} as Record, cited: [] as string[] } } }; } it("accepts real explicit same-turn delivery and actual blue result", () => expect(gradeActiveSteering(valid()).passed).toBe(true)); it("accepts an actual visible provider rejection with retained one-time normal delivery", () => { @@ -81,7 +83,8 @@ it("binds only the requested website-delivery document written by the consuming // Retained runner-opencode attempt (gha-43077703607): the CEO relayed the direction as its own comment on Morgan's task. // It queues beside the person's message and reaches Morgan through normal admission; it is not the person's to steer. it("accepts a same-turn steer when the chat lead's own relay stays queued for one normal continuation", () => { - const o = valid(); Object.assign(o.state, { leadRelayCount: 1, ordinaryConsumerCount: 1 }); + const o = valid(); Object.assign(o.state, { leadRelayCount: 1, ordinaryConsumerCount: 1, continuationInputBound: true }); + o.state.queuedComments.push({ id: "relay", updatedAt: "2026-11-13T04:02:58.671Z", body: "Use a blue palette." }); o.state.queueAfterAction.entries = [{ comment: { id: "relay", body: "Use a blue palette.", authorAgentId: "ceo" } } as never]; o.runs.push({ id: "relay-consumer", status: "succeeded" }); expect(gradeActiveSteering(o).passed).toBe(true); o.state.ordinaryConsumerCount = 0; expect(gradeActiveSteering(o).passed).toBe(false); @@ -95,6 +98,7 @@ it("rejects an extra run after a same-turn steer when no relay was queued", () = }); it("accepts an unsupported runtime whose continuation also consumes the chat lead's relay", () => { const o = unsupported(); Object.assign(o.state, { leadRelayCount: 1 }); + o.state.queuedComments.push({ id: "relay", updatedAt: "2026-11-13T04:02:58.671Z", body: "Use a blue palette." }); o.state.queueAfterAction.entries.push({ comment: { id: "relay", body: "Use a blue palette.", authorAgentId: "ceo" } } as never); expect(gradeActiveSteering(o).passed).toBe(true); }); @@ -129,3 +133,26 @@ describe("the final website may come from either of the worker's runs, once, aft const twice = unsupported(); Object.assign(twice.state, { ordinaryConsumerCount: 2 }); expect(gradeActiveSteering(twice).passed).toBe(false); }); }); + +const relay = { id: "relay", updatedAt: "2026-11-13T04:02:58.671Z", body: "Morgan, the person asked for a blue design." }; +function citedInTurn() { + const o = unsupported(); o.runs.pop(); + Object.assign(o.state, { ordinaryConsumerCount: 0, continuationInputBound: false, leadRelayCount: 1 }); + o.state.queuedComments = [o.state.queuedComments[0]!, relay]; + o.state.workerCommentReceipts = { reads: { blue: "2026-11-13T04:01:38.950Z", relay: "2026-11-13T04:02:58.671Z" }, cited: ["relay", "blue"] }; + return o; +} +it("accepts the retained cohort-10 shape: the running turn read and cited both queued messages, built blue, and no continuation ran", () => expect(gradeActiveSteering(citedInTurn()).passed).toBe(true)); +it("accepts the target handled in turn while the uncited relay reaches one continuation", () => { + const o = citedInTurn(); o.state.workerCommentReceipts.cited = ["blue"]; o.state.ordinaryConsumerCount = 1; o.state.continuationInputBound = true; o.runs.push({ id: "ordinary", status: "succeeded" }); + expect(gradeActiveSteering(o).passed).toBe(true); +}); +it.each([ + ["reads the direction without citing it and starts no continuation", (o: ReturnType) => { o.state.workerCommentReceipts.cited = ["relay"]; }], + ["cites a revision it never read", (o: ReturnType) => { o.state.workerCommentReceipts.reads = { relay: relay.updatedAt }; }], + ["read an older revision of an edited direction", (o: ReturnType) => { o.state.queuedComments[0]!.updatedAt = "2026-11-13T04:03:30.000Z"; }], + ["cites the direction but never builds a blue page", (o: ReturnType) => { o.state.actualBlueArtifact = false; }], + ["cites the direction and still replays it in a continuation", (o: ReturnType) => { o.state.ordinaryConsumerCount = 1; o.state.continuationInputBound = false; o.runs.push({ id: "ordinary", status: "succeeded" }); }], + ["handles it in turn and admits an extra run", (o: ReturnType) => { o.runs.push({ id: "extra", status: "succeeded" }); }], + ["loses the relayed copy from the observed queue", (o: ReturnType) => { o.state.queuedComments = [o.state.queuedComments[0]!]; }], +])("rejects a running turn that %s", (_label, mutate) => { const o = citedInTurn(); mutate(o); expect(gradeActiveSteering(o).passed).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.ts b/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.ts index 93bf1473169..65d14b5ac56 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.ts @@ -8,7 +8,8 @@ import { checks, type Entity } from "./internal-support.js"; import { probeAgentPermission, cleanupIdentityProbes } from "./identity-support.js"; import { busyRuntimeWorker, releaseRuntimeWorker } from "./runtime-busy-worker.js"; import { renderRuntimeArtifact } from "./runtime-rendered-artifact.js"; -import { runtimeQueueCases } from "./runtime-queue.js"; +import { runCommentReceipts, runtimeQueueCases, settledQueue } from "./runtime-queue.js"; +import { continuationCarriesOwed, owedAfterTurn, type CommentReceipts, type QueuedFollowUp } from "./runtime-cited-delivery.js"; import { deliveryRevisionBound } from "./runtime-interruption.js"; import { runtimeDefinition, observeRuntime, runtimeObservation } from "./runtime-support.js"; import type { ChatCatalogContext } from "../chat-catalog-contract.js"; @@ -59,16 +60,20 @@ export function gradeActiveSteering(value: unknown) { c.add("human-only-control-without-forged-delivery", s.agentSteerStatus === 403 && (unsupported ? s.humanSteerAttempted === true : s.humanSteerSelected === true), "Only the person's own action can steer the message; the agent's attempt is refused."); const retained = after?.entries.some(row => row.comment.id === s.targetId && row.comment.body === direction) === true; + const queued = Array.isArray(s.queuedComments) ? s.queuedComments as QueuedFollowUp[] : [], receipts = s.workerCommentReceipts as CommentReceipts | undefined; + const queuedComplete = queued.filter(row => row.id === s.targetId).length === 1 && queued.length === 1 + Number(s.leadRelayCount ?? 0) && new Set(queued.map(row => row.id)).size === queued.length; + const owed = owedAfterTurn(queued.filter(row => !(accepted && row.id === s.targetId)), receipts); + const consumed = queuedComplete && (owed.length ? s.ordinaryConsumerCount === 1 && s.continuationInputBound === true : s.ordinaryConsumerCount === 0); c.add("delivery-distinct-from-queue-with-no-loss", s.steerObservedBranch === branch && (accepted - ? s.acknowledgedSameRun === true && after?.entries.every(row => row.comment.authorAgentId === o.leadId) === true && s.steerAuditBound === true && s.idempotentRetryStatus === 200 && s.ordinaryConsumerCount === (Number(s.leadRelayCount) ? 1 : 0) + ? s.acknowledgedSameRun === true && after?.entries.every(row => row.comment.authorAgentId === o.leadId) === true && s.steerAuditBound === true && s.idempotentRetryStatus === 200 && consumed : branch === "unsupported-queued-normal-continuation" ? s.steerStatus === 409 && s.steerFailureCode === "steering_unsupported" && s.steeredActivityCount === 0 && retained && s.unsupportedStateRetained === true - && s.ordinaryConsumerCount === 1 && s.continuationInputBound === true + && consumed : branch === "provider-rejected-normal-continuation" && ["steering_rejected", "steering_unsupported"].includes(String(s.steerFailureCode)) && s.steerStatus === 409 && s.providerFailureVisible === true - && s.failureRetainedMessage === true && retained && s.supportedContinuationVisible === true && s.ordinaryConsumerCount === 1 && s.continuationInputBound === true), - "A durable same-run ACK is required for delivery; a runtime that cannot steer (or a provider rejection) instead keeps the exact message visibly queued, records no steering, and one normal continuation consumes it."); + && s.failureRetainedMessage === true && retained && s.supportedContinuationVisible === true && consumed), + "A durable same-run ACK is required for steering; a runtime that cannot steer (or a provider rejection) instead keeps the exact message visibly queued and records no steering. Every queued message is delivered once: the running turn read it and cited it as handled, or one normal continuation receives each message not handled in turn and none that was."); c.add("actual-blue-output-correct-native-owner", s.actualBlueArtifact === true && s.artifactRegisteredAfterDirection === true && s.deliveryRevisionBound === true && s.workerTaskDone === true && s.queueEmptyAtEnd === true - && o.runs.length === (accepted && !Number(s.leadRelayCount) ? 2 : 3) && o.runs.every(row => row.status === "succeeded") && o.tasks.length === 1, + && o.runs.length === 2 + (owed.length ? 1 : 0) && o.runs.every(row => row.status === "succeeded") && o.tasks.length === 1, "The website the worker registered last, after the direction was saved, renders blue (whichever of its runs applied the direction), and the delivery document is current at a run of that work, with no duplicate work or hidden run."); return c.result(); } @@ -97,9 +102,9 @@ export const runtimeActiveSteeringCases = [runtimeDefinition("steering.steer-act await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); return { ...state, humanId: session.user.id, queueBefore: before, targetId, ...controls }; }, gradeActiveSteering, { - authentication: "human-session", expectedRunCount: 3, minimumRunCount: 2, attemptTimeoutMs: 720_000, oracleVersion: "paperclip.chat-catalog.runtime.steer-active.v8", + authentication: "human-session", expectedRunCount: 3, minimumRunCount: 2, attemptTimeoutMs: 720_000, oracleVersion: "paperclip.chat-catalog.runtime.steer-active.v9", setupPolicy: runtimeQueueCases[0]!.setupPolicy, verifySetup: runtimeQueueCases[0]!.verifySetup, - sourceFiles: ["chat-catalog-domains/runtime-active-steering.ts", "chat-catalog-domains/runtime-rendered-artifact.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-interruption.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "registered-artifact.ts", "chat-scenarios/steering.yaml"], + sourceFiles: ["chat-catalog-domains/runtime-active-steering.ts", "chat-catalog-domains/runtime-rendered-artifact.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-cited-delivery.ts", "chat-catalog-domains/runtime-interruption.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "registered-artifact.ts", "chat-scenarios/steering.yaml"], async act(ctx, state) { await pollUntil({ label: "Canonical steering guidance while the original native owner remains active", deadlineAt: Date.now() + 150_000, load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; }, @@ -112,7 +117,7 @@ export const runtimeActiveSteeringCases = [runtimeDefinition("steering.steer-act const human = before.entries.filter(row => !row.comment.authorAgentId), relays = before.entries.filter(row => row.comment.authorAgentId === ctx.input.fixtures.agent.id); if (human.length !== 1 || human[0]?.comment.id !== targetId || human.length + relays.length !== before.entries.length || before.targetRunId !== state.workerRunId) throw new Error("Canonical guidance lost or duplicated the saved human direction"); - state.leadRelayCount = relays.length; + state.leadRelayCount = relays.length; state.leadRelayIds = relays.map(row => row.comment.id); const mutation = { queueId: before.queueId, revision: before.revision, targetRunId: before.targetRunId }; state.agentSteerStatus = await probeAgentPermission(ctx, `${endpoint}/${targetId}/steer`, "POST", mutation); await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" }); @@ -157,16 +162,21 @@ export const runtimeActiveSteeringCases = [runtimeDefinition("steering.steer-act await writeFile((state.gate as { gate: string }).gate, state.steerStatus === 200 ? `The human brief is ready. ${build} Use the actual design direction you received, then complete normally. Preserve saved human requirements for normal admission; do not create tasks or publish externally.` : `The human brief is ready. ${build} Then finish this current turn normally. Keep human requirements queued by Paperclip intact for normal delivery; when one arrives, apply it to website.html, register the updated file, update the delivery document, and complete the task. Do not create tasks or publish externally.`, "utf8"); - const finished = await pollUntil({ label: "Actual selected steering or preserved normal continuation produces its website", deadlineAt: Date.now() + 240_000, + const done = (v: { runs: Array<{ status: string }>; queue: IssueQueuedCommentQueue; task: Entity }) => v.task.status === "done" && v.queue.entries.length === 0 && v.runs.every(row => row.status === "succeeded"); + const settledInTurn = settledQueue((v: { runs: Array<{ status: string }>; queue: IssueQueuedCommentQueue; task: Entity }) => done(v) && v.runs.length === 2, 30_000); + const finished = await pollUntil({ label: "Actual selected steering, in-turn handling or preserved normal continuation produces its website", deadlineAt: Date.now() + 240_000, load: async () => ({ runs: await ctx.allRuns(), queue: await ctx.input.api.get(endpoint), task: await ctx.input.api.get(`/api/issues/${task.id}`) }), - accept: v => v.task.status === "done" && v.queue.entries.length === 0 && v.runs.length === (state.steerStatus === 200 && !Number(state.leadRelayCount) ? 2 : 3) && v.runs.every(row => row.status === "succeeded"), + accept: v => settledInTurn(v) || done(v) && v.runs.length === 3, reject: v => v.runs.length > 3 || v.runs.some(row => ["failed", "cancelled", "timed_out"].includes(row.status)) ? "Steered work failed or admitted an undeclared execution" : undefined }); const successors = finished.runs.filter(row => row.agentId === (state.worker as Entity).id && row.id !== state.workerRunId); - state.ordinaryConsumerCount = successors.length; state.outputRunId = state.steerStatus === 200 ? state.workerRunId : successors[0]?.id; - if (state.steerStatus === 200 && Number(state.leadRelayCount) && successors.length === 1) state.relayConsumerRunId = successors[0]!.id; + state.ordinaryConsumerCount = successors.length; state.outputRunId = state.steerStatus === 200 || !successors.length ? state.workerRunId : successors[0]!.id; + if (successors.length === 1) state.relayConsumerRunId = successors[0]!.id; + const comments = await ctx.input.api.get(`/api/issues/${task.id}/comments?order=asc`), queuedIds = [targetId, ...(state.leadRelayIds as string[])]; + const queued = queuedIds.map(id => comments.find(row => row.id === id)).filter((row): row is Entity => !!row).map(row => ({ id: String(row.id), updatedAt: row.updatedAt, body: String(row.body ?? "") })); + const receipts = await runCommentReceipts(ctx, String(state.workerRunId)); state.queuedComments = queued; state.workerCommentReceipts = receipts; if (successors.length === 1) { - const full = await ctx.input.api.get<{ contextSnapshot?: { wakeCommentIds?: string[] }; runnerProfileJson?: { nativeExecutionInput?: { task?: { prompt?: string } } } }>(`/api/heartbeat-runs/${successors[0]!.id}`); - state.continuationInputBound = full.contextSnapshot?.wakeCommentIds?.filter(id => id === targetId).length === 1 && full.runnerProfileJson?.nativeExecutionInput?.task?.prompt?.includes(direction) === true; + const full = await ctx.input.api.get<{ contextSnapshot?: { wakeCommentIds?: string[]; paperclipWake?: { comments?: Array<{ id?: string; readByEarlierTurn?: boolean }> } }; runnerProfileJson?: { nativeExecutionInput?: { task?: { prompt?: string } } } }>(`/api/heartbeat-runs/${successors[0]!.id}`); + state.continuationInputBound = continuationCarriesOwed(queued.filter(row => !(state.steerStatus === 200 && row.id === targetId)), receipts, { contextSnapshot: full.contextSnapshot, prompt: full.runnerProfileJson?.nativeExecutionInput?.task?.prompt }); } state.workerTaskDone = finished.task.status === "done"; state.queueEmptyAtEnd = finished.queue.entries.length === 0; await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-cited-delivery.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-cited-delivery.test.ts new file mode 100644 index 00000000000..1420ff0b4dd --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-cited-delivery.test.ts @@ -0,0 +1,28 @@ +import { expect, it } from "vitest"; +import { continuationCarriesOwed, owedAfterTurn } from "./runtime-cited-delivery.js"; + +const font = { id: "3b2395dc-11b1-43dd-95ff-56c2fcdfb5bd", updatedAt: "2026-11-13T04:01:28.647Z", body: "Use a readable font." }; +const contact = { id: "3aa8c723-5c96-4346-85a8-a4992694307e", updatedAt: "2026-11-13T04:03:45.909Z", body: "Add a contact section.\nLabel it." }; +const read = { [font.id]: font.updatedAt, [contact.id]: contact.updatedAt }; +const wake = (ids: string[], earlier: string[] = []) => ({ wakeCommentIds: ids, paperclipWake: { comments: ids.map(id => ({ id, readByEarlierTurn: earlier.includes(id) })) } }); + +it("owes nothing that the turn read at its current revision and cited", () => expect(owedAfterTurn([font, contact], { reads: read, cited: [contact.id, font.id] })).toEqual([])); +it("owes a read that was not cited, a citation without a read, and a stale read", () => { + expect(owedAfterTurn([font, contact], { reads: read, cited: [font.id] })).toEqual([contact]); + expect(owedAfterTurn([font, contact], { reads: { [font.id]: font.updatedAt }, cited: [font.id, contact.id] })).toEqual([contact]); + expect(owedAfterTurn([{ ...font, updatedAt: "2026-11-13T04:05:00.000Z" }], { reads: read, cited: [font.id] })).toHaveLength(1); +}); +it("accepts one continuation carrying each owed comment once, JSON-escaped, with an earlier-read warning", () => { + const prompt = JSON.stringify({ comments: [contact.body] }); + expect(continuationCarriesOwed([font, contact], { reads: read, cited: [font.id] }, { contextSnapshot: wake([contact.id], [contact.id]), prompt })).toBe(true); + expect(continuationCarriesOwed([font, contact], { reads: {}, cited: [] }, { contextSnapshot: wake([font.id, contact.id]), prompt: `${font.body} ${prompt}` })).toBe(true); +}); +it.each([ + ["replays a comment already handled in turn", wake([font.id, contact.id], [contact.id]), `${font.body} ${JSON.stringify(contact.body)}`], + ["omits the owed comment", wake([], []), JSON.stringify(contact.body)], + ["delivers it twice", { wakeCommentIds: [contact.id, contact.id], paperclipWake: { comments: [{ id: contact.id, readByEarlierTurn: true }] } }, JSON.stringify(contact.body)], + ["does not warn that the earlier turn read it", wake([contact.id]), JSON.stringify(contact.body)], + ["lacks its text in the prompt", wake([contact.id], [contact.id]), "Use the brief."], +])("rejects a continuation that %s", (_label, contextSnapshot, prompt) => + expect(continuationCarriesOwed([font, contact], { reads: read, cited: [font.id] }, { contextSnapshot, prompt })).toBe(false)); +it("rejects a continuation when nothing was owed", () => expect(continuationCarriesOwed([font], { reads: read, cited: [font.id] }, { contextSnapshot: wake([]), prompt: "" })).toBe(false)); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-cited-delivery.ts b/tests/runner-e2e/chat-catalog-domains/runtime-cited-delivery.ts new file mode 100644 index 00000000000..a1824138f39 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-cited-delivery.ts @@ -0,0 +1,23 @@ +import { handledInTurn, readCurrentRevision, wakeMarksEarlierRead } from "./runtime-queue.js"; + +export type QueuedFollowUp = { id: string; updatedAt?: unknown; body?: string }; +export type CommentReceipts = { reads?: Record; cited?: string[] }; +export type ContinuationInput = { + contextSnapshot?: { wakeCommentIds?: string[]; paperclipWake?: { comments?: Array<{ id?: string; readByEarlierTurn?: boolean }> } }; + prompt?: string; +}; + +export function owedAfterTurn(queued: readonly T[], receipts: CommentReceipts | undefined) { + return queued.filter(row => !handledInTurn(receipts, row)); +} + +function carries(prompt: string | undefined, body: string | undefined) { + return !!prompt && !!body?.trim() && (prompt.includes(body) || prompt.includes(JSON.stringify(body).slice(1, -1))); +} + +export function continuationCarriesOwed(queued: readonly QueuedFollowUp[], receipts: CommentReceipts | undefined, run: ContinuationInput | undefined) { + const owed = owedAfterTurn(queued, receipts), handled = queued.filter(row => !owed.includes(row)), ids = run?.contextSnapshot?.wakeCommentIds ?? []; + return owed.length > 0 && owed.every(row => ids.filter(id => id === row.id).length === 1 && carries(run?.prompt, row.body) + && (!readCurrentRevision(receipts?.reads, row) || wakeMarksEarlierRead(run?.contextSnapshot, row.id))) + && handled.every(row => !ids.includes(row.id)); +} diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-discard.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-discard.test.ts index 63a68cbe290..bdf01b62cc8 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-discard.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-discard.test.ts @@ -12,7 +12,10 @@ function valid() { queryLinkedTask: true, targetPendingAfterReply: true, consumedLabelSaved: true, consumedBeforeCancellation: true, consumedStillReadable: true, lateDiscardStatus: 409, latePhaseAttributed: true, lateReply: "The later request was already consumed. I sent new direction to remove that section; its original history remains intact.", - cancelDirectionRunBound: true, finalQueueEmpty: true, finalPlanWithoutContact: true, initialRevisionRetained: true } }; + cancelDirectionRunBound: true, finalQueueEmpty: true, finalPlanWithoutContact: true, initialRevisionRetained: true, + workerRunId: "initial", secondContactId: "second-contact", consumerRunId: "contact", firstDeliveryRunCount: 3, continuationConsumerCount: 1, continuationInputBound: true, + queuedFollowUps: [{ id: "font", updatedAt: "2026-11-13T04:01:28.647Z" }, { id: "second-contact", updatedAt: "2026-11-13T04:03:45.909Z" }], + workerCommentReceipts: { reads: {} as Record, cited: [] as string[] } } }; } it("accepts genuine pending discard and distinct consumed-input cancellation as new direction", () => expect(gradeDiscard(valid()).passed).toBe(true)); it("accepts natural human-control and subsequent consumed-input guidance", () => { const o = valid(); o.state.queryReply = "That waiting message is yours. Open its queue menu and remove it with Discard."; o.state.lateReply = "Morgan processed the second request before you cancelled it. A follow-up now asks for the contact section to be removed; the delivered input stays in history."; expect(gradeDiscard(o).passed).toBe(true); }); @@ -94,3 +97,29 @@ it("fails when no labelled section was ever saved, so removal cannot be shown", const unlabelled = consumedPlan.replace(`## Contact section (label: ${label})`, "## Contact section"); expect(documentContactRemoved([...revs(initialPlan)].map(row => row.id === "r2" ? { ...row, body: unlabelled } : row), "final", "consumer", label)).toBe(false); }); + +function citedInTurn() { + const o = valid(); o.runs = o.runs.filter(row => row.id !== "contact"); + Object.assign(o.state, { consumerRunId: "initial", firstDeliveryRunCount: 2, continuationConsumerCount: 0, continuationInputBound: false, consumedBeforeCancellation: false }); + o.state.workerCommentReceipts = { reads: { "second-contact": "2026-11-13T04:03:45.909Z", font: "2026-11-13T04:01:28.647Z" }, cited: ["font", "second-contact"] }; + return o; +} +it("accepts the retained cohort-10 shape: the running turn read and cited both follow-ups, saved the contact label, and no continuation ran", () => expect(gradeDiscard(citedInTurn()).passed).toBe(true)); +it("accepts the later request handled in turn while the uncited font note reaches one continuation", () => { + const o = citedInTurn(); o.state.workerCommentReceipts.cited = ["second-contact"]; Object.assign(o.state, { firstDeliveryRunCount: 3, continuationConsumerCount: 1, continuationInputBound: true }); + o.runs.splice(2, 0, { id: "font-consumer", status: "succeeded" }); expect(gradeDiscard(o).passed).toBe(true); +}); +it.each([ + ["reads the later request without citing it and starts no continuation", (o: ReturnType) => { o.state.workerCommentReceipts.cited = ["font"]; }], + ["cites a request it never read", (o: ReturnType) => { delete o.state.workerCommentReceipts.reads["second-contact"]; }], + ["cites the request but never saves the labelled contact section", (o: ReturnType) => { o.state.consumedLabelSaved = false; }], + ["cites the request and still replays it in a continuation", (o: ReturnType) => { Object.assign(o.state, { firstDeliveryRunCount: 3, continuationConsumerCount: 1 }); o.runs.splice(2, 0, { id: "replay", status: "succeeded" }); }], + ["handles both in turn and admits an extra later run", (o: ReturnType) => { o.runs.push({ id: "extra", status: "succeeded" }); }], + ["cites the discarded request as handled", (o: ReturnType) => { o.state.workerCommentReceipts.cited.push("pending-contact"); }], + ["credits the later request to a run other than the one that handled it", (o: ReturnType) => { o.state.consumerRunId = "new-direction"; }], +])("rejects a running turn that %s", (_label, mutate) => { const o = citedInTurn(); mutate(o); expect(gradeDiscard(o).passed).toBe(false); }); +it.each([ + ["no continuation consumes the owed follow-ups", (o: ReturnType) => { o.state.continuationConsumerCount = 0; }], + ["the continuation input lacks an owed follow-up", (o: ReturnType) => { o.state.continuationInputBound = false; }], + ["the continuation never received the later request", (o: ReturnType) => { o.state.consumedBeforeCancellation = false; }], +])("rejects the continuation path when %s", (_label, mutate) => { const o = valid(); mutate(o); expect(gradeDiscard(o).passed).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-discard.ts b/tests/runner-e2e/chat-catalog-domains/runtime-discard.ts index 24219b1d364..3311de4d728 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-discard.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-discard.ts @@ -9,7 +9,8 @@ import { checks, hasLink, type Entity } from "./internal-support.js"; import { probeAgentPermission, cleanupIdentityProbes } from "./identity-support.js"; import { releaseRuntimeWorker } from "./runtime-busy-worker.js"; import { humanQueuedRuntime, probeConsumedHumanQueue } from "./runtime-queue-controls.js"; -import { runtimeQueueCases } from "./runtime-queue.js"; +import { handledInTurn, runCommentReceipts, runtimeQueueCases, settledQueue } from "./runtime-queue.js"; +import { continuationCarriesOwed, owedAfterTurn, type CommentReceipts, type QueuedFollowUp } from "./runtime-cited-delivery.js"; import { runtimeDefinition, runtimeObservation, observeRuntime } from "./runtime-support.js"; const contactText = "Also add a contact section to the website."; @@ -49,19 +50,26 @@ export function gradeDiscard(value: unknown) { c.add("chat-guides-supported-human-discard", String(s.queryReply ?? "").trim().length > 0 && s.queryLinkedTask === true && s.targetPendingAfterReply === true && s.agentDiscardStatus === 403, "The reply links the task whose queue holds the request, the request is still pending after the reply, and the agent cannot impersonate the person's discard."); - c.add("already-delivered-input-cannot-be-erased", s.consumedBeforeCancellation === true && s.consumedStillReadable === true + const followUps = Array.isArray(s.queuedFollowUps) ? s.queuedFollowUps as QueuedFollowUp[] : [], receipts = s.workerCommentReceipts as CommentReceipts | undefined; + const second = followUps.find(row => row.id === s.secondContactId), owed = owedAfterTurn(followUps, receipts); + const firstDelivery = followUps.length === 2 && !!second && !receipts?.cited?.includes(String(s.targetId)) && (owed.length + ? s.firstDeliveryRunCount === 3 && s.continuationConsumerCount === 1 && s.continuationInputBound === true + : s.firstDeliveryRunCount === 2 && s.continuationConsumerCount === 0); + c.add("later-input-delivered-once-before-cancellation", firstDelivery && (handledInTurn(receipts, second) ? s.consumerRunId === s.workerRunId : owed.includes(second!) && s.consumedBeforeCancellation === true), + "Each saved follow-up is delivered once before the cancellation: the running turn read it and cited it as handled, or one ordinary continuation receives every follow-up not handled in turn and none that was. The discarded request is never cited as handled."); + c.add("already-delivered-input-cannot-be-erased", s.consumedStillReadable === true && s.lateDiscardStatus === 409 && s.latePhaseAttributed === true && String(s.lateReply ?? "").trim().length > 0, "The consumed request stays readable and cannot be discarded late; the later answer is attributed to its own run."); c.add("new-direction-reaches-exactly-one-native-continuation", s.cancelDirectionRunBound === true && s.finalQueueEmpty === true && s.consumedLabelSaved === true && s.finalPlanWithoutContact === true - && s.initialRevisionRetained === true && o.runs.length === 5 && o.runs.every(row => row.status === "succeeded") && o.tasks.length === 1 + && s.initialRevisionRetained === true && o.runs.length === Number(s.firstDeliveryRunCount) + 2 && o.runs.every(row => row.status === "succeeded") && o.tasks.length === 1 && o.tasks[0]?.id === (s.task as Entity).id && o.tasks[0]?.status === "done", "A distinct saved instruction reaches one real later worker, updates the saved plan and preserves prior work/history without replaying the discarded request."); return c.result(); } export const runtimeDiscardCases = [runtimeDefinition("steering.discard-queued-message", ctx => humanQueuedRuntime(ctx, contactText), gradeDiscard, { - authentication: "human-session", expectedHumanMessageCount: 2, expectedRunCount: 5, attemptTimeoutMs: 1_020_000, oracleVersion: "paperclip.chat-catalog.runtime-discard.v6", - sourceFiles: ["chat-catalog-domains/runtime-discard.ts", "chat-catalog-domains/runtime-queue-controls.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "chat-scenarios/steering.yaml"], + authentication: "human-session", expectedHumanMessageCount: 2, expectedRunCount: 5, minimumRunCount: 4, attemptTimeoutMs: 1_020_000, oracleVersion: "paperclip.chat-catalog.runtime-discard.v7", + sourceFiles: ["chat-catalog-domains/runtime-discard.ts", "chat-catalog-domains/runtime-queue-controls.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-cited-delivery.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "chat-scenarios/steering.yaml"], setupPolicy: runtimeQueueCases[0]!.setupPolicy, verifySetup: runtimeQueueCases[0]!.verifySetup, async act(ctx, state) { await pollUntil({ label: "Human discard guidance while Morgan is genuinely active", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000), @@ -84,22 +92,32 @@ export const runtimeDiscardCases = [runtimeDefinition("steering.discard-queued-m state.discardVisible = true; state.discardedCommentStatus = (await ctx.input.page.request.get(`${root}/comments/${targetId}`)).status(); await ctx.input.capture("actual-human-contact-discard", "Only the eligible contact request has left the real task queue", "runtime-discard.png"); // This new, explicit human request is a distinct later instruction. It - // deliberately reaches normal native delivery before its cancellation. + // deliberately reaches delivery before its cancellation. const secondText = `I have changed my mind: add a contact section to the saved website plan, labelled ${ctx.input.nonce}. Keep the accessible-font requirement and finish this task.`; await submitTaskReply(ctx.input.page, secondText); const secondQueue = await pollUntil({ label: "Distinct human contact request genuinely queued", deadlineAt: Date.now() + 20_000, load: () => ctx.input.api.get(queuePath), accept: q => q.entries.length === 2 && q.entries.some(row => row.comment.body === secondText) }); const second = secondQueue.entries.find(row => row.comment.body === secondText)!; state.secondContactId = second.comment.id; - await writeFile((state.gate as { gate: string }).gate, "The human brief is ready: save an initial website-plan task document, retaining accessible font guidance, then finish this turn normally. Leave saved follow-ups for their ordinary continuation.", "utf8"); + await writeFile((state.gate as { gate: string }).gate, "The human brief is ready: save an initial website-plan task document, retaining accessible font guidance, then finish this turn normally.", "utf8"); + type Delivery = { runs: Array<{ status: string }>; task: Entity; queue: IssueQueuedCommentQueue }; + const delivered = (v: Delivery) => v.runs.every(row => row.status === "succeeded") && v.task.status === "done" && v.queue.entries.length === 0; + const settledInTurn = settledQueue((v: Delivery) => delivered(v) && v.runs.length === 2, 30_000); const consumed = await pollUntil({ label: "Real later contact request consumed before cancellation", deadlineAt: Date.now() + 240_000, load: async () => ({ runs: await ctx.allRuns(), task: await ctx.input.api.get(root), queue: await ctx.input.api.get(queuePath) }), - accept: v => v.runs.length === 3 && v.runs.every(row => row.status === "succeeded") && v.task.status === "done" && v.queue.entries.length === 0 }); - const consumer = consumed.runs.find(row => row.agentId === (state.worker as Entity).id && row.id !== state.workerRunId)!; - state.consumerRunId = consumer.id; - const detail = await ctx.input.api.get<{ contextSnapshot?: { wakeCommentIds?: string[] }; runnerProfileJson?: { nativeExecutionInput?: { task?: { prompt?: string } } } }>(`/api/heartbeat-runs/${consumer.id}`); - state.consumedBeforeCancellation = detail.contextSnapshot?.wakeCommentIds?.includes(second.comment.id) === true - && promptCarries(detail.runnerProfileJson?.nativeExecutionInput?.task?.prompt, secondText); + accept: v => settledInTurn(v) || delivered(v) && v.runs.length === 3, + reject: v => v.runs.length > 3 || v.runs.some(row => ["failed", "cancelled", "timed_out"].includes(row.status)) ? "The saved follow-ups failed or reached more than one continuation" : undefined }); + const taskComments = await ctx.input.api.get(`${root}/comments?order=asc`); + const followUps = secondQueue.entries.map(row => taskComments.find(comment => comment.id === row.comment.id)).filter((row): row is Entity => !!row) + .map(row => ({ id: String(row.id), updatedAt: row.updatedAt, body: String(row.body ?? "") })); + const receipts = await runCommentReceipts(ctx, String(state.workerRunId)); state.queuedFollowUps = followUps; state.workerCommentReceipts = receipts; + const successors = consumed.runs.filter(row => row.agentId === (state.worker as Entity).id && row.id !== state.workerRunId); + state.firstDeliveryRunCount = consumed.runs.length; state.continuationConsumerCount = successors.length; + const detail = successors[0] && await ctx.input.api.get<{ contextSnapshot?: { wakeCommentIds?: string[]; paperclipWake?: { comments?: Array<{ id?: string; readByEarlierTurn?: boolean }> } }; runnerProfileJson?: { nativeExecutionInput?: { task?: { prompt?: string } } } }>(`/api/heartbeat-runs/${successors[0].id}`); + const prompt = detail?.runnerProfileJson?.nativeExecutionInput?.task?.prompt; + state.continuationInputBound = !!detail && continuationCarriesOwed(followUps, receipts, { contextSnapshot: detail.contextSnapshot, prompt }); + state.consumerRunId = handledInTurn(receipts, followUps.find(row => row.id === second.comment.id)) ? state.workerRunId : successors[0]?.id; + state.consumedBeforeCancellation = !!detail && detail.contextSnapshot?.wakeCommentIds?.includes(second.comment.id) === true && promptCarries(prompt, secondText); state.discardedNeverDelivered = consumed.runs.every(row => !((row.contextSnapshot?.wakeCommentIds as string[] | undefined) ?? []).includes(targetId)); state.lateDiscardStatus = await probeConsumedHumanQueue(ctx, String(state.humanId), `${queuePath}/${second.comment.id}`, "DELETE", { queueId: secondQueue.queueId, revision: secondQueue.revision }); await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); @@ -108,8 +126,8 @@ export const runtimeDiscardCases = [runtimeDefinition("steering.discard-queued-m await sendChatMessage(ctx.input.page, message, chatCatalogNativePhaseRemainingBudget(ctx)); const final = await pollUntil({ label: "One real new-direction continuation after already-delivered cancellation", deadlineAt: Date.now() + 300_000, load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns(), task: await ctx.input.api.get(root), queue: await ctx.input.api.get(queuePath) }; }, - accept: v => v.source?.conversationState === "waiting" && v.runs.length === 5 && v.runs.every(row => row.status === "succeeded") && v.task.status === "done" && v.queue.entries.length === 0, - reject: v => v.runs.length > 5 || v.runs.some(row => ["failed", "cancelled", "timed_out"].includes(row.status)) ? "The declared later cancellation failed or duplicated execution" : undefined }); + accept: v => v.source?.conversationState === "waiting" && v.runs.length === priorIds.length + 2 && v.runs.every(row => row.status === "succeeded") && v.task.status === "done" && v.queue.entries.length === 0, + reject: v => v.runs.length > priorIds.length + 2 || v.runs.some(row => ["failed", "cancelled", "timed_out"].includes(row.status)) ? "The declared later cancellation failed or duplicated execution" : undefined }); const later = final.runs.filter(row => !priorIds.includes(row.id)), lead = later.find(row => row.agentId === ctx.input.fixtures.agent.id), worker = later.find(row => row.agentId === (state.worker as Entity).id); const sourceComments = await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/comments?order=asc`), human = sourceComments.filter(row => !row.authorAgentId && row.body === message); const replies = sourceComments.filter(row => row.createdByRunId === lead?.id && row.authorAgentId === ctx.input.fixtures.agent.id); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-queue.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-queue.test.ts index 11f665b3465..7a6997b9f03 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-queue.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-queue.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect } from "vitest"; import { deflateRawSync } from "node:zlib"; -import { gradeQueuedFollowup, historyReadRevisions, keyboardMoveQueuedMessageUp, promptCarriesComment, readCurrentRevision, resultCitedCommentIds, zipEntryTexts } from "./runtime-queue.js"; +import { gradeQueuedFollowup, historyReadRevisions, keyboardMoveQueuedMessageUp, promptCarriesComment, queuedFollowUpBrief, readCurrentRevision, resultCitedCommentIds, zipEntryTexts } from "./runtime-queue.js"; /** A minimal zip with one deflated and one stored entry, laid out as zip tools write it. */ function zip(entries: Array<{ name: string; text: string; deflate: boolean }>) { const locals: Buffer[] = [], centrals: Buffer[] = []; let offset = 0; @@ -127,3 +127,12 @@ it("reads comment citations only from the run's accepted result", () => { expect(resultCitedCommentIds([{ eventType: "run.result.proposed", payload: { prpEvent: { payload: { result: { evidence: [{ ref: `comment:${id}` }] } } } } }])).toEqual([]); expect(resultCitedCommentIds([accepted({ evidence: [{ ref: `document:comment:${id}` }, { ref: "comment:not-a-comment" }] })])).toEqual([]); }); + +describe("queued follow-up brief", () => { + const brief = queuedFollowUpBrief("Original website brief n1"); + it("gives the worker a complete website brief bound to its reference", () => { + expect(brief).toContain("reference Original website brief n1"); + for (const part of ["Acorn Notes", "website.html", "Research notes your whole team can find", "Request early access", "Free for teams", "hello@acornnotes.example"]) expect(brief).toContain(part); + }); + it("does not pre-complete the queued contact requirement", () => expect(brief).not.toMatch(/contact/i)); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-queue.ts b/tests/runner-e2e/chat-catalog-domains/runtime-queue.ts index 783fb937726..69987bd7b14 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-queue.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-queue.ts @@ -127,6 +127,18 @@ export function replyNamesTask(reply: string, task: Entity) { function savedRequirement(queue: IssueQueuedCommentQueue | undefined, leadId: string) { return queue?.entries.length === 1 && queue.entries[0]!.comment.authorAgentId === leadId ? queue.entries[0] : undefined; } +export function queuedFollowUpBrief(marker: string) { + return [ + `The human brief is now ready. Website brief, reference ${marker}:`, + "- Product: Acorn Notes, a shared notebook app for small research teams. It launches in beta next month.", + "- Page: one static website.html page with a light background and a readable sans-serif font.", + "- Hero: the headline \"Research notes your whole team can find\", the line \"Shared notebooks, full-text search across experiments, and one-click citation export\", and a \"Request early access\" button.", + "- Features: three short items: shared notebooks, search across every experiment, citation export to BibTeX.", + "- Pricing: \"Free for teams of up to ten during the beta.\"", + "- Company details: hello@acornnotes.example, +1 555 0142, 12 Elm Street, Portland, OR 97201.", + `Save this brief as an initial task document that includes the exact reference ${marker}, then finish this current turn normally. Keep all prior user requirements queued by Paperclip intact for normal delivery; do not create tasks or external effects.`, + ].join("\n"); +} export function gradeQueuedFollowup(value: unknown) { const c = checks(); const o = runtimeObservation(value); c.add("real-queue-and-delivery-evidence", !!o, "The actual task queue, saved requirement, real initial worker and later consuming native input are required."); if (!o) return c.result(); @@ -150,7 +162,7 @@ export function gradeQueuedFollowup(value: unknown) { return c.result(); } export const runtimeQueueCases = [runtimeDefinition("steering.queue-follow-up", busyRuntimeWorker, gradeQueuedFollowup, { - oracleVersion: "paperclip.chat-catalog.runtime-queue.v3", + oracleVersion: "paperclip.chat-catalog.runtime-queue.v4", sourceFiles: ["chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/steering.yaml"], expectedRunCount: 3, attemptTimeoutMs: 720_000, setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["lead-ready", "real-busy-worker", "worker-task-scope"] }, @@ -181,7 +193,7 @@ export const runtimeQueueCases = [runtimeDefinition("steering.queue-follow-up", await ctx.input.capture("actual-contact-followup-queue", "Contact requirement remains visibly queued behind real active work", "runtime-contact-queue.png"); await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); const marker = `Original website brief ${ctx.input.nonce}`; state.briefMarker = marker; - await writeFile((state.gate as { gate: string }).gate, `The human brief is now ready: save an initial task document with the exact reference ${marker}, then finish this current turn normally. Keep all prior user requirements queued by Paperclip intact for normal delivery; do not create tasks or external effects.`, "utf8"); + await writeFile((state.gate as { gate: string }).gate, queuedFollowUpBrief(marker), "utf8"); const delivered = await pollUntil({ label: "Normal successor receives actual queued contact requirement", deadlineAt: Date.now() + 240_000, load: async () => ({ task: await ctx.input.api.get(`/api/issues/${task.id}`), runs: await ctx.allRuns(), queue: await ctx.input.api.get(`/api/issues/${task.id}/queued-comments`) }), accept: value => value.task.status === "done" && value.runs.length === 3 && value.runs.every(run => run.status === "succeeded") && value.queue.entries.length === 0,