diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 53434a786bf..3b5b747c1e8 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -4994,7 +4994,7 @@ "chat-flow.ts", "chat-scenarios/secrets.yaml" ], - "oracleVersion": "paperclip.chat-catalog.company-secret-controller-baseline.v21", + "oracleVersion": "paperclip.chat-catalog.company-secret-controller-baseline.v22", "expectedRunCount": 3, "minimumRunCount": 2, "expectedHumanMessageCount": 2, @@ -5080,7 +5080,7 @@ "../../ui/src/pages/secrets/SetMyUserSecretDialog.tsx", "chat-scenarios/secrets.yaml" ], - "oracleVersion": "paperclip.chat-catalog.personal-secret.v20", + "oracleVersion": "paperclip.chat-catalog.personal-secret.v21", "expectedRunCount": 3, "expectedHumanMessageCount": 2, "minimumRunCount": 3, @@ -5489,14 +5489,16 @@ "chat-catalog-domains/runtime-discard.ts", "chat-catalog-domains/runtime-queue-controls.ts", "chat-catalog-domains/runtime-queue.ts", + "chat-catalog-domains/runtime-cited-delivery.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "chat-scenarios/steering.yaml" ], - "oracleVersion": "paperclip.chat-catalog.runtime-discard.v6", + "oracleVersion": "paperclip.chat-catalog.runtime-discard.v7", "coverageGaps": [], "expectedRunCount": 5, + "minimumRunCount": 4, "expectedHumanMessageCount": 2, "turnTimeoutMs": 120000, "attemptTimeoutMs": 1020000, @@ -5622,7 +5624,7 @@ "chat-catalog-domains/runtime-support.ts", "chat-scenarios/steering.yaml" ], - "oracleVersion": "paperclip.chat-catalog.runtime-queue.v3", + "oracleVersion": "paperclip.chat-catalog.runtime-queue.v4", "coverageGaps": [], "expectedRunCount": 3, "turnTimeoutMs": 120000, @@ -5644,6 +5646,7 @@ "chat-catalog-domains/runtime-active-steering.ts", "chat-catalog-domains/runtime-rendered-artifact.ts", "chat-catalog-domains/runtime-queue.ts", + "chat-catalog-domains/runtime-cited-delivery.ts", "chat-catalog-domains/runtime-interruption.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", @@ -5651,7 +5654,7 @@ "registered-artifact.ts", "chat-scenarios/steering.yaml" ], - "oracleVersion": "paperclip.chat-catalog.runtime.steer-active.v8", + "oracleVersion": "paperclip.chat-catalog.runtime.steer-active.v9", "coverageGaps": [], "expectedRunCount": 3, "minimumRunCount": 2, @@ -6002,7 +6005,7 @@ "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/identity-support.ts" ], - "oracleVersion": "paperclip.chat-catalog.internal.private-task.v2", + "oracleVersion": "paperclip.chat-catalog.internal.private-task.v3", "coverageGaps": [], "expectedRunCount": 3, "minimumRunCount": 2, diff --git a/tests/runner-e2e/chat-catalog-domains/external-company-secret-create-dialog.test.ts b/tests/runner-e2e/chat-catalog-domains/external-company-secret-create-dialog.test.ts index e8c3060bd7c..eb1a5fd2c0c 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-company-secret-create-dialog.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-company-secret-create-dialog.test.ts @@ -12,21 +12,24 @@ it.skipIf(process.env.PAPERCLIP_CHAT_EVAL_SECRET_ROUTE_CANARY !== "1")("waits fo const code = stripTypeScriptTypes(`async function operation(input:any,page:any){${source.slice(start, end)}}`); const operation = new Function("validCompanySecretMetadata","canonicalCompanySecretKey",`${code};return operation`)(validCompanySecretMetadata,canonicalCompanySecretKey) as (input: unknown, page: Page) => Promise<{ secretId: string; storageClean: boolean }>; const ui = await readFile(new URL("../../../ui/src/pages/Secrets.tsx", import.meta.url), "utf8"); - expect(ui).toContain('{editingDefinition ? "Edit user-provided secret" : "Create secret"}'); + for (const anchor of ['ariaLabel="Who provides the value"', 'title: "Organization"', 'htmlFor="new-secret-name">Name', 'htmlFor="new-secret-value">Value', 'htmlFor="new-secret-vault">Stored in', 'aria-expanded={createMoreOpen}', 'More options', ': "Create secret"}']) expect(ui).toContain(anchor); + expect(ui).toContain('{secretValueProvider === "user" ? "Environment variable" : "Key"}'); expect(ui).toContain("setCreateOpen(false)"); expect(ui).toContain("setDetailSelection(result.item.id)"); const browser = await chromium.launch({ headless: true, ...(process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE ? { executablePath: process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE } : {}) }); const companyId = "11111111-1111-4111-8111-111111111111", secretId = "22222222-2222-4222-8222-222222222222"; try { - for (const scenario of ["created", "requested metadata", "wrong stored metadata", "failed POST", "missing form"]) { + for (const scenario of ["created", "requested metadata", "several vaults", "wrong stored metadata", "wrong storage", "failed POST", "missing form", "retired form"]) { const page = await browser.newPage(); page.setDefaultTimeout(700); const companyMetadata=scenario==="requested metadata"||scenario==="wrong stored metadata"?{name:"Deployment API Key",key:"DEPLOYMENT_API_KEY"}:undefined,expectedName=companyMetadata?.name??"Deployment API key"; await page.route("**/*", async route => { - if (route.request().method() === "POST") {const posted=route.request().postDataJSON();return route.fulfill({ status: scenario === "failed POST" ? 400 : 201, contentType: "application/json", body: JSON.stringify({ id:secretId,companyId,name:posted.name,key:scenario==="wrong stored metadata"?"other-key":canonicalCompanySecretKey(posted.key),scope:"company",status:"active" }) });} - await route.fulfill({ contentType: "text/html; charset=utf-8", body: `` }); + if (route.request().method() === "POST") {const posted=route.request().postDataJSON();return route.fulfill({ status: scenario === "failed POST" ? 400 : 201, contentType: "application/json", body: JSON.stringify({ id:secretId,companyId,name:posted.name,key:scenario==="wrong stored metadata"?"other-key":canonicalCompanySecretKey(posted.key),scope:"company",status:"active",provider:scenario==="wrong storage"?"aws_secrets_manager":"local_encrypted" }) });} + const vaults=scenario==="several vaults"?``:""; + const form=scenario==="retired form"?``:``; + await route.fulfill({ contentType: "text/html; charset=utf-8", body: `${form}` }); }); const result = operation({ mode: "company-secret-create", companyId, setupURL: "http://127.0.0.1:3300/RUN/company/settings/secrets", credential: "synthetic-private-value",companyMetadata }, page); - if (scenario === "created"||scenario==="requested metadata") { + if (scenario === "created"||scenario==="requested metadata"||scenario==="several vaults") { expect(await result).toMatchObject({ secretId, storageClean: true,...(companyMetadata?{requestedCompanyName:"Deployment API Key",requestedCompanyKey:"DEPLOYMENT_API_KEY",savedCompanyName:"Deployment API Key",savedCompanyKey:"deployment_api_key"}:{}) }); expect(await page.getByRole("dialog", { name: expectedName, exact: true }).isVisible()).toBe(true); } else await expect(result).rejects.toThrow(); diff --git a/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts b/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts index 167d5259e30..d7a9291079a 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts @@ -215,7 +215,7 @@ export const companySecretExternalCases=[definition("secrets.configure-company-s }, optionalCompanySecretBindingApproval:true, expectedCompanySecretBindingApproval(value){if(!gradeCompanySecret(value).passed)return;return companySecretApprovalProof(value);}, - oracleVersion:"paperclip.chat-catalog.company-secret-controller-baseline.v21",expectedRunCount:3,minimumRunCount:2,expectedHumanMessageCount:2,attemptTimeoutMs:480000, + oracleVersion:"paperclip.chat-catalog.company-secret-controller-baseline.v22",expectedRunCount:3,minimumRunCount:2,expectedHumanMessageCount:2,attemptTimeoutMs:480000, sourceFiles:["../../server/src/routes/secrets.ts","../../packages/paperclip-runner/src/cli/opencode-app-server-proxy.ts","../../packages/paperclip-runner/src/cli/opencode-proxy-finish-diagnostics.ts","../../packages/paperclip-runner/src/protocol-actions/request-human-input.ts","chat-catalog-domains/external-company-secret.ts","chat-catalog-domains/external-company-secret-confirmation.ts","chat-catalog-domains/external-secret-proposal.ts","chat-catalog-domains/external-secret-question.ts","chat-catalog-domains/external-secret-route.ts","chat-catalog-domains/external-company-secret-menu.ts","chat-catalog-contract.ts","chat-catalog-flow.ts","chat-catalog-domains/external-private-browser.ts","chat-catalog-domains/external-private-browser-worker.ts","chat-catalog-domains/external-credential-language.ts","chat-catalog-domains/external-secret-scan.ts","chat-catalog-domains/external-support.ts","chat-catalog-domains/internal-support.ts","chat-catalog-domains/identity-support.ts","live-fixtures.ts","../../server/src/services/secrets.ts","../../server/src/services/native-runtime/runner-api-catalog.ts","../../server/src/routes/issues.ts","run-observations.ts","chat-flow.ts","chat-scenarios/secrets.yaml"], async act(ctx,state){ const deadline=chatCatalogOriginalPhaseDeadline(ctx),remaining=()=>remainingIdentityObservationBudget(ctx); diff --git a/tests/runner-e2e/chat-catalog-domains/external-personal-secret-entry-dialog.test.ts b/tests/runner-e2e/chat-catalog-domains/external-personal-secret-entry-dialog.test.ts index accfbeb7555..29e27167a91 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-personal-secret-entry-dialog.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-personal-secret-entry-dialog.test.ts @@ -29,3 +29,20 @@ it.skipIf(process.env.PAPERCLIP_CHAT_EVAL_SECRET_ROUTE_CANARY !== "1")("observes expect(persisted).toBe(false); } finally { await browser.close(); } }, 10000); +it.skipIf(process.env.PAPERCLIP_CHAT_EVAL_SECRET_ROUTE_CANARY !== "1")("binds the private entry form to the fixture definition by the dialog heading the product renders", async () => { + const { readFile } = await import("node:fs/promises"); + const ui = await readFile(new URL("../../../ui/src/pages/secrets/SetMyUserSecretDialog.tsx", import.meta.url), "utf8"); + const worker = await readFile(new URL("./external-private-browser-worker.ts", import.meta.url), "utf8"); + expect(ui).toContain('{existingSecret ? "Replace your value" : "Set your value"}'); + expect(ui).toContain('{definition ? ` for ${definition.name}` : ""}'); + const check = worker.match(/dialog\.getByRole\('heading',\{name:'([^']+)',exact:true\}\)\.isVisible\(\)\)throw new Error\('Wrong personal credential form'\)/); + expect(check?.[1]).toBe("Set your value for Personal publishing credential"); + const browser = await chromium.launch({ headless: true, ...process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE ? { executablePath: process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE } : {} }), page = await browser.newPage(); + try { + for (const [heading, bound] of [["Set your value for Personal publishing credential", true], ["Set your value for Shared publishing credential", false], ["Replace your value for Personal publishing credential", false]] as const) { + await page.setContent(`

${heading}

`); + const dialog = personalSecretEntryDialog(page); + expect(await dialog.getByRole("heading", { name: check![1], exact: true }).isVisible()).toBe(bound); + } + } finally { await browser.close(); } +}, 10000); diff --git a/tests/runner-e2e/chat-catalog-domains/external-personal-secret.test.ts b/tests/runner-e2e/chat-catalog-domains/external-personal-secret.test.ts index 47d9469c950..49b00e2b6aa 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-personal-secret.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-personal-secret.test.ts @@ -338,7 +338,7 @@ it("keeps a confirmation dependent on actual worker delivery",()=>{ }); it("declares the prospective personal acknowledgment calibration without changing its bounds", () => { - expect(personalSecretExternalCases[0]).toMatchObject({ oracleVersion: "paperclip.chat-catalog.personal-secret.v20", + expect(personalSecretExternalCases[0]).toMatchObject({ oracleVersion: "paperclip.chat-catalog.personal-secret.v21", expectedRunCount: 3, minimumRunCount: 3, expectedHumanMessageCount: 2, attemptTimeoutMs: 660000 }); }); // Full cohort 3: the recorded guidance linked My secrets and excluded chat; a variant that also asks diff --git a/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts b/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts index 56351ccbbe7..034c1ac2f95 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts @@ -489,7 +489,7 @@ export const personalSecretExternalCases = [ if (!o.state.initialQuestionId) return; return { interactionId: String(o.state.initialQuestionId), sourceRunId: String(o.state.initialRunId), userId: String(o.state.currentPersonId) }; }, - oracleVersion: "paperclip.chat-catalog.personal-secret.v20", + oracleVersion: "paperclip.chat-catalog.personal-secret.v21", setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["current-owner-missing-required-personal-value"] }, expectedRunCount: 3, minimumRunCount: 3, diff --git a/tests/runner-e2e/chat-catalog-domains/external-private-browser-worker.ts b/tests/runner-e2e/chat-catalog-domains/external-private-browser-worker.ts index 50157010d80..82fa7e02182 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-private-browser-worker.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-private-browser-worker.ts @@ -225,7 +225,7 @@ async function main() { await page.getByRole('listitem').filter({hasText:entry.definition.key}).getByRole('button',{name:'Set value',exact:true}).click(); } else await page.getByRole('button',{name:'Set my value',exact:true}).click(); const dialog=personalSecretEntryDialog(page);await dialog.getByRole('heading',{name:/^Set your value/}).waitFor(); - if(!await dialog.getByText('Personal publishing credential',{exact:true}).isVisible())throw new Error('Wrong personal credential form'); + if(!await dialog.getByRole('heading',{name:'Set your value for Personal publishing credential',exact:true}).isVisible())throw new Error('Wrong personal credential form'); await dialog.getByRole('textbox').fill(input.credential); const created=page.waitForResponse(response=>new URL(response.url()).pathname===`/api/companies/${input.companyId}/me/user-secrets`&&response.request().method()==='POST'&&response.status()===201); await dialog.getByRole('button',{name:'Save value',exact:true}).click();const secret=await(await created).json() as {id:string;ownerUserId:string;userSecretDefinitionId:string}; @@ -237,16 +237,17 @@ async function main() { if(!validCompanySecretMetadata(input.companyMetadata))throw new Error("Invalid requested company secret metadata"); const requestedName=input.companyMetadata?.name??"Deployment API key",requestedKey=input.companyMetadata?.key; if(!input.companyId||!/^[a-f0-9-]{36}$/i.test(input.companyId)||!new URL(input.setupURL).pathname.endsWith("/company/settings/secrets"))throw new Error("Invalid finite company secret operation"); - await page.goto(input.setupURL,{waitUntil:"domcontentloaded"});await page.getByRole("button",{name:"New secret",exact:true}).click(); - const dialog=page.getByRole("dialog",{name:"Create secret",exact:true});await dialog.getByLabel("Name",{exact:true}).fill(requestedName); - if(requestedKey){await dialog.getByRole("button",{name:"Edit",exact:true}).click();await dialog.getByLabel("Key",{exact:true}).fill(requestedKey);} - await dialog.getByLabel("Provider",{exact:true}).selectOption("local_encrypted"); + await page.goto(input.setupURL,{waitUntil:"domcontentloaded"});await page.getByRole("button",{name:"New secret",exact:true}).first().click(); + const dialog=page.getByRole("dialog").filter({has:page.getByRole("radiogroup",{name:"Who provides the value",exact:true})});await dialog.getByLabel("Name",{exact:true}).fill(requestedName); + await dialog.getByRole("radiogroup",{name:"Who provides the value",exact:true}).getByRole("radio",{name:/^Organization\b/}).click(); + if(requestedKey){const more=dialog.getByRole("button",{name:"More options",exact:true});if(await more.getAttribute("aria-expanded")!=="true")await more.click();await dialog.getByLabel("Key",{exact:true}).fill(requestedKey);} + const storedIn=dialog.getByLabel("Stored in",{exact:true});if(await storedIn.count()){const local=await storedIn.locator("option").evaluateAll(options=>options.map(option=>(option as HTMLOptionElement).value).filter(value=>value.startsWith("local_encrypted:")));if(local.length!==1)throw new Error("Paperclip-encrypted storage is not a single choice");await storedIn.selectOption(local[0]!);} await dialog.getByLabel("Value",{exact:true}).fill(input.credential); const created=page.waitForResponse(response=>new URL(response.url()).pathname===`/api/companies/${input.companyId}/secrets`&&response.request().method()==="POST"&&response.status()===201); await dialog.getByRole("button",{name:"Create secret",exact:true}).click();const response=await created,posted=response.request().postDataJSON() as {name?:string;key?:string}; if(posted.name!==requestedName||requestedKey&&posted.key!==requestedKey)throw new Error("Outgoing company secret metadata differs from the actual request"); - const submittedKey=requestedKey??posted.key??requestedName,secret=await response.json() as {id:string;companyId:string;name:string;key:string;scope:string;status:string}; - if(secret.companyId!==input.companyId||secret.scope!=="company"||secret.status!=="active"||secret.name!==requestedName||secret.key!==canonicalCompanySecretKey(submittedKey))throw new Error("Saved company secret metadata differs from the actual request"); + const submittedKey=requestedKey??posted.key??requestedName,secret=await response.json() as {id:string;companyId:string;name:string;key:string;scope:string;status:string;provider:string}; + if(secret.companyId!==input.companyId||secret.provider!=="local_encrypted"||secret.scope!=="company"||secret.status!=="active"||secret.name!==requestedName||secret.key!==canonicalCompanySecretKey(submittedKey))throw new Error("Saved company secret metadata differs from the actual request"); if(!/^[a-f0-9-]{36}$/i.test(secret.id))throw new Error("Invalid created secret identity");await dialog.waitFor({state:"hidden"}); await page.getByText(requestedName,{exact:true}).first().waitFor(); const storageClean=await page.evaluate(secret=>!JSON.stringify([Object.entries(localStorage),Object.entries(sessionStorage),document.body.innerText,location.href]).includes(secret),input.credential); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-private-delegation.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-private-delegation.test.ts new file mode 100644 index 00000000000..3dd6036dbef --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/internal-private-delegation.test.ts @@ -0,0 +1,24 @@ +import { expect, it } from "vitest"; +import { privateDelegationSettled } from "./internal-private-delegation.js"; + +const waiting = { conversationState: "waiting" }; +const lead = "ceo", worker = "morgan"; +const retained = () => [{ agentId: lead, status: "succeeded" }, { agentId: worker, status: "succeeded" }, { agentId: lead, status: "running" }]; + +it("keeps waiting while the completion follow-up from the retained cohort-10 attempt is still running", () => + expect(privateDelegationSettled({ source: waiting, runs: retained(), leadId: lead, workerId: worker })).toBe("unsettled")); +it("is settled once the completion follow-up has reported back", () => { + const runs = retained(); runs[2]!.status = "succeeded"; + expect(privateDelegationSettled({ source: waiting, runs, leadId: lead, workerId: worker })).toBe("reported"); +}); +it("waits out a quiet window when the worker finished and no follow-up has appeared yet", () => + expect(privateDelegationSettled({ source: waiting, runs: retained().slice(0, 2), leadId: lead, workerId: worker })).toBe("quiet")); +it.each([ + ["the worker is still running", (runs: ReturnType) => { runs[1]!.status = "running"; runs.pop(); }], + ["the worker never ran", (runs: ReturnType) => { runs.splice(1, 1); }], + ["the follow-up was cancelled", (runs: ReturnType) => { runs[2]!.status = "cancelled"; }], +])("is not settled when %s", (_label, mutate) => { const runs = retained(); mutate(runs); expect(privateDelegationSettled({ source: waiting, runs, leadId: lead, workerId: worker })).toBe("unsettled"); }); +it("is not settled while the conversation is still working", () => { + const runs = retained(); runs[2]!.status = "succeeded"; + expect(privateDelegationSettled({ source: { conversationState: "working" }, runs, leadId: lead, workerId: worker })).toBe("unsettled"); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-private-delegation.ts b/tests/runner-e2e/chat-catalog-domains/internal-private-delegation.ts index 9f2bc73c9a1..14112c0f6ef 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-private-delegation.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-private-delegation.ts @@ -1,5 +1,6 @@ import { expect } from "@playwright/test"; -import type { ChatCatalogCase } from "../chat-catalog-contract.js"; +import { pollUntil } from "../api.js"; +import { chatCatalogOriginalPhaseDeadline, type ChatCatalogCase, type ChatCatalogContext } from "../chat-catalog-contract.js"; import { fixtureWorker, makeDefinition } from "./internal-definition.js"; import { openStandingChat } from "./identity-support.js"; import { agentStatus, cleanupPrivacyProbes, mainPerson, observerPerson, personRead } from "./internal-privacy.js"; @@ -35,6 +36,28 @@ export function gradePrivateDelegation(value: unknown) { "Actual worker execution belongs to the private work task instead of the standing conversation."); return c.result(); } +const followUpFailure = "Private delegated work or its completion follow-up failed"; +export function privateDelegationSettled(input: { source?: { conversationState?: string | null }; runs: Array<{ agentId: string; status: string }>; leadId: string; workerId: string }) { + if (input.source?.conversationState !== "waiting" || !input.runs.every(run => run.status === "succeeded") || !input.runs.some(run => run.agentId === input.workerId)) return "unsettled"; + return input.runs.filter(run => run.agentId === input.leadId).length >= 2 ? "reported" : "quiet"; +} +async function settleDelegatedCompletion(ctx: ChatCatalogContext, workerId: string) { + const original = chatCatalogOriginalPhaseDeadline(ctx), deadlineAt = Math.min(original === undefined ? Number.POSITIVE_INFINITY : original - 15_000, Date.now() + 120_000); + let quietSince: number | undefined; + try { + await pollUntil({ label: "Private delegated work and its completion follow-up settled", deadlineAt, intervalMs: 1000, + load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; }, + accept: ({ source, runs }) => { + const state = privateDelegationSettled({ source, runs, leadId: ctx.input.fixtures.agent.id, workerId }); + if (state === "unsettled") { quietSince = undefined; return false; } + if (state === "reported") return true; + quietSince ??= Date.now(); return Date.now() - quietSince >= 30_000; + }, + reject: ({ runs }) => runs.some(run => ["failed", "cancelled", "timed_out"].includes(run.status)) ? followUpFailure : undefined }); + } catch (error) { + if (error instanceof Error && error.message.includes(followUpFailure)) throw error; + } +} function privateDelegation(): ChatCatalogCase { let observer: Awaited> | undefined; const base = makeDefinition("tasks.private-task", async ctx => { @@ -49,7 +72,7 @@ function privateDelegation(): ChatCatalogCase { await putDocument(ctx, source.id, `# Confidential launch brief\n${confidentialMarker}. Keep the announcement inside Private Launch and its authorized team.\n`); return { project: p, worker, ownerId, observerId: observer.id, sourceId: source.id, confidentialMarker }; }, gradePrivateDelegation); - return { ...base, oracleVersion: "paperclip.chat-catalog.internal.private-task.v2", authentication: "human-session", expectedRunCount: 3, minimumRunCount: 2, + return { ...base, oracleVersion: "paperclip.chat-catalog.internal.private-task.v3", authentication: "human-session", expectedRunCount: 3, minimumRunCount: 2, runPolicy: { kind: "source-settled-with-active-work", maximumActiveWorkerRuns: 1 }, sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-private-delegation.ts", "chat-catalog-domains/internal-privacy.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/identity-support.ts"], async observe(ctx, state): Promise { @@ -71,6 +94,7 @@ function privateDelegation(): ChatCatalogCase { observerSearchResults: JSON.stringify(search.results ?? []) }; }, async verifyResult(ctx, state, value) { + await settleDelegatedCompletion(ctx, entity(state, "worker").id); await base.verifyResult(ctx, state, value); if (!observer) throw new Error("Missing authenticated observer browser"); const o = value as PrivateDelegationObservation; const saved = o.tasks[0]!; const page = await observer.context.newPage(); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.test.ts index 5e88c65f23e..60f3e491a97 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.test.ts @@ -10,7 +10,9 @@ function valid() { queryReply: "Use Steer beside your saved message to send this direction to Morgan's active turn; waiting in the queue is separate from delivery.", steerObservedBranch: "same-run-acknowledged", steerStatus: 200, acknowledgedSameRun: true, steerAuditBound: true, idempotentRetryStatus: 200, ordinaryConsumerCount: 0, actualBlueArtifact: true, artifactRegisteredAfterDirection: true, deliveryRevisionBound: true, workerTaskDone: true, queueEmptyAtEnd: true, - steerFailureCode: "" as string | undefined, providerFailureVisible: false, failureRetainedMessage: false, supportedContinuationVisible: false, continuationInputBound: false } }; + steerFailureCode: "" as string | undefined, providerFailureVisible: false, failureRetainedMessage: false, supportedContinuationVisible: false, continuationInputBound: false, + leadRelayCount: 0, queuedComments: [{ id: "blue", updatedAt: "2026-11-13T04:01:38.950Z", body: "Switch to a blue design while you are working." }] as Array<{ id: string; updatedAt: string; body: string }>, + workerCommentReceipts: { reads: {} as Record, cited: [] as string[] } } }; } it("accepts real explicit same-turn delivery and actual blue result", () => expect(gradeActiveSteering(valid()).passed).toBe(true)); it("accepts an actual visible provider rejection with retained one-time normal delivery", () => { @@ -81,7 +83,8 @@ it("binds only the requested website-delivery document written by the consuming // Retained runner-opencode attempt (gha-43077703607): the CEO relayed the direction as its own comment on Morgan's task. // It queues beside the person's message and reaches Morgan through normal admission; it is not the person's to steer. it("accepts a same-turn steer when the chat lead's own relay stays queued for one normal continuation", () => { - const o = valid(); Object.assign(o.state, { leadRelayCount: 1, ordinaryConsumerCount: 1 }); + const o = valid(); Object.assign(o.state, { leadRelayCount: 1, ordinaryConsumerCount: 1, continuationInputBound: true }); + o.state.queuedComments.push({ id: "relay", updatedAt: "2026-11-13T04:02:58.671Z", body: "Use a blue palette." }); o.state.queueAfterAction.entries = [{ comment: { id: "relay", body: "Use a blue palette.", authorAgentId: "ceo" } } as never]; o.runs.push({ id: "relay-consumer", status: "succeeded" }); expect(gradeActiveSteering(o).passed).toBe(true); o.state.ordinaryConsumerCount = 0; expect(gradeActiveSteering(o).passed).toBe(false); @@ -95,6 +98,7 @@ it("rejects an extra run after a same-turn steer when no relay was queued", () = }); it("accepts an unsupported runtime whose continuation also consumes the chat lead's relay", () => { const o = unsupported(); Object.assign(o.state, { leadRelayCount: 1 }); + o.state.queuedComments.push({ id: "relay", updatedAt: "2026-11-13T04:02:58.671Z", body: "Use a blue palette." }); o.state.queueAfterAction.entries.push({ comment: { id: "relay", body: "Use a blue palette.", authorAgentId: "ceo" } } as never); expect(gradeActiveSteering(o).passed).toBe(true); }); @@ -129,3 +133,26 @@ describe("the final website may come from either of the worker's runs, once, aft const twice = unsupported(); Object.assign(twice.state, { ordinaryConsumerCount: 2 }); expect(gradeActiveSteering(twice).passed).toBe(false); }); }); + +const relay = { id: "relay", updatedAt: "2026-11-13T04:02:58.671Z", body: "Morgan, the person asked for a blue design." }; +function citedInTurn() { + const o = unsupported(); o.runs.pop(); + Object.assign(o.state, { ordinaryConsumerCount: 0, continuationInputBound: false, leadRelayCount: 1 }); + o.state.queuedComments = [o.state.queuedComments[0]!, relay]; + o.state.workerCommentReceipts = { reads: { blue: "2026-11-13T04:01:38.950Z", relay: "2026-11-13T04:02:58.671Z" }, cited: ["relay", "blue"] }; + return o; +} +it("accepts the retained cohort-10 shape: the running turn read and cited both queued messages, built blue, and no continuation ran", () => expect(gradeActiveSteering(citedInTurn()).passed).toBe(true)); +it("accepts the target handled in turn while the uncited relay reaches one continuation", () => { + const o = citedInTurn(); o.state.workerCommentReceipts.cited = ["blue"]; o.state.ordinaryConsumerCount = 1; o.state.continuationInputBound = true; o.runs.push({ id: "ordinary", status: "succeeded" }); + expect(gradeActiveSteering(o).passed).toBe(true); +}); +it.each([ + ["reads the direction without citing it and starts no continuation", (o: ReturnType) => { o.state.workerCommentReceipts.cited = ["relay"]; }], + ["cites a revision it never read", (o: ReturnType) => { o.state.workerCommentReceipts.reads = { relay: relay.updatedAt }; }], + ["read an older revision of an edited direction", (o: ReturnType) => { o.state.queuedComments[0]!.updatedAt = "2026-11-13T04:03:30.000Z"; }], + ["cites the direction but never builds a blue page", (o: ReturnType) => { o.state.actualBlueArtifact = false; }], + ["cites the direction and still replays it in a continuation", (o: ReturnType) => { o.state.ordinaryConsumerCount = 1; o.state.continuationInputBound = false; o.runs.push({ id: "ordinary", status: "succeeded" }); }], + ["handles it in turn and admits an extra run", (o: ReturnType) => { o.runs.push({ id: "extra", status: "succeeded" }); }], + ["loses the relayed copy from the observed queue", (o: ReturnType) => { o.state.queuedComments = [o.state.queuedComments[0]!]; }], +])("rejects a running turn that %s", (_label, mutate) => { const o = citedInTurn(); mutate(o); expect(gradeActiveSteering(o).passed).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.ts b/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.ts index 93bf1473169..65d14b5ac56 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.ts @@ -8,7 +8,8 @@ import { checks, type Entity } from "./internal-support.js"; import { probeAgentPermission, cleanupIdentityProbes } from "./identity-support.js"; import { busyRuntimeWorker, releaseRuntimeWorker } from "./runtime-busy-worker.js"; import { renderRuntimeArtifact } from "./runtime-rendered-artifact.js"; -import { runtimeQueueCases } from "./runtime-queue.js"; +import { runCommentReceipts, runtimeQueueCases, settledQueue } from "./runtime-queue.js"; +import { continuationCarriesOwed, owedAfterTurn, type CommentReceipts, type QueuedFollowUp } from "./runtime-cited-delivery.js"; import { deliveryRevisionBound } from "./runtime-interruption.js"; import { runtimeDefinition, observeRuntime, runtimeObservation } from "./runtime-support.js"; import type { ChatCatalogContext } from "../chat-catalog-contract.js"; @@ -59,16 +60,20 @@ export function gradeActiveSteering(value: unknown) { c.add("human-only-control-without-forged-delivery", s.agentSteerStatus === 403 && (unsupported ? s.humanSteerAttempted === true : s.humanSteerSelected === true), "Only the person's own action can steer the message; the agent's attempt is refused."); const retained = after?.entries.some(row => row.comment.id === s.targetId && row.comment.body === direction) === true; + const queued = Array.isArray(s.queuedComments) ? s.queuedComments as QueuedFollowUp[] : [], receipts = s.workerCommentReceipts as CommentReceipts | undefined; + const queuedComplete = queued.filter(row => row.id === s.targetId).length === 1 && queued.length === 1 + Number(s.leadRelayCount ?? 0) && new Set(queued.map(row => row.id)).size === queued.length; + const owed = owedAfterTurn(queued.filter(row => !(accepted && row.id === s.targetId)), receipts); + const consumed = queuedComplete && (owed.length ? s.ordinaryConsumerCount === 1 && s.continuationInputBound === true : s.ordinaryConsumerCount === 0); c.add("delivery-distinct-from-queue-with-no-loss", s.steerObservedBranch === branch && (accepted - ? s.acknowledgedSameRun === true && after?.entries.every(row => row.comment.authorAgentId === o.leadId) === true && s.steerAuditBound === true && s.idempotentRetryStatus === 200 && s.ordinaryConsumerCount === (Number(s.leadRelayCount) ? 1 : 0) + ? s.acknowledgedSameRun === true && after?.entries.every(row => row.comment.authorAgentId === o.leadId) === true && s.steerAuditBound === true && s.idempotentRetryStatus === 200 && consumed : branch === "unsupported-queued-normal-continuation" ? s.steerStatus === 409 && s.steerFailureCode === "steering_unsupported" && s.steeredActivityCount === 0 && retained && s.unsupportedStateRetained === true - && s.ordinaryConsumerCount === 1 && s.continuationInputBound === true + && consumed : branch === "provider-rejected-normal-continuation" && ["steering_rejected", "steering_unsupported"].includes(String(s.steerFailureCode)) && s.steerStatus === 409 && s.providerFailureVisible === true - && s.failureRetainedMessage === true && retained && s.supportedContinuationVisible === true && s.ordinaryConsumerCount === 1 && s.continuationInputBound === true), - "A durable same-run ACK is required for delivery; a runtime that cannot steer (or a provider rejection) instead keeps the exact message visibly queued, records no steering, and one normal continuation consumes it."); + && s.failureRetainedMessage === true && retained && s.supportedContinuationVisible === true && consumed), + "A durable same-run ACK is required for steering; a runtime that cannot steer (or a provider rejection) instead keeps the exact message visibly queued and records no steering. Every queued message is delivered once: the running turn read it and cited it as handled, or one normal continuation receives each message not handled in turn and none that was."); c.add("actual-blue-output-correct-native-owner", s.actualBlueArtifact === true && s.artifactRegisteredAfterDirection === true && s.deliveryRevisionBound === true && s.workerTaskDone === true && s.queueEmptyAtEnd === true - && o.runs.length === (accepted && !Number(s.leadRelayCount) ? 2 : 3) && o.runs.every(row => row.status === "succeeded") && o.tasks.length === 1, + && o.runs.length === 2 + (owed.length ? 1 : 0) && o.runs.every(row => row.status === "succeeded") && o.tasks.length === 1, "The website the worker registered last, after the direction was saved, renders blue (whichever of its runs applied the direction), and the delivery document is current at a run of that work, with no duplicate work or hidden run."); return c.result(); } @@ -97,9 +102,9 @@ export const runtimeActiveSteeringCases = [runtimeDefinition("steering.steer-act await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); return { ...state, humanId: session.user.id, queueBefore: before, targetId, ...controls }; }, gradeActiveSteering, { - authentication: "human-session", expectedRunCount: 3, minimumRunCount: 2, attemptTimeoutMs: 720_000, oracleVersion: "paperclip.chat-catalog.runtime.steer-active.v8", + authentication: "human-session", expectedRunCount: 3, minimumRunCount: 2, attemptTimeoutMs: 720_000, oracleVersion: "paperclip.chat-catalog.runtime.steer-active.v9", setupPolicy: runtimeQueueCases[0]!.setupPolicy, verifySetup: runtimeQueueCases[0]!.verifySetup, - sourceFiles: ["chat-catalog-domains/runtime-active-steering.ts", "chat-catalog-domains/runtime-rendered-artifact.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-interruption.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "registered-artifact.ts", "chat-scenarios/steering.yaml"], + sourceFiles: ["chat-catalog-domains/runtime-active-steering.ts", "chat-catalog-domains/runtime-rendered-artifact.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-cited-delivery.ts", "chat-catalog-domains/runtime-interruption.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "registered-artifact.ts", "chat-scenarios/steering.yaml"], async act(ctx, state) { await pollUntil({ label: "Canonical steering guidance while the original native owner remains active", deadlineAt: Date.now() + 150_000, load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; }, @@ -112,7 +117,7 @@ export const runtimeActiveSteeringCases = [runtimeDefinition("steering.steer-act const human = before.entries.filter(row => !row.comment.authorAgentId), relays = before.entries.filter(row => row.comment.authorAgentId === ctx.input.fixtures.agent.id); if (human.length !== 1 || human[0]?.comment.id !== targetId || human.length + relays.length !== before.entries.length || before.targetRunId !== state.workerRunId) throw new Error("Canonical guidance lost or duplicated the saved human direction"); - state.leadRelayCount = relays.length; + state.leadRelayCount = relays.length; state.leadRelayIds = relays.map(row => row.comment.id); const mutation = { queueId: before.queueId, revision: before.revision, targetRunId: before.targetRunId }; state.agentSteerStatus = await probeAgentPermission(ctx, `${endpoint}/${targetId}/steer`, "POST", mutation); await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" }); @@ -157,16 +162,21 @@ export const runtimeActiveSteeringCases = [runtimeDefinition("steering.steer-act await writeFile((state.gate as { gate: string }).gate, state.steerStatus === 200 ? `The human brief is ready. ${build} Use the actual design direction you received, then complete normally. Preserve saved human requirements for normal admission; do not create tasks or publish externally.` : `The human brief is ready. ${build} Then finish this current turn normally. Keep human requirements queued by Paperclip intact for normal delivery; when one arrives, apply it to website.html, register the updated file, update the delivery document, and complete the task. Do not create tasks or publish externally.`, "utf8"); - const finished = await pollUntil({ label: "Actual selected steering or preserved normal continuation produces its website", deadlineAt: Date.now() + 240_000, + const done = (v: { runs: Array<{ status: string }>; queue: IssueQueuedCommentQueue; task: Entity }) => v.task.status === "done" && v.queue.entries.length === 0 && v.runs.every(row => row.status === "succeeded"); + const settledInTurn = settledQueue((v: { runs: Array<{ status: string }>; queue: IssueQueuedCommentQueue; task: Entity }) => done(v) && v.runs.length === 2, 30_000); + const finished = await pollUntil({ label: "Actual selected steering, in-turn handling or preserved normal continuation produces its website", deadlineAt: Date.now() + 240_000, load: async () => ({ runs: await ctx.allRuns(), queue: await ctx.input.api.get(endpoint), task: await ctx.input.api.get(`/api/issues/${task.id}`) }), - accept: v => v.task.status === "done" && v.queue.entries.length === 0 && v.runs.length === (state.steerStatus === 200 && !Number(state.leadRelayCount) ? 2 : 3) && v.runs.every(row => row.status === "succeeded"), + accept: v => settledInTurn(v) || done(v) && v.runs.length === 3, reject: v => v.runs.length > 3 || v.runs.some(row => ["failed", "cancelled", "timed_out"].includes(row.status)) ? "Steered work failed or admitted an undeclared execution" : undefined }); const successors = finished.runs.filter(row => row.agentId === (state.worker as Entity).id && row.id !== state.workerRunId); - state.ordinaryConsumerCount = successors.length; state.outputRunId = state.steerStatus === 200 ? state.workerRunId : successors[0]?.id; - if (state.steerStatus === 200 && Number(state.leadRelayCount) && successors.length === 1) state.relayConsumerRunId = successors[0]!.id; + state.ordinaryConsumerCount = successors.length; state.outputRunId = state.steerStatus === 200 || !successors.length ? state.workerRunId : successors[0]!.id; + if (successors.length === 1) state.relayConsumerRunId = successors[0]!.id; + const comments = await ctx.input.api.get(`/api/issues/${task.id}/comments?order=asc`), queuedIds = [targetId, ...(state.leadRelayIds as string[])]; + const queued = queuedIds.map(id => comments.find(row => row.id === id)).filter((row): row is Entity => !!row).map(row => ({ id: String(row.id), updatedAt: row.updatedAt, body: String(row.body ?? "") })); + const receipts = await runCommentReceipts(ctx, String(state.workerRunId)); state.queuedComments = queued; state.workerCommentReceipts = receipts; if (successors.length === 1) { - const full = await ctx.input.api.get<{ contextSnapshot?: { wakeCommentIds?: string[] }; runnerProfileJson?: { nativeExecutionInput?: { task?: { prompt?: string } } } }>(`/api/heartbeat-runs/${successors[0]!.id}`); - state.continuationInputBound = full.contextSnapshot?.wakeCommentIds?.filter(id => id === targetId).length === 1 && full.runnerProfileJson?.nativeExecutionInput?.task?.prompt?.includes(direction) === true; + const full = await ctx.input.api.get<{ contextSnapshot?: { wakeCommentIds?: string[]; paperclipWake?: { comments?: Array<{ id?: string; readByEarlierTurn?: boolean }> } }; runnerProfileJson?: { nativeExecutionInput?: { task?: { prompt?: string } } } }>(`/api/heartbeat-runs/${successors[0]!.id}`); + state.continuationInputBound = continuationCarriesOwed(queued.filter(row => !(state.steerStatus === 200 && row.id === targetId)), receipts, { contextSnapshot: full.contextSnapshot, prompt: full.runnerProfileJson?.nativeExecutionInput?.task?.prompt }); } state.workerTaskDone = finished.task.status === "done"; state.queueEmptyAtEnd = finished.queue.entries.length === 0; await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-cited-delivery.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-cited-delivery.test.ts new file mode 100644 index 00000000000..1420ff0b4dd --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-cited-delivery.test.ts @@ -0,0 +1,28 @@ +import { expect, it } from "vitest"; +import { continuationCarriesOwed, owedAfterTurn } from "./runtime-cited-delivery.js"; + +const font = { id: "3b2395dc-11b1-43dd-95ff-56c2fcdfb5bd", updatedAt: "2026-11-13T04:01:28.647Z", body: "Use a readable font." }; +const contact = { id: "3aa8c723-5c96-4346-85a8-a4992694307e", updatedAt: "2026-11-13T04:03:45.909Z", body: "Add a contact section.\nLabel it." }; +const read = { [font.id]: font.updatedAt, [contact.id]: contact.updatedAt }; +const wake = (ids: string[], earlier: string[] = []) => ({ wakeCommentIds: ids, paperclipWake: { comments: ids.map(id => ({ id, readByEarlierTurn: earlier.includes(id) })) } }); + +it("owes nothing that the turn read at its current revision and cited", () => expect(owedAfterTurn([font, contact], { reads: read, cited: [contact.id, font.id] })).toEqual([])); +it("owes a read that was not cited, a citation without a read, and a stale read", () => { + expect(owedAfterTurn([font, contact], { reads: read, cited: [font.id] })).toEqual([contact]); + expect(owedAfterTurn([font, contact], { reads: { [font.id]: font.updatedAt }, cited: [font.id, contact.id] })).toEqual([contact]); + expect(owedAfterTurn([{ ...font, updatedAt: "2026-11-13T04:05:00.000Z" }], { reads: read, cited: [font.id] })).toHaveLength(1); +}); +it("accepts one continuation carrying each owed comment once, JSON-escaped, with an earlier-read warning", () => { + const prompt = JSON.stringify({ comments: [contact.body] }); + expect(continuationCarriesOwed([font, contact], { reads: read, cited: [font.id] }, { contextSnapshot: wake([contact.id], [contact.id]), prompt })).toBe(true); + expect(continuationCarriesOwed([font, contact], { reads: {}, cited: [] }, { contextSnapshot: wake([font.id, contact.id]), prompt: `${font.body} ${prompt}` })).toBe(true); +}); +it.each([ + ["replays a comment already handled in turn", wake([font.id, contact.id], [contact.id]), `${font.body} ${JSON.stringify(contact.body)}`], + ["omits the owed comment", wake([], []), JSON.stringify(contact.body)], + ["delivers it twice", { wakeCommentIds: [contact.id, contact.id], paperclipWake: { comments: [{ id: contact.id, readByEarlierTurn: true }] } }, JSON.stringify(contact.body)], + ["does not warn that the earlier turn read it", wake([contact.id]), JSON.stringify(contact.body)], + ["lacks its text in the prompt", wake([contact.id], [contact.id]), "Use the brief."], +])("rejects a continuation that %s", (_label, contextSnapshot, prompt) => + expect(continuationCarriesOwed([font, contact], { reads: read, cited: [font.id] }, { contextSnapshot, prompt })).toBe(false)); +it("rejects a continuation when nothing was owed", () => expect(continuationCarriesOwed([font], { reads: read, cited: [font.id] }, { contextSnapshot: wake([]), prompt: "" })).toBe(false)); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-cited-delivery.ts b/tests/runner-e2e/chat-catalog-domains/runtime-cited-delivery.ts new file mode 100644 index 00000000000..a1824138f39 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-cited-delivery.ts @@ -0,0 +1,23 @@ +import { handledInTurn, readCurrentRevision, wakeMarksEarlierRead } from "./runtime-queue.js"; + +export type QueuedFollowUp = { id: string; updatedAt?: unknown; body?: string }; +export type CommentReceipts = { reads?: Record; cited?: string[] }; +export type ContinuationInput = { + contextSnapshot?: { wakeCommentIds?: string[]; paperclipWake?: { comments?: Array<{ id?: string; readByEarlierTurn?: boolean }> } }; + prompt?: string; +}; + +export function owedAfterTurn(queued: readonly T[], receipts: CommentReceipts | undefined) { + return queued.filter(row => !handledInTurn(receipts, row)); +} + +function carries(prompt: string | undefined, body: string | undefined) { + return !!prompt && !!body?.trim() && (prompt.includes(body) || prompt.includes(JSON.stringify(body).slice(1, -1))); +} + +export function continuationCarriesOwed(queued: readonly QueuedFollowUp[], receipts: CommentReceipts | undefined, run: ContinuationInput | undefined) { + const owed = owedAfterTurn(queued, receipts), handled = queued.filter(row => !owed.includes(row)), ids = run?.contextSnapshot?.wakeCommentIds ?? []; + return owed.length > 0 && owed.every(row => ids.filter(id => id === row.id).length === 1 && carries(run?.prompt, row.body) + && (!readCurrentRevision(receipts?.reads, row) || wakeMarksEarlierRead(run?.contextSnapshot, row.id))) + && handled.every(row => !ids.includes(row.id)); +} diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-discard.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-discard.test.ts index 63a68cbe290..bdf01b62cc8 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-discard.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-discard.test.ts @@ -12,7 +12,10 @@ function valid() { queryLinkedTask: true, targetPendingAfterReply: true, consumedLabelSaved: true, consumedBeforeCancellation: true, consumedStillReadable: true, lateDiscardStatus: 409, latePhaseAttributed: true, lateReply: "The later request was already consumed. I sent new direction to remove that section; its original history remains intact.", - cancelDirectionRunBound: true, finalQueueEmpty: true, finalPlanWithoutContact: true, initialRevisionRetained: true } }; + cancelDirectionRunBound: true, finalQueueEmpty: true, finalPlanWithoutContact: true, initialRevisionRetained: true, + workerRunId: "initial", secondContactId: "second-contact", consumerRunId: "contact", firstDeliveryRunCount: 3, continuationConsumerCount: 1, continuationInputBound: true, + queuedFollowUps: [{ id: "font", updatedAt: "2026-11-13T04:01:28.647Z" }, { id: "second-contact", updatedAt: "2026-11-13T04:03:45.909Z" }], + workerCommentReceipts: { reads: {} as Record, cited: [] as string[] } } }; } it("accepts genuine pending discard and distinct consumed-input cancellation as new direction", () => expect(gradeDiscard(valid()).passed).toBe(true)); it("accepts natural human-control and subsequent consumed-input guidance", () => { const o = valid(); o.state.queryReply = "That waiting message is yours. Open its queue menu and remove it with Discard."; o.state.lateReply = "Morgan processed the second request before you cancelled it. A follow-up now asks for the contact section to be removed; the delivered input stays in history."; expect(gradeDiscard(o).passed).toBe(true); }); @@ -94,3 +97,29 @@ it("fails when no labelled section was ever saved, so removal cannot be shown", const unlabelled = consumedPlan.replace(`## Contact section (label: ${label})`, "## Contact section"); expect(documentContactRemoved([...revs(initialPlan)].map(row => row.id === "r2" ? { ...row, body: unlabelled } : row), "final", "consumer", label)).toBe(false); }); + +function citedInTurn() { + const o = valid(); o.runs = o.runs.filter(row => row.id !== "contact"); + Object.assign(o.state, { consumerRunId: "initial", firstDeliveryRunCount: 2, continuationConsumerCount: 0, continuationInputBound: false, consumedBeforeCancellation: false }); + o.state.workerCommentReceipts = { reads: { "second-contact": "2026-11-13T04:03:45.909Z", font: "2026-11-13T04:01:28.647Z" }, cited: ["font", "second-contact"] }; + return o; +} +it("accepts the retained cohort-10 shape: the running turn read and cited both follow-ups, saved the contact label, and no continuation ran", () => expect(gradeDiscard(citedInTurn()).passed).toBe(true)); +it("accepts the later request handled in turn while the uncited font note reaches one continuation", () => { + const o = citedInTurn(); o.state.workerCommentReceipts.cited = ["second-contact"]; Object.assign(o.state, { firstDeliveryRunCount: 3, continuationConsumerCount: 1, continuationInputBound: true }); + o.runs.splice(2, 0, { id: "font-consumer", status: "succeeded" }); expect(gradeDiscard(o).passed).toBe(true); +}); +it.each([ + ["reads the later request without citing it and starts no continuation", (o: ReturnType) => { o.state.workerCommentReceipts.cited = ["font"]; }], + ["cites a request it never read", (o: ReturnType) => { delete o.state.workerCommentReceipts.reads["second-contact"]; }], + ["cites the request but never saves the labelled contact section", (o: ReturnType) => { o.state.consumedLabelSaved = false; }], + ["cites the request and still replays it in a continuation", (o: ReturnType) => { Object.assign(o.state, { firstDeliveryRunCount: 3, continuationConsumerCount: 1 }); o.runs.splice(2, 0, { id: "replay", status: "succeeded" }); }], + ["handles both in turn and admits an extra later run", (o: ReturnType) => { o.runs.push({ id: "extra", status: "succeeded" }); }], + ["cites the discarded request as handled", (o: ReturnType) => { o.state.workerCommentReceipts.cited.push("pending-contact"); }], + ["credits the later request to a run other than the one that handled it", (o: ReturnType) => { o.state.consumerRunId = "new-direction"; }], +])("rejects a running turn that %s", (_label, mutate) => { const o = citedInTurn(); mutate(o); expect(gradeDiscard(o).passed).toBe(false); }); +it.each([ + ["no continuation consumes the owed follow-ups", (o: ReturnType) => { o.state.continuationConsumerCount = 0; }], + ["the continuation input lacks an owed follow-up", (o: ReturnType) => { o.state.continuationInputBound = false; }], + ["the continuation never received the later request", (o: ReturnType) => { o.state.consumedBeforeCancellation = false; }], +])("rejects the continuation path when %s", (_label, mutate) => { const o = valid(); mutate(o); expect(gradeDiscard(o).passed).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-discard.ts b/tests/runner-e2e/chat-catalog-domains/runtime-discard.ts index 24219b1d364..3311de4d728 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-discard.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-discard.ts @@ -9,7 +9,8 @@ import { checks, hasLink, type Entity } from "./internal-support.js"; import { probeAgentPermission, cleanupIdentityProbes } from "./identity-support.js"; import { releaseRuntimeWorker } from "./runtime-busy-worker.js"; import { humanQueuedRuntime, probeConsumedHumanQueue } from "./runtime-queue-controls.js"; -import { runtimeQueueCases } from "./runtime-queue.js"; +import { handledInTurn, runCommentReceipts, runtimeQueueCases, settledQueue } from "./runtime-queue.js"; +import { continuationCarriesOwed, owedAfterTurn, type CommentReceipts, type QueuedFollowUp } from "./runtime-cited-delivery.js"; import { runtimeDefinition, runtimeObservation, observeRuntime } from "./runtime-support.js"; const contactText = "Also add a contact section to the website."; @@ -49,19 +50,26 @@ export function gradeDiscard(value: unknown) { c.add("chat-guides-supported-human-discard", String(s.queryReply ?? "").trim().length > 0 && s.queryLinkedTask === true && s.targetPendingAfterReply === true && s.agentDiscardStatus === 403, "The reply links the task whose queue holds the request, the request is still pending after the reply, and the agent cannot impersonate the person's discard."); - c.add("already-delivered-input-cannot-be-erased", s.consumedBeforeCancellation === true && s.consumedStillReadable === true + const followUps = Array.isArray(s.queuedFollowUps) ? s.queuedFollowUps as QueuedFollowUp[] : [], receipts = s.workerCommentReceipts as CommentReceipts | undefined; + const second = followUps.find(row => row.id === s.secondContactId), owed = owedAfterTurn(followUps, receipts); + const firstDelivery = followUps.length === 2 && !!second && !receipts?.cited?.includes(String(s.targetId)) && (owed.length + ? s.firstDeliveryRunCount === 3 && s.continuationConsumerCount === 1 && s.continuationInputBound === true + : s.firstDeliveryRunCount === 2 && s.continuationConsumerCount === 0); + c.add("later-input-delivered-once-before-cancellation", firstDelivery && (handledInTurn(receipts, second) ? s.consumerRunId === s.workerRunId : owed.includes(second!) && s.consumedBeforeCancellation === true), + "Each saved follow-up is delivered once before the cancellation: the running turn read it and cited it as handled, or one ordinary continuation receives every follow-up not handled in turn and none that was. The discarded request is never cited as handled."); + c.add("already-delivered-input-cannot-be-erased", s.consumedStillReadable === true && s.lateDiscardStatus === 409 && s.latePhaseAttributed === true && String(s.lateReply ?? "").trim().length > 0, "The consumed request stays readable and cannot be discarded late; the later answer is attributed to its own run."); c.add("new-direction-reaches-exactly-one-native-continuation", s.cancelDirectionRunBound === true && s.finalQueueEmpty === true && s.consumedLabelSaved === true && s.finalPlanWithoutContact === true - && s.initialRevisionRetained === true && o.runs.length === 5 && o.runs.every(row => row.status === "succeeded") && o.tasks.length === 1 + && s.initialRevisionRetained === true && o.runs.length === Number(s.firstDeliveryRunCount) + 2 && o.runs.every(row => row.status === "succeeded") && o.tasks.length === 1 && o.tasks[0]?.id === (s.task as Entity).id && o.tasks[0]?.status === "done", "A distinct saved instruction reaches one real later worker, updates the saved plan and preserves prior work/history without replaying the discarded request."); return c.result(); } export const runtimeDiscardCases = [runtimeDefinition("steering.discard-queued-message", ctx => humanQueuedRuntime(ctx, contactText), gradeDiscard, { - authentication: "human-session", expectedHumanMessageCount: 2, expectedRunCount: 5, attemptTimeoutMs: 1_020_000, oracleVersion: "paperclip.chat-catalog.runtime-discard.v6", - sourceFiles: ["chat-catalog-domains/runtime-discard.ts", "chat-catalog-domains/runtime-queue-controls.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "chat-scenarios/steering.yaml"], + authentication: "human-session", expectedHumanMessageCount: 2, expectedRunCount: 5, minimumRunCount: 4, attemptTimeoutMs: 1_020_000, oracleVersion: "paperclip.chat-catalog.runtime-discard.v7", + sourceFiles: ["chat-catalog-domains/runtime-discard.ts", "chat-catalog-domains/runtime-queue-controls.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-cited-delivery.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "chat-scenarios/steering.yaml"], setupPolicy: runtimeQueueCases[0]!.setupPolicy, verifySetup: runtimeQueueCases[0]!.verifySetup, async act(ctx, state) { await pollUntil({ label: "Human discard guidance while Morgan is genuinely active", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000), @@ -84,22 +92,32 @@ export const runtimeDiscardCases = [runtimeDefinition("steering.discard-queued-m state.discardVisible = true; state.discardedCommentStatus = (await ctx.input.page.request.get(`${root}/comments/${targetId}`)).status(); await ctx.input.capture("actual-human-contact-discard", "Only the eligible contact request has left the real task queue", "runtime-discard.png"); // This new, explicit human request is a distinct later instruction. It - // deliberately reaches normal native delivery before its cancellation. + // deliberately reaches delivery before its cancellation. const secondText = `I have changed my mind: add a contact section to the saved website plan, labelled ${ctx.input.nonce}. Keep the accessible-font requirement and finish this task.`; await submitTaskReply(ctx.input.page, secondText); const secondQueue = await pollUntil({ label: "Distinct human contact request genuinely queued", deadlineAt: Date.now() + 20_000, load: () => ctx.input.api.get(queuePath), accept: q => q.entries.length === 2 && q.entries.some(row => row.comment.body === secondText) }); const second = secondQueue.entries.find(row => row.comment.body === secondText)!; state.secondContactId = second.comment.id; - await writeFile((state.gate as { gate: string }).gate, "The human brief is ready: save an initial website-plan task document, retaining accessible font guidance, then finish this turn normally. Leave saved follow-ups for their ordinary continuation.", "utf8"); + await writeFile((state.gate as { gate: string }).gate, "The human brief is ready: save an initial website-plan task document, retaining accessible font guidance, then finish this turn normally.", "utf8"); + type Delivery = { runs: Array<{ status: string }>; task: Entity; queue: IssueQueuedCommentQueue }; + const delivered = (v: Delivery) => v.runs.every(row => row.status === "succeeded") && v.task.status === "done" && v.queue.entries.length === 0; + const settledInTurn = settledQueue((v: Delivery) => delivered(v) && v.runs.length === 2, 30_000); const consumed = await pollUntil({ label: "Real later contact request consumed before cancellation", deadlineAt: Date.now() + 240_000, load: async () => ({ runs: await ctx.allRuns(), task: await ctx.input.api.get(root), queue: await ctx.input.api.get(queuePath) }), - accept: v => v.runs.length === 3 && v.runs.every(row => row.status === "succeeded") && v.task.status === "done" && v.queue.entries.length === 0 }); - const consumer = consumed.runs.find(row => row.agentId === (state.worker as Entity).id && row.id !== state.workerRunId)!; - state.consumerRunId = consumer.id; - const detail = await ctx.input.api.get<{ contextSnapshot?: { wakeCommentIds?: string[] }; runnerProfileJson?: { nativeExecutionInput?: { task?: { prompt?: string } } } }>(`/api/heartbeat-runs/${consumer.id}`); - state.consumedBeforeCancellation = detail.contextSnapshot?.wakeCommentIds?.includes(second.comment.id) === true - && promptCarries(detail.runnerProfileJson?.nativeExecutionInput?.task?.prompt, secondText); + accept: v => settledInTurn(v) || delivered(v) && v.runs.length === 3, + reject: v => v.runs.length > 3 || v.runs.some(row => ["failed", "cancelled", "timed_out"].includes(row.status)) ? "The saved follow-ups failed or reached more than one continuation" : undefined }); + const taskComments = await ctx.input.api.get(`${root}/comments?order=asc`); + const followUps = secondQueue.entries.map(row => taskComments.find(comment => comment.id === row.comment.id)).filter((row): row is Entity => !!row) + .map(row => ({ id: String(row.id), updatedAt: row.updatedAt, body: String(row.body ?? "") })); + const receipts = await runCommentReceipts(ctx, String(state.workerRunId)); state.queuedFollowUps = followUps; state.workerCommentReceipts = receipts; + const successors = consumed.runs.filter(row => row.agentId === (state.worker as Entity).id && row.id !== state.workerRunId); + state.firstDeliveryRunCount = consumed.runs.length; state.continuationConsumerCount = successors.length; + const detail = successors[0] && await ctx.input.api.get<{ contextSnapshot?: { wakeCommentIds?: string[]; paperclipWake?: { comments?: Array<{ id?: string; readByEarlierTurn?: boolean }> } }; runnerProfileJson?: { nativeExecutionInput?: { task?: { prompt?: string } } } }>(`/api/heartbeat-runs/${successors[0].id}`); + const prompt = detail?.runnerProfileJson?.nativeExecutionInput?.task?.prompt; + state.continuationInputBound = !!detail && continuationCarriesOwed(followUps, receipts, { contextSnapshot: detail.contextSnapshot, prompt }); + state.consumerRunId = handledInTurn(receipts, followUps.find(row => row.id === second.comment.id)) ? state.workerRunId : successors[0]?.id; + state.consumedBeforeCancellation = !!detail && detail.contextSnapshot?.wakeCommentIds?.includes(second.comment.id) === true && promptCarries(prompt, secondText); state.discardedNeverDelivered = consumed.runs.every(row => !((row.contextSnapshot?.wakeCommentIds as string[] | undefined) ?? []).includes(targetId)); state.lateDiscardStatus = await probeConsumedHumanQueue(ctx, String(state.humanId), `${queuePath}/${second.comment.id}`, "DELETE", { queueId: secondQueue.queueId, revision: secondQueue.revision }); await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); @@ -108,8 +126,8 @@ export const runtimeDiscardCases = [runtimeDefinition("steering.discard-queued-m await sendChatMessage(ctx.input.page, message, chatCatalogNativePhaseRemainingBudget(ctx)); const final = await pollUntil({ label: "One real new-direction continuation after already-delivered cancellation", deadlineAt: Date.now() + 300_000, load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns(), task: await ctx.input.api.get(root), queue: await ctx.input.api.get(queuePath) }; }, - accept: v => v.source?.conversationState === "waiting" && v.runs.length === 5 && v.runs.every(row => row.status === "succeeded") && v.task.status === "done" && v.queue.entries.length === 0, - reject: v => v.runs.length > 5 || v.runs.some(row => ["failed", "cancelled", "timed_out"].includes(row.status)) ? "The declared later cancellation failed or duplicated execution" : undefined }); + accept: v => v.source?.conversationState === "waiting" && v.runs.length === priorIds.length + 2 && v.runs.every(row => row.status === "succeeded") && v.task.status === "done" && v.queue.entries.length === 0, + reject: v => v.runs.length > priorIds.length + 2 || v.runs.some(row => ["failed", "cancelled", "timed_out"].includes(row.status)) ? "The declared later cancellation failed or duplicated execution" : undefined }); const later = final.runs.filter(row => !priorIds.includes(row.id)), lead = later.find(row => row.agentId === ctx.input.fixtures.agent.id), worker = later.find(row => row.agentId === (state.worker as Entity).id); const sourceComments = await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/comments?order=asc`), human = sourceComments.filter(row => !row.authorAgentId && row.body === message); const replies = sourceComments.filter(row => row.createdByRunId === lead?.id && row.authorAgentId === ctx.input.fixtures.agent.id); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-queue.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-queue.test.ts index 11f665b3465..7a6997b9f03 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-queue.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-queue.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect } from "vitest"; import { deflateRawSync } from "node:zlib"; -import { gradeQueuedFollowup, historyReadRevisions, keyboardMoveQueuedMessageUp, promptCarriesComment, readCurrentRevision, resultCitedCommentIds, zipEntryTexts } from "./runtime-queue.js"; +import { gradeQueuedFollowup, historyReadRevisions, keyboardMoveQueuedMessageUp, promptCarriesComment, queuedFollowUpBrief, readCurrentRevision, resultCitedCommentIds, zipEntryTexts } from "./runtime-queue.js"; /** A minimal zip with one deflated and one stored entry, laid out as zip tools write it. */ function zip(entries: Array<{ name: string; text: string; deflate: boolean }>) { const locals: Buffer[] = [], centrals: Buffer[] = []; let offset = 0; @@ -127,3 +127,12 @@ it("reads comment citations only from the run's accepted result", () => { expect(resultCitedCommentIds([{ eventType: "run.result.proposed", payload: { prpEvent: { payload: { result: { evidence: [{ ref: `comment:${id}` }] } } } } }])).toEqual([]); expect(resultCitedCommentIds([accepted({ evidence: [{ ref: `document:comment:${id}` }, { ref: "comment:not-a-comment" }] })])).toEqual([]); }); + +describe("queued follow-up brief", () => { + const brief = queuedFollowUpBrief("Original website brief n1"); + it("gives the worker a complete website brief bound to its reference", () => { + expect(brief).toContain("reference Original website brief n1"); + for (const part of ["Acorn Notes", "website.html", "Research notes your whole team can find", "Request early access", "Free for teams", "hello@acornnotes.example"]) expect(brief).toContain(part); + }); + it("does not pre-complete the queued contact requirement", () => expect(brief).not.toMatch(/contact/i)); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-queue.ts b/tests/runner-e2e/chat-catalog-domains/runtime-queue.ts index 783fb937726..69987bd7b14 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-queue.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-queue.ts @@ -127,6 +127,18 @@ export function replyNamesTask(reply: string, task: Entity) { function savedRequirement(queue: IssueQueuedCommentQueue | undefined, leadId: string) { return queue?.entries.length === 1 && queue.entries[0]!.comment.authorAgentId === leadId ? queue.entries[0] : undefined; } +export function queuedFollowUpBrief(marker: string) { + return [ + `The human brief is now ready. Website brief, reference ${marker}:`, + "- Product: Acorn Notes, a shared notebook app for small research teams. It launches in beta next month.", + "- Page: one static website.html page with a light background and a readable sans-serif font.", + "- Hero: the headline \"Research notes your whole team can find\", the line \"Shared notebooks, full-text search across experiments, and one-click citation export\", and a \"Request early access\" button.", + "- Features: three short items: shared notebooks, search across every experiment, citation export to BibTeX.", + "- Pricing: \"Free for teams of up to ten during the beta.\"", + "- Company details: hello@acornnotes.example, +1 555 0142, 12 Elm Street, Portland, OR 97201.", + `Save this brief as an initial task document that includes the exact reference ${marker}, then finish this current turn normally. Keep all prior user requirements queued by Paperclip intact for normal delivery; do not create tasks or external effects.`, + ].join("\n"); +} export function gradeQueuedFollowup(value: unknown) { const c = checks(); const o = runtimeObservation(value); c.add("real-queue-and-delivery-evidence", !!o, "The actual task queue, saved requirement, real initial worker and later consuming native input are required."); if (!o) return c.result(); @@ -150,7 +162,7 @@ export function gradeQueuedFollowup(value: unknown) { return c.result(); } export const runtimeQueueCases = [runtimeDefinition("steering.queue-follow-up", busyRuntimeWorker, gradeQueuedFollowup, { - oracleVersion: "paperclip.chat-catalog.runtime-queue.v3", + oracleVersion: "paperclip.chat-catalog.runtime-queue.v4", sourceFiles: ["chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/steering.yaml"], expectedRunCount: 3, attemptTimeoutMs: 720_000, setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["lead-ready", "real-busy-worker", "worker-task-scope"] }, @@ -181,7 +193,7 @@ export const runtimeQueueCases = [runtimeDefinition("steering.queue-follow-up", await ctx.input.capture("actual-contact-followup-queue", "Contact requirement remains visibly queued behind real active work", "runtime-contact-queue.png"); await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); const marker = `Original website brief ${ctx.input.nonce}`; state.briefMarker = marker; - await writeFile((state.gate as { gate: string }).gate, `The human brief is now ready: save an initial task document with the exact reference ${marker}, then finish this current turn normally. Keep all prior user requirements queued by Paperclip intact for normal delivery; do not create tasks or external effects.`, "utf8"); + await writeFile((state.gate as { gate: string }).gate, queuedFollowUpBrief(marker), "utf8"); const delivered = await pollUntil({ label: "Normal successor receives actual queued contact requirement", deadlineAt: Date.now() + 240_000, load: async () => ({ task: await ctx.input.api.get(`/api/issues/${task.id}`), runs: await ctx.allRuns(), queue: await ctx.input.api.get(`/api/issues/${task.id}/queued-comments`) }), accept: value => value.task.status === "done" && value.runs.length === 3 && value.runs.every(run => run.status === "succeeded") && value.queue.entries.length === 0,