From cb31ebccf0411f3e260159b1d39e6c047a39f4e9 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 28 Oct 2026 09:57:44 -0500 Subject: [PATCH] test(e2e): preserve secret proposal original observation clock --- tests/runner-e2e/chat-catalog-contract.ts | 1 + .../external-secret-proposal.ts | 13 +++++++--- .../chat-catalog-original-clock.test.ts | 2 +- .../chat-secret-proposal-clock.test.ts | 25 +++++++++++++++++++ 4 files changed, 36 insertions(+), 5 deletions(-) create mode 100644 tests/runner-e2e/chat-secret-proposal-clock.test.ts diff --git a/tests/runner-e2e/chat-catalog-contract.ts b/tests/runner-e2e/chat-catalog-contract.ts index 2c7de8cb352..64d8efd5976 100644 --- a/tests/runner-e2e/chat-catalog-contract.ts +++ b/tests/runner-e2e/chat-catalog-contract.ts @@ -26,6 +26,7 @@ export interface ChatCatalogContext { /** These exact selected OpenCode/local journeys use the original flow clock for * native observation phases. Other cases/lanes keep their authored phase caps. */ const originalClockExecutionIds = new Set([ + "chat-features.runner-opencode.local.secrets.secret-proposal", "chat-features.runner-opencode.local.developer-labs.long-thread-performance", "chat-features.runner-opencode.local.questions-and-approvals.approve-tool-once", "chat-features.runner-opencode.local.runs-adapters.reset-session", diff --git a/tests/runner-e2e/chat-catalog-domains/external-secret-proposal.ts b/tests/runner-e2e/chat-catalog-domains/external-secret-proposal.ts index b22e551f0a1..a2300bec7b9 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-secret-proposal.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-secret-proposal.ts @@ -1,3 +1,4 @@ +import {chatCatalogNativePhaseDeadline,chatCatalogNativePhaseIsTimely,chatCatalogNativePhaseRemainingBudget} from "../chat-catalog-contract.js"; import {randomBytes} from "node:crypto"; import {expect} from "@playwright/test"; import type {RequestConfirmationInteraction} from "../../../packages/shared/src/types/issue.js"; @@ -74,14 +75,18 @@ export function gradeSecretProposal(value:unknown){ c.add("phase-specific-truthful-outcomes",o.phaseReplies.some(reply=>reply.runId===o.state.denialReplyRunId&&truthfulProposalPhase(reply.body,false))&&o.phaseReplies.some(reply=>reply.runId===o.state.approvalReplyRunId&&truthfulProposalPhase(reply.body,true)),"Denial and approval explanations belong to their own actual successful continuation runs, rather than an aggregate transcript rescuing the wrong phase."); c.add("secret-identity-and-private-bytes-preserved",sameSecretIdentity(o.state.secretBefore as CompanySecret,o.state.secretAfter as CompanySecret)&&o.state.secretScanComplete===true&&o.state.secretLeaked===false,"Value identity/version is unchanged and private scans include actual documents/revisions/artifacts and raw native events before redaction.");return c.result(); } -async function settled(ctx:ChatCatalogContext,count:number){return pollUntil({label:"actual proposal phase native settlement",deadlineAt:Date.now()+150000,intervalMs:500,load:async()=>{await ctx.refreshIssue();return ctx.allRuns();},accept:runs=>runs.length===count&&runs.every(run=>run.status==="succeeded")&&ctx.issue()?.conversationState==="waiting",reject:runs=>runs.length>count||runs.some(run=>["failed","cancelled","timed_out"].includes(run.status))?"Proposal phase admitted extra or unsuccessful native work":undefined});} +async function settled(ctx:ChatCatalogContext,count:number){return pollUntil({label:"actual proposal phase native settlement",deadlineAt:chatCatalogNativePhaseDeadline(ctx,Date.now()+150000),intervalMs:500,load:async()=>{await ctx.refreshIssue();return ctx.allRuns();},accept:runs=>chatCatalogNativePhaseIsTimely(ctx)&&runs.length===count&&runs.every(run=>run.status==="succeeded")&&ctx.issue()?.conversationState==="waiting",reject:runs=>runs.length>count||runs.some(run=>["failed","cancelled","timed_out"].includes(run.status))?"Proposal phase admitted extra or unsuccessful native work":undefined});} +export function secretProposalDecisionClickOptions(ctx:ChatCatalogContext){ + const remaining=chatCatalogNativePhaseRemainingBudget(ctx); + return remaining?{timeout:Math.min(30_000,remaining())}:undefined; +} async function review(ctx:ChatCatalogContext,row:SecretProposalView,approve:boolean){ const source=ctx.input.page.url();await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/secrets`,{waitUntil:"domcontentloaded"});await ctx.input.page.getByRole("tab",{name:/^Proposals/}).click(); await expect(ctx.input.page.getByText("PUBLISHING_API_KEY",{exact:true})).toBeVisible();await expect(ctx.input.page.getByText("Deployment publishing key",{exact:true}).first()).toBeVisible(); const pending=(await proposals(ctx)).filter(item=>item.status==="pending");if(pending.length!==1||pending[0]!.id!==row.id||!pending[0]!.viewerCanApprove)throw new Error("Actual human cannot review the sole intended pending binding proposal"); await ctx.input.page.getByRole("button",{name:approve?"Approve":"Reject",exact:true}).click();const dialog=ctx.input.page.getByRole("dialog"); - if(approve){await expect(dialog.getByRole("heading",{name:"Approve binding",exact:true})).toBeVisible();await expect(dialog).toContainText("PUBLISHING_API_KEY");await dialog.getByRole("button",{name:"Approve binding",exact:true}).click();} - else{await expect(dialog.getByRole("heading",{name:"Reject proposal",exact:true})).toBeVisible();await dialog.getByRole("textbox",{name:"Reason",exact:true}).fill("Do not grant this publishing binding yet.");await dialog.getByRole("button",{name:"Reject",exact:true}).click();} + if(approve){await expect(dialog.getByRole("heading",{name:"Approve binding",exact:true})).toBeVisible();await expect(dialog).toContainText("PUBLISHING_API_KEY");await dialog.getByRole("button",{name:"Approve binding",exact:true}).click(secretProposalDecisionClickOptions(ctx));} + else{await expect(dialog.getByRole("heading",{name:"Reject proposal",exact:true})).toBeVisible();await dialog.getByRole("textbox",{name:"Reason",exact:true}).fill("Do not grant this publishing binding yet.");await dialog.getByRole("button",{name:"Reject",exact:true}).click(secretProposalDecisionClickOptions(ctx));} await expect(dialog).not.toBeVisible();await ctx.input.page.goto(source,{waitUntil:"domcontentloaded"}); } export const secretProposalExternalCases=[definition("secrets.secret-proposal",async ctx=>{ @@ -99,7 +104,7 @@ export const secretProposalExternalCases=[definition("secrets.secret-proposal",a await review(ctx,first,false);state.firstReviewVisible=true;const denial=await settled(ctx,2);state.denialReplyRunId=denial.find(run=>run.id!==first.originRunId)!.id; state.first=(await proposals(ctx)).find(row=>row.id===first.id);state.denialBindingCount=(await bindings(ctx,String(state.secretId))).length;const deniedAgent=await ctx.input.api.get(`/api/agents/${state.morganId}`);state.denialConfigAbsent=!(deniedAgent.adapterConfig.env as Record|undefined)?.PUBLISHING_API_KEY; if((await proposals(ctx)).length!==1||state.denialBindingCount!==0||state.denialConfigAbsent!==true)throw new Error("Denial spawned another proposal or changed the consumer binding"); - await sendChatMessage(ctx.input.page,ctx.scenario.query);const repeated=await settled(ctx,3),secondPending=(await proposals(ctx)).filter(row=>row.status==="pending");if(secondPending.length!==1||!requested(secondPending[0]!,state,ctx.issue()!.id)||secondPending[0]!.id===first.id||denial.some(run=>run.id===secondPending[0]!.originRunId))throw new Error("Explicit repeated human request did not produce one new intended proposal"); + await sendChatMessage(ctx.input.page,ctx.scenario.query,chatCatalogNativePhaseRemainingBudget(ctx));const repeated=await settled(ctx,3),secondPending=(await proposals(ctx)).filter(row=>row.status==="pending");if(secondPending.length!==1||!requested(secondPending[0]!,state,ctx.issue()!.id)||secondPending[0]!.id===first.id||denial.some(run=>run.id===secondPending[0]!.originRunId))throw new Error("Explicit repeated human request did not produce one new intended proposal"); const second=secondPending[0]!;await recordPendingProposalDisplay(ctx,state,second);await review(ctx,second,true);state.correctHumanReviewed=true;const approved=await settled(ctx,4);state.approvalReplyRunId=approved.find(run=>!repeated.some(prior=>prior.id===run.id))!.id;state.second=(await proposals(ctx)).find(row=>row.id===second.id); }, async observe(ctx,state){ diff --git a/tests/runner-e2e/chat-catalog-original-clock.test.ts b/tests/runner-e2e/chat-catalog-original-clock.test.ts index 05e7745b9fb..9e27d2f4b0f 100644 --- a/tests/runner-e2e/chat-catalog-original-clock.test.ts +++ b/tests/runner-e2e/chat-catalog-original-clock.test.ts @@ -11,7 +11,7 @@ function context(scenarioId = "cli-operations.owned-dev-instance", deadline = 87 } afterEach(() => { vi.useRealTimers(); vi.restoreAllMocks(); vi.clearAllMocks(); }); describe("selected native phases share the once-created original flow clock", () => { - it.each(["developer-labs.long-thread-performance", "cli-operations.owned-dev-instance", "recovery.controller-restart", "routines.run-now", "task-coordination.independent-review", "steering.edit-queued-message", "tasks.website-without-project", "inbox-search.archive-partial-failure", "questions-and-approvals.approve-tool-once", "runs-adapters.reset-session"])("%s uses absolute remaining outer budget rather than resetting its shorter cap", scenario => { + it.each(["secrets.secret-proposal", "developer-labs.long-thread-performance", "cli-operations.owned-dev-instance", "recovery.controller-restart", "routines.run-now", "task-coordination.independent-review", "steering.edit-queued-message", "tasks.website-without-project", "inbox-search.archive-partial-failure", "questions-and-approvals.approve-tool-once", "runs-adapters.reset-session"])("%s uses absolute remaining outer budget rather than resetting its shorter cap", scenario => { vi.useFakeTimers(); vi.setSystemTime(800_000); const ctx = context(scenario); expect(chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000)).toBe(870_000); vi.setSystemTime(850_000); expect(chatCatalogNativePhaseDeadline(ctx, Date.now() + 180_000)).toBe(870_000); diff --git a/tests/runner-e2e/chat-secret-proposal-clock.test.ts b/tests/runner-e2e/chat-secret-proposal-clock.test.ts new file mode 100644 index 00000000000..2ecf90e2692 --- /dev/null +++ b/tests/runner-e2e/chat-secret-proposal-clock.test.ts @@ -0,0 +1,25 @@ +import { afterEach, expect, it, vi } from "vitest"; +import type { ChatCatalogContext } from "./chat-catalog-contract.js"; +const poll = vi.hoisted(() => vi.fn()); +vi.mock("./api.js", async original => ({ ...await original(), pollUntil: poll })); +import { secretProposalExternalCases, secretProposalDecisionClickOptions } from "./chat-catalog-domains/external-secret-proposal.js"; +afterEach(() => { vi.useRealTimers(); vi.clearAllMocks(); }); +function context() { return { observationDeadlineAt: 810_000, scenario: { id: "secrets.secret-proposal" }, input: { execution: { id: "chat-features.runner-opencode.local.secrets.secret-proposal" } }, refreshIssue: async () => {}, issue: () => ({ conversationState: "waiting" }) } as unknown as ChatCatalogContext; } +it("actual proposal settlement uses the setup-inclusive810s clock and rejects running or late results", async () => { + vi.useFakeTimers(); vi.setSystemTime(200_000); const ctx = context(); poll.mockRejectedValue(new Error("captured settlement")); + await expect(secretProposalExternalCases[0]!.act!(ctx, {})).rejects.toThrow("captured settlement"); + const options = poll.mock.calls[0]![0]; expect(options.deadlineAt).toBe(810_000); + expect(options.accept([{ status: "succeeded" }])).toBe(true); expect(options.accept([{ status: "running" }])).toBe(false); + expect(options.reject([{ status: "succeeded" }, { status: "succeeded" }])).toMatch(/extra/); + vi.setSystemTime(810_000); expect(options.accept([{ status: "succeeded" }])).toBe(false); +}); +it("expired original clock cannot start another native observation", async () => { + vi.useFakeTimers(); vi.setSystemTime(810_000); + await expect(secretProposalExternalCases[0]!.act!(context(), {})).rejects.toThrow(/exhausted/); expect(poll).not.toHaveBeenCalled(); +}); +it("real human decision keeps the30s UI cap and fails closed after elapsed composer or review work", () => { + vi.useFakeTimers(); vi.setSystemTime(200_000); const ctx = context(); expect(secretProposalDecisionClickOptions(ctx)).toEqual({ timeout: 30_000 }); + vi.setSystemTime(809_999); expect(secretProposalDecisionClickOptions(ctx)).toEqual({ timeout: 1 }); + vi.setSystemTime(810_000); expect(() => secretProposalDecisionClickOptions(ctx)).toThrow(/exhausted/); + expect(ctx.observationDeadlineAt).toBe(810_000); +});