From c75fba8e771a0083c8a2903ebc69e46019767667 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 3 Nov 2026 00:51:31 -0600 Subject: [PATCH] test(evals): observe personal-secret reply within original clock Bind the current person's canonical source and native reply before secure human entry. Preserve the actual original660s definition and all independent secret/privacy phases. Co-Authored-By: Paperclip --- ...-personal-secret-original-deadline.test.ts | 49 +++++++++++++++++++ .../external-personal-secret.ts | 31 +++++++++++- 2 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/external-personal-secret-original-deadline.test.ts diff --git a/tests/runner-e2e/chat-catalog-domains/external-personal-secret-original-deadline.test.ts b/tests/runner-e2e/chat-catalog-domains/external-personal-secret-original-deadline.test.ts new file mode 100644 index 00000000000..9a5bb00b285 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-personal-secret-original-deadline.test.ts @@ -0,0 +1,49 @@ +import { afterEach, expect, it, vi } from "vitest"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +const port = vi.hoisted(() => ({ observation: {} as Record })); +vi.mock("../codex-subscription-fixture.js", () => ({ codexSubscriptionAgentConfig: (value: unknown) => value })); +vi.mock("./external-support.js", async original => ({ ...await original(), observeExternal: async () => port.observation })); +import { personalSecretExternalCases } from "./external-personal-secret.js"; +afterEach(() => { vi.useRealTimers(); vi.restoreAllMocks(); }); +const start = 10_000, deadline = start + 630_000; +type Fault = "failed" | "cancelled" | "foreign actor" | "foreign company" | "foreign source" | "foreign task context" | "foreign comment" | "foreign person" | "foreign reply" | "no-secret-question" | "no work" | "late read"; +function fixture(fault?: Fault, budget: number | "missing" = deadline) { + vi.useFakeTimers(); vi.setSystemTime(start); + let reachedHumanPhase = false; + const query = "Set up my personal publishing credential for this task."; + const body = fault === "no-secret-question" ? "Which project should I work on?" : "Open secure My secrets settings and set your own personal publishing value. Never paste your key here."; + const run = { id: "initial", companyId: fault === "foreign company" ? "foreign" : "company", agentId: fault === "foreign actor" ? "morgan" : "ceo", runtimeMode: "native", + status: "succeeded", contextSnapshot: { issueId: fault === "foreign source" ? "foreign" : "source", ...(fault === "foreign task context" ? { taskId: "foreign" } : {}), commentId: fault === "foreign comment" ? "foreign" : "human" } }; + const comments = [{ id: "human", companyId: "company", issueId: "source", authorUserId: fault === "foreign person" ? "riley" : "person", body: query }, + { id: "reply", companyId: "company", issueId: "source", authorAgentId: "ceo", createdByRunId: "initial", body }]; + const source = { id: "source", companyId: "company", conversationState: "waiting" }; + const allRuns = async () => fault === "no work" ? [] : [{ ...run, status: fault === "failed" || fault === "cancelled" ? fault : Date.now() < (fault ? start : start + 190_000) ? "running" : "succeeded" }]; + const ctx = { ...(budget === "missing" ? {} : { observationDeadlineAt: budget }), scenario: { id: "secrets.personal-secret", query }, + input: { fixtures: { company: { id: "company", issuePrefix: "ACORN" }, agent: { id: "ceo" } }, api: { get: async (path: string) => { + if (path === "/api/issues/source/comments?order=asc") { if (fault === "late read") vi.setSystemTime(deadline + 1); return comments; } + if (path === "/api/companies/company/me/user-secrets") { reachedHumanPhase = true; throw Error("NEXT_REAL_HUMAN_PHASE"); } + throw Error("Unexpected read"); + } } }, issue: () => source, refreshIssue: async () => {}, allRuns, + } as unknown as ChatCatalogContext; + port.observation = { companyId: "company", phaseReplies: [{ runId: fault === "foreign reply" ? "foreign" : "initial", commentId: "reply", body }], links: [] }; + const state = { sourceIssueId: "source", currentPersonId: "person", definitionId: "definition" }; + async function act(ms = 191_000) { + const pending = personalSecretExternalCases[0]!.act!(ctx, state).then(() => undefined, (error: Error) => error); + await vi.advanceTimersByTimeAsync(ms); return pending; + } + return { ctx, act, reachedHumanPhase: () => reachedHumanPhase }; +} +it("actual personal-secret act admits the canonical native response after190s within the original deadline, before any secure-entry work", async () => { + const f = fixture(); const error = await f.act(); + expect(error?.message).toBe("NEXT_REAL_HUMAN_PHASE"); expect(f.reachedHumanPhase()).toBe(true); + expect(f.ctx.observationDeadlineAt).toBe(deadline); expect(personalSecretExternalCases[0]!.attemptTimeoutMs).toBe(660_000); +}); +it.each(["failed", "cancelled", "foreign actor", "foreign company", "foreign source", "foreign task context", "foreign comment", "foreign person", "foreign reply", "no-secret-question", "late read"] as const)("rejects %s before private human entry", async fault => { + const f = fixture(fault); expect(await f.act()).toBeInstanceOf(Error); expect(f.reachedHumanPhase()).toBe(false); +}); +it.each(["missing", NaN, Infinity, start, start - 1] as const)("rejects invalid or exhausted original deadline %s", async budget => { + const f = fixture(undefined, budget); expect(await f.act(1000)).toBeInstanceOf(Error); expect(f.reachedHumanPhase()).toBe(false); +}); +it("never invents work when the canonical native response is absent", async () => { + const f = fixture("no work"); expect(await f.act(631_000)).toBeInstanceOf(Error); expect(f.reachedHumanPhase()).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts b/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts index 00d8be92c96..aeaed2f3d5d 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts @@ -313,6 +313,7 @@ export const personalSecretExternalCases = [ description: `This request concerns publishing task ${task.identifier ?? task.id}, assigned to Morgan. It is in backlog because the current responsible person has not set the required Personal publishing credential definition ${definition.id} (PERSONAL_PUBLISHING_KEY). Morgan's required PUBLISHING_API_KEY user-secret reference is already configured. Riley is another company member with their own separate personal value.`, }); return { + sourceIssueId: chat.id, morganReadyBeforeRequiredBinding: true, morganUnboundConfigHash: personalSecretUnboundConfigHash(agent.adapterConfig), definitionId: definition.id, @@ -441,11 +442,37 @@ export const personalSecretExternalCases = [ }; }, async act(ctx, state) { - await waitForReply(ctx); + const remaining = () => remainingIdentityObservationBudget(ctx); + remaining(); + await waitForReply(ctx, ctx.observationDeadlineAt); + remaining(); + const source = ctx.issue(); + if (!source || source.id !== state.sourceIssueId || source.companyId !== ctx.input.fixtures.company.id) + throw new Error("Personal-secret initial response has the wrong conversation source"); + const [runs, comments] = await Promise.all([ + ctx.allRuns(), + ctx.input.api.get(`/api/issues/${source.id}/comments?order=asc`, { timeout: remaining() }), + ]); + remaining(); + const canonical = comments.filter(comment => comment.companyId === source.companyId && comment.issueId === source.id + && comment.authorUserId === state.currentPersonId && !comment.authorAgentId && !comment.createdByRunId + && comment.body?.trim() === ctx.scenario.query.trim()); + const run = runs[0]; + if (ctx.issue()?.id !== source.id || source.conversationState !== "waiting" || runs.length !== 1 || !run + || run.runtimeMode !== "native" || run.companyId !== source.companyId || run.agentId !== ctx.input.fixtures.agent.id + || run.status !== "succeeded" || run.contextSnapshot?.issueId !== source.id || canonical.length !== 1 + || (run.contextSnapshot.taskId !== undefined && run.contextSnapshot.taskId !== source.id) + || (run.nativeIssueId != null && run.nativeIssueId !== source.id) + || run.contextSnapshot?.commentId !== canonical[0]?.id) + throw new Error("Personal-secret initial response is not bound to its current person's canonical native request"); const fixture = fixtures.get(ctx)!, initial = await observeExternal(ctx, state), reply = initial.phaseReplies.at(-1); - if (!reply || !personalSecretGuidance(reply.body)) + remaining(); + if (initial.companyId !== source.companyId || !reply || reply.runId !== run.id + || !comments.some(comment => comment.id === reply.commentId && comment.companyId === source.companyId + && comment.issueId === source.id && comment.authorAgentId === run.agentId && comment.createdByRunId === run.id + && comment.body?.trim() === reply.body.trim()) || !personalSecretGuidance(reply.body)) throw new Error( "Personal setup must direct the current person's secure publishing value before human action", );