From 2c99c248e839f41b520fc2d7e3702169ac6cb0a2 Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 5 Nov 2026 19:42:50 -0600 Subject: [PATCH] test(evals): declare API discovery ranking provenance Co-Authored-By: Paperclip --- tests/runner-e2e/chat-catalog-descriptors.json | 4 +++- .../chat-catalog-domains/external-company-secret.ts | 2 +- .../runner-e2e/chat-catalog-domains/internal-work-timeline.ts | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 04b238253c0..90a29eddd2c 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -256,7 +256,8 @@ "chat-catalog-domains/internal-routine-audit.ts", "../e2e/fetch-allowed-port.ts", "../../server/src/services/native-runtime/status-decision-committer.ts", - "../../server/src/services/work-timeline.ts" + "../../server/src/services/work-timeline.ts", + "../../server/src/services/native-runtime/runner-api-catalog.ts" ], "oracleVersion": "paperclip.chat-catalog.work-timeline.v11", "coverageGaps": [], @@ -4906,6 +4907,7 @@ "chat-catalog-domains/identity-support.ts", "live-fixtures.ts", "../../server/src/services/secrets.ts", + "../../server/src/services/native-runtime/runner-api-catalog.ts", "../../server/src/routes/issues.ts", "run-observations.ts", "chat-flow.ts", diff --git a/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts b/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts index 36e74ccf343..1227d9aee42 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts @@ -203,7 +203,7 @@ export const companySecretExternalCases=[definition("secrets.configure-company-s optionalCompanySecretBindingApproval:true, expectedCompanySecretBindingApproval(value){if(!gradeCompanySecret(value).passed)return;return companySecretApprovalProof(value);}, oracleVersion:"paperclip.chat-catalog.company-secret-controller-baseline.v13",expectedRunCount:3,minimumRunCount:2,expectedHumanMessageCount:2,attemptTimeoutMs:480000, - sourceFiles:["chat-catalog-domains/external-company-secret.ts","chat-catalog-domains/external-secret-proposal.ts","chat-catalog-domains/external-secret-question.ts","chat-catalog-domains/external-secret-route.ts","chat-catalog-domains/external-company-secret-menu.ts","chat-catalog-contract.ts","chat-catalog-flow.ts","chat-catalog-domains/external-private-browser.ts","chat-catalog-domains/external-private-browser-worker.ts","chat-catalog-domains/external-credential-language.ts","chat-catalog-domains/external-secret-scan.ts","chat-catalog-domains/external-support.ts","chat-catalog-domains/internal-support.ts","chat-catalog-domains/identity-support.ts","live-fixtures.ts","../../server/src/services/secrets.ts","../../server/src/routes/issues.ts","run-observations.ts","chat-flow.ts","chat-scenarios/secrets.yaml"], + sourceFiles:["chat-catalog-domains/external-company-secret.ts","chat-catalog-domains/external-secret-proposal.ts","chat-catalog-domains/external-secret-question.ts","chat-catalog-domains/external-secret-route.ts","chat-catalog-domains/external-company-secret-menu.ts","chat-catalog-contract.ts","chat-catalog-flow.ts","chat-catalog-domains/external-private-browser.ts","chat-catalog-domains/external-private-browser-worker.ts","chat-catalog-domains/external-credential-language.ts","chat-catalog-domains/external-secret-scan.ts","chat-catalog-domains/external-support.ts","chat-catalog-domains/internal-support.ts","chat-catalog-domains/identity-support.ts","live-fixtures.ts","../../server/src/services/secrets.ts","../../server/src/services/native-runtime/runner-api-catalog.ts","../../server/src/routes/issues.ts","run-observations.ts","chat-flow.ts","chat-scenarios/secrets.yaml"], async act(ctx,state){ const deadline=chatCatalogOriginalPhaseDeadline(ctx),remaining=()=>remainingIdentityObservationBudget(ctx); await waitForReply(ctx,deadline);remaining(); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-work-timeline.ts b/tests/runner-e2e/chat-catalog-domains/internal-work-timeline.ts index 0e71f4e841a..f1e06a54979 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-work-timeline.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-work-timeline.ts @@ -93,7 +93,7 @@ function timeline(): ChatCatalogCase { await ctx.input.evidence("chat-catalog-real-launch-timeline.json", { ...state, verification: verifier.receipts, events }); return state; }, gradeWorkTimeline); return { ...base, oracleVersion: "paperclip.chat-catalog.work-timeline.v11", authentication: "human-session", expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 720000, - sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-work-timeline.ts", "chat-catalog-domains/internal-timeline-narrative.ts", "chat-catalog-domains/internal-timeline-verifier.ts", "chat-catalog-domains/internal-privacy.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/internal-routine-audit.ts", "../e2e/fetch-allowed-port.ts", "../../server/src/services/native-runtime/status-decision-committer.ts", "../../server/src/services/work-timeline.ts"], + sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-work-timeline.ts", "chat-catalog-domains/internal-timeline-narrative.ts", "chat-catalog-domains/internal-timeline-verifier.ts", "chat-catalog-domains/internal-privacy.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/internal-routine-audit.ts", "../e2e/fetch-allowed-port.ts", "../../server/src/services/native-runtime/status-decision-committer.ts", "../../server/src/services/work-timeline.ts","../../server/src/services/native-runtime/runner-api-catalog.ts"], async observe(ctx, state): Promise { return { ...await observeInternal(ctx, state), verification: structuredClone(verifier?.receipts ?? []), timelineEvents: await ctx.input.api.get(`/api/issues/${entity(state, "task").id}/activity`), hiddenStatus: await agentStatus(ctx, ctx.input.fixtures.agent.id, `/api/issues/${entity(state, "hiddenTask").id}`) }; }, async verifyResult(ctx, state, value) { await base.verifyResult(ctx, state, value); const source = ctx.input.page.url(); try { await ctx.input.page.goto(new URL(`/${ctx.input.fixtures.company.issuePrefix}/issues/${entity(state, "prepTask").identifier}`, ctx.input.api.baseURL).href); await expect(ctx.input.page.getByRole("heading", { name: entity(state, "prepTask").title!, exact: true })).toBeVisible(); } finally { await ctx.input.page.goto(source); await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); } }, async cleanup(ctx) { try { await verifier?.close(); } finally { verifier = undefined; await cleanupPrivacyProbes(ctx); await owner?.dispose(); owner = undefined; } },