From e03ce0e27d0775b1a3cc72ba85d4ed443e25a165 Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 5 Nov 2026 14:54:51 -0600 Subject: [PATCH] test(evals): version the changed secure-entry language consumers Co-Authored-By: Paperclip --- tests/runner-e2e/chat-catalog-descriptors.json | 8 ++++---- .../chat-catalog-domains/external-secret-delivery.ts | 2 +- .../chat-catalog-domains/external-secret-proposal.ts | 2 +- .../chat-catalog-domains/external-secure-key.ts | 2 +- .../chat-catalog-domains/external-vault-reference.ts | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 3298e8fc4a7..54a95c0a16a 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -2599,7 +2599,7 @@ "chat-catalog-domains/identity-support.ts", "chat-scenarios/connection-setup.yaml" ], - "oracleVersion": "paperclip.chat-catalog.external.v4", + "oracleVersion": "paperclip.chat-catalog.external.v5", "expectedRunCount": 1, "turnTimeoutMs": 120000, "attemptTimeoutMs": 420000, @@ -4947,7 +4947,7 @@ "../../packages/plugins/sdk/dist/index.js", "chat-scenarios/secrets.yaml" ], - "oracleVersion": "paperclip.chat-catalog.external.v4", + "oracleVersion": "paperclip.chat-catalog.external.v5", "expectedRunCount": 3, "expectedHumanMessageCount": 1, "minimumRunCount": 3, @@ -5021,7 +5021,7 @@ "../../server/src/services/secret-proposal-notifications.ts", "chat-scenarios/secrets.yaml" ], - "oracleVersion": "paperclip.chat-catalog.external.v4", + "oracleVersion": "paperclip.chat-catalog.external.v5", "expectedRunCount": 4, "expectedHumanMessageCount": 4, "minimumRunCount": 4, @@ -5069,7 +5069,7 @@ "../../ui/src/pages/secrets/ImportFromVaultDialog.tsx", "chat-scenarios/secrets.yaml" ], - "oracleVersion": "paperclip.chat-catalog.external.v4", + "oracleVersion": "paperclip.chat-catalog.external.v5", "expectedRunCount": 3, "expectedHumanMessageCount": 2, "minimumRunCount": 2, diff --git a/tests/runner-e2e/chat-catalog-domains/external-secret-delivery.ts b/tests/runner-e2e/chat-catalog-domains/external-secret-delivery.ts index da6df71ae1b..d312b933f53 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-secret-delivery.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-secret-delivery.ts @@ -298,7 +298,7 @@ export const secretDeliveryExternalCases = [ }, gradeSecretDelivery, { - oracleVersion: "paperclip.chat-catalog.external.v4", authentication: "human-session", + oracleVersion: "paperclip.chat-catalog.external.v5", authentication: "human-session", expectedRunCount: 3, minimumRunCount: 3, expectedHumanMessageCount: 1, diff --git a/tests/runner-e2e/chat-catalog-domains/external-secret-proposal.ts b/tests/runner-e2e/chat-catalog-domains/external-secret-proposal.ts index de07ba803e1..72a3d5d141d 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-secret-proposal.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-secret-proposal.ts @@ -94,7 +94,7 @@ export const secretProposalExternalCases=[definition("secrets.secret-proposal",a const worker=await identityWorker(ctx,"Morgan"),secret=await ctx.input.api.postSensitive(`${root(ctx)}/secrets`,{name:"Deployment publishing key",key:"deployment-publishing",provider:"local_encrypted",value}); try{const source=await openStandingChat(ctx),session=await ctx.input.api.get<{user:{id:string}}>("/api/auth/get-session");if(!session.user.id||session.user.id==="local-board")throw new Error("Actual authorized human review identity is required");await ctx.input.api.patch(`/api/issues/${source.id}`,{description:`The publishing credential already exists as company secret ${secret.id}, Deployment publishing key. Morgan (${worker.id}) needs the environment reference PUBLISHING_API_KEY to publish the site. No value is supplied in chat and no binding exists yet.`});return {companyId:ctx.input.fixtures.company.id,leadId:ctx.input.fixtures.agent.id,morganId:worker.id,secretId:secret.id,secretBefore:secret,sourceId:source.id,personId:session.user.id,originalQuery:ctx.scenario.query};}catch(error){await ctx.input.api.delete(`/api/secrets/${secret.id}`,{allowNotFound:true});privateValues.delete(ctx);throw error;} },gradeSecretProposal,{ - oracleVersion: "paperclip.chat-catalog.external.v4", authentication:"human-session",expectedRunCount:4,minimumRunCount:4,expectedHumanMessageCount:4,attemptTimeoutMs:840000, + oracleVersion: "paperclip.chat-catalog.external.v5", authentication:"human-session",expectedRunCount:4,minimumRunCount:4,expectedHumanMessageCount:4,attemptTimeoutMs:840000, sourceFiles:["chat-catalog-domains/external-secret-proposal.ts","chat-catalog-domains/external-secret-scan.ts","chat-catalog-domains/external-credential-language.ts","chat-catalog-domains/identity-support.ts","chat-catalog-domains/external-support.ts","chat-catalog-domains/internal-support.ts","chat-flow.ts","run-observations.ts","../../server/src/routes/secrets.ts","../../server/src/services/secret-proposals.ts","../../server/src/services/secret-proposal-notifications.ts","chat-scenarios/secrets.yaml"], async act(ctx,state){ const initial=await settled(ctx,1),pending=(await proposals(ctx)).filter(row=>row.status==="pending");if(pending.length!==1||!requested(pending[0]!,state,ctx.issue()!.id)||pending[0]!.originRunId!==initial[0]!.id)throw new Error("Actual model must request exactly the intended reference binding before human review"); diff --git a/tests/runner-e2e/chat-catalog-domains/external-secure-key.ts b/tests/runner-e2e/chat-catalog-domains/external-secure-key.ts index 6630114876b..0454e57bc68 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-secure-key.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-secure-key.ts @@ -47,7 +47,7 @@ export const secureKeyExternalCases = [definition("connection-setup.secure-api-k return { serverUrl: provider.url, personId: factualContextAttachment.createdByUserId, factualContextAttachment, setupPath: `/${ctx.input.fixtures.company.issuePrefix}/apps/byo` }; } catch (error) { fixtures.delete(ctx); await provider.close(); throw error; } }, gradeSecureKey, { - oracleVersion: "paperclip.chat-catalog.external.v4", authentication: "human-session", + oracleVersion: "paperclip.chat-catalog.external.v5", authentication: "human-session", queryAttachments: [{ stateKey: "factualContextAttachment", filename: "provider-account.md" }], attemptTimeoutMs: 420_000, sourceFiles: ["chat-catalog-domains/external-secure-key.ts", "chat-query-attachments.ts", "../../ui/src/App.tsx", "../../ui/src/pages/apps/app-connect-policy.ts", "../../ui/src/features/connections/ConnectionSetupFlow.tsx", "chat-catalog-domains/external-private-browser.ts", "chat-catalog-domains/external-private-browser-worker.ts", "chat-catalog-domains/external-credential-language.ts", "chat-catalog-domains/external-provider.ts", "chat-catalog-domains/external-secret-scan.ts", "run-observations.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "chat-scenarios/connection-setup.yaml"], diff --git a/tests/runner-e2e/chat-catalog-domains/external-vault-reference.ts b/tests/runner-e2e/chat-catalog-domains/external-vault-reference.ts index 4be804749dc..94318e310ca 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-vault-reference.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-vault-reference.ts @@ -426,7 +426,7 @@ export const vaultReferenceExternalCases = [ }, gradeVaultReference, { - oracleVersion: "paperclip.chat-catalog.external.v4", authentication: "human-session", + oracleVersion: "paperclip.chat-catalog.external.v5", authentication: "human-session", expectedRunCount: 3, minimumRunCount: 2, expectedHumanMessageCount: 2,