test(evals): declare exact private ChatGPT human setup prerequisites

tests/runner-e2e/chat-catalog-domains/external-chatgpt-private.test.ts

@@ -0,0 +1,109 @@
+import { describe, it, expect } from "vitest";
+import {
+  runtimePrivateRequirements,
+  validateRuntimePrivateRequirement,
+  runtimePrivateReferenceNames,
+} from "../chat-runtime-private-requirements.js";
+import {
+  selectedRuntimeInputs,
+  runtimePrivateSpec,
+} from "./runtime-private.js";
+import type { ChatCatalogContext } from "../chat-catalog-contract.js";
+const id = "assistant-connections.connect-external-assistant" as const;
+describe("exact named ChatGPT operator private requirement", () => {
+  it("declares only selected operator/account authorization metadata and no API credential surrogate", () => {
+    const spec = runtimePrivateRequirements[id];
+    expect(spec).toEqual({
+      provider: "chatgpt",
+      fields: ["operatorRef", "chatgptAuthorizationRef", "operatorMode"],
+      credentials: [],
+    });
+    expect(
+      validateRuntimePrivateRequirement(
+        id,
+        spec.provider,
+        spec.fields,
+        spec.credentials,
+      ),
+    ).toBe(true);
+    expect(runtimePrivateReferenceNames("chatgpt")).toEqual([]);
+    expect(runtimePrivateSpec(id).requiredCredentials).toEqual([]);
+  });
+  it("rejects unrelated cases, extra implicit credentials and undeclared session seams", () => {
+    const spec = runtimePrivateRequirements[id];
+    for (const args of [
+      ["assistant-connections.use-scoped-client", spec.fields, []],
+      [id, [...spec.fields, "sessionJson"], []],
+      [id, spec.fields, ["OPENAI_API_KEY"]],
+    ] as const)
+      expect(() =>
+        validateRuntimePrivateRequirement(args[0], "chatgpt", args[1], args[2]),
+      ).toThrow();
+  });
+  it("requires the exact selected disposable provider config; never an ambient or other-provider authorization", () => {
+    const configuration = {
+      schemaVersion: 1,
+      providers: {
+        chatgpt: {
+          disposable: true,
+          resourceIds: ["authorized-synthetic-chatgpt"],
+          prerequisites: {
+            operatorRef: "qa-operator",
+            chatgptAuthorizationRef: "authorized-synthetic-chatgpt",
+            operatorMode: "interactive-private-ui",
+          },
+        },
+      },
+    };
+    const context = {
+      scenario: { id },
+      input: {
+        privateInputs: {
+          configuration: () => configuration,
+          credentials: {},
+          read: () => undefined,
+        },
+      },
+    } as unknown as ChatCatalogContext;
+    expect(selectedRuntimeInputs(context, id).values.operatorRef).toBe(
+      "qa-operator",
+    );
+    for (const variant of [
+      undefined,
+      { ...configuration, providers: {} },
+      {
+        ...configuration,
+        providers: {
+          chatgpt: { ...configuration.providers.chatgpt, disposable: false },
+        },
+      },
+    ])
+      expect(() =>
+        selectedRuntimeInputs(
+          {
+            ...context,
+            input: {
+              ...context.input,
+              privateInputs: {
+                ...context.input.privateInputs!,
+                configuration: () => variant,
+              },
+            },
+          },
+          id,
+        ),
+      ).toThrow();
+    expect(() =>
+      selectedRuntimeInputs(
+        {
+          ...context,
+          scenario: {
+            ...context.scenario,
+            id: "assistant-connections.decline-consent",
+          },
+        },
+        id,
+      ),
+    ).toThrow();
+  });
+});

tests/runner-e2e/chat-runtime-private-requirements.ts

@@ -9,6 +9,7 @@ export const runtimePrivateRequirements = {
   "execution-environments.add-computer": { provider: "ssh", fields: ["hostname", "port", "login", "hostKeySha256", "knownHosts", "allowedRemoteRoot", "privateKeyEnv", "qualifiedHarness"], credentials: [] },
   "execution-environments.test-computer": { provider: "ssh", fields: ["hostname", "port", "login", "hostKeySha256", "knownHosts", "allowedRemoteRoot", "privateKeyEnv", "qualifiedHarness"], credentials: [] },
   "execution-environments.choose-capable-target": { provider: "ssh", fields: ["hostname", "port", "login", "hostKeySha256", "knownHosts", "allowedRemoteRoot", "privateKeyEnv", "qualifiedHarness"], credentials: [] },
+  "assistant-connections.connect-external-assistant": { provider: "chatgpt", fields: ["operatorRef", "chatgptAuthorizationRef", "operatorMode"], credentials: [] },
   "agents.existing-grok-bot": { provider: "grok", fields: ["botId", "endpoint", "challengeMethod", "credentialEnvName", "operatorRef", "botAuthorizationRef", "operatorMode"], credentials: [] },
   "agents.hire-claude-code": { provider: "claude", fields: ["targetRef", "harnessExecutable", "model", "accountBindingRef"], credentials: ["ANTHROPIC_API_KEY"] },
   "agents.hire-needs-account": { provider: "claude", fields: ["targetRef", "harnessExecutable", "model", "accountBindingRef"], credentials: ["ANTHROPIC_API_KEY"] },
@@ -16,7 +17,7 @@ export const runtimePrivateRequirements = {
   "runs-adapters.upgrade-runner": { provider: "claude", fields: ["targetRef", "harnessExecutable", "model", "accountBindingRef", "legacySettingsRef"], credentials: ["ANTHROPIC_API_KEY"] },
 } as const;
 export type RuntimePrivateProvider = typeof runtimePrivateRequirements[keyof typeof runtimePrivateRequirements]["provider"];
-export const runtimePrivateProviders: readonly string[] = ["daytona", "ssh", "grok", "claude"];
+export const runtimePrivateProviders: readonly string[] = ["daytona", "ssh", "grok", "claude", "chatgpt"];
 export function validateRuntimePrivateRequirement(scenarioId: string | undefined, provider: string, fields: readonly string[], credentials: readonly string[]) {
   if (!runtimePrivateProviders.includes(provider)) return false;
   const spec = runtimePrivateRequirements[scenarioId as keyof typeof runtimePrivateRequirements];