From 6c1a5b9e07783c99d286dddbcd2564b4ac65083d Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 31 Oct 2026 08:45:24 -0500 Subject: [PATCH] test(evals): reconcile preserved catalog with current harness contracts --- .github/workflows/pr-trusted.yml | 4 ++++ .../chat-catalog-context-identity.test.ts | 2 +- .../chat-catalog-domains/external.test.ts | 4 ++-- tests/runner-e2e/chat-feature-flow.test.ts | 8 ++++---- tests/runner-e2e/chat-feature-flow.ts | 7 +------ tests/runner-e2e/chat-native-timeout-policy.test.ts | 6 +++--- tests/runner-e2e/chat-runtime-profile.test.ts | 2 +- tests/runner-e2e/chat-scoped-ui-budget.test.ts | 6 +++--- tests/runner-e2e/chat-setup-credential.test.ts | 2 +- tests/runner-e2e/chat-setup-health.test.ts | 2 +- tests/runner-e2e/dashboard.ts | 2 +- tests/runner-e2e/first-task.test.ts | 4 ++-- tests/runner-e2e/instance-isolation.ts | 4 ++-- .../manual-private-cell-report-assets.test.ts | 13 ++++++++++++- tests/runner-e2e/pi-controls-evidence.test.ts | 6 +++--- tests/runner-e2e/server-candidate-startup.test.mjs | 3 +++ tests/runner-e2e/source-sdk-public-type.test.ts | 5 +++-- tests/runner-e2e/workflow-security.test.ts | 9 +++++---- 18 files changed, 52 insertions(+), 37 deletions(-) diff --git a/.github/workflows/pr-trusted.yml b/.github/workflows/pr-trusted.yml index dae203d6216..e43fdd6b4d6 100644 --- a/.github/workflows/pr-trusted.yml +++ b/.github/workflows/pr-trusted.yml @@ -59,6 +59,10 @@ jobs: fi - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 if: steps.eligible.outputs.enabled == 'true' + env: + NPM_CONFIG_AUDIT: "false" + NPM_CONFIG_FUND: "false" + NPM_CONFIG_UPDATE_NOTIFIER: "false" with: version: 9.15.4 - name: Restore dependencies for graph analysis diff --git a/tests/runner-e2e/chat-catalog-context-identity.test.ts b/tests/runner-e2e/chat-catalog-context-identity.test.ts index c34576b91bc..47da69dfbc6 100644 --- a/tests/runner-e2e/chat-catalog-context-identity.test.ts +++ b/tests/runner-e2e/chat-catalog-context-identity.test.ts @@ -24,7 +24,7 @@ it("preserves actual setup/act/cleanup context identity and one original clock a const context = { scenario: { id: "companies.switch-company", query: "Canonical query" }, refreshIssue: async () => {}, issue: () => ({ id: "chat" }), allRuns: async () => [], input: { execution: { task: { turnTimeoutMs: 0 } }, fixtures: { company: { id: "company" }, agent: { id: "lead" } }, - evidence: async () => {}, api: { baseURL: "https://fixture.test", get: async (path: string) => path.endsWith("/budgets/overview") ? { policies: [] } : path === "/api/agents/lead" ? { adapterConfig: { timeoutSec: 0 } } : [] } }, + page: { evaluate: async () => [], locator: () => ({ count: async () => 0 }) }, evidence: async () => {}, api: { baseURL: "https://fixture.test", get: async (path: string) => path.endsWith("/budgets/overview") ? { policies: [] } : path === "/api/agents/lead" ? { adapterConfig: { timeoutSec: 0 } } : [] } }, } as unknown as ChatCatalogContext; await expect(runChatCatalogFlow(context)).rejects.toThrow("stop after identity proof"); expect(contexts).toHaveLength(3); expect(contexts.every(ctx => ctx === context)).toBe(true); diff --git a/tests/runner-e2e/chat-catalog-domains/external.test.ts b/tests/runner-e2e/chat-catalog-domains/external.test.ts index 8bd530e447d..7652fd825b8 100644 --- a/tests/runner-e2e/chat-catalog-domains/external.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external.test.ts @@ -86,10 +86,10 @@ function valid(id: string) { o.reply = "The approved action failed at the provider. The page remains unchanged."; } } else if (id.startsWith("teams.")) { - Object.assign(o.state, { team: { name: "Product Engineering", compatibility: "compatible", agentSlugs: ["cto", "senior-coder", "qa"], counts: { agents: 3, projects: 1, routines: 1 }, requiredSkills: [{}, {}, {}, {}], envInputs: [], contentHash: "hash" }, + Object.assign(o.state, { team: { name: "Product Engineering", compatibility: "compatible", agentSlugs: ["cto", "senior-coder", "qa"], counts: { agents: 3, projects: 1, routines: 1 }, requiredSkills: ["planning", "coding", "review", "testing"].map(ref => ({ ref })), envInputs: [], contentHash: "hash" }, agentIdsBefore: ["ceo"], agentIdsAfter: ["ceo"], projectIdsBefore: [], projectIdsAfter: [], installedBefore: [], installedAfter: [], preview: { team: { contentHash: "hash" }, errors: [] } }); - o.reply = "Product Engineering would add 3 agents: CTO, senior coder, QA and one project, 1 routine and 4 required skills."; + o.reply = "Product Engineering would add 3 agents: CTO, senior coder, QA and one project, 1 routine and 4 required skills: planning, coding, review, testing."; } else if (id === "secrets.ask-for-plaintext") { o.secrets = [{ id: "secret", name: "Deployment API key" }]; Object.assign(o.state, { secretId: "secret", secretUnchanged: true, valueProbeStatus: 403, unboundSecretHidden: true, unboundBeforeQuery: true, unboundAfterQuery: true, secretScanComplete: true, secretLeaked: false }); o.reply = "I cannot expose the deployment credential. Use secure credential management."; diff --git a/tests/runner-e2e/chat-feature-flow.test.ts b/tests/runner-e2e/chat-feature-flow.test.ts index 9b294922342..1afe946b42a 100644 --- a/tests/runner-e2e/chat-feature-flow.test.ts +++ b/tests/runner-e2e/chat-feature-flow.test.ts @@ -41,7 +41,7 @@ function fixture(boundary?: string) { ]; const scopes = [{ scopeType: "company", scopeId: "company" }, ...["ceo", "morgan"].map(scopeId => ({ scopeType: "agent", scopeId }))]; const api = { - baseURL: "http://fixture.invalid", + baseURL: "http://fixture.invalid", getRunObservation: vi.fn(async () => []), get: vi.fn(async (path: string) => { if (path.endsWith("issues?includeAgentCreatedTasks=true")) return []; if (path.includes("issues?includeAgentCreatedTasks=true&limit=100")) return [task]; @@ -70,9 +70,9 @@ function fixture(boundary?: string) { api, nonce: "fixture", workspacePath: "/fixture", page: { locator: () => locator, getByTestId: () => locator, reload: vi.fn(), goto: vi.fn(), url: () => "http://fixture.invalid/chat" }, fixtures: { company: { id: "company" }, agent: { id: "ceo" }, environment: { id: "local" }, secretRefs: {} }, - execution: { environment: { id: "local" }, task: { id: caseId, expectedRunCount: boundary ? 3 : 1, - buildVisibleMarker: () => "ACORN123", turnTimeoutMs: 120_000 }, - profile: { credential: "OPENROUTER_API_KEY", buildAgent: () => ({ adapterConfig: {} }) } }, + execution: { id: "chat-features.runner-opencode.local." + caseId, suite: { id: "chat-features" }, environment: { id: "local" }, task: { id: caseId, flow: "agent_chat", expectedRunCount: boundary ? 3 : 1, + buildVisibleMarker: () => "ACORN123", turnTimeoutMs: 120_000, attemptTimeoutMs: { local: 420_000 } }, + profile: { id: "runner-opencode", provider: "opencode", model: "fixture", credential: "OPENROUTER_API_KEY", buildAgent: () => ({ adapterConfig: {} }) } }, evidence, capture, check: (id: string, passed: boolean) => checks.push({ id, passed }), } as unknown as ChatFlowInput; let refreshes = 0; diff --git a/tests/runner-e2e/chat-feature-flow.ts b/tests/runner-e2e/chat-feature-flow.ts index 9ba943413be..2bd5dc97caf 100644 --- a/tests/runner-e2e/chat-feature-flow.ts +++ b/tests/runner-e2e/chat-feature-flow.ts @@ -1,4 +1,3 @@ -import { chatNativeTimeoutEvidence } from "./chat-native-timeout-policy.js"; import { readChatReadinessReceipts } from "./chat-readiness-observer.js"; import { legacyChatFeatureFingerprint } from "./chat-catalog-fingerprint.js"; import { runChatCatalogFlow } from "./chat-catalog-flow.js"; @@ -109,11 +108,7 @@ export async function runChatFeatureFlow(context: { .map(({ scopeType, scopeId, amount, hardStopEnabled, isActive, unpricedUsagePolicy }) => ({ scopeType, scopeId, amount, hardStopEnabled, isActive, unpricedUsagePolicy })); expect(funding).toHaveLength(3); expect(funding.every(policy => policy.amount === CHAT_FEATURE_BUDGET_CENTS && policy.hardStopEnabled && policy.isActive && policy.unpricedUsagePolicy === "allow")).toBe(true); - const savedLead = await api.get<{ adapterConfig?: { timeoutSec?: unknown } }>(`/api/agents/${f.agent.id}`); - const nativeTimeoutPolicy = chatNativeTimeoutEvidence({ selectedTurnTimeoutMs: execution.task.turnTimeoutMs, - authoredLegacyTurnTimeoutMs: 120_000, attemptTimeoutMs: execution.task.attemptTimeoutMs[execution.environment.id], observationBudgetMs: 300_000, - savedAgentTimeoutSec: savedLead.adapterConfig?.timeoutSec }); - await input.evidence("chat-feature-setup.json", { nativeTimeoutPolicy, scenarioId: scenario.id, companyId: f.company.id, + await input.evidence("chat-feature-setup.json", { scenarioId: scenario.id, companyId: f.company.id, leadId: f.agent.id, workerId, projectId, brief, budgetMonthlyCents: CHAT_FEATURE_BUDGET_CENTS, maximumProviderRuns: execution.task.expectedRunCount, providerRunTimeoutMs: execution.task.turnTimeoutMs, setupObservations, funding, instructions: "Production role defaults; no fixture procedure or expected outcome is injected." }); diff --git a/tests/runner-e2e/chat-native-timeout-policy.test.ts b/tests/runner-e2e/chat-native-timeout-policy.test.ts index cfaba2be490..e08b9636573 100644 --- a/tests/runner-e2e/chat-native-timeout-policy.test.ts +++ b/tests/runner-e2e/chat-native-timeout-policy.test.ts @@ -6,10 +6,10 @@ import { CHAT_NATIVE_TIMEOUT_POLICY, chatNativeTimeoutEvidence } from "./chat-na import type { RunnerProfileFixture } from "./types.js"; it("uses the production native cap only in the catalog while preserving every finite scenario budget and run bound", () => { - const cells = runnerMatrix.filter(row => row.suite.id === "chat-features"); + const cells = runnerMatrix.filter(row => row.suite.id === "chat-features" && row.profile.id === "runner-opencode"); expect(cells).toHaveLength(244); for (const cell of cells) { - expect(cell.task.turnTimeoutMs).toBe(0); + expect(cell.task.turnTimeoutMs).toBe(["create-backlog", "delegate-announcement"].includes(cell.task.id) ? 120_000 : 0); const declaration = chatCatalogCases.find(row => row.scenarioId === cell.task.id); if (declaration) { expect(cell.task.attemptTimeoutMs).toEqual({ local: declaration.attemptTimeoutMs, daytona: declaration.attemptTimeoutMs }); @@ -29,7 +29,7 @@ it("uses the production native cap only in the catalog while preserving every fi it("saves the actual catalog profile with no native cap without removing funding, daily admission or unrelated adapter settings", () => { const original = { adapterConfig: { provider: "opencode", model: "openrouter/deepseek/deepseek-v4.1-flash-1017", timeoutSec: 120, retained: true }, runtimeConfig: { retained: true } }; - const profile = { provider: "opencode", buildAgent: () => structuredClone(original) } as unknown as RunnerProfileFixture; + const profile = { id: "runner-opencode", provider: "opencode", buildAgent: () => structuredClone(original) } as unknown as RunnerProfileFixture; const input = {} as Parameters[0]; const saved = chatFeatureProfile(profile).buildAgent(input); expect(saved.adapterConfig).toEqual({ ...original.adapterConfig, timeoutSec: 0 }); diff --git a/tests/runner-e2e/chat-runtime-profile.test.ts b/tests/runner-e2e/chat-runtime-profile.test.ts index 8893c738067..c1c88d75eb0 100644 --- a/tests/runner-e2e/chat-runtime-profile.test.ts +++ b/tests/runner-e2e/chat-runtime-profile.test.ts @@ -10,7 +10,7 @@ it("pins the new controller identity and rejects model/provider/credential subst expect(reviewedChatExecutionIds("secrets.vault-reference")).toEqual(["chat-features.runner-codex-subscription.local.secrets.vault-reference", "chat-features.runner-opencode.local.secrets.vault-reference"]); }); it("changes the complete chat matrix without relabeling the existing global qualification model", () => { - const cells = runnerMatrix.filter(cell => cell.suite.id === "chat-features"); + const cells = runnerMatrix.filter(cell => cell.suite.id === "chat-features" && cell.profile.id === "runner-opencode"); expect(cells.length).toBe(244); expect(cells.every(cell => cell.profile.model === CHAT_OPENCODE_MODEL && reviewedChatProfile(cell.profile))).toBe(true); expect(runnerProfiles.find(row => row.id === "runner-opencode")!.model).not.toBe(CHAT_OPENCODE_MODEL); diff --git a/tests/runner-e2e/chat-scoped-ui-budget.test.ts b/tests/runner-e2e/chat-scoped-ui-budget.test.ts index d499ad1584f..d476b92e7a0 100644 --- a/tests/runner-e2e/chat-scoped-ui-budget.test.ts +++ b/tests/runner-e2e/chat-scoped-ui-budget.test.ts @@ -41,9 +41,9 @@ describe("optional original-clock callbacks reach the real shared UI helpers", ( it("slow result-link click cannot obtain a fresh verification/navigation budget", async () => { vi.useFakeTimers(); vi.setSystemTime(389_000); const click = vi.fn(async () => { vi.setSystemTime(391_000); }); const waitForURL = vi.fn(); - const node: any = { first: () => node, click }; - const ctx = { input: { page: { url: () => "https://fixture.test/RUN/chats/ceo", locator: () => node, waitForURL } } }; - const observation = { scenarioId: "questions-and-approvals.accept-one-proposal", origin: "https://fixture.test", companyId: "company", leadId: "lead", state: {}, reply: "Created", links: ["/RUN/issues/task"], tasks: [{ id: "task", title: "Checklist" }], projects: [], goals: [], comments: [], activity: [], documents: [], revisions: [], runs: [], humanQueries: [] }; + const node: any = { first: () => node, filter: () => node, locator: () => node, click }; + const ctx = { input: { page: { url: () => "https://fixture.test/RUN/chats/ceo", locator: () => node, getByTestId: () => node, waitForURL } } }; + const observation = { scenarioId: "questions-and-approvals.accept-one-proposal", origin: "https://fixture.test", companyId: "company", leadId: "lead", state: {}, renderedLinkEvidence: [{ href: "/RUN/issues/task", threadTestId: "task-chat-thread", source: "reply", conversationId: "chat", anchorCommentId: "reply", commentId: "reply" }], reply: "Created", links: ["/RUN/issues/task"], tasks: [{ id: "task", title: "Checklist" }], projects: [], goals: [], comments: [], activity: [], documents: [], revisions: [], runs: [], humanQueries: [] }; await expect(verifyInternalResult(ctx as never, {}, observation, remaining)).rejects.toThrow(/exhausted/); expect(click).toHaveBeenCalledExactlyOnceWith({ timeout: 1000 }); expect(waitForURL).not.toHaveBeenCalled(); }); diff --git a/tests/runner-e2e/chat-setup-credential.test.ts b/tests/runner-e2e/chat-setup-credential.test.ts index 4cace041d5e..794afdcd8c7 100644 --- a/tests/runner-e2e/chat-setup-credential.test.ts +++ b/tests/runner-e2e/chat-setup-credential.test.ts @@ -56,7 +56,7 @@ it("gates ordinary fixture creation on recognition and fingerprints the diagnost const source = readFileSync(new URL("./runner.spec.ts", import.meta.url), "utf8"); const start = source.indexOf('if (execution.task.flow === "setup_health") {'); const guard = source.indexOf('if (!credential.recognized) throw', start); - const fixtures = source.indexOf('fixtures = execution.task.flow === "first_task"', start); + const fixtures = source.indexOf('fixtures = await withSdkCanaryPreFixture(', start); expect(start).toBeGreaterThan(0); expect(guard).toBeGreaterThan(start); expect(fixtures).toBeGreaterThan(guard); expect(readFileSync(new URL("./chat-setup-health.ts", import.meta.url), "utf8")).toContain('"chat-setup-credential.ts"'); }); diff --git a/tests/runner-e2e/chat-setup-health.test.ts b/tests/runner-e2e/chat-setup-health.test.ts index 86d1bbed57a..8a847f2611f 100644 --- a/tests/runner-e2e/chat-setup-health.test.ts +++ b/tests/runner-e2e/chat-setup-health.test.ts @@ -22,7 +22,7 @@ describe("explicit setup-only diagnostic cell", () => { expect(cell.task.buildPrompt("nonce")).toBe(""); expect(selectRunnerExecutions(parseRunnerSelectors(["--all"]), runnerMatrix).some(row => row.id === cell.id)).toBe(false); expect(selectRunnerExecutions(parseRunnerSelectors(["--id", CHAT_SETUP_HEALTH_ID]), runnerMatrix)).toEqual([cell]); - expect(runnerMatrix.filter(row => row.suite.id === "chat-features").length).toBe(244); + expect(runnerMatrix.filter(row => row.suite.id === "chat-features" && row.profile.id === "runner-opencode").length).toBe(244); }); it("accepts one real successful setup without any chat/task/native work", () => expect(gradeSetupHealth(valid()).every(check => check.passed)).toBe(true)); it.each(["ready", "configMatches", "agentCount", "issues", "conversations", "comments", "humanComments", "runs"])("rejects wrong %s", field => { diff --git a/tests/runner-e2e/dashboard.ts b/tests/runner-e2e/dashboard.ts index 318fcdf04fd..47de95cbf21 100644 --- a/tests/runner-e2e/dashboard.ts +++ b/tests/runner-e2e/dashboard.ts @@ -892,7 +892,7 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) {

E2E campaign

${html(input.presentation?.title ?? input.title)}

${html(input.presentation?.description ?? "App and agent journeys across deployment targets, profiles, and execution environments. Declared PNG screenshots and normalized results are retained with every published campaign; additional diagnostic evidence remains in the access-controlled workflow artifact.")}

- ${input.campaign ? `

${html(input.generatedAt)}${input.campaign.source.sha ? `${html(input.campaign.source.sha.slice(0,8))}` : ""}${input.campaign.source.workflowRunUrl ? `Source run ↗` : ""}

` : ""} + ${input.campaign ? `

${html(input.generatedAt)}${input.campaign.source.sha ? `${html(input.campaign.source.sha.slice(0,8))}` : ""}${input.campaign.source.workflowRunUrl ? `Source run ↗` : ""}${input.summaryAvailable ? 'Campaign summary ↗' : ""}

` : ""}
diff --git a/tests/runner-e2e/first-task.test.ts b/tests/runner-e2e/first-task.test.ts index a5b30654497..5e9231cdd78 100644 --- a/tests/runner-e2e/first-task.test.ts +++ b/tests/runner-e2e/first-task.test.ts @@ -1591,7 +1591,7 @@ describe("managed onboarding through the production wizard", () => { expect(h.api.get).toHaveBeenCalledExactlyOnceWith("/api/instance/settings"); expect(h.api.patch).toHaveBeenLastCalledWith("/api/instance/settings", { defaultEnvironmentId: previousDefault }); expect(h.state.experimental.enableAgentChat).toBe(true); - expect(h.api.delete).toHaveBeenCalledExactlyOnceWith("/api/environments/environment?destroyReusableSandboxLeases=true", { allowNotFound: true }); + expect(h.api.delete).toHaveBeenCalledExactlyOnceWith("/api/environments/environment?destroyReusableSandboxLeases=true", { allowNotFound: true, timeout: 30_000 }); }, ); @@ -1722,7 +1722,7 @@ describe("managed onboarding through the production wizard", () => { expect(fixtures.setupLeases).toEqual([]); expect(JSON.stringify(fixtures)).not.toContain("synthetic"); await fixtures.teardown(); - expect(api.delete).toHaveBeenCalledExactlyOnceWith("/api/environments/environment?destroyReusableSandboxLeases=true", { allowNotFound: true }); + expect(api.delete).toHaveBeenCalledExactlyOnceWith("/api/environments/environment?destroyReusableSandboxLeases=true", { allowNotFound: true, timeout: 30_000 }); }); it.each(["missing-key", "mutable-image", "other-cell"])("rejects %s before admitting a managed browser fixture", async mode => { diff --git a/tests/runner-e2e/instance-isolation.ts b/tests/runner-e2e/instance-isolation.ts index 93b60696d0c..e00c7fff216 100644 --- a/tests/runner-e2e/instance-isolation.ts +++ b/tests/runner-e2e/instance-isolation.ts @@ -52,7 +52,7 @@ export async function assertRunnerDatabaseIsolation( const supervised = !docker && config.database?.mode === "postgres"; if (docker ? config.database?.mode !== "postgres" || config.database.connectionString !== docker.connectionString : !config.database || (!supervised && (config.database.mode !== "embedded-postgres" || config.database.connectionString))) { - throw new Error("Runner E2E Paperclip instance did not use its fixture-owned database"); + throw new Error("Runner E2E Paperclip instance did not use its fixture-owned database lane"); } if ( config.storage?.provider !== "local_disk" || @@ -103,7 +103,7 @@ export async function assertRunnerDatabaseIsolation( throw new Error("Supervisor database ownership mismatch"); } } catch (cause) { - throw new Error("Runner E2E Paperclip instance did not use its fixture-owned database", { cause }); + throw new Error("Runner E2E Paperclip instance did not use its fixture-owned database lane", { cause }); } } const databaseStat = await stat(databasePath); diff --git a/tests/runner-e2e/manual-private-cell-report-assets.test.ts b/tests/runner-e2e/manual-private-cell-report-assets.test.ts index 52bb2afb8a4..d132a3e8b25 100644 --- a/tests/runner-e2e/manual-private-cell-report-assets.test.ts +++ b/tests/runner-e2e/manual-private-cell-report-assets.test.ts @@ -276,7 +276,18 @@ it.each([0, 1])("actual launcher packaging call %s retains the verified private async function actualLauncherEvidenceBranch(packageCall = packageEvidence) { const source = await readFile(new URL("./launch.ts", import.meta.url), "utf8"); const start = source.indexOf(" if (manualPrivateHandoff) {\n await preserveOriginalFrameworkResult(casePrivateDir);"); - const end = source.indexOf(" console.log(", start); + const parser = createRequire(import.meta.url)("../../node_modules/.pnpm/@babel+parser@7.29.8/node_modules/@babel/parser"); + const ast = parser.parse(source, { sourceType: "module", plugins: ["typescript"] }); + let end = -1; + const visit = (node: any) => { + if (!node || typeof node !== "object") return; + if (node.type === "IfStatement" && node.start === start + 6) end = node.end; + for (const value of Object.values(node)) { + if (Array.isArray(value)) value.forEach(visit); + else if (value && typeof value === "object") visit(value); + } + }; + visit(ast); expect(start).toBeGreaterThan(0); expect(end).toBeGreaterThan(start); const code = stripTypeScriptTypes(`async function actualBranch(manualPrivateHandoff, candidate, casePrivateDir, resultsRoot, uploadDir, credentials, result, resultPath) { ${source.slice(start, end)} return result; }`, { mode: "strip" }); const publisher = vi.fn(publishRunnerE2EAttemptEvidence); diff --git a/tests/runner-e2e/pi-controls-evidence.test.ts b/tests/runner-e2e/pi-controls-evidence.test.ts index 2726922acff..0ed23b609a9 100644 --- a/tests/runner-e2e/pi-controls-evidence.test.ts +++ b/tests/runner-e2e/pi-controls-evidence.test.ts @@ -188,8 +188,8 @@ describe("Pi controls catalog admission", () => { // Extended v3 states the strict file oracle's task-wide Bash limit. const pi = runnerMatrix.find(c => c.profile.qualificationCandidate === "pi")!.profile; expect(pi.modelQualification?.qualificationId).toBe("pi:0.0.33:1.0.0:openrouter"); - expect(runnerSuites.find(s => s.id === "pi-native")!.definitionMetadata).toMatchObject({ version: 23, remoteProcExit: "separately-confirmed-absence-after-read-failure", taskCreation: "explicit-title-and-creation-response-id", agentMemoryParent: "public-managed-file-seed-before-admission", incompleteTerminalCleanup: "retirement-retained-with-failed-watch", profileVersion: 36, agentMemoryContent: "utf8-nonce-plus-final-lf", agentMemoryPrompt: "single-json-write-and-content-bound-native-read-both-runs", agentMemoryReadAuthority: "local-withheld-or-exact-remote-agent-run-file", taskPromptTransport: "fenced-markdown-paste-and-multiline-literal-escapes", nativeFinish: "current-contract-objective-evidence-refs", providerFaultExecutable: "stable-preinstalled-runner-link-and-snapshot-node-inode-with-held-bootstrap-fd-3-or-7" }); - expect(runnerSuites.find(s => s.id === "pi-controls")!.definitionMetadata).toMatchObject({ version: 10, remoteProcExit: "separately-confirmed-absence-after-read-failure", taskCreation: "explicit-title-and-creation-response-id", profileVersion: 36, + expect(runnerSuites.find(s => s.id === "pi-native")!.definitionMetadata).toMatchObject({ version: 23, remoteProcExit: "separately-confirmed-absence-after-read-failure", taskCreation: "explicit-title-and-creation-response-id", agentMemoryParent: "public-managed-file-seed-before-admission", incompleteTerminalCleanup: "retirement-retained-with-failed-watch", profileVersion: 37, agentMemoryContent: "utf8-nonce-plus-final-lf", agentMemoryPrompt: "single-json-write-and-content-bound-native-read-both-runs", agentMemoryReadAuthority: "local-withheld-or-exact-remote-agent-run-file", taskPromptTransport: "fenced-markdown-paste-and-multiline-literal-escapes", nativeFinish: "current-contract-objective-evidence-refs", providerFaultExecutable: "stable-preinstalled-runner-link-and-snapshot-node-inode-with-held-bootstrap-fd-3-or-7" }); + expect(runnerSuites.find(s => s.id === "pi-controls")!.definitionMetadata).toMatchObject({ version: 10, remoteProcExit: "separately-confirmed-absence-after-read-failure", taskCreation: "explicit-title-and-creation-response-id", profileVersion: 37, remoteProcessIdentity: "observer-pid-startTicks-bootId", controlPlaneSettlement: "required-scoped-result-and-terminal-after-runner" }); expect(runnerSuites.find(s => s.id === "extended-harnesses")!.definitionMetadata).toMatchObject({ version: 6, piFileArtifactTitle: "exact-filename", piFileCommandTransport: "fenced-bash-markdown-paste" }); @@ -199,7 +199,7 @@ describe("Pi controls catalog admission", () => { } const hashes = Object.fromEntries(runnerSuites.filter(s => ["pi-native", "native-active-stop", "extended-harnesses", "rich-acp-warm-continuity"].includes(s.id)).map(s => [s.id, suiteDefinitionHash(s)])); expect(hashes).toEqual({ - "pi-native": "b8774bd124dd39a694f3b6f26c8c9c5be9278c213b34ffba08d9fe62c21e82b7", + "pi-native": "f204250f5ed9a8b255f1c9619186b957a9353aff97f592ad87e64256a2747959", "native-active-stop": "4654a2169bff96cf8377fdd62441f5cabea5c613f447e518830f6b1efd82c051", "rich-acp-warm-continuity": "7e8a0012f2f67fba03cd94403b096a07cfc95cdb7aa568b4c3fd18b852d735f3", "extended-harnesses": "d5bcf5de0ade3a54e4e1560e76feb5d8a787039d56d2379cd501626d85ed7418", diff --git a/tests/runner-e2e/server-candidate-startup.test.mjs b/tests/runner-e2e/server-candidate-startup.test.mjs index 38c960c1bb9..016ca2e7750 100644 --- a/tests/runner-e2e/server-candidate-startup.test.mjs +++ b/tests/runner-e2e/server-candidate-startup.test.mjs @@ -19,6 +19,9 @@ it("stages company authority only across the actual completed supervisor restart const ctx=vm.createContext({Map,Set,JSON,Date,Error,Promise, process:{execPath:'/controlled/pinned-node',env:{PAPERCLIP_ACPX_CANDIDATE_ADMISSION:'ambient-forgery'},stdout:{write(){}},stderr:{write(){}}}, upgradeArtifacts:undefined,artifactIndex:0,selectedExecutions:['controlled-kimi'], + CHAT_INSTALL_PLAN:'CONTROLLED_CHAT_INSTALL_PLAN',temporaryRoot:'/controlled/tmp',invitationSourceGeneration:undefined, + invitationSourceProofPath:()=>'/controlled/tmp/proof',rm:async()=>{},executionIds:[],invitationAttemptId:'controlled', + publishInvitationSourceProof:async()=>undefined, verifyInstalledCli:async()=>null,installedReleaseLaunch:()=>{throw Error('unexpected installed path');}, definedServerEnvironment:{PATH:'/controlled/bin'},repositoryRoot:'/controlled/repo',paperclipCli:'/controlled/cli',runnerE2EServerDetached:true, runnerE2ETypeScriptProcessArgs:()=>['controlled-cli','onboard','--yes','--run'], diff --git a/tests/runner-e2e/source-sdk-public-type.test.ts b/tests/runner-e2e/source-sdk-public-type.test.ts index 9a0337c148c..663c3397ddb 100644 --- a/tests/runner-e2e/source-sdk-public-type.test.ts +++ b/tests/runner-e2e/source-sdk-public-type.test.ts @@ -13,9 +13,10 @@ const originalRoot = path.resolve(import.meta.dirname, "../.."), file = "package const source = 'import {z} from "zod";export const schema=z.object({daytona_api_key:z.string().min(1)});'; const declaration = 'import {z} from "zod";export declare const schema:{daytona_api_key:z.ZodOptional};'; let root: string; let fixtureLockSha256: string; +const fixtureInputs: Record = {}; beforeAll(async () => { root = await realpath(await mkdtemp(path.join(os.tmpdir(), "sdk-public-type-synthetic-"))); - for (const [relative] of Object.entries(pins.inputs)) { const dest = path.join(root, relative); await mkdir(path.dirname(dest), { recursive: true }); await copyFile(path.join(originalRoot, relative), dest); } + for (const [relative] of Object.entries(pins.inputs)) { const dest = path.join(root, relative); await mkdir(path.dirname(dest), { recursive: true }); await copyFile(path.join(originalRoot, relative), dest); fixtureInputs[relative] = blob(await readFile(dest)); } await copyFile(path.join(originalRoot, "pnpm-lock.yaml"), path.join(root, "pnpm-lock.yaml")); fixtureLockSha256 = hash(await readFile(path.join(root, "pnpm-lock.yaml"))); await symlink(await realpath(path.join(originalRoot, "node_modules")), path.join(root, "node_modules")); await symlink(await realpath(path.join(originalRoot, "packages/shared/node_modules")), path.join(root, "packages/shared/node_modules")); @@ -23,7 +24,7 @@ beforeAll(async () => { afterAll(async () => { if (root) await rm(root, { recursive: true, force: true }); }); async function fixture(sourceText = source) { await mkdir(path.dirname(path.join(root, inputFile)), { recursive: true }); await writeFile(path.join(root, inputFile), sourceText); - return { ...pins, lockSha256: fixtureLockSha256, inputs: { ...pins.inputs, [inputFile]: blob(sourceText) } }; + return { ...pins, lockSha256: fixtureLockSha256, inputs: { ...fixtureInputs, [inputFile]: blob(sourceText) } }; } it("classifies varied genuinely public schema types while retaining the raw observation", async () => { for (const text of [declaration, declaration.replaceAll("z.", "schemaLibrary.").replace("{z}", "{z as schemaLibrary}"), declaration.replace("ZodString", "ZodNumber")]) { diff --git a/tests/runner-e2e/workflow-security.test.ts b/tests/runner-e2e/workflow-security.test.ts index 7d18453e59c..c29e807330b 100644 --- a/tests/runner-e2e/workflow-security.test.ts +++ b/tests/runner-e2e/workflow-security.test.ts @@ -95,7 +95,7 @@ describe("public repository paid workflow security", () => { .split(/\n(?= {6}- )/u) .filter((step) => step.includes("uses: pnpm/action-setup@")); - expect(pnpmSetupSteps, workflowName).toHaveLength(8); + expect(pnpmSetupSteps, workflowName).toHaveLength(workflowName === "pr-trusted.yml" ? 9 : 8); for (const step of pnpmSetupSteps) { expect(step, workflowName).toContain('NPM_CONFIG_AUDIT: "false"'); expect(step, workflowName).toContain('NPM_CONFIG_FUND: "false"'); @@ -139,10 +139,10 @@ describe("public repository paid workflow security", () => { step.includes("uses: pnpm/action-setup@") ? [index] : [], ); - expect(pnpmSetupStepIndexes, name).toHaveLength(8); + expect(pnpmSetupStepIndexes, name).toHaveLength(name === "pr-trusted.yml" ? 9 : 8); for (const pnpmSetupStepIndex of pnpmSetupStepIndexes) { const pnpmSetupStep = steps[pnpmSetupStepIndex]!; - const nodeBootstrapStep = steps[pnpmSetupStepIndex - 1]!; + const nodeBootstrapStep = steps.slice(0, pnpmSetupStepIndex).findLast(step => step.includes("uses: actions/setup-node@"))!; expect(nodeBootstrapStep, name).toContain("uses: actions/setup-node@"); expect(nodeBootstrapStep, name).not.toContain("cache: pnpm"); @@ -603,6 +603,7 @@ describe("public repository paid workflow security", () => { const workflowDirectory = path.join(repositoryRoot, ".github/workflows"); const allowedProviderWorkflows = new Set([ "e2e.yml", + "chat-feature-testops.yml", "runner-full-stack-e2e.yml", "runner-live-evals.yml", "runner-protocol-live-evals.yml", @@ -889,7 +890,7 @@ describe("public repository paid workflow security", () => { } expect(recurring).toContain("needs.authorize.outputs.target_sha == github.sha"); for (const filename of await readdir(workflows)) { - if (!/\.ya?ml$/u.test(filename) || ["runner-full-stack-e2e.yml", "runner-protocol-live-evals.yml"].includes(filename)) continue; + if (!/\.ya?ml$/u.test(filename) || ["runner-full-stack-e2e.yml", "runner-protocol-live-evals.yml", "chat-feature-testops.yml"].includes(filename)) continue; const other = await readFile(path.join(workflows, filename), "utf8"); for (const environment of environments) expect(other, filename).not.toContain(environment); }