diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 9da51889668..11cb2492839 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -6,7 +6,12 @@ The explicit-only `automatic-repository` suite is an early repair check for native Codex and Claude on local and Daytona execution. One browser request must stage its managed repository, continue without a timer or another user message, and deliver a registered download. The oracle compares downloaded -JSON with the actual checkout and the original task file. Both admissions +JSON with the actual checkout and the original task file. The source admission +creates an unpredictable reference and registers its bytes before setup. The +observer must download that source-attributed artifact while only the first +admission is running, before the successor starts. A missing original, late +observation, or different successor-created reference fails qualification. The +successor is instructed to stop on a missing file and never recreate it. Both admissions must retain one workspace and commit their native results. The fixture uses production hire instructions and records the selected catalog model. These four development cells do not qualify Slack, staging configurations, migration, @@ -16,6 +21,9 @@ Select it explicitly with `--suite automatic-repository`. Every execution has one attempt, two expected provider admissions, and 1,000-cent company and agent hard stops. Automatic infrastructure retries are disabled. Existing campaign reports retain failures, timings, usage, runtime costs, and source identity. +Source-server startup prepares the existing shared/plugin SDK build dependencies +before importing native tool authority. Installed-package checks never execute +checkout build helpers. ## Live provider connection journeys diff --git a/tests/runner-e2e/automatic-repository-flow.ts b/tests/runner-e2e/automatic-repository-flow.ts index a2e28680c2f..49aec15ab83 100644 --- a/tests/runner-e2e/automatic-repository-flow.ts +++ b/tests/runner-e2e/automatic-repository-flow.ts @@ -24,9 +24,11 @@ export async function runAutomaticRepositoryFlow(input: { const files = automaticRepositoryFiles(nonce); let issue: Row | undefined; let runs: Row[] = []; - const observation: AutomaticRepositoryObservation = { reference: files.reference, issueDone: false, runCount: 0, + const observation: AutomaticRepositoryObservation = { reference: "", issueDone: false, runCount: 0, successfulCommittedRuns: 0, sourceContinuation: null, successorSource: null, workspaceIds: [], readyRepository: false, - savedContext: null, actualReadme: null, actualCommit: null, downloaded: null, downloadVerified: false, + savedContext: null, firstSavedContext: null, firstContextAttributed: false, + firstContextObservedAt: null, successorStartedAt: null, + actualReadme: null, actualCommit: null, downloaded: null, downloadVerified: false, visibleDelivery: false, pendingInteractions: -1, monitorScheduled: false }; async function refresh() { if (!issue) return; @@ -41,6 +43,17 @@ export async function runAutomaticRepositoryFlow(input: { run.nativePhase === "committed" && run.resultJson?.finalizationPhase === "committed").length; observation.sourceContinuation = runs[0]?.resultJson?.nativeResult?.continuation?.kind ?? null; observation.successorSource = runs[1]?.contextSnapshot?.source ?? null; + observation.successorStartedAt = runs[1]?.startedAt ? Date.parse(runs[1].startedAt) : null; + if (runs.length === 1 && runs[0]?.status === "running" && observation.firstContextObservedAt === null) { + // Observe the source artifact while its admission is still running. A + // successor-created file must never become the baseline for retention. + const original = await readRegisteredArtifacts(api, issue.id, runs[0].id, [files.context]).catch(() => null); + if (original?.[0]) { + observation.firstSavedContext = original[0].content; + observation.firstContextAttributed = original[0].contentVerified; + observation.firstContextObservedAt = Date.now(); + } + } observation.workspaceIds = runs.map(run => run.runnerProfileJson?.nativeExecutionInput?.binding?.executionWorkspaceId ?? null); observation.monitorScheduled ||= Boolean(issue.monitorNextCheckAt || runs.some(run => String(run.resultJson?.finalizationReasonCode).startsWith("scheduled_monitor_"))); } @@ -86,6 +99,9 @@ export async function runAutomaticRepositoryFlow(input: { observation.actualCommit = (await git(["rev-parse", "HEAD"])).trim(); observation.actualReadme = await git(["show", "HEAD:README"]); observation.readyRepository = repositories[0].pinnedCommit === observation.actualCommit; + const [original] = await readRegisteredArtifacts(api, issue.id, runs[0]!.id, [files.context]); + observation.firstContextAttributed &&= original.contentVerified && original.content === observation.firstSavedContext; + if (/^[a-f0-9]{64}\n$/.test(original.content)) observation.reference = original.content.trimEnd(); const context = await api.get(`/api/issues/${issue.id}/file-resources/content?workspace=execution&path=${encodeURIComponent(files.context)}`); observation.savedContext = context.resource?.workspaceKind === "execution_workspace" && typeof context.content?.data === "string" ? context.content.data : null; const [artifact] = await readRegisteredArtifacts(api, issue.id, runs[1]!.id, [files.result]); diff --git a/tests/runner-e2e/automatic-repository.test.ts b/tests/runner-e2e/automatic-repository.test.ts index 58edb86ed3a..5bee95ef762 100644 --- a/tests/runner-e2e/automatic-repository.test.ts +++ b/tests/runner-e2e/automatic-repository.test.ts @@ -2,10 +2,13 @@ import { expect, it } from "vitest"; import { gradeAutomaticRepository, type AutomaticRepositoryObservation } from "./automatic-repository.js"; const sha = "a".repeat(40); -const valid: AutomaticRepositoryObservation = { reference: "REFERENCE-fresh", issueDone: true, runCount: 2, +const reference = "c".repeat(64); +const valid: AutomaticRepositoryObservation = { reference, issueDone: true, runCount: 2, successfulCommittedRuns: 2, sourceContinuation: "same_agent", successorSource: "native_status_decision", workspaceIds: ["workspace", "workspace"], - readyRepository: true, savedContext: "REFERENCE-fresh\n", actualReadme: "Hello World!\n", actualCommit: sha, - downloaded: { reference: "REFERENCE-fresh", readme: "Hello World!\n", commit: sha }, downloadVerified: true, + readyRepository: true, savedContext: `${reference}\n`, firstSavedContext: `${reference}\n`, firstContextAttributed: true, + firstContextObservedAt: 1000, successorStartedAt: 2000, + actualReadme: "Hello World!\n", actualCommit: sha, + downloaded: { reference, readme: "Hello World!\n", commit: sha }, downloadVerified: true, visibleDelivery: true, pendingInteractions: 0, monitorScheduled: false }; it("accepts independent checkout, retained context, and accessible registered output", () => { @@ -19,7 +22,10 @@ it("rejects unexpected result fields rather than trusting a model's extra claims it.each>([ { runCount: 1 }, { successfulCommittedRuns: 1 }, { sourceContinuation: "monitor" }, { successorSource: "issue.comment" }, - { workspaceIds: ["old", "new"] }, { savedContext: null }, { readyRepository: false }, { downloaded: null }, + { workspaceIds: ["old", "new"] }, { savedContext: null }, { firstSavedContext: null }, { firstContextAttributed: false }, + { firstSavedContext: `${"d".repeat(64)}\n` }, { firstSavedContext: "REFERENCE-fresh\n" }, + { firstContextObservedAt: null }, { firstContextObservedAt: 2000 }, { firstContextObservedAt: 3000 }, { successorStartedAt: null }, + { readyRepository: false }, { downloaded: null }, { downloaded: { ...valid.downloaded, reference: "REFERENCE-stale" } }, { downloaded: { ...valid.downloaded, readme: "I inspected it and all checks pass" } }, { downloaded: { ...valid.downloaded, commit: "b".repeat(40) } }, diff --git a/tests/runner-e2e/automatic-repository.ts b/tests/runner-e2e/automatic-repository.ts index 2b71cc690ac..0873d68b884 100644 --- a/tests/runner-e2e/automatic-repository.ts +++ b/tests/runner-e2e/automatic-repository.ts @@ -3,7 +3,7 @@ import { readFileSync } from "node:fs"; import type { RunnerTaskFixture } from "./types.js"; export function automaticRepositoryFiles(nonce: string) { - return { context: `setup-context-${nonce}.txt`, result: `repository-result-${nonce}.json`, reference: `REFERENCE-${nonce}` }; + return { context: `setup-context-${nonce}.txt`, result: `repository-result-${nonce}.json` }; } export const automaticRepositoryTask: RunnerTaskFixture = { @@ -17,9 +17,10 @@ export const automaticRepositoryTask: RunnerTaskFixture = { const files = automaticRepositoryFiles(nonce); return [ "Verify a repository in this projectless task and deliver the result. Do not create a project or another task.", - `Before repository setup, save ${files.context} in the task directory with exactly ${JSON.stringify(`${files.reference}\n`)}. The final newline is an actual LF.`, + `Before repository setup, generate a fresh unpredictable 32-byte hexadecimal reference using the operating system random source. Save that reference followed by one LF in ${files.context} in the task directory. Do not include its value in comments or the final answer.`, + `Before requesting repository preparation, register ${files.context} as a downloadable artifact with that exact title. This records the first admission's original bytes.`, "Use the task's managed repository capability to prepare https://github.com/octocat/Hello-World. Keep the task directory as the root.", - "When the checkout is ready, read its committed README and HEAD commit. Read the saved context file again; preserve its original reference.", + "When the checkout is ready, read its committed README and HEAD commit. Read the saved context file again; preserve its original reference. If the file is missing or invalid, report the failure and stop. Never recreate, overwrite, or recover it from an attachment or earlier conversation.", `Write ${files.result} as JSON with exactly these fields: reference (the saved reference without its trailing LF), readme (the complete committed README bytes as text), and commit (the full HEAD SHA).`, `Register ${files.result} as a downloadable artifact work product with that exact title. Deliver its accessible link and finish the verification. Repository setup alone does not complete this request.`, ].join("\n"); @@ -37,6 +38,10 @@ export interface AutomaticRepositoryObservation { workspaceIds: Array; readyRepository: boolean; savedContext: string | null; + firstSavedContext: string | null; + firstContextAttributed: boolean; + firstContextObservedAt: number | null; + successorStartedAt: number | null; actualReadme: string | null; actualCommit: string | null; downloaded: { reference?: unknown; readme?: unknown; commit?: unknown } | null; @@ -55,7 +60,10 @@ export function gradeAutomaticRepository(value: AutomaticRepositoryObservation) "Repository setup must continue through the saved same-agent wake without a timer or user message."), check("retained-workspace", value.workspaceIds.length === 2 && Boolean(value.workspaceIds[0]) && value.workspaceIds.every(id => id === value.workspaceIds[0]) && value.readyRepository, "Both admissions must retain the same task workspace and its prepared repository."), - check("retained-context", value.savedContext === `${value.reference}\n`, "The original task file must survive setup byte for byte."), + check("retained-context", value.firstContextAttributed && value.firstContextObservedAt !== null && value.successorStartedAt !== null && + value.firstContextObservedAt < value.successorStartedAt && /^[a-f0-9]{64}\n$/.test(value.firstSavedContext ?? "") && + value.firstSavedContext === value.savedContext && value.savedContext === `${value.reference}\n`, + "The first admission's independently downloaded unpredictable reference must survive setup byte for byte."), check("independent-result", value.downloadVerified && value.downloaded !== null && Object.keys(value.downloaded).sort().join(",") === "commit,readme,reference" && value.downloaded.reference === value.reference && typeof value.actualReadme === "string" && value.actualReadme.length > 0 && value.downloaded?.readme === value.actualReadme && diff --git a/tests/runner-e2e/server.ts b/tests/runner-e2e/server.ts index 1a5ecf34b44..29c44b87d33 100644 --- a/tests/runner-e2e/server.ts +++ b/tests/runner-e2e/server.ts @@ -4,6 +4,7 @@ import { runnerMatrix } from "./catalog.js"; import { createRunnerE2EServerStopper, runnerE2EServerDetached } from "./server-stop.js"; import { runnerE2ETypeScriptProcessArgs } from "./web-server-command.js"; import { qualifyLegacyClaudeCli } from "./legacy-claude-cli.js"; +import { prepareSourceServerDependencies } from "./source-startup.js"; import { spawn, type ChildProcess } from "node:child_process"; import { createWriteStream } from "node:fs"; import { mkdir, readFile, rename, writeFile } from "node:fs/promises"; @@ -98,6 +99,8 @@ const definedServerEnvironment = Object.fromEntries( ), ); +await prepareSourceServerDependencies(repositoryRoot, definedServerEnvironment, usesInstalledCli(process.env)); + await Promise.all([ mkdir(path.dirname(logPath), { recursive: true }), mkdir(controlDirectory, { recursive: true, mode: 0o700 }), diff --git a/tests/runner-e2e/source-startup.test.ts b/tests/runner-e2e/source-startup.test.ts new file mode 100644 index 00000000000..9f167629eba --- /dev/null +++ b/tests/runner-e2e/source-startup.test.ts @@ -0,0 +1,31 @@ +import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, expect, it } from "vitest"; +import { prepareSourceServerDependencies } from "./source-startup.js"; + +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); +async function fixture(source: string) { + const root = await mkdtemp(path.join(os.tmpdir(), "paperclip-source-startup-")); roots.push(root); + await mkdir(path.join(root, "scripts")); + await writeFile(path.join(root, "scripts/ensure-plugin-build-deps.mjs"), source); + return root; +} + +it("prepares missing source dependencies before the entry can import them", async () => { + const root = await fixture(`import {writeFileSync} from 'node:fs'; writeFileSync('dependency.mjs', 'export default 42');`); + await expect(readFile(path.join(root, "dependency.mjs"))).rejects.toThrow(); + await prepareSourceServerDependencies(root, {}, false); + expect(await readFile(path.join(root, "dependency.mjs"), "utf8")).toBe("export default 42"); +}); + +it("fails before server admission when dependency preparation fails", async () => { + const root = await fixture("process.exit(7)"); + await expect(prepareSourceServerDependencies(root, {}, false)).rejects.toMatchObject({ code: 7 }); +}); + +it("does not execute checkout helpers for an installed package", async () => { + const root = await fixture("throw new Error('must not execute checkout code')"); + await expect(prepareSourceServerDependencies(root, {}, true)).resolves.toBeUndefined(); +}); diff --git a/tests/runner-e2e/source-startup.ts b/tests/runner-e2e/source-startup.ts new file mode 100644 index 00000000000..88fdefcbbc7 --- /dev/null +++ b/tests/runner-e2e/source-startup.ts @@ -0,0 +1,18 @@ +import { execFile } from "node:child_process"; +import path from "node:path"; +import { promisify } from "node:util"; + +/** The source entry imports tool authority before the CLI can prepare these packages. */ +export async function prepareSourceServerDependencies( + repositoryRoot: string, + environment: NodeJS.ProcessEnv, + installed: boolean, +) { + if (installed) return; + await promisify(execFile)(process.execPath, [path.join(repositoryRoot, "scripts/ensure-plugin-build-deps.mjs")], { + cwd: repositoryRoot, + env: environment, + timeout: 120_000, + maxBuffer: 1024 * 1024, + }); +}