diff --git a/tests/runner-e2e/CATALOG-PRESERVATION.md b/tests/runner-e2e/CATALOG-PRESERVATION.md index 679a392dddf..c06fe7380a3 100644 --- a/tests/runner-e2e/CATALOG-PRESERVATION.md +++ b/tests/runner-e2e/CATALOG-PRESERVATION.md @@ -94,3 +94,14 @@ and durable retry deadlines. It also contains event-driven issue/receipt recover and the guard against a native run changing its own task status. These are upstream product changes, not qualifications of the preserved scenarios. Fresh pilots remain bound to their exact source. + +## TestOps completion pilot + +The dedicated main-only `chat-completion-testops.yml` workflow may select only +`projects.project-work`, `projects.project-defaults`, and +`instance-operations.restart-service` on the native OpenCode 1017 profile. +It needs the TestOps admission marker, a real job-run identity, attempt one, +and the same source-bound AWS host verification as the existing campaign. +The entry point preserves original queries, grading, clocks and zero retries. +Registration is separate from a real TestOps pass and verified result upload. +The broader catalog remains on this integration branch. diff --git a/tests/runner-e2e/chat-remote-host.test.ts b/tests/runner-e2e/chat-remote-host.test.ts index f3085b340bc..9b481d43a0f 100644 --- a/tests/runner-e2e/chat-remote-host.test.ts +++ b/tests/runner-e2e/chat-remote-host.test.ts @@ -22,6 +22,43 @@ async function fixture(lifecycle = "spot") { receipt: async () => JSON.parse(await readFile(path.join(root, "chat-remote-host.json"), "utf8")) }; } describe("remote-only chat host qualification without native/provider calls", () => { + it.each(["projects.project-work", "projects.project-defaults", "instance-operations.restart-service"])("admits only the bound TestOps pilot cell %s on the actual AWS host", async scenario => { + const f = await fixture(); + f.env.GITHUB_WORKFLOW_REF = "paperclipai/paperclip/.github/workflows/chat-completion-testops.yml@refs/heads/master"; + Object.assign(f.env, { PAPERCLIP_CHAT_COMPLETION_TESTOPS: "v1", ALLURE_JOB_RUN_ID: "77", GITHUB_RUN_ATTEMPT: "1" }); + f.executions[0]!.id = `chat-features.runner-opencode.local.${scenario}`; + f.executions[0]!.task.id = scenario; + expect(await prepareChatRemoteHost(f)).toMatchObject({ databaseLane: "embedded-postgres", maxAutomaticRetries: 0 }); + expect(await f.receipt()).toMatchObject({ workflowRef: f.env.GITHUB_WORKFLOW_REF, evaluationAdmitted: false, + testOps: { admission: "v1", allureJobRunId: "77" } }); + }); + it.each(["foreign-cell", "wrong-task-id", "missing-admission", "missing-jobrun", "replay", "candidate-workflow"])("rejects TestOps %s before AWS metadata or inference", async kind => { + const f = await fixture(); + f.env.GITHUB_WORKFLOW_REF = "paperclipai/paperclip/.github/workflows/chat-completion-testops.yml@refs/heads/master"; + const extra = { PAPERCLIP_CHAT_COMPLETION_TESTOPS: "v1", ALLURE_JOB_RUN_ID: "77" }; + Object.assign(f.env, extra); f.executions[0]!.id = "chat-features.runner-opencode.local.projects.project-work"; + f.executions[0]!.task.id = "projects.project-work"; + if (kind === "foreign-cell") f.executions[0]!.id = "chat-features.runner-opencode.local.tasks.delete"; + if (kind === "wrong-task-id") f.executions[0]!.task.id = "foreign"; + if (kind === "missing-admission") Object.assign(f.env, { PAPERCLIP_CHAT_COMPLETION_TESTOPS: undefined }); + if (kind === "missing-jobrun") Object.assign(f.env, { ALLURE_JOB_RUN_ID: undefined }); + if (kind === "replay") f.env.GITHUB_RUN_ATTEMPT = "2"; + if (kind === "candidate-workflow") f.env.GITHUB_WORKFLOW_REF = f.env.GITHUB_WORKFLOW_REF.replace("refs/heads/master", "refs/heads/candidate"); + await expect(prepareChatRemoteHost(f)).rejects.toThrow("trusted-github-linux-source-required"); + expect(f.fetch).not.toHaveBeenCalled(); + }); + it.each(["00077", "9007199254740992", "12345678901234567"])("rejects the same unsafe job identity %s as the dedicated dispatcher", async jobRunId => { + const f = await fixture(); f.env.GITHUB_WORKFLOW_REF = "paperclipai/paperclip/.github/workflows/chat-completion-testops.yml@refs/heads/master"; + Object.assign(f.env, { PAPERCLIP_CHAT_COMPLETION_TESTOPS: "v1", ALLURE_JOB_RUN_ID: jobRunId }); + f.executions[0]!.task.id = "projects.project-work"; f.executions[0]!.id = `chat-features.runner-opencode.local.${f.executions[0]!.task.id}`; + await expect(prepareChatRemoteHost(f)).rejects.toThrow("trusted-github-linux-source-required"); expect(f.fetch).not.toHaveBeenCalled(); + }); + it("admits the maximum safe job identity and retains it exactly", async () => { + const f = await fixture(); f.env.GITHUB_WORKFLOW_REF = "paperclipai/paperclip/.github/workflows/chat-completion-testops.yml@refs/heads/master"; + Object.assign(f.env, { PAPERCLIP_CHAT_COMPLETION_TESTOPS: "v1", ALLURE_JOB_RUN_ID: "9007199254740991" }); + f.executions[0]!.task.id = "projects.project-work"; f.executions[0]!.id = `chat-features.runner-opencode.local.${f.executions[0]!.task.id}`; + await prepareChatRemoteHost(f); expect(await f.receipt()).toMatchObject({ testOps: { admission: "v1", allureJobRunId: "9007199254740991" } }); + }); it("requires real Spot metadata and retains only closed source-bound host proof", async () => { const f = await fixture(); const policy = await prepareChatRemoteHost(f); const receipt = await f.receipt(); expect(policy).toMatchObject({ databaseLane: "embedded-postgres", maxAutomaticRetries: 0, maxParallel: 2 }); diff --git a/tests/runner-e2e/chat-remote-host.ts b/tests/runner-e2e/chat-remote-host.ts index af715fd775d..202b147f4b3 100644 --- a/tests/runner-e2e/chat-remote-host.ts +++ b/tests/runner-e2e/chat-remote-host.ts @@ -7,6 +7,9 @@ import { reviewedChatProfile } from "./chat-runtime-profile.js"; import type { MatrixExecution } from "./types.js"; const workflow = "paperclipai/paperclip/.github/workflows/runner-full-stack-e2e.yml@refs/heads/master"; +const testOpsWorkflow = "paperclipai/paperclip/.github/workflows/chat-completion-testops.yml@refs/heads/master"; +const testOpsPilotIds = new Set(["projects.project-work", "projects.project-defaults", "instance-operations.restart-service"] + .map(scenario => `chat-features.runner-opencode.local.${scenario}`)); type Selection = Pick; type Environment = Record; export interface RemoteHostDependencies { @@ -32,12 +35,19 @@ export async function prepareChatRemoteHost(input: { executions: readonly Select const persist = () => writeFile(filename, `${JSON.stringify(receipt, null, 2)}\n`, { mode: 0o600 }); let reason = "trusted-github-linux-source-required"; try { - if (env.GITHUB_ACTIONS !== "true" || env.GITHUB_REPOSITORY !== "paperclipai/paperclip" || env.GITHUB_REF !== "refs/heads/master" || env.GITHUB_WORKFLOW_REF !== workflow + const testOpsPilot = env.GITHUB_WORKFLOW_REF === testOpsWorkflow && env.PAPERCLIP_CHAT_COMPLETION_TESTOPS === "v1" + && env.GITHUB_RUN_ATTEMPT === "1" && /^[1-9][0-9]{0,15}$/.test(env.ALLURE_JOB_RUN_ID ?? "") + && Number.isSafeInteger(Number(env.ALLURE_JOB_RUN_ID)) + && input.executions.every(execution => testOpsPilotIds.has(execution.id) + && execution.id === `chat-features.runner-opencode.local.${execution.task.id}`); + if (env.GITHUB_ACTIONS !== "true" || env.GITHUB_REPOSITORY !== "paperclipai/paperclip" || env.GITHUB_REF !== "refs/heads/master" + || env.GITHUB_WORKFLOW_REF !== workflow && !testOpsPilot || !/^\d+$/.test(env.GITHUB_RUN_ID ?? "") || !/^\d+$/.test(env.GITHUB_RUN_ATTEMPT ?? "") || deps.platform !== "linux" || deps.arch !== "x64") throw new Error(reason); if (input.executions.some(e => (e.suite.id !== "chat-features" && (e.suite.id !== CHAT_SETUP_HEALTH_SUITE || e.id !== CHAT_SETUP_HEALTH_ID)) || e.profile.id !== "runner-opencode" || !reviewedChatProfile(e.profile) || e.environment.id !== "local" || e.task.automaticRetryPolicy !== "single_attempt")) throw new Error(reason); const source = await deps.sourceRevision(input.repositoryRoot); if (!/^[a-f0-9]{40}$/.test(source) || env.PAPERCLIP_RUNNER_E2E_SOURCE_SHA !== source) throw new Error(reason); - Object.assign(receipt, { workflowRef: workflow, workflowRunId: env.GITHUB_RUN_ID, workflowAttempt: Number(env.GITHUB_RUN_ATTEMPT), sourceRevision: source }); + Object.assign(receipt, { workflowRef: env.GITHUB_WORKFLOW_REF, workflowRunId: env.GITHUB_RUN_ID, workflowAttempt: Number(env.GITHUB_RUN_ATTEMPT), sourceRevision: source }); + if (testOpsPilot) Object.assign(receipt, { testOps: { admission: "v1", allureJobRunId: env.ALLURE_JOB_RUN_ID } }); reason = "aws-instance-identity-unavailable"; const tokenResponse = await deps.fetch("http://169.254.169.254/latest/api/token", { method: "PUT", redirect: "error", headers: { "X-aws-ec2-metadata-token-ttl-seconds": "60" }, signal: AbortSignal.timeout(5000) }); if (!tokenResponse.ok) throw new Error(reason);