diff --git a/tests/runner-e2e/chat-remote-campaign.test.ts b/tests/runner-e2e/chat-remote-campaign.test.ts new file mode 100644 index 00000000000..ea285683af8 --- /dev/null +++ b/tests/runner-e2e/chat-remote-campaign.test.ts @@ -0,0 +1,58 @@ +import { createHash } from "node:crypto"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { applyRemoteCell, validateRemoteCell, remoteCellUsage, type RemoteArtifactDocument, type RemoteCampaign } from "./chat-remote-campaign.js"; +import { DATABASE_LANE_ENV, POSTGRES_ARTIFACT_ENV, POSTGRES_IMAGE_ID_ENV, POSTGRES_AMD64_IMAGE, POSTGRES_AMD64_IMAGE_IDS, POSTGRES_VERSION } from "./docker-postgres.js"; +import { CHAT_OPENCODE_MODEL } from "./chat-runtime-profile.js"; +import type { CampaignAttempt } from "./chat-campaign.js"; +const digest = (x: string) => createHash("sha256").update(x).digest("hex"); +const source = "a".repeat(40), hash = "b".repeat(64), runId = "12345"; +const execution = "chat-features.runner-opencode.local.create-backlog"; +const wrap = (name: string, value: unknown): RemoteArtifactDocument => { const bytes = JSON.stringify(value); return { path: `tests/runner-e2e/results/remote/${name}`, bytes, sha256: digest(bytes) }; }; +function fixture() { + const result: any = { schema: "paperclip.runner-e2e.result/v2", executionId: execution, attempt: 1, status: "failed", cleanup: "passed", profileId: "runner-opencode", provider: "opencode", model: CHAT_OPENCODE_MODEL, environmentId: "local", source: { sha: source, workflowRunUrl: `https://github.com/paperclipai/paperclip/actions/runs/${runId}` }, accountingIdentity: { campaignId: "actual-workflow-generated", attemptId: "c".repeat(32) }, suiteDefinitionHash: hash, startedAt: "2026-10-27T14:00:00Z", finishedAt: "2026-10-27T14:01:00Z", durationMs: 60000, + databaseIdentity: { lane: "owned-docker-postgres", image: POSTGRES_AMD64_IMAGE, imageId: POSTGRES_AMD64_IMAGE_IDS[0], architecture: "amd64", imageVersion: POSTGRES_VERSION, artifactVerified: true, contextSha256: hash, engineSha256: hash, containerId: "d".repeat(64), cluster: { version: "18.6 (Debian)", systemIdentifier: "12345" }, status: "removed" } }; + const host: any = { schema: "paperclip.chat-remote-host/v1", status: "qualified-host-only", scope: "before-evaluation", evaluationAdmitted: false, spotVerified: true, lifecycle: "spot", architecture: "x86_64", region: "us-east-1", workflowRef: "paperclipai/paperclip/.github/workflows/runner-full-stack-e2e.yml@refs/heads/master", workflowRunId: runId, workflowAttempt: 1, sourceRevision: source, executionIds: [execution], automaticRetries: 0, dockerEngineSha256: hash, postgres: { image: POSTGRES_AMD64_IMAGE, imageId: POSTGRES_AMD64_IMAGE_IDS[0], architecture: "amd64", version: POSTGRES_VERSION } }; + const definition: any = { scenarioId: "tasks.create-backlog", definitionDigest: hash, graderDigest: hash, suiteDefinitionHash: hash }; + const input = () => ({ intent: { campaignId: "remote-campaign", sourceRevision: source, selected: [{ scenarioId: "tasks.create-backlog", executionId: execution, remoteOrdinal: 1 as const }] }, provenance: { repository: "paperclipai/paperclip" as const, workflow: "runner-full-stack-e2e.yml" as const, workflowRef: "refs/heads/master" as const, runId, runAttempt: 1, artifactId: "998", archiveSha256: hash, targetSourceRevision: source }, result: wrap("result.json", result), host: wrap("chat-remote-host.json", host), definition: wrap("chat-catalog-definition.json", definition) }); + return { result, host, definition, input }; +} +afterEach(() => vi.unstubAllEnvs()); +describe("remote Actions cell import", () => { + it("retains actual cell facts independently of any reporting-job conclusion", () => { + const f = fixture(), cell = validateRemoteCell(f.input()); + expect(cell.result.status).toBe("failed"); expect(cell.result.accountingIdentity?.campaignId).toBe("actual-workflow-generated"); expect(cell.databaseLane).toMatchObject({ architecture: "amd64" }); + }); + it.each([ + ["OnDemand", (f: ReturnType) => { f.host.lifecycle = "on-demand"; }], + ["wrong source", (f: ReturnType) => { f.result.source.sha = "e".repeat(40); }], + ["wrong model", (f: ReturnType) => { f.result.model = "gpt-5.6"; }], + ["wrong workflow", (f: ReturnType) => { f.host.workflowRef = "fork/workflow"; }], + ["wrong database", (f: ReturnType) => { f.result.databaseIdentity.architecture = "arm64"; }], + ["mixed engine", (f: ReturnType) => { f.result.databaseIdentity.engineSha256 = "e".repeat(64); }], + ["wrong scenario", (f: ReturnType) => { f.definition.scenarioId = "tasks.other"; }], + ["missing accounting", (f: ReturnType) => { delete f.result.accountingIdentity; }], + ] as const)("rejects %s without recording a measurement", (_name, mutate) => { const f = fixture(); mutate(f); expect(() => validateRemoteCell(f.input())).toThrow("Remote import:"); }); + it("rejects changed bytes, foreign result paths and unreviewed Actions reruns", () => { + const f = fixture(), changed = f.input(); changed.result.bytes += " "; expect(() => validateRemoteCell(changed)).toThrow("bytes changed"); + const foreign = f.input(); foreign.result.path = "../result.json"; expect(() => validateRemoteCell(foreign)).toThrow("artifact path"); + const rerun = f.input(); rerun.provenance.runAttempt = 2; expect(() => validateRemoteCell(rerun)).toThrow("job rerun"); + }); + it("keeps unknown billing unknown and reported OpenRouter billing separate", () => { + const f = fixture(); expect(remoteCellUsage(f.result)).toEqual({ costUsd: null, tokens: null, coverage: "unavailable" }); + f.result.billing = { llm: { totalTokens: 123, reportedCostUsd: 0.02, costStatus: "reported" } }; + expect(remoteCellUsage(f.result)).toEqual({ costUsd: 0.02, tokens: 123, coverage: "reported" }); + }); + it("imports exactly once through existing campaign guards and never changes historical input", () => { + vi.stubEnv(DATABASE_LANE_ENV, "owned-docker-postgres"); vi.stubEnv(POSTGRES_ARTIFACT_ENV, "linux-amd64"); vi.stubEnv(POSTGRES_IMAGE_ID_ENV, POSTGRES_AMD64_IMAGE_IDS[0]); + const f = fixture(), cell = validateRemoteCell(f.input()); + const campaign = { campaignId: "remote-campaign", scenarios: [{ id: cell.scenarioId, executionId: execution, readiness: "implemented", authoringStatus: "complete", productPolicy: { lane: "source-build", sourceBaselineRevision: source }, attempts: [] }] } as unknown as RemoteCampaign; + const attempt: CampaignAttempt = { ordinal: 1, executionId: execution, outcome: "running", startedAt: f.result.startedAt, finishedAt: null, sourceRevision: source, workingTreeDigest: hash, definitionDigest: hash, graderDigest: hash, suiteDefinitionDigest: hash, profile: "runner-opencode", model: CHAT_OPENCODE_MODEL, environment: "local", automaticRetries: 0, correction: null, classification: null, observedFailure: null, evidencePaths: [], usage: { costUsd: null, tokens: null, coverage: "unavailable" }, cleanup: "pending", corePrDependencies: [], databaseLane: cell.databaseLane }; + const qualification = { attempt, completion: { outcome: "failed" as const, finishedAt: f.result.finishedAt, definitionDigest: hash, graderDigest: hash, evidencePaths: [cell.resultPath], cleanup: "passed" as const, usage: attempt.usage, classification: "candidate_failure", observedFailure: null, corePrDependencies: [] } }; + const imported = applyRemoteCell(campaign, cell, qualification); + expect(() => applyRemoteCell({ ...campaign, campaignId: "historical-local" }, cell, qualification)).toThrow("historical or foreign campaign"); + expect(campaign.scenarios[0]!.attempts).toHaveLength(0); expect(imported.campaign.scenarios[0]!.attempts).toHaveLength(1); + expect(applyRemoteCell(imported.campaign, cell, qualification).duplicate).toBe(true); + expect(() => applyRemoteCell(imported.campaign, { ...cell, resultSha256: "e".repeat(64) }, qualification)).toThrow("duplicate ordinal"); + expect(() => applyRemoteCell(campaign, cell, { ...qualification, completion: { ...qualification.completion, outcome: "passed" } })).toThrow("outcome or cleanup rewritten"); + }); +}); diff --git a/tests/runner-e2e/chat-remote-campaign.ts b/tests/runner-e2e/chat-remote-campaign.ts new file mode 100644 index 00000000000..b68d98b3a3d --- /dev/null +++ b/tests/runner-e2e/chat-remote-campaign.ts @@ -0,0 +1,110 @@ +import { createHash } from "node:crypto"; +import { completeCampaignAttempt, reserveCampaignAttempt, type CampaignAttempt, type ChatCampaign } from "./chat-campaign.js"; +import { databaseIdentityMatches, POSTGRES_AMD64_IMAGE, POSTGRES_AMD64_IMAGE_IDS, POSTGRES_VERSION, type DatabaseFingerprintIdentity } from "./docker-postgres.js"; +import { CHAT_OPENCODE_MODEL, CHAT_OPENCODE_PROFILE } from "./chat-runtime-profile.js"; +import type { RunnerE2EResult } from "./types.js"; + +const sha = (bytes: string) => createHash("sha256").update(bytes).digest("hex"); +const hex = (value: unknown): value is string => typeof value === "string" && /^[a-f0-9]{64}$/.test(value); +function requireProof(condition: unknown, message: string): asserts condition { if (!condition) throw new Error(`Remote import: ${message}`); } +export interface RemoteCellIntent { + campaignId: string; sourceRevision: string; + selected: Array<{ scenarioId: string; executionId: string; remoteOrdinal: 1 | 2 }>; +} +/** Obtained by the coordinator through authenticated Actions run/artifact APIs, not from + * files inside an untrusted archive. A reporting job's conclusion is deliberately absent. */ +export interface RemoteArtifactProvenance { + repository: "paperclipai/paperclip"; workflow: "runner-full-stack-e2e.yml"; workflowRef: "refs/heads/master"; + runId: string; runAttempt: number; artifactId: string; archiveSha256: string; + targetSourceRevision: string; +} +export interface RemoteArtifactDocument { path: string; bytes: string; sha256: string } +export interface ValidatedRemoteCell { + schema: "paperclip.chat-remote-cell-import/v1"; + intentCampaignId: string; importKey: string; measurementKey: string; artifactSha256: string; + scenarioId: string; ordinal: 1 | 2; sourceRevision: string; + resultPath: string; resultSha256: string; hostSha256: string; definitionSha256: string; + result: RunnerE2EResult; definitionDigest: string; graderDigest: string; + databaseLane: DatabaseFingerprintIdentity; +} +function document(value: RemoteArtifactDocument): Record { + requireProof(value.path.startsWith("tests/runner-e2e/results/") && !value.path.split("/").includes("..") && !value.path.includes("\\"), "non-retained artifact path"); + requireProof(hex(value.sha256) && sha(value.bytes) === value.sha256, "artifact document bytes changed"); + const parsed = JSON.parse(value.bytes); + requireProof(parsed && typeof parsed === "object" && !Array.isArray(parsed), "invalid artifact document"); + return parsed; +} +/** Validates measured cell facts only. Final pass qualification still uses the existing + * product/build/native, definition replay, privacy and cleanup campaign gates below. */ +export function validateRemoteCell(input: { + intent: RemoteCellIntent; provenance: RemoteArtifactProvenance; + result: RemoteArtifactDocument; host: RemoteArtifactDocument; definition: RemoteArtifactDocument; +}): ValidatedRemoteCell { + const { intent, provenance: p } = input; + requireProof(!!intent.campaignId.trim(), "missing new campaign identity"); + requireProof(p.repository === "paperclipai/paperclip" && p.workflow === "runner-full-stack-e2e.yml" && p.workflowRef === "refs/heads/master", "foreign workflow"); + requireProof(/^\d+$/.test(p.runId) && p.runAttempt === 1 && /^\d+$/.test(p.artifactId) && hex(p.archiveSha256), "missing artifact identity or unreviewed job rerun"); + requireProof(/^[a-f0-9]{40}$/.test(intent.sourceRevision) && p.targetSourceRevision === intent.sourceRevision, "target source mismatch"); + const result = document(input.result) as RunnerE2EResult, host = document(input.host), definition = document(input.definition); + const selected = intent.selected.filter(row => row.executionId === result.executionId); + requireProof(selected.length === 1, "execution not uniquely selected"); + const selection = selected[0]!; + requireProof([1, 2].includes(selection.remoteOrdinal) && result.attempt === 1, "automatic retry or invalid remote ordinal"); + requireProof(result.schema === "paperclip.runner-e2e.result/v2" && ["passed", "failed"].includes(result.status), "missing terminal cell result"); + requireProof(result.profileId === CHAT_OPENCODE_PROFILE && result.provider === "opencode" && result.model === CHAT_OPENCODE_MODEL && result.environmentId === "local", "provider/model/profile mismatch"); + requireProof(result.source?.sha === intent.sourceRevision && result.source.workflowRunUrl === `https://github.com/paperclipai/paperclip/actions/runs/${p.runId}`, "result source/run mismatch"); + requireProof(!!result.accountingIdentity?.campaignId && /^[a-f0-9]{32}$/.test(result.accountingIdentity?.attemptId ?? ""), "missing actual framework accounting identity"); + requireProof(Number.isFinite(Date.parse(result.startedAt)) && Date.parse(result.finishedAt) >= Date.parse(result.startedAt) && Number.isFinite(result.durationMs) && result.durationMs >= 0, "invalid actual timing"); + requireProof(host.schema === "paperclip.chat-remote-host/v1" && host.status === "qualified-host-only" && host.scope === "before-evaluation" && host.evaluationAdmitted === false + && host.spotVerified === true && host.lifecycle === "spot" && host.architecture === "x86_64" && host.region === "us-east-1", "host not qualified actual Spot"); + requireProof(host.workflowRef === "paperclipai/paperclip/.github/workflows/runner-full-stack-e2e.yml@refs/heads/master" + && host.workflowRunId === p.runId && host.workflowAttempt === p.runAttempt && host.sourceRevision === intent.sourceRevision + && Array.isArray(host.executionIds) && host.executionIds.includes(result.executionId) && host.automaticRetries === 0 && hex(host.dockerEngineSha256), "host/result provenance mismatch"); + requireProof(host.postgres?.image === POSTGRES_AMD64_IMAGE && POSTGRES_AMD64_IMAGE_IDS.includes(host.postgres?.imageId) && host.postgres.architecture === "amd64" && host.postgres.version === POSTGRES_VERSION, "foreign database artifact"); + const databaseLane = { lane: "owned-docker-postgres" as const, image: host.postgres.image, imageId: host.postgres.imageId, version: host.postgres.version, architecture: "amd64" as const }; + requireProof(databaseIdentityMatches(databaseLane, result.databaseIdentity) && result.databaseIdentity?.engineSha256 === host.dockerEngineSha256, "cell database identity mismatch"); + requireProof(definition.scenarioId === selection.scenarioId && hex(definition.definitionDigest) && hex(definition.graderDigest) + && definition.suiteDefinitionHash === result.suiteDefinitionHash && hex(result.suiteDefinitionHash), "definition/result mismatch"); + return { schema: "paperclip.chat-remote-cell-import/v1", intentCampaignId: intent.campaignId, importKey: `${p.runId}/${p.runAttempt}/${result.executionId}`, + measurementKey: `${result.accountingIdentity!.campaignId}/${result.accountingIdentity!.attemptId}/${result.executionId}`, + artifactSha256: p.archiveSha256, scenarioId: selection.scenarioId, ordinal: selection.remoteOrdinal, sourceRevision: intent.sourceRevision, + resultPath: input.result.path, resultSha256: input.result.sha256, hostSha256: input.host.sha256, definitionSha256: input.definition.sha256, + result, definitionDigest: definition.definitionDigest, graderDigest: definition.graderDigest, databaseLane }; +} +export function remoteCellUsage(result: RunnerE2EResult): CampaignAttempt["usage"] { + const usage = result.billing?.llm; + return { tokens: usage && Number.isFinite(usage.totalTokens) ? usage.totalTokens : null, + costUsd: usage && ["reported", "partial"].includes(usage.costStatus) && Number.isFinite(usage.reportedCostUsd) ? usage.reportedCostUsd : null, + coverage: usage?.costStatus ?? "unavailable" }; +} +export interface RemoteImportRecord { importKey: string; measurementKey: string; resultSha256: string; artifactSha256: string; ordinal: 1 | 2; scenarioId: string } +export type RemoteCampaign = ChatCampaign & { remoteImports?: RemoteImportRecord[] }; +/** Pure transaction: caller is the sole writer and atomically persists this returned + * existing campaign ledger. A failed gate never partially reserves an ordinal. + * Framework accounting campaign IDs are retained, not rewritten to the review intent ID. */ +export function applyRemoteCell(campaign: RemoteCampaign, cell: ValidatedRemoteCell, qualification: { + attempt: CampaignAttempt; completion: Parameters[1]; +}): { campaign: RemoteCampaign; duplicate: boolean } { + requireProof(campaign.campaignId === cell.intentCampaignId, "cannot import into historical or foreign campaign"); + const records = campaign.remoteImports ?? []; + const previous = records.find(row => row.importKey === cell.importKey || row.measurementKey === cell.measurementKey || row.scenarioId === cell.scenarioId && row.ordinal === cell.ordinal); + if (previous) { + requireProof(previous.importKey === cell.importKey && previous.measurementKey === cell.measurementKey && previous.resultSha256 === cell.resultSha256 && previous.artifactSha256 === cell.artifactSha256, "changed or duplicate ordinal measurement"); + return { campaign, duplicate: true }; + } + const { attempt, completion } = qualification; + requireProof(completion.usage.tokens === remoteCellUsage(cell.result).tokens && completion.usage.costUsd === remoteCellUsage(cell.result).costUsd && completion.usage.coverage === remoteCellUsage(cell.result).coverage, "usage differs from actual cell billing"); + requireProof(attempt.ordinal === cell.ordinal && attempt.executionId === cell.result.executionId && attempt.sourceRevision === cell.sourceRevision + && attempt.startedAt === cell.result.startedAt && completion.finishedAt === cell.result.finishedAt + && attempt.profile === cell.result.profileId && attempt.model === cell.result.model + && attempt.definitionDigest === cell.definitionDigest && attempt.graderDigest === cell.graderDigest + && attempt.suiteDefinitionDigest === cell.result.suiteDefinitionHash + && JSON.stringify(attempt.databaseLane) === JSON.stringify(cell.databaseLane), "qualification does not describe measured cell"); + requireProof(completion.outcome === cell.result.status && completion.cleanup === cell.result.cleanup && completion.evidencePaths.includes(cell.resultPath), "measured outcome or cleanup rewritten"); + const next = structuredClone(campaign), scenario = next.scenarios.find(row => row.id === cell.scenarioId); + requireProof(scenario, "scenario absent from frozen campaign"); + reserveCampaignAttempt(scenario, structuredClone(attempt)); + completeCampaignAttempt(scenario, structuredClone(completion)); + next.remoteImports = [...records, { importKey: cell.importKey, measurementKey: cell.measurementKey, resultSha256: cell.resultSha256, artifactSha256: cell.artifactSha256, ordinal: cell.ordinal, scenarioId: cell.scenarioId }]; + return { campaign: next, duplicate: false }; +}