mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-04 19:28:46 +01:00
Validate remote chat cell provenance before campaign import
This commit is contained in:
Dotta committed
1 parent
6175603fd3
commit
4e93c384ed
2 files changed
+168
No files matched your search
@@ -0,0 +1,58 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { applyRemoteCell, validateRemoteCell, remoteCellUsage, type RemoteArtifactDocument, type RemoteCampaign } from "./chat-remote-campaign.js";
|
||||
import { DATABASE_LANE_ENV, POSTGRES_ARTIFACT_ENV, POSTGRES_IMAGE_ID_ENV, POSTGRES_AMD64_IMAGE, POSTGRES_AMD64_IMAGE_IDS, POSTGRES_VERSION } from "./docker-postgres.js";
|
||||
import { CHAT_OPENCODE_MODEL } from "./chat-runtime-profile.js";
|
||||
import type { CampaignAttempt } from "./chat-campaign.js";
|
||||
const digest = (x: string) => createHash("sha256").update(x).digest("hex");
|
||||
const source = "a".repeat(40), hash = "b".repeat(64), runId = "12345";
|
||||
const execution = "chat-features.runner-opencode.local.create-backlog";
|
||||
const wrap = (name: string, value: unknown): RemoteArtifactDocument => { const bytes = JSON.stringify(value); return { path: `tests/runner-e2e/results/remote/${name}`, bytes, sha256: digest(bytes) }; };
|
||||
function fixture() {
|
||||
const result: any = { schema: "paperclip.runner-e2e.result/v2", executionId: execution, attempt: 1, status: "failed", cleanup: "passed", profileId: "runner-opencode", provider: "opencode", model: CHAT_OPENCODE_MODEL, environmentId: "local", source: { sha: source, workflowRunUrl: `https://github.com/paperclipai/paperclip/actions/runs/${runId}` }, accountingIdentity: { campaignId: "actual-workflow-generated", attemptId: "c".repeat(32) }, suiteDefinitionHash: hash, startedAt: "2026-10-27T14:00:00Z", finishedAt: "2026-10-27T14:01:00Z", durationMs: 60000,
|
||||
databaseIdentity: { lane: "owned-docker-postgres", image: POSTGRES_AMD64_IMAGE, imageId: POSTGRES_AMD64_IMAGE_IDS[0], architecture: "amd64", imageVersion: POSTGRES_VERSION, artifactVerified: true, contextSha256: hash, engineSha256: hash, containerId: "d".repeat(64), cluster: { version: "18.6 (Debian)", systemIdentifier: "12345" }, status: "removed" } };
|
||||
const host: any = { schema: "paperclip.chat-remote-host/v1", status: "qualified-host-only", scope: "before-evaluation", evaluationAdmitted: false, spotVerified: true, lifecycle: "spot", architecture: "x86_64", region: "us-east-1", workflowRef: "paperclipai/paperclip/.github/workflows/runner-full-stack-e2e.yml@refs/heads/master", workflowRunId: runId, workflowAttempt: 1, sourceRevision: source, executionIds: [execution], automaticRetries: 0, dockerEngineSha256: hash, postgres: { image: POSTGRES_AMD64_IMAGE, imageId: POSTGRES_AMD64_IMAGE_IDS[0], architecture: "amd64", version: POSTGRES_VERSION } };
|
||||
const definition: any = { scenarioId: "tasks.create-backlog", definitionDigest: hash, graderDigest: hash, suiteDefinitionHash: hash };
|
||||
const input = () => ({ intent: { campaignId: "remote-campaign", sourceRevision: source, selected: [{ scenarioId: "tasks.create-backlog", executionId: execution, remoteOrdinal: 1 as const }] }, provenance: { repository: "paperclipai/paperclip" as const, workflow: "runner-full-stack-e2e.yml" as const, workflowRef: "refs/heads/master" as const, runId, runAttempt: 1, artifactId: "998", archiveSha256: hash, targetSourceRevision: source }, result: wrap("result.json", result), host: wrap("chat-remote-host.json", host), definition: wrap("chat-catalog-definition.json", definition) });
|
||||
return { result, host, definition, input };
|
||||
}
|
||||
afterEach(() => vi.unstubAllEnvs());
|
||||
describe("remote Actions cell import", () => {
|
||||
it("retains actual cell facts independently of any reporting-job conclusion", () => {
|
||||
const f = fixture(), cell = validateRemoteCell(f.input());
|
||||
expect(cell.result.status).toBe("failed"); expect(cell.result.accountingIdentity?.campaignId).toBe("actual-workflow-generated"); expect(cell.databaseLane).toMatchObject({ architecture: "amd64" });
|
||||
});
|
||||
it.each([
|
||||
["OnDemand", (f: ReturnType<typeof fixture>) => { f.host.lifecycle = "on-demand"; }],
|
||||
["wrong source", (f: ReturnType<typeof fixture>) => { f.result.source.sha = "e".repeat(40); }],
|
||||
["wrong model", (f: ReturnType<typeof fixture>) => { f.result.model = "gpt-5.6"; }],
|
||||
["wrong workflow", (f: ReturnType<typeof fixture>) => { f.host.workflowRef = "fork/workflow"; }],
|
||||
["wrong database", (f: ReturnType<typeof fixture>) => { f.result.databaseIdentity.architecture = "arm64"; }],
|
||||
["mixed engine", (f: ReturnType<typeof fixture>) => { f.result.databaseIdentity.engineSha256 = "e".repeat(64); }],
|
||||
["wrong scenario", (f: ReturnType<typeof fixture>) => { f.definition.scenarioId = "tasks.other"; }],
|
||||
["missing accounting", (f: ReturnType<typeof fixture>) => { delete f.result.accountingIdentity; }],
|
||||
] as const)("rejects %s without recording a measurement", (_name, mutate) => { const f = fixture(); mutate(f); expect(() => validateRemoteCell(f.input())).toThrow("Remote import:"); });
|
||||
it("rejects changed bytes, foreign result paths and unreviewed Actions reruns", () => {
|
||||
const f = fixture(), changed = f.input(); changed.result.bytes += " "; expect(() => validateRemoteCell(changed)).toThrow("bytes changed");
|
||||
const foreign = f.input(); foreign.result.path = "../result.json"; expect(() => validateRemoteCell(foreign)).toThrow("artifact path");
|
||||
const rerun = f.input(); rerun.provenance.runAttempt = 2; expect(() => validateRemoteCell(rerun)).toThrow("job rerun");
|
||||
});
|
||||
it("keeps unknown billing unknown and reported OpenRouter billing separate", () => {
|
||||
const f = fixture(); expect(remoteCellUsage(f.result)).toEqual({ costUsd: null, tokens: null, coverage: "unavailable" });
|
||||
f.result.billing = { llm: { totalTokens: 123, reportedCostUsd: 0.02, costStatus: "reported" } };
|
||||
expect(remoteCellUsage(f.result)).toEqual({ costUsd: 0.02, tokens: 123, coverage: "reported" });
|
||||
});
|
||||
it("imports exactly once through existing campaign guards and never changes historical input", () => {
|
||||
vi.stubEnv(DATABASE_LANE_ENV, "owned-docker-postgres"); vi.stubEnv(POSTGRES_ARTIFACT_ENV, "linux-amd64"); vi.stubEnv(POSTGRES_IMAGE_ID_ENV, POSTGRES_AMD64_IMAGE_IDS[0]);
|
||||
const f = fixture(), cell = validateRemoteCell(f.input());
|
||||
const campaign = { campaignId: "remote-campaign", scenarios: [{ id: cell.scenarioId, executionId: execution, readiness: "implemented", authoringStatus: "complete", productPolicy: { lane: "source-build", sourceBaselineRevision: source }, attempts: [] }] } as unknown as RemoteCampaign;
|
||||
const attempt: CampaignAttempt = { ordinal: 1, executionId: execution, outcome: "running", startedAt: f.result.startedAt, finishedAt: null, sourceRevision: source, workingTreeDigest: hash, definitionDigest: hash, graderDigest: hash, suiteDefinitionDigest: hash, profile: "runner-opencode", model: CHAT_OPENCODE_MODEL, environment: "local", automaticRetries: 0, correction: null, classification: null, observedFailure: null, evidencePaths: [], usage: { costUsd: null, tokens: null, coverage: "unavailable" }, cleanup: "pending", corePrDependencies: [], databaseLane: cell.databaseLane };
|
||||
const qualification = { attempt, completion: { outcome: "failed" as const, finishedAt: f.result.finishedAt, definitionDigest: hash, graderDigest: hash, evidencePaths: [cell.resultPath], cleanup: "passed" as const, usage: attempt.usage, classification: "candidate_failure", observedFailure: null, corePrDependencies: [] } };
|
||||
const imported = applyRemoteCell(campaign, cell, qualification);
|
||||
expect(() => applyRemoteCell({ ...campaign, campaignId: "historical-local" }, cell, qualification)).toThrow("historical or foreign campaign");
|
||||
expect(campaign.scenarios[0]!.attempts).toHaveLength(0); expect(imported.campaign.scenarios[0]!.attempts).toHaveLength(1);
|
||||
expect(applyRemoteCell(imported.campaign, cell, qualification).duplicate).toBe(true);
|
||||
expect(() => applyRemoteCell(imported.campaign, { ...cell, resultSha256: "e".repeat(64) }, qualification)).toThrow("duplicate ordinal");
|
||||
expect(() => applyRemoteCell(campaign, cell, { ...qualification, completion: { ...qualification.completion, outcome: "passed" } })).toThrow("outcome or cleanup rewritten");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,110 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { completeCampaignAttempt, reserveCampaignAttempt, type CampaignAttempt, type ChatCampaign } from "./chat-campaign.js";
|
||||
import { databaseIdentityMatches, POSTGRES_AMD64_IMAGE, POSTGRES_AMD64_IMAGE_IDS, POSTGRES_VERSION, type DatabaseFingerprintIdentity } from "./docker-postgres.js";
|
||||
import { CHAT_OPENCODE_MODEL, CHAT_OPENCODE_PROFILE } from "./chat-runtime-profile.js";
|
||||
import type { RunnerE2EResult } from "./types.js";
|
||||
|
||||
const sha = (bytes: string) => createHash("sha256").update(bytes).digest("hex");
|
||||
const hex = (value: unknown): value is string => typeof value === "string" && /^[a-f0-9]{64}$/.test(value);
|
||||
function requireProof(condition: unknown, message: string): asserts condition { if (!condition) throw new Error(`Remote import: ${message}`); }
|
||||
export interface RemoteCellIntent {
|
||||
campaignId: string; sourceRevision: string;
|
||||
selected: Array<{ scenarioId: string; executionId: string; remoteOrdinal: 1 | 2 }>;
|
||||
}
|
||||
/** Obtained by the coordinator through authenticated Actions run/artifact APIs, not from
|
||||
* files inside an untrusted archive. A reporting job's conclusion is deliberately absent. */
|
||||
export interface RemoteArtifactProvenance {
|
||||
repository: "paperclipai/paperclip"; workflow: "runner-full-stack-e2e.yml"; workflowRef: "refs/heads/master";
|
||||
runId: string; runAttempt: number; artifactId: string; archiveSha256: string;
|
||||
targetSourceRevision: string;
|
||||
}
|
||||
export interface RemoteArtifactDocument { path: string; bytes: string; sha256: string }
|
||||
export interface ValidatedRemoteCell {
|
||||
schema: "paperclip.chat-remote-cell-import/v1";
|
||||
intentCampaignId: string; importKey: string; measurementKey: string; artifactSha256: string;
|
||||
scenarioId: string; ordinal: 1 | 2; sourceRevision: string;
|
||||
resultPath: string; resultSha256: string; hostSha256: string; definitionSha256: string;
|
||||
result: RunnerE2EResult; definitionDigest: string; graderDigest: string;
|
||||
databaseLane: DatabaseFingerprintIdentity;
|
||||
}
|
||||
function document(value: RemoteArtifactDocument): Record<string, any> {
|
||||
requireProof(value.path.startsWith("tests/runner-e2e/results/") && !value.path.split("/").includes("..") && !value.path.includes("\\"), "non-retained artifact path");
|
||||
requireProof(hex(value.sha256) && sha(value.bytes) === value.sha256, "artifact document bytes changed");
|
||||
const parsed = JSON.parse(value.bytes);
|
||||
requireProof(parsed && typeof parsed === "object" && !Array.isArray(parsed), "invalid artifact document");
|
||||
return parsed;
|
||||
}
|
||||
/** Validates measured cell facts only. Final pass qualification still uses the existing
|
||||
* product/build/native, definition replay, privacy and cleanup campaign gates below. */
|
||||
export function validateRemoteCell(input: {
|
||||
intent: RemoteCellIntent; provenance: RemoteArtifactProvenance;
|
||||
result: RemoteArtifactDocument; host: RemoteArtifactDocument; definition: RemoteArtifactDocument;
|
||||
}): ValidatedRemoteCell {
|
||||
const { intent, provenance: p } = input;
|
||||
requireProof(!!intent.campaignId.trim(), "missing new campaign identity");
|
||||
requireProof(p.repository === "paperclipai/paperclip" && p.workflow === "runner-full-stack-e2e.yml" && p.workflowRef === "refs/heads/master", "foreign workflow");
|
||||
requireProof(/^\d+$/.test(p.runId) && p.runAttempt === 1 && /^\d+$/.test(p.artifactId) && hex(p.archiveSha256), "missing artifact identity or unreviewed job rerun");
|
||||
requireProof(/^[a-f0-9]{40}$/.test(intent.sourceRevision) && p.targetSourceRevision === intent.sourceRevision, "target source mismatch");
|
||||
const result = document(input.result) as RunnerE2EResult, host = document(input.host), definition = document(input.definition);
|
||||
const selected = intent.selected.filter(row => row.executionId === result.executionId);
|
||||
requireProof(selected.length === 1, "execution not uniquely selected");
|
||||
const selection = selected[0]!;
|
||||
requireProof([1, 2].includes(selection.remoteOrdinal) && result.attempt === 1, "automatic retry or invalid remote ordinal");
|
||||
requireProof(result.schema === "paperclip.runner-e2e.result/v2" && ["passed", "failed"].includes(result.status), "missing terminal cell result");
|
||||
requireProof(result.profileId === CHAT_OPENCODE_PROFILE && result.provider === "opencode" && result.model === CHAT_OPENCODE_MODEL && result.environmentId === "local", "provider/model/profile mismatch");
|
||||
requireProof(result.source?.sha === intent.sourceRevision && result.source.workflowRunUrl === `https://github.com/paperclipai/paperclip/actions/runs/${p.runId}`, "result source/run mismatch");
|
||||
requireProof(!!result.accountingIdentity?.campaignId && /^[a-f0-9]{32}$/.test(result.accountingIdentity?.attemptId ?? ""), "missing actual framework accounting identity");
|
||||
requireProof(Number.isFinite(Date.parse(result.startedAt)) && Date.parse(result.finishedAt) >= Date.parse(result.startedAt) && Number.isFinite(result.durationMs) && result.durationMs >= 0, "invalid actual timing");
|
||||
requireProof(host.schema === "paperclip.chat-remote-host/v1" && host.status === "qualified-host-only" && host.scope === "before-evaluation" && host.evaluationAdmitted === false
|
||||
&& host.spotVerified === true && host.lifecycle === "spot" && host.architecture === "x86_64" && host.region === "us-east-1", "host not qualified actual Spot");
|
||||
requireProof(host.workflowRef === "paperclipai/paperclip/.github/workflows/runner-full-stack-e2e.yml@refs/heads/master"
|
||||
&& host.workflowRunId === p.runId && host.workflowAttempt === p.runAttempt && host.sourceRevision === intent.sourceRevision
|
||||
&& Array.isArray(host.executionIds) && host.executionIds.includes(result.executionId) && host.automaticRetries === 0 && hex(host.dockerEngineSha256), "host/result provenance mismatch");
|
||||
requireProof(host.postgres?.image === POSTGRES_AMD64_IMAGE && POSTGRES_AMD64_IMAGE_IDS.includes(host.postgres?.imageId) && host.postgres.architecture === "amd64" && host.postgres.version === POSTGRES_VERSION, "foreign database artifact");
|
||||
const databaseLane = { lane: "owned-docker-postgres" as const, image: host.postgres.image, imageId: host.postgres.imageId, version: host.postgres.version, architecture: "amd64" as const };
|
||||
requireProof(databaseIdentityMatches(databaseLane, result.databaseIdentity) && result.databaseIdentity?.engineSha256 === host.dockerEngineSha256, "cell database identity mismatch");
|
||||
requireProof(definition.scenarioId === selection.scenarioId && hex(definition.definitionDigest) && hex(definition.graderDigest)
|
||||
&& definition.suiteDefinitionHash === result.suiteDefinitionHash && hex(result.suiteDefinitionHash), "definition/result mismatch");
|
||||
return { schema: "paperclip.chat-remote-cell-import/v1", intentCampaignId: intent.campaignId, importKey: `${p.runId}/${p.runAttempt}/${result.executionId}`,
|
||||
measurementKey: `${result.accountingIdentity!.campaignId}/${result.accountingIdentity!.attemptId}/${result.executionId}`,
|
||||
artifactSha256: p.archiveSha256, scenarioId: selection.scenarioId, ordinal: selection.remoteOrdinal, sourceRevision: intent.sourceRevision,
|
||||
resultPath: input.result.path, resultSha256: input.result.sha256, hostSha256: input.host.sha256, definitionSha256: input.definition.sha256,
|
||||
result, definitionDigest: definition.definitionDigest, graderDigest: definition.graderDigest, databaseLane };
|
||||
}
|
||||
export function remoteCellUsage(result: RunnerE2EResult): CampaignAttempt["usage"] {
|
||||
const usage = result.billing?.llm;
|
||||
return { tokens: usage && Number.isFinite(usage.totalTokens) ? usage.totalTokens : null,
|
||||
costUsd: usage && ["reported", "partial"].includes(usage.costStatus) && Number.isFinite(usage.reportedCostUsd) ? usage.reportedCostUsd : null,
|
||||
coverage: usage?.costStatus ?? "unavailable" };
|
||||
}
|
||||
export interface RemoteImportRecord { importKey: string; measurementKey: string; resultSha256: string; artifactSha256: string; ordinal: 1 | 2; scenarioId: string }
|
||||
export type RemoteCampaign = ChatCampaign & { remoteImports?: RemoteImportRecord[] };
|
||||
/** Pure transaction: caller is the sole writer and atomically persists this returned
|
||||
* existing campaign ledger. A failed gate never partially reserves an ordinal.
|
||||
* Framework accounting campaign IDs are retained, not rewritten to the review intent ID. */
|
||||
export function applyRemoteCell(campaign: RemoteCampaign, cell: ValidatedRemoteCell, qualification: {
|
||||
attempt: CampaignAttempt; completion: Parameters<typeof completeCampaignAttempt>[1];
|
||||
}): { campaign: RemoteCampaign; duplicate: boolean } {
|
||||
requireProof(campaign.campaignId === cell.intentCampaignId, "cannot import into historical or foreign campaign");
|
||||
const records = campaign.remoteImports ?? [];
|
||||
const previous = records.find(row => row.importKey === cell.importKey || row.measurementKey === cell.measurementKey || row.scenarioId === cell.scenarioId && row.ordinal === cell.ordinal);
|
||||
if (previous) {
|
||||
requireProof(previous.importKey === cell.importKey && previous.measurementKey === cell.measurementKey && previous.resultSha256 === cell.resultSha256 && previous.artifactSha256 === cell.artifactSha256, "changed or duplicate ordinal measurement");
|
||||
return { campaign, duplicate: true };
|
||||
}
|
||||
const { attempt, completion } = qualification;
|
||||
requireProof(completion.usage.tokens === remoteCellUsage(cell.result).tokens && completion.usage.costUsd === remoteCellUsage(cell.result).costUsd && completion.usage.coverage === remoteCellUsage(cell.result).coverage, "usage differs from actual cell billing");
|
||||
requireProof(attempt.ordinal === cell.ordinal && attempt.executionId === cell.result.executionId && attempt.sourceRevision === cell.sourceRevision
|
||||
&& attempt.startedAt === cell.result.startedAt && completion.finishedAt === cell.result.finishedAt
|
||||
&& attempt.profile === cell.result.profileId && attempt.model === cell.result.model
|
||||
&& attempt.definitionDigest === cell.definitionDigest && attempt.graderDigest === cell.graderDigest
|
||||
&& attempt.suiteDefinitionDigest === cell.result.suiteDefinitionHash
|
||||
&& JSON.stringify(attempt.databaseLane) === JSON.stringify(cell.databaseLane), "qualification does not describe measured cell");
|
||||
requireProof(completion.outcome === cell.result.status && completion.cleanup === cell.result.cleanup && completion.evidencePaths.includes(cell.resultPath), "measured outcome or cleanup rewritten");
|
||||
const next = structuredClone(campaign), scenario = next.scenarios.find(row => row.id === cell.scenarioId);
|
||||
requireProof(scenario, "scenario absent from frozen campaign");
|
||||
reserveCampaignAttempt(scenario, structuredClone(attempt));
|
||||
completeCampaignAttempt(scenario, structuredClone(completion));
|
||||
next.remoteImports = [...records, { importKey: cell.importKey, measurementKey: cell.measurementKey, resultSha256: cell.resultSha256, artifactSha256: cell.artifactSha256, ordinal: cell.ordinal, scenarioId: cell.scenarioId }];
|
||||
return { campaign: next, duplicate: false };
|
||||
}
|
||||
Reference in new issue
Block a user