diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 8c0243c4fa9..34f3183846b 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -3544,3 +3544,14 @@ database and storage; failed private evidence is retained under the usual policy This proves preservation during initial enrollment, which has an offline step. Use the separate rolling deployment suites for uninterrupted deployment proof. + +## Canonical private framework references + +The protected private-cell reader accepts legacy evidence and the source-owned +compact Playwright JSON representation. It validates both sealed inventories and +all compact attachment hashes against the original tree. After those checks, it +materializes exact referenced files only in the derived `selected-runner-e2e` +tree. The sealed original and final inventories remain unchanged. The existing +normalizer and TestOps exporter consume the derived files. Registered raw JSON +remains subject to deep JSON and decoded-body privacy checks regardless of its +text suffix. No existing physical byte, entry, ZIP, time, or archive bound changes. diff --git a/tests/runner-e2e/chat-full-catalog-testops-manifest.json b/tests/runner-e2e/chat-full-catalog-testops-manifest.json index 547674dfb96..2f1ea7ee87e 100644 --- a/tests/runner-e2e/chat-full-catalog-testops-manifest.json +++ b/tests/runner-e2e/chat-full-catalog-testops-manifest.json @@ -1,6 +1,6 @@ { "schema": "paperclip.chat-full-catalog-testops-manifest/v1", - "targetSha": "c6cc59282a45ed3c79880a220f7f60aa3ac1d6a4", + "targetSha": "155dd1bc1c4b85ae107f6a0cd429720caa281367", "metadataSourceSha": "de0941835cc8f0d01c982574ee258e110babdbeb", "profile": "runner-opencode", "model": "openrouter/deepseek/deepseek-v4.1-flash-1017", @@ -172,7 +172,7 @@ "baseTargetSha": "3b66c3c144360ddf32714a0742fd774b8ceafb16", "previousTargetSha": "ab2019e976076ec0eb60b026844f3d9211cb0208", "definitionSourceSha": "de0941835cc8f0d01c982574ee258e110babdbeb", - "candidateDescriptorSha256": "f83a36c8aff7c933dd6b2d9f9613c1c5778e9bf742a1098c1e1abe5d9a3349b4", + "candidateDescriptorSha256": "e026d5c000102f8d50397dc59055d187388304f91dbca9a6ccfbe747004cd16d", "changedDefinitionFiles": [ { "path": "tests/runner-e2e/chat-scenarios/companies.yaml", @@ -195,7 +195,7 @@ "sha256": "33e98171de19c3151391164ebf266b53806f387164ad3c4e35f600044d4c1db2" } ], - "conditions": "New candidate measurements only. Original definitions, prior failed attempts and source6 ab201 remain historical. This target retains that catalog and integrates merged runtime fixes #18711, #18713 and #18718, with the prior-run restoration condition adapted to its existing owner. It adds the reviewed previous-question locator, attributed human-only denial and conditional-outcome oracle, and actor-sensitive timeline narrative/recorded-reassignment fixes. Closed finish diagnostics retain an effective schema digest, hashed call correlation and bounded type/category enums only. Historical MCP timeout and continuation-shape causes remain unresolved; no new all-ten PASS is claimed. Queries, model, human/native counts and original clocks remain fixed; the earlier Studio144 explicit 2..3 accounting stays. Core PRs18717/18734/18742 are now merged. SDK115 completion remains UNKNOWN and the other five SDK cases remain held. Full244 runtime catalog is not merged to master and the196/244 gate is unchanged.", + "conditions": "New candidate measurements only. Original definitions, prior failed attempts and source6 ab201 remain historical. This target retains that catalog and integrates merged runtime fixes #18711, #18713 and #18718, with the prior-run restoration condition adapted to its existing owner. It adds the reviewed previous-question locator, attributed human-only denial and conditional-outcome oracle, and actor-sensitive timeline narrative/recorded-reassignment fixes. Closed finish diagnostics retain an effective schema digest, hashed call correlation and bounded type/category enums only. Historical MCP timeout and continuation-shape causes remain unresolved; no new all-ten PASS is claimed. Queries, model, human/native counts and original clocks remain fixed; the earlier Studio144 explicit 2..3 accounting stays. Core PRs18717/18734/18742 are now merged. SDK115 completion remains UNKNOWN and the other five SDK cases remain held. Full244 runtime catalog is not merged to master and the196/244 gate is unchanged. This successor binds the seven reviewed scenario repairs plus framework evidence: canonical skill navigation, original announcement phase clock, attributed timeline context, bound hire revision history, credential warning and configuration-question admission, completed team visibility, and canonical private framework evidence. Twelve declaration fields across eleven cases change only oracle versions and skill source-file provenance. All original requests, setup, human/native bounds, model and clocks remain fixed. Historical failed attempts and missing-original UNKNOWN grades remain unchanged; no uniform-ten PASS is claimed. The existing SDK506/efa qualification is reused only for byte-identical registered source inputs.", "changedFields": [ { "key": "activity.work-timeline", @@ -232,7 +232,7 @@ "key": "activity.work-timeline", "field": "oracleVersion", "previous": "paperclip.chat-catalog.internal.v1", - "current": "paperclip.chat-catalog.work-timeline.v6" + "current": "paperclip.chat-catalog.work-timeline.v7" }, { "key": "activity.work-timeline", @@ -513,7 +513,7 @@ "key": "navigation-preferences.dismiss-announcement", "field": "oracleVersion", "previous": "paperclip.chat-catalog.identity.v1", - "current": "paperclip.chat-catalog.announcement-dismissal.v2" + "current": "paperclip.chat-catalog.announcement-dismissal.v3" }, { "key": "questions-and-approvals.duplicate-answer", @@ -544,7 +544,7 @@ "key": "questions-and-approvals.formal-hire-revision", "field": "oracleVersion", "previous": "paperclip.chat-catalog.hire-revision.native1017.v2", - "current": "paperclip.chat-catalog.hire-revision.native1017.v3" + "current": "paperclip.chat-catalog.hire-revision.native1017.v4" }, { "key": "questions-and-approvals.multi-part-question", @@ -635,7 +635,7 @@ "key": "secrets.configure-company-secret", "field": "oracleVersion", "previous": "paperclip.chat-catalog.external.v1", - "current": "paperclip.chat-catalog.company-secret-controller-baseline.v5" + "current": "paperclip.chat-catalog.company-secret-controller-baseline.v6" }, { "key": "secrets.personal-secret", @@ -694,7 +694,7 @@ "key": "secrets.personal-secret", "field": "oracleVersion", "previous": "paperclip.chat-catalog.external.v1", - "current": "paperclip.chat-catalog.personal-secret.v6" + "current": "paperclip.chat-catalog.personal-secret.v7" }, { "key": "skills.author-skill", @@ -713,14 +713,16 @@ "chat-catalog-domains/integrator-skill-versions.ts", "chat-catalog-domains/identity-support.ts", "../../ui/src/components/MarkdownEditor.tsx", - "../../ui/src/pages/SkillStudio.tsx" + "../../ui/src/pages/SkillStudio.tsx", + "../../ui/src/pages/CompanySkills.tsx", + "../../ui/src/lib/company-skill-routes.ts" ] }, { "key": "skills.author-skill", "field": "oracleVersion", "previous": "paperclip.chat-catalog.integrator.v1", - "current": "paperclip.chat-catalog.skill-authoring.v2" + "current": "paperclip.chat-catalog.skill-authoring.v3" }, { "key": "skills.author-skill", @@ -904,25 +906,31 @@ "key": "connection-setup.secure-api-key", "field": "oracleVersion", "previous": "paperclip.chat-catalog.external.v1", - "current": "paperclip.chat-catalog.external.v2" + "current": "paperclip.chat-catalog.external.v3" }, { "key": "secrets.least-scope-delivery", "field": "oracleVersion", "previous": "paperclip.chat-catalog.external.v1", - "current": "paperclip.chat-catalog.external.v2" + "current": "paperclip.chat-catalog.external.v3" }, { "key": "secrets.secret-proposal", "field": "oracleVersion", "previous": "paperclip.chat-catalog.external.v1", - "current": "paperclip.chat-catalog.external.v2" + "current": "paperclip.chat-catalog.external.v3" }, { "key": "secrets.vault-reference", "field": "oracleVersion", "previous": "paperclip.chat-catalog.external.v1", - "current": "paperclip.chat-catalog.external.v2" + "current": "paperclip.chat-catalog.external.v3" + }, + { + "key": "teams.install-team", + "field": "oracleVersion", + "previous": "paperclip.chat-catalog.external.v1", + "current": "paperclip.chat-catalog.team-install.v2" } ], "candidateDescriptorSnapshot": "fixtures/chat-repair-candidate-descriptors.json", @@ -962,7 +970,7 @@ "key": "activity.work-timeline", "field": "oracleVersion", "previous": "paperclip.chat-catalog.internal.v1", - "current": "paperclip.chat-catalog.work-timeline.v6" + "current": "paperclip.chat-catalog.work-timeline.v7" }, { "key": "agents.choose-primary", @@ -1213,7 +1221,7 @@ "key": "navigation-preferences.dismiss-announcement", "field": "oracleVersion", "previous": "paperclip.chat-catalog.identity.v1", - "current": "paperclip.chat-catalog.announcement-dismissal.v2" + "current": "paperclip.chat-catalog.announcement-dismissal.v3" }, { "key": "questions-and-approvals.duplicate-answer", @@ -1244,7 +1252,7 @@ "key": "questions-and-approvals.formal-hire-revision", "field": "oracleVersion", "previous": "paperclip.chat-catalog.hire-revision.native1017.v2", - "current": "paperclip.chat-catalog.hire-revision.native1017.v3" + "current": "paperclip.chat-catalog.hire-revision.native1017.v4" }, { "key": "questions-and-approvals.multi-part-question", @@ -1335,7 +1343,7 @@ "key": "secrets.configure-company-secret", "field": "oracleVersion", "previous": "paperclip.chat-catalog.external.v1", - "current": "paperclip.chat-catalog.company-secret-controller-baseline.v5" + "current": "paperclip.chat-catalog.company-secret-controller-baseline.v6" }, { "key": "secrets.personal-secret", @@ -1405,7 +1413,7 @@ "key": "secrets.personal-secret", "field": "oracleVersion", "previous": "paperclip.chat-catalog.external.v1", - "current": "paperclip.chat-catalog.personal-secret.v6" + "current": "paperclip.chat-catalog.personal-secret.v7" }, { "key": "skills.author-skill", @@ -1424,14 +1432,16 @@ "chat-catalog-domains/integrator-skill-versions.ts", "chat-catalog-domains/identity-support.ts", "../../ui/src/components/MarkdownEditor.tsx", - "../../ui/src/pages/SkillStudio.tsx" + "../../ui/src/pages/SkillStudio.tsx", + "../../ui/src/pages/CompanySkills.tsx", + "../../ui/src/lib/company-skill-routes.ts" ] }, { "key": "skills.author-skill", "field": "oracleVersion", "previous": "paperclip.chat-catalog.integrator.v1", - "current": "paperclip.chat-catalog.skill-authoring.v2" + "current": "paperclip.chat-catalog.skill-authoring.v3" }, { "key": "skills.enable-for-agent", @@ -1659,25 +1669,31 @@ "key": "connection-setup.secure-api-key", "field": "oracleVersion", "previous": "paperclip.chat-catalog.external.v1", - "current": "paperclip.chat-catalog.external.v2" + "current": "paperclip.chat-catalog.external.v3" }, { "key": "secrets.least-scope-delivery", "field": "oracleVersion", "previous": "paperclip.chat-catalog.external.v1", - "current": "paperclip.chat-catalog.external.v2" + "current": "paperclip.chat-catalog.external.v3" }, { "key": "secrets.secret-proposal", "field": "oracleVersion", "previous": "paperclip.chat-catalog.external.v1", - "current": "paperclip.chat-catalog.external.v2" + "current": "paperclip.chat-catalog.external.v3" }, { "key": "secrets.vault-reference", "field": "oracleVersion", "previous": "paperclip.chat-catalog.external.v1", - "current": "paperclip.chat-catalog.external.v2" + "current": "paperclip.chat-catalog.external.v3" + }, + { + "key": "teams.install-team", + "field": "oracleVersion", + "previous": "paperclip.chat-catalog.external.v1", + "current": "paperclip.chat-catalog.team-install.v2" } ], "runtimeConditions": { @@ -1793,9 +1809,9 @@ }, "candidateLauncherSnapshot": { "path": "fixtures/chat-repair-candidate-launcher.txt", - "targetSha": "c6cc59282a45ed3c79880a220f7f60aa3ac1d6a4", + "targetSha": "155dd1bc1c4b85ae107f6a0cd429720caa281367", "sourcePath": "tests/runner-e2e/launch.ts", - "gitBlobOid": "dba1467e211611f37937e9e1fc4c0c5be4329cbc" + "gitBlobOid": "c88df15f509b2ac8638af826647a97d45f16bd1f" } } }, @@ -2336,7 +2352,7 @@ "activity/timeline" ], "checks": null, - "oracleVersion": "paperclip.chat-catalog.work-timeline.v6", + "oracleVersion": "paperclip.chat-catalog.work-timeline.v7", "sourceFiles": [ "chat-catalog-domains/internal-definition.ts", "chat-catalog-domains/internal-support.ts", @@ -6558,7 +6574,7 @@ "connection-setup/credentials" ], "checks": null, - "oracleVersion": "paperclip.chat-catalog.external.v2", + "oracleVersion": "paperclip.chat-catalog.external.v3", "sourceFiles": [ "chat-catalog-domains/external-secure-key.ts", "chat-query-attachments.ts", @@ -8550,7 +8566,7 @@ "navigation-preferences/announcements" ], "checks": null, - "oracleVersion": "paperclip.chat-catalog.announcement-dismissal.v2", + "oracleVersion": "paperclip.chat-catalog.announcement-dismissal.v3", "sourceFiles": [ "chat-catalog-domains/identity-navigation-boundaries.ts", "chat-catalog-domains/identity-auth.ts", @@ -9863,7 +9879,7 @@ "agents/hire" ], "checks": null, - "oracleVersion": "paperclip.chat-catalog.hire-revision.native1017.v3", + "oracleVersion": "paperclip.chat-catalog.hire-revision.native1017.v4", "sourceFiles": [ "chat-catalog-domains/identity-hire-revision.ts", "chat-catalog-domains/identity-hire-model-choice.ts", @@ -11128,7 +11144,7 @@ "connection-setup/credentials" ], "checks": null, - "oracleVersion": "paperclip.chat-catalog.company-secret-controller-baseline.v5", + "oracleVersion": "paperclip.chat-catalog.company-secret-controller-baseline.v6", "sourceFiles": [ "chat-catalog-domains/external-company-secret.ts", "chat-catalog-domains/external-secret-question.ts", @@ -11180,7 +11196,7 @@ "secrets/runtime-delivery" ], "checks": null, - "oracleVersion": "paperclip.chat-catalog.external.v2", + "oracleVersion": "paperclip.chat-catalog.external.v3", "sourceFiles": [ "chat-catalog-domains/external-secret-delivery.ts", "chat-catalog-domains/external-secret-endpoint.ts", @@ -11244,7 +11260,7 @@ "secrets/user-secrets" ], "checks": null, - "oracleVersion": "paperclip.chat-catalog.personal-secret.v6", + "oracleVersion": "paperclip.chat-catalog.personal-secret.v7", "sourceFiles": [ "chat-catalog-domains/external-personal-secret.ts", "chat-catalog-domains/external-secret-question.ts", @@ -11305,7 +11321,7 @@ "secrets/proposals" ], "checks": null, - "oracleVersion": "paperclip.chat-catalog.external.v2", + "oracleVersion": "paperclip.chat-catalog.external.v3", "sourceFiles": [ "chat-catalog-domains/external-secret-proposal.ts", "chat-catalog-domains/external-secret-scan.ts", @@ -11368,7 +11384,7 @@ "secrets/vault-import" ], "checks": null, - "oracleVersion": "paperclip.chat-catalog.external.v2", + "oracleVersion": "paperclip.chat-catalog.external.v3", "sourceFiles": [ "chat-catalog-domains/external-vault-reference.ts", "chat-catalog-domains/external-fixtures.ts", @@ -11426,7 +11442,7 @@ "skills/authoring" ], "checks": null, - "oracleVersion": "paperclip.chat-catalog.skill-authoring.v2", + "oracleVersion": "paperclip.chat-catalog.skill-authoring.v3", "sourceFiles": [ "chat-catalog-domains/integrator-support.ts", "chat-catalog-domains/internal-support.ts", @@ -11435,7 +11451,9 @@ "chat-catalog-domains/integrator-skill-versions.ts", "chat-catalog-domains/identity-support.ts", "../../ui/src/components/MarkdownEditor.tsx", - "../../ui/src/pages/SkillStudio.tsx" + "../../ui/src/pages/SkillStudio.tsx", + "../../ui/src/pages/CompanySkills.tsx", + "../../ui/src/lib/company-skill-routes.ts" ], "screenshot": "final-state.png", "evidenceKind": "private-final-state-png", @@ -13222,7 +13240,7 @@ "agents/hire" ], "checks": null, - "oracleVersion": "paperclip.chat-catalog.external.v1", + "oracleVersion": "paperclip.chat-catalog.team-install.v2", "sourceFiles": [ "chat-catalog-domains/external-team-install.ts", "chat-catalog-domains/external-team-conditional.ts", @@ -13645,5 +13663,5 @@ "boundaryTarget": null } ], - "launcherSha256": "2f6f1a3c5f646020424477c10ca44ce95b1fa9d46dfc28998698580d09c4fadf" + "launcherSha256": "2b9a6953ee51f0ea0dbbd475b0491d9a6c39a1d6835eeaa912e0dbb7cf987724" } diff --git a/tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs b/tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs index a7853cc9aa8..630967fdadc 100644 --- a/tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs +++ b/tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs @@ -103,7 +103,7 @@ test('repair source metadata binds the corrected candidate and explicit changed const repair = manifest.source.repairMetadata; assert.equal(repair.baseTargetSha, '3b66c3c144360ddf32714a0742fd774b8ceafb16'); assert.equal(repair.previousTargetSha, 'ab2019e976076ec0eb60b026844f3d9211cb0208'); - assert.equal(manifest.targetSha, 'c6cc59282a45ed3c79880a220f7f60aa3ac1d6a4'); + assert.equal(manifest.targetSha, '155dd1bc1c4b85ae107f6a0cd429720caa281367'); assert.equal(repair.definitionSourceSha, manifest.metadataSourceSha); assert.equal(repair.candidateDescriptorSnapshot, 'fixtures/chat-repair-candidate-descriptors.json'); const descriptorBytes = readFileSync(new URL(repair.candidateDescriptorSnapshot, import.meta.url)); @@ -311,14 +311,18 @@ test('repair source metadata binds the corrected candidate and explicit changed [ "secrets.vault-reference", "oracleVersion" + ], + [ + "teams.install-team", + "oracleVersion" ] ]); for (const change of repair.changedFields) { assert.deepEqual(row(change.key)[change.field], change.current); assert.notDeepEqual(change.current, change.previous); } - assert.equal(repair.descriptorChanges.length, 48); - assert.equal(new Set(repair.descriptorChanges.map(change => change.key)).size, 32); + assert.equal(repair.descriptorChanges.length, 49); + assert.equal(new Set(repair.descriptorChanges.map(change => change.key)).size, 33); assert.deepEqual([...new Set(repair.descriptorChanges.map(change => change.field))].sort(), ['expectedRunCount', 'minimumRunCount', 'oracleVersion', 'setupPolicy', 'sourceFiles']); for (const change of repair.descriptorChanges) { assert.notDeepEqual(change.current, change.previous); diff --git a/tests/runner-e2e/evidence.ts b/tests/runner-e2e/evidence.ts index 96157985355..08b369414ef 100644 --- a/tests/runner-e2e/evidence.ts +++ b/tests/runner-e2e/evidence.ts @@ -1,3 +1,4 @@ +import { PRIVATE_FRAMEWORK_REPORT, validatePrivateFrameworkReport, inspectPrivateFrameworkMetadata } from "./private-framework-report.js"; import { spawn } from "node:child_process"; import { mkdir, @@ -164,6 +165,9 @@ export async function packageEvidence(input: { if (input.privateCell && standard.length > 4096) throw new Error("Private cell evidence exceeds entry allowance"); const privateSecrets = input.privateCell ? normalizedSecrets(input.secrets).flatMap(value => [value, encodeURIComponent(value), Buffer.from(value).toString("base64"), Buffer.from(value).toString("base64url")]) : []; + const compact = input.privateCell?.layout === "original" && standard.includes(PRIVATE_FRAMEWORK_REPORT) + ? await validatePrivateFrameworkReport(input.privateDir) : undefined; + const registeredAttachments = new Set(compact?.attachments.map(row => row.path) ?? []); let privateBytes = 0; for (const relative of standard) { const source = path.join(input.privateDir, relative); @@ -192,12 +196,21 @@ export async function packageEvidence(input: { continue; } else if (TEXT_EXTENSIONS.has(extension)) { const raw = await readFile(source, "utf8"); + if (input.privateCell && inspectPrivateFrameworkMetadata(relative.split(path.sep).join("/"), raw, input.secrets)) { + assertSecretFree(raw, input.secrets, "private framework metadata"); + await copyFile(source, destination); files.push(relative); continue; + } const parsed = extension === ".json" ? JSON.parse(raw) : null; const leak = extension === ".json" ? findSecretLeakInJsonValues(parsed, input.secrets) : findSecretLeak(raw, input.secrets); if (leak) leaks.push({ file: relative, reason: leak }); + if (registeredAttachments.has(relative.split(path.sep).join("/"))) { + if (extension === ".json" ? JSON.stringify(sanitizeJson(parsed, input.secrets)) !== JSON.stringify(parsed) : redactText(raw, input.secrets) !== raw) throw new Error("Private framework attachment requires redaction"); + if (!leak) { await copyFile(source, destination); files.push(relative); } + continue; + } // Redacting an already serialized JSON string can change escape // boundaries around shell commands. Sanitize parsed values instead so // uploaded snapshots stay valid JSON. @@ -243,8 +256,8 @@ export async function packageEvidence(input: { const missing = input.expectPassScreenshot ? [ - ...REQUIRED_PASS_FILES.filter((required) => !files.includes(required)), - ...(files.some( + ...REQUIRED_PASS_FILES.filter((required) => !files.includes(required) && !(compact && required === path.join("html-report", "index.html"))), + ...(compact || files.some( (file) => file.startsWith(`blob-report${path.sep}`) && file.endsWith(".zip"), ) diff --git a/tests/runner-e2e/fixtures/chat-repair-candidate-descriptors.json b/tests/runner-e2e/fixtures/chat-repair-candidate-descriptors.json index d14ae9b3a2f..466429a09cf 100644 --- a/tests/runner-e2e/fixtures/chat-repair-candidate-descriptors.json +++ b/tests/runner-e2e/fixtures/chat-repair-candidate-descriptors.json @@ -258,7 +258,7 @@ "../../server/src/services/native-runtime/status-decision-committer.ts", "../../server/src/services/work-timeline.ts" ], - "oracleVersion": "paperclip.chat-catalog.work-timeline.v6", + "oracleVersion": "paperclip.chat-catalog.work-timeline.v7", "coverageGaps": [], "expectedRunCount": 3, "minimumRunCount": 3, @@ -2599,7 +2599,7 @@ "chat-catalog-domains/identity-support.ts", "chat-scenarios/connection-setup.yaml" ], - "oracleVersion": "paperclip.chat-catalog.external.v2", + "oracleVersion": "paperclip.chat-catalog.external.v3", "expectedRunCount": 1, "turnTimeoutMs": 120000, "attemptTimeoutMs": 420000, @@ -3587,7 +3587,7 @@ "../../ui/src/pages/Projects.tsx", "../../ui/src/components/AnnouncementWell.tsx" ], - "oracleVersion": "paperclip.chat-catalog.announcement-dismissal.v2", + "oracleVersion": "paperclip.chat-catalog.announcement-dismissal.v3", "expectedRunCount": 1, "turnTimeoutMs": 120000, "attemptTimeoutMs": 300000, @@ -4209,7 +4209,7 @@ "chat-rendered-links.ts", "chat-scenarios/questions-and-approvals.yaml" ], - "oracleVersion": "paperclip.chat-catalog.hire-revision.native1017.v3", + "oracleVersion": "paperclip.chat-catalog.hire-revision.native1017.v4", "expectedRunCount": 1, "turnTimeoutMs": 120000, "attemptTimeoutMs": 300000, @@ -4905,7 +4905,7 @@ "chat-flow.ts", "chat-scenarios/secrets.yaml" ], - "oracleVersion": "paperclip.chat-catalog.company-secret-controller-baseline.v5", + "oracleVersion": "paperclip.chat-catalog.company-secret-controller-baseline.v6", "expectedRunCount": 2, "expectedHumanMessageCount": 2, "turnTimeoutMs": 120000, @@ -4944,7 +4944,7 @@ "../../packages/plugins/sdk/dist/index.js", "chat-scenarios/secrets.yaml" ], - "oracleVersion": "paperclip.chat-catalog.external.v2", + "oracleVersion": "paperclip.chat-catalog.external.v3", "expectedRunCount": 3, "expectedHumanMessageCount": 1, "minimumRunCount": 3, @@ -4981,7 +4981,7 @@ "../../ui/src/pages/secrets/SetMyUserSecretDialog.tsx", "chat-scenarios/secrets.yaml" ], - "oracleVersion": "paperclip.chat-catalog.personal-secret.v6", + "oracleVersion": "paperclip.chat-catalog.personal-secret.v7", "expectedRunCount": 3, "expectedHumanMessageCount": 2, "minimumRunCount": 3, @@ -5013,7 +5013,7 @@ "../../server/src/services/secret-proposal-notifications.ts", "chat-scenarios/secrets.yaml" ], - "oracleVersion": "paperclip.chat-catalog.external.v2", + "oracleVersion": "paperclip.chat-catalog.external.v3", "expectedRunCount": 4, "expectedHumanMessageCount": 4, "minimumRunCount": 4, @@ -5061,7 +5061,7 @@ "../../ui/src/pages/secrets/ImportFromVaultDialog.tsx", "chat-scenarios/secrets.yaml" ], - "oracleVersion": "paperclip.chat-catalog.external.v2", + "oracleVersion": "paperclip.chat-catalog.external.v3", "expectedRunCount": 3, "expectedHumanMessageCount": 2, "minimumRunCount": 2, @@ -5081,9 +5081,11 @@ "chat-catalog-domains/integrator-skill-versions.ts", "chat-catalog-domains/identity-support.ts", "../../ui/src/components/MarkdownEditor.tsx", - "../../ui/src/pages/SkillStudio.tsx" + "../../ui/src/pages/SkillStudio.tsx", + "../../ui/src/pages/CompanySkills.tsx", + "../../ui/src/lib/company-skill-routes.ts" ], - "oracleVersion": "paperclip.chat-catalog.skill-authoring.v2", + "oracleVersion": "paperclip.chat-catalog.skill-authoring.v3", "expectedRunCount": 1, "expectedHumanMessageCount": 1, "turnTimeoutMs": 120000, @@ -6013,7 +6015,7 @@ "chat-flow.ts", "chat-scenarios/teams.yaml" ], - "oracleVersion": "paperclip.chat-catalog.external.v1", + "oracleVersion": "paperclip.chat-catalog.team-install.v2", "expectedRunCount": 2, "expectedHumanMessageCount": 2, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/fixtures/chat-repair-candidate-launcher.txt b/tests/runner-e2e/fixtures/chat-repair-candidate-launcher.txt index dba1467e211..c88df15f509 100644 --- a/tests/runner-e2e/fixtures/chat-repair-candidate-launcher.txt +++ b/tests/runner-e2e/fixtures/chat-repair-candidate-launcher.txt @@ -18,6 +18,7 @@ import { readUpgradeArtifacts, verifyUpgradeArtifact, upgradeArtifactKey } from import { recoverWorkerTiming } from "./worker-timing.js"; import { openEvalOperations, type EvalOperations } from "./operations.js"; import { readRecoveredFeedback } from "./feedback-recovery.js"; +import { PRIVATE_FRAMEWORK_REPORT, referenceSealedFrameworkAttachments } from "./private-framework-report.js"; import { readInterruptedAttemptTrace } from "./trace-recovery.js"; import { EVAL_OBSERVABILITY_SECRET_NAMES, evalBraintrustConfig, finalizeEvalAttemptTrace } from "./braintrust.js"; import { finalizeEvalAttemptThroughRelay } from "./eval-finalization-relay.js"; @@ -53,7 +54,7 @@ import { packageEvidence, publishRunnerE2EAttemptEvidence } from "./evidence.js" import { prepareRunnerE2ECampaignDirectory, resolveRunnerE2EResultsRoot } from "./results-root.js"; import { privateCellEvidenceOwner } from "./manual-private-report-assets.js"; import { readManualPrivateHandoff, sealManualResultEvidence, preserveOriginalFrameworkResult, sealStandardOriginalEvidence, type ManualPrivateHandoff } from "./manual-private-handoff.js"; -import { requiresPrivateManualEvidence } from "./manual-private-retention.js"; +import { requiresPrivateManualEvidence, manualCellKind } from "./manual-private-retention.js"; import { copySharedEvidence } from "./shared-evidence.js"; import { classifyFailure } from "./failure-classifier.js"; import { mustPreserveRecoveryState, shouldKeepFailedDiagnostics } from "./cleanup-verification.js"; @@ -941,6 +942,8 @@ async function runAttempt(input: { manualDiagnostic!.stage = "seal-original"; await sealStandardOriginalEvidence({ context: manualPrivateHandoff, executionId: candidate.id, source: casePrivateDir, evidence: originalEvidence, secrets: credentials, expectPassScreenshot: originalFeaturePassed }); + if (evidence.files.includes(PRIVATE_FRAMEWORK_REPORT)) await referenceSealedFrameworkAttachments(uploadDir, + path.join(manualPrivateHandoff.root, "originals", manualCellKind(candidate.id)), candidate.id); } else { result = await publishRunnerE2EAttemptEvidence({ result, diff --git a/tests/runner-e2e/manual-private-transfer.ts b/tests/runner-e2e/manual-private-transfer.ts index 3c70f966486..2f92800d4d8 100644 --- a/tests/runner-e2e/manual-private-transfer.ts +++ b/tests/runner-e2e/manual-private-transfer.ts @@ -1,3 +1,4 @@ +import { PRIVATE_FRAMEWORK_REPORT, validatePrivateFrameworkReport, materializePrivateFrameworkAttachments } from "./private-framework-report.js"; import { createHash } from "node:crypto"; import { appendFile, cp, lstat, mkdir, mkdtemp, readFile, readdir, realpath, writeFile } from "node:fs/promises"; import path from "node:path"; @@ -95,6 +96,18 @@ export async function validateSealedInventory(directory: string, inventory: stri } await visit(""); if (actual.size !== expected.size) throw new Error("Sealed evidence is incomplete"); + if (treeKind === "standard") for (const row of receipt.files) { + if (row.path === PRIVATE_FRAMEWORK_REPORT || row.path.endsWith(`/${PRIVATE_FRAMEWORK_REPORT}`)) { + const cell = path.join(directory, row.path.slice(0, -PRIVATE_FRAMEWORK_REPORT.length)); + if (id !== undefined) await validatePrivateFrameworkReport(cell); + else { + const metadata = JSON.parse(await readFile(path.join(directory, row.path), "utf8")); + const owner = metadata.canonicalExecutionId; + if (!context || typeof owner !== "string" || !context.executionIds.includes(owner)) throw new Error("Unbound canonical framework evidence"); + await validatePrivateFrameworkReport(cell, path.join(context.root, "originals", manualCellKind(owner)), owner); + } + } + } return receipt; } export async function validateSealedOriginals(root: string, id: string, context?: ManualPrivateHandoff) { @@ -158,11 +171,22 @@ async function downloadCell(env: NodeJS.ProcessEnv) { const originals = await validateSealedOriginals(directory, id, { ...identity, root: directory }), summary = cellCompleteness(id, originals); if (hash(JSON.stringify(identity.evidenceCompleteness)) !== hash(JSON.stringify(summary)) || hash(JSON.stringify(receipt.evidenceCompleteness)) !== hash(JSON.stringify(summary))) throw new Error("Private evidence completeness receipt differs from archive"); - await validateSealedInventory(path.join(directory, "sealed-results", "html-report"), "../result-evidence-inventory.json", undefined, "standard", { ...identity, root: directory }); - await cp(path.join(directory, "sealed-results", "html-report"), path.join(selected, manualCellKind(id)), { recursive: true, errorOnExist: true, force: false }); + await stageSealedPrivateCell(directory, selected, id, { ...identity, root: directory }); await writeFile(path.join(root, `manual-cell-completeness-${index}.json`), JSON.stringify(summary), { mode: 0o600, flag: "wx" }); } } +export async function stageSealedPrivateCell(directory: string, selected: string, id: string, context: ManualPrivateHandoff) { + await validateSealedOriginals(directory, id, context); + const inventory = await validateSealedInventory(path.join(directory, "sealed-results", "html-report"), "../result-evidence-inventory.json", undefined, "standard", context); + const destination = path.join(selected, manualCellKind(id)); + await cp(path.join(directory, "sealed-results", "html-report"), destination, { recursive: true, errorOnExist: true, force: false }); + for (const row of inventory.files) { + if (row.path === PRIVATE_FRAMEWORK_REPORT || row.path.endsWith(`/${PRIVATE_FRAMEWORK_REPORT}`)) { + const cell = path.join(destination, row.path.slice(0, -PRIVATE_FRAMEWORK_REPORT.length)); + await materializePrivateFrameworkAttachments(cell, path.join(directory, "originals", manualCellKind(id)), id); + } + } +} async function verifyDownloads(env: NodeJS.ProcessEnv) { const { approval } = request(env), root = await realpath(env.PRIVATE_REPORT_ROOT ?? ""); const cells: PrivateEvidenceCompleteness["cells"] = []; diff --git a/tests/runner-e2e/private-framework-report.test.ts b/tests/runner-e2e/private-framework-report.test.ts new file mode 100644 index 00000000000..d908e6a2277 --- /dev/null +++ b/tests/runner-e2e/private-framework-report.test.ts @@ -0,0 +1,194 @@ +import { catalogTargetSha, fullCatalogCases } from "./chat-full-catalog-testops-manifest.mjs"; +import { exportCompletionPilot } from "./chat-completion-testops-export.js"; +import { exportPilotAllure } from "./chat-completion-testops-results.mjs"; +import type { ManualPrivateHandoff } from "./manual-private-handoff.js"; +import { afterAll, expect, it } from "vitest"; +import { createHash } from "node:crypto"; +import { execFile } from "node:child_process"; +import { createRequire } from "node:module"; +import { cp, mkdtemp, mkdir, readFile, realpath, rm, symlink, writeFile } from "node:fs/promises"; +import path from "node:path"; +import os from "node:os"; +import { promisify } from "node:util"; +import { copyPrivateFrameworkReport, inspectPrivateFrameworkMetadata, PRIVATE_FRAMEWORK_ORIGINAL_REPORT, PRIVATE_FRAMEWORK_REPORT, referenceSealedFrameworkAttachments, validatePrivateFrameworkReport } from "./private-framework-report.js"; +const roots: string[] = []; +afterAll(async () => { for (const root of roots) await rm(root, { recursive: true, force: true }); }); +async function fixture() { + const root = await mkdtemp(path.join(await realpath(os.tmpdir()), "framework-json-control-")); roots.push(root); + const source = path.join(root, "capture"), cell = path.join(source, "cases/activity.agent-feedback"), output = path.join(source, "playwright-output/owned"); + await mkdir(cell, { recursive: true }); await mkdir(output, { recursive: true }); + await writeFile(path.join(output, "trace.zip"), "synthetic canonical trace"); + const report = { config: { version: "1.62.1" }, errors: [], suites: [{ specs: [{ tests: [{ results: [{ status: "failed", error: { message: "actual failure" }, errors: [{ message: "actual failure" }], stdout: [], stderr: [], steps: [{ title: "owned failed step", error: { message: "actual failure" } }], attachments: [{ name: "trace", contentType: "application/zip", path: path.join(output, "trace.zip") }] }] }] }], suites: [] }] }; + await writeFile(path.join(source, "framework-report.json"), JSON.stringify(report)); + await cp(path.join(source, "playwright-output"), path.join(cell, "playwright-output"), { recursive: true }); + return { root, source, cell, report }; +} +it("keeps official failed status/errors/steps and binds one canonical attachment across original and result trees", async () => { + const f = await fixture(); await writeFile(path.join(f.cell, "playwright-output/unregistered.log"), "unexplained evidence retained"); await copyPrivateFrameworkReport(f.source, f.cell, []); + const value = await validatePrivateFrameworkReport(f.cell); + expect((value.report.suites as any)[0].specs[0].tests[0].results[0]).toMatchObject({ status: "failed", error: { message: "actual failure" }, steps: [{ title: "owned failed step" }] }); + const original = path.join(f.root, "original"); await cp(f.cell, original, { recursive: true }); + await referenceSealedFrameworkAttachments(f.cell, original, "owned"); + await expect(readFile(path.join(f.cell, "playwright-output/owned/trace.zip"))).rejects.toMatchObject({ code: "ENOENT" }); + expect(await readFile(path.join(f.cell, "playwright-output/unregistered.log"), "utf8")).toBe("unexplained evidence retained"); + expect(await readFile(path.join(original, "playwright-output/owned/trace.zip"), "utf8")).toBe("synthetic canonical trace"); + await expect(validatePrivateFrameworkReport(f.cell, original, "owned")).resolves.toMatchObject({ canonicalExecutionId: "owned" }); + await expect(validatePrivateFrameworkReport(f.cell, original, "foreign")).rejects.toThrow(); + await writeFile(path.join(original, "playwright-output/owned/trace.zip"), "tampered"); + await expect(validatePrivateFrameworkReport(f.cell, original, "owned")).rejects.toThrow(); +}); +it.each(["empty", "malformed", "foreign", "missing", "secret", "body-secret", "foreign-version", "unknown-status"])("refuses %s framework evidence", async kind => { + const f = await fixture(); + if (kind === "foreign-version") f.report.config.version = "0.0.0"; + if (kind === "unknown-status") f.report.suites[0]!.specs[0]!.tests[0]!.results[0]!.status = "invented"; + if (kind === "empty") f.report.suites = []; + if (kind === "foreign") f.report.suites[0]!.specs[0]!.tests[0]!.results[0]!.attachments[0]!.path = path.join(f.root, "outside.zip"); + if (kind === "missing") await rm(path.join(f.cell, "playwright-output/owned/trace.zip")); + if (kind === "secret") f.report.errors = [{ message: "synthetic-private-credential" }] as any; + if (kind === "body-secret") (f.report.suites[0]!.specs[0]!.tests[0]!.results[0]!.attachments[0] as any).body = Buffer.from("synthetic-private-credential").toString("base64"); + await writeFile(path.join(f.source, "framework-report.json"), kind === "malformed" ? "{" : JSON.stringify(f.report)); + await expect(copyPrivateFrameworkReport(f.source, f.cell, ["synthetic-private-credential"])).rejects.toThrow(); +}); +it("uses the real official Playwright JSON producer with failed errors, user steps, and automatic trace", async () => { + const f = await fixture(), tests = path.join(f.root, "tests"); await mkdir(tests); + const require = createRequire(import.meta.url), pkg = require.resolve("@playwright/test/package.json"), entry = require.resolve("@playwright/test"); + await writeFile(path.join(tests, "failed.spec.cjs"), `const {test,expect}=require(${JSON.stringify(entry)});test('original failed result',async()=>{await test.step('owned failed step',async()=>expect('actual').toBe('expected'));});`); + const config = path.join(tests, "config.cjs"); + await writeFile(config, `module.exports=${JSON.stringify({ testDir: tests, testMatch: "failed.spec.cjs", workers: 1, outputDir: path.join(f.source, "playwright-output"), use: { trace: "retain-on-failure" }, reporter: [["json", { outputFile: path.join(f.source, "framework-report.json") }]] })};`); + await promisify(execFile)(process.execPath, [path.join(path.dirname(pkg), "cli.js"), "test", "--config", config], { cwd: tests, env: { PATH: process.env.PATH, CI: "1", NODE_OPTIONS: "", NODE_PATH: "" }, timeout: 30000 }).then(() => { throw new Error("Original should fail"); }, error => { expect(error.code).toBe(1); }); + await rm(path.join(f.cell, "playwright-output"), { recursive: true }); + await cp(path.join(f.source, "playwright-output"), path.join(f.cell, "playwright-output"), { recursive: true }); + const produced = await readFile(path.join(f.source, "framework-report.json")); + await copyPrivateFrameworkReport(f.source, f.cell, []); + expect(await readFile(path.join(f.cell, "snapshots/playwright-framework-original.txt"))).toEqual(produced); + const retained = await validatePrivateFrameworkReport(f.cell), result = (retained.report.suites as any)[0].specs[0].tests[0].results[0]; + expect(result.status).toBe("failed"); expect(result.errors.length).toBeGreaterThan(0); expect(result.steps[0].title).toBe("owned failed step"); + expect(retained.attachments.some(a => a.path.endsWith("/trace.zip"))).toBe(true); +}); +it("keeps the unchanged 256MiB packaging/archive caps with one canonical large attachment", async () => { + const { packageEvidence } = await import("./evidence.js"); + const { inspectCellReportProducer, privateCellEvidenceOwner } = await import("./manual-private-report-assets.js"); + const { archivePrivateEvidence } = await import("./private-history.js"); + const { sealStandardOriginalEvidence, sealManualResultEvidence } = await import("./manual-private-handoff.js"); + const { validateSealedOriginals, validateSealedInventory, stageSealedPrivateCell } = await import("./manual-private-transfer.js"); + const { MANUAL_PRIVATE_PROTOCOL, manualCellKind } = await import("./manual-private-retention.js"); + const f = await fixture(), id = "chat-features.runner-opencode.local.activity.agent-feedback"; + const context: ManualPrivateHandoff = { root: await (await import("node:fs/promises")).realpath(f.root), private: true as const, protocol: MANUAL_PRIVATE_PROTOCOL, + runId: "123", attempt: "1", workflowSha: "a".repeat(40), targetSha: catalogTargetSha, executionIds: [id], cellReportProducer: await inspectCellReportProducer() }; + const zip = path.join(f.root, "input.zip"); + await promisify(execFile)("python3", ["-c", "import zipfile,sys;z=zipfile.ZipFile(sys.argv[1],'w',compression=zipfile.ZIP_STORED);f=z.open('synthetic.trace','w');[f.write(b'0'*1048576) for _ in range(140)];f.close();z.close()", zip], { timeout: 30000 }); + const tests = path.join(f.root, "budget-tests"); await mkdir(tests); + const require = createRequire(import.meta.url), pkg = require.resolve("@playwright/test/package.json"), entry = require.resolve("@playwright/test"); + await writeFile(path.join(tests, "failed.spec.cjs"), `const {test,expect}=require(${JSON.stringify(entry)});test('owned failed large attachment',async({},info)=>{await info.attach('owned trace',{path:${JSON.stringify(zip)},contentType:'application/zip'});await test.step('actual failed assertion',async()=>expect(1).toBe(2));});`); + const config = path.join(tests, "config.cjs"); + await writeFile(config, `module.exports=${JSON.stringify({ testDir: tests, testMatch: "failed.spec.cjs", workers: 1, use: { trace: "retain-on-failure" }, metadata: { diagnostic: "The --token option requires a value" }, outputDir: path.join(f.source, "playwright-output"), reporter: [["json", { outputFile: path.join(f.source, "framework-report.json") }], ["html", { outputFolder: path.join(f.source, "html-report"), open: "never" }], ["blob", { outputDir: path.join(f.source, "blob-report") }]] })};`); + await promisify(execFile)(process.execPath, [path.join(path.dirname(pkg), "cli.js"), "test", "--config", config], { cwd: tests, timeout: 30000, env: { PATH: process.env.PATH, CI: "1", NODE_OPTIONS: "", NODE_PATH: "" } }).then(() => { throw new Error("Original should fail"); }, error => { expect(error.code).toBe(1); }); + await rm(path.join(f.cell, "playwright-output"), { recursive: true }); + for (const folder of ["playwright-output", "html-report", "blob-report"]) await cp(path.join(f.source, folder), path.join(f.cell, folder), { recursive: true }); + const owner = await privateCellEvidenceOwner(context, id), prefix = `gha-123-1-${id}/chat-features/runner-opencode/local/activity.agent-feedback/attempt-1`; + const upload = path.join(context.root, "results", prefix); + const input = { privateDir: f.cell, uploadDir: upload, secrets: [], expectPassScreenshot: false, privateCell: { owner, layout: "original" as const } }; + await expect(packageEvidence(input)).rejects.toThrow("Private cell evidence exceeds byte allowance"); + for (const folder of ["html-report", "blob-report"]) await rm(path.join(f.cell, folder), { recursive: true }); + await copyPrivateFrameworkReport(f.source, f.cell, []); + const png = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/l9sAAAAASUVORK5CYII=", "base64"); + await writeFile(path.join(f.cell, "final-state.png"), png); + const descriptor = fullCatalogCases.find(row => row.executionId === id)!; + await writeFile(path.join(f.cell, "result.json"), JSON.stringify({ schema: "paperclip.runner-e2e.result/v2", executionId: id, suiteId: "chat-features", suiteDefinitionHash: "c".repeat(64), + source: { sha: catalogTargetSha, ref: "refs/heads/master", workflowRunUrl: "https://github.com/paperclipai/paperclip/actions/runs/123" }, attempt: 1, status: "failed", error: "original failed assertion", cleanup: "passed", evidenceValid: false, + runIds: [], profileId: "runner-opencode", environmentId: "local", caseId: descriptor.scenario, provider: "opencode", model: "openrouter/deepseek/deepseek-v4.1-flash-1017", runtimeMode: "native", + startedAt: "2026-11-01T00:00:00Z", finishedAt: "2026-11-01T00:00:01Z", durationMs: 1000, matcherResults: [], screenshots: [] })); + const rawFramework = await readFile(path.join(f.cell, "snapshots/playwright-framework-original.txt")); + const { redactText } = await import("./redaction.js"); expect(redactText(rawFramework.toString("utf8"), [])).not.toBe(rawFramework.toString("utf8")); + const evidence = await packageEvidence(input); expect(evidence.leaks).toEqual([]); + expect(await readFile(path.join(upload, "snapshots/playwright-framework-original.txt"))).toEqual(rawFramework); + await sealStandardOriginalEvidence({ context, executionId: id, source: f.cell, evidence, secrets: [], expectPassScreenshot: false }); + const original = path.join(context.root, "originals", manualCellKind(id)); + await referenceSealedFrameworkAttachments(upload, original, id); + await sealManualResultEvidence(context, []); + expect((await validateSealedOriginals(context.root, id, context)).complete).toBe(true); + expect((await validateSealedInventory(path.join(context.root, "sealed-results/html-report"), "../result-evidence-inventory.json", undefined, "standard", context)).complete).toBe(true); + const bundle = path.join(context.root, "bundle"); await mkdir(bundle); + await cp(path.join(context.root, "originals"), path.join(bundle, "originals"), { recursive: true }); + await cp(path.join(context.root, "sealed-results"), path.join(bundle, "sealed-results"), { recursive: true }); + const archive = path.join(context.root, "retained.zip"), restored = path.join(context.root, "restored"); + await archivePrivateEvidence("pack", bundle, archive); await archivePrivateEvidence("unpack", archive, restored); + const downloaded = { ...context, root: restored }; + await expect(validateSealedOriginals(restored, id, downloaded)).resolves.toMatchObject({ complete: true }); + await expect(validateSealedInventory(path.join(restored, "sealed-results/html-report"), "../result-evidence-inventory.json", undefined, "standard", downloaded)).resolves.toMatchObject({ complete: true }); + expect(JSON.parse(await readFile(path.join(restored, "originals", manualCellKind(id), "result.json"), "utf8")).status).toBe("failed"); + const selected = path.join(f.root, "selected-runner-e2e"); await mkdir(selected); + const sealedInventory = await readFile(path.join(restored, "sealed-results/result-evidence-inventory.json")); + await stageSealedPrivateCell(restored, selected, id, downloaded); + const derived = path.join(selected, manualCellKind(id), prefix); + for (const attachment of (await validatePrivateFrameworkReport(derived, path.join(restored, "originals", manualCellKind(id)), id)).attachments) { + expect(createHash("sha256").update(await readFile(path.join(derived, attachment.path))).digest("hex")).toBe(attachment.sha256); + } + expect(await readFile(path.join(restored, "sealed-results/result-evidence-inventory.json"))).toEqual(sealedInventory); + const report = path.join(f.root, "normalized"), repository = path.resolve(import.meta.dirname, "../.."); + const cli = createRequire(path.join(repository, "cli/package.json")).resolve("tsx/cli"); + const reportProcess = await promisify(execFile)(process.execPath, ["--import", path.join(path.dirname(cli), "loader.mjs"), path.join(repository, "tests/runner-e2e/report.ts")], { cwd: repository, timeout: 30000, env: { PATH: process.env.PATH, CI: "1", NODE_OPTIONS: "", NODE_PATH: "", + PAPERCLIP_RUNNER_E2E_REPORT_ROOT: selected, PAPERCLIP_RUNNER_E2E_REPORT_OUT: report, PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: JSON.stringify([id]), + PAPERCLIP_E2E_CAMPAIGN_ID: "gha-123-1", PAPERCLIP_RUNNER_E2E_SOURCE_SHA: catalogTargetSha, PAPERCLIP_RUNNER_E2E_SOURCE_REF: "refs/heads/master" } }).then(() => 0, error => error.code); + expect(reportProcess).toBe(1); + const binding = { runId: "123", attempt: "1", workflowSha: "a".repeat(40), targetSha: catalogTargetSha, catalog: "full" as const }; + const projection = path.join(f.root, "projection"); + const exported = await exportCompletionPilot(report, projection, binding, [id]); + expect(exported.privateEvidence[0]!.finalScreenshot!.sha256).toBe(createHash("sha256").update(png).digest("hex")); + const allure = await exportPilotAllure(projection, path.join(f.root, "allure"), binding, { schema: "paperclip.chat-completion-testops-selection/v1", ...binding, projectId: 1, planId: 10, jobId: 11, launchId: 12, allureJobRunId: "77", cases: [{ ...descriptor, caseId: 100 }] }); + expect(allure[0]!.status).toBe("failed"); + expect(JSON.parse(await readFile(path.join(projection, "normalized-results.json"), "utf8")).results[0].status).toBe("failed"); +}, 30000); + +it.each(["literal", "escaped", "base64-body", "escaped-json-body"])("deep-scans registered immutable raw JSON text: %s", async kind => { + const f = await fixture(), canary = "synthetic-private-credential"; + const attachment = f.report.suites[0]!.specs[0]!.tests[0]!.results[0]!.attachments[0]! as any; + if (kind === "literal" || kind === "escaped") f.report.errors = [{ message: canary }] as any; + else { + attachment.contentType = kind === "base64-body" ? "text/plain" : "application/json"; + attachment.body = Buffer.from(kind === "base64-body" ? canary : JSON.stringify({ value: canary }).replace("s", "\\u0073")).toString("base64"); + } + let raw = JSON.stringify(f.report); + if (kind === "escaped") raw = raw.replace(canary, "\\u0073" + canary.slice(1)); + expect(() => inspectPrivateFrameworkMetadata(PRIVATE_FRAMEWORK_ORIGINAL_REPORT, raw, [canary])).toThrow(); +}); +it("retains and checks a clean official inline text attachment without losing its body", async () => { + const f = await fixture(); const attachment = f.report.suites[0]!.specs[0]!.tests[0]!.results[0]!.attachments[0]! as any; + attachment.contentType = "text/plain"; attachment.body = Buffer.from("safe diagnostic").toString("base64"); + await writeFile(path.join(f.source, "framework-report.json"), JSON.stringify(f.report)); + await copyPrivateFrameworkReport(f.source, f.cell, []); + const retained = await validatePrivateFrameworkReport(f.cell); + expect((retained.report.suites as any)[0].specs[0].tests[0].results[0].attachments[0].body).toBe(attachment.body); +}); + +it("rejects a symbolic attachment and changed final framework outcome", async () => { + const f = await fixture(), file = path.join(f.cell, "playwright-output/owned/trace.zip"); + await rm(file); await symlink(path.join(f.source, "playwright-output/owned/trace.zip"), file); + await expect(copyPrivateFrameworkReport(f.source, f.cell, [])).rejects.toThrow(); + await rm(file); await cp(path.join(f.source, "playwright-output/owned/trace.zip"), file); + await copyPrivateFrameworkReport(f.source, f.cell, []); + const original = path.join(f.root, "original"); await cp(f.cell, original, { recursive: true }); + await referenceSealedFrameworkAttachments(f.cell, original, "owned"); + const sidecar = JSON.parse(await readFile(path.join(f.cell, PRIVATE_FRAMEWORK_REPORT), "utf8")); + sidecar.report.suites[0].specs[0].tests[0].results[0].status = "passed"; + await writeFile(path.join(f.cell, PRIVATE_FRAMEWORK_REPORT), JSON.stringify(sidecar)); + await expect(validatePrivateFrameworkReport(f.cell, original, "owned")).rejects.toThrow(); +}); + +it("rejects unrecognized credentials in a decoded plain-text body", async () => { + const f = await fixture(), attachment = f.report.suites[0]!.specs[0]!.tests[0]!.results[0]!.attachments[0]! as any; + attachment.contentType = "text/plain"; attachment.body = Buffer.from('{"password":"opaque-value"}').toString("base64"); + await writeFile(path.join(f.source, "framework-report.json"), JSON.stringify(f.report)); + await expect(copyPrivateFrameworkReport(f.source, f.cell, [])).rejects.toThrow(); +}); +it("rejects unrecognized command credentials in a registered plain-text file", async () => { + const f = await fixture(), file = path.join(f.cell, "snapshots/owned.log"); await mkdir(path.dirname(file), { recursive: true }); + await writeFile(file, "--token opaque-value"); + f.report.suites[0]!.specs[0]!.tests[0]!.results[0]!.attachments[0]!.path = file; + await writeFile(path.join(f.source, "framework-report.json"), JSON.stringify(f.report)); await copyPrivateFrameworkReport(f.source, f.cell, []); + const { packageEvidence } = await import("./evidence.js"); + const { inspectCellReportProducer, privateCellEvidenceOwner } = await import("./manual-private-report-assets.js"); + const id = "chat-features.runner-opencode.local.activity.agent-feedback"; + const owner = await privateCellEvidenceOwner({ root: f.root, private: true, runId: "123", attempt: "1", executionIds: [id], cellReportProducer: await inspectCellReportProducer() }, id); + await expect(packageEvidence({ privateDir: f.cell, uploadDir: path.join(f.root, "upload"), secrets: [], expectPassScreenshot: false, privateCell: { owner, layout: "original" } })).rejects.toThrow("requires redaction"); +}); diff --git a/tests/runner-e2e/private-framework-report.ts b/tests/runner-e2e/private-framework-report.ts new file mode 100644 index 00000000000..16c5be60f88 --- /dev/null +++ b/tests/runner-e2e/private-framework-report.ts @@ -0,0 +1,173 @@ +import { constants } from "node:fs"; +import { createRequire } from "node:module"; +import { createHash } from "node:crypto"; +import { copyFile, lstat, mkdir, readFile, realpath, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { assertSecretFree, findSecretLeak, findSecretLeakInJsonValues, normalizedSecrets, sanitizeJson, redactText } from "./redaction.js"; + +export const PRIVATE_FRAMEWORK_ORIGINAL_REPORT = "snapshots/playwright-framework-original.txt"; +export const PRIVATE_FRAMEWORK_REPORT = "snapshots/playwright-framework.json"; +const producerVersion: string = createRequire(import.meta.url)("@playwright/test/package.json").version; +const LIMIT = 256 * 1024 * 1024; +const hash = (data: Buffer) => createHash("sha256").update(data).digest("hex"); +const fail = (): never => { throw new Error("Invalid private framework evidence"); }; +type Row = { path: string; bytes: number; sha256: string }; +type FrameworkReport = { schema: string; sourceReportSha256: string; report: Record; attachments: Row[]; canonicalExecutionId?: string }; +async function regular(root: string, relative: string, limit = LIMIT) { + if (!relative || relative.includes("\\") || relative.split("/").some(p => !p || p === "." || p === "..") || /[\x00-\x1f:]/.test(relative)) fail(); + let current = root; + for (const part of relative.split("/")) { current = path.join(current, part); const entry = await lstat(current); if (entry.isSymbolicLink()) fail(); } + const info = await lstat(current); + if (!info.isFile() || info.size > limit || !(await realpath(current)).startsWith(`${await realpath(root)}${path.sep}`)) fail(); + const data = await readFile(current), after = await lstat(current); + if (data.length !== info.size || after.ino !== info.ino || after.mtimeMs !== info.mtimeMs || after.ctimeMs !== info.ctimeMs) fail(); + return data; +} +function results(report: Record) { + if (!report.config || typeof report.config !== "object" || !Array.isArray(report.suites) || !report.suites.length || !Array.isArray(report.errors)) fail(); + if ((report.config as Record).version !== producerVersion) fail(); + const found: Array> = []; let visited = 0; + const suite = (value: any) => { + if (++visited > 4096 || !value || (value.specs !== undefined && !Array.isArray(value.specs)) || (value.suites !== undefined && !Array.isArray(value.suites))) fail(); + for (const spec of value.specs ?? []) { + if (!spec || !Array.isArray(spec.tests) || !spec.tests.length) fail(); + for (const test of spec.tests) { + if (!test || !Array.isArray(test.results) || !test.results.length) fail(); + for (const result of test.results) { + if (++visited > 4096 || !result || !["passed", "failed", "timedOut", "skipped", "interrupted"].includes(result.status) + || !Array.isArray(result.attachments) || !Array.isArray(result.errors) || !Array.isArray(result.stdout) || !Array.isArray(result.stderr)) fail(); + found.push(result); + } + } + } + for (const child of value.suites ?? []) suite(child); + }; + for (const value of report.suites as unknown[]) suite(value); + if (!found.length) fail(); + return found; +} +export function inspectPrivateFrameworkMetadata(relative: string, raw: string, secrets: readonly string[]) { + const original = relative === PRIVATE_FRAMEWORK_ORIGINAL_REPORT || relative.endsWith(`/${PRIVATE_FRAMEWORK_ORIGINAL_REPORT}`) || relative === "framework-report.json"; + const sidecar = relative === PRIVATE_FRAMEWORK_REPORT || relative.endsWith(`/${PRIVATE_FRAMEWORK_REPORT}`); + if (!original && !sidecar) return false; + if (Buffer.byteLength(raw) > 128 * 1024 * 1024) fail(); + const parsed = JSON.parse(raw), report = original ? parsed : parsed.report; + const expanded = normalizedSecrets(secrets).flatMap(v => [v, encodeURIComponent(v), Buffer.from(v).toString("base64"), Buffer.from(v).toString("base64url")]); + if (findSecretLeakInJsonValues(parsed, expanded) || JSON.stringify(sanitizeJson(parsed, secrets)) !== JSON.stringify(parsed)) fail(); + for (const result of results(report)) for (const attachment of result.attachments as any[]) { + if (!attachment || typeof attachment.name !== "string" || typeof attachment.contentType !== "string") fail(); + if (attachment.body !== undefined) { + if (typeof attachment.body !== "string" || !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(attachment.body)) fail(); + const body = Buffer.from(attachment.body, "base64"), text = body.toString("utf8"); + if (!["text/plain", "application/json"].includes(attachment.contentType) || !Buffer.from(text).equals(body) || findSecretLeak(body, expanded)) fail(); + if (attachment.contentType === "text/plain" && redactText(text, secrets) !== text) fail(); + if (attachment.contentType === "application/json" && (findSecretLeak(JSON.stringify(JSON.parse(text)), expanded) || JSON.stringify(sanitizeJson(JSON.parse(text), secrets)) !== JSON.stringify(JSON.parse(text)))) fail(); + } + } + return true; +} +export async function copyPrivateFrameworkReport(source: string, destination: string, secrets: readonly string[]) { + const raw = await regular(source, "framework-report.json", 128 * 1024 * 1024); + if (!Buffer.from(raw.toString("utf8")).equals(raw)) fail(); + assertSecretFree(raw.toString("utf8"), secrets, "private framework metadata"); + const report = JSON.parse(raw.toString("utf8")); + const attachments = new Map(); + inspectPrivateFrameworkMetadata("framework-report.json", raw.toString("utf8"), secrets); + for (const result of results(report)) for (const attachment of result.attachments as any[]) { + if (!attachment || typeof attachment.name !== "string" || typeof attachment.contentType !== "string") fail(); + if (attachment.path === undefined) { if (attachment.body === undefined) fail(); continue; } + if (typeof attachment.path !== "string" || !path.isAbsolute(attachment.path)) fail(); + const absolute = path.resolve(attachment.path), output = path.join(source, "playwright-output"); + let relative: string; + if (absolute.startsWith(`${path.resolve(destination)}${path.sep}`)) relative = path.relative(destination, absolute); + else if (absolute.startsWith(`${path.resolve(output)}${path.sep}`)) relative = path.relative(source, absolute); + else fail(); + relative = relative!.split(path.sep).join("/"); + const data = await regular(absolute.startsWith(`${path.resolve(destination)}${path.sep}`) ? destination : source, relative); + if (relative === "result.json") { + relative = "snapshots/playwright-attached-result.json"; + await mkdir(path.dirname(path.join(destination, relative)), { recursive: true, mode: 0o700 }); + await writeFile(path.join(destination, relative), data, { flag: "wx", mode: 0o600 }); + } else { + const copied = await regular(destination, relative); + if (!copied.equals(data)) fail(); + } + attachment.path = relative; + const row = { path: relative, bytes: data.length, sha256: hash(data) }; + const previous = attachments.get(relative); if (previous && JSON.stringify(previous) !== JSON.stringify(row)) fail(); + attachments.set(relative, row); + } + const sidecar: FrameworkReport = { schema: "paperclip.private-framework-json/v1", sourceReportSha256: hash(raw), report, + attachments: [...attachments.values()].sort((a, b) => a.path.localeCompare(b.path)) }; + await mkdir(path.join(destination, "snapshots"), { recursive: true, mode: 0o700 }); + await writeFile(path.join(destination, PRIVATE_FRAMEWORK_ORIGINAL_REPORT), raw, { flag: "wx", mode: 0o600 }); + await writeFile(path.join(destination, PRIVATE_FRAMEWORK_REPORT), `${JSON.stringify(sidecar)}\n`, { flag: "wx", mode: 0o600 }); +} +export async function validatePrivateFrameworkReport(directory: string, canonical = directory, executionId?: string) { + const raw = await regular(directory, PRIVATE_FRAMEWORK_REPORT, 128 * 1024 * 1024); + const value = JSON.parse(raw.toString("utf8")) as FrameworkReport; + const keys = Object.keys(value).sort().join(); + if (keys !== (executionId ? "attachments,canonicalExecutionId,report,schema,sourceReportSha256" : "attachments,report,schema,sourceReportSha256") + || value.schema !== "paperclip.private-framework-json/v1" || !/^[0-9a-f]{64}$/.test(value.sourceReportSha256) + || executionId && value.canonicalExecutionId !== executionId || !Array.isArray(value.attachments) || value.attachments.length > 4096) fail(); + if (executionId) { + const original = await validatePrivateFrameworkReport(canonical); + const { canonicalExecutionId: _, ...retained } = value; + if (JSON.stringify(retained) !== JSON.stringify(original)) fail(); + } + const originalRaw = await regular(directory, PRIVATE_FRAMEWORK_ORIGINAL_REPORT, 128 * 1024 * 1024); + if (hash(originalRaw) !== value.sourceReportSha256) fail(); + inspectPrivateFrameworkMetadata(PRIVATE_FRAMEWORK_ORIGINAL_REPORT, originalRaw.toString("utf8"), []); + const authoritative = JSON.parse(originalRaw.toString("utf8")); + const originalResults = results(authoritative), retainedResults = results(value.report); + if (originalResults.length !== retainedResults.length) fail(); + for (let i = 0; i < originalResults.length; i++) { + const originalAttachments = originalResults[i]!.attachments as any[], retainedAttachments = retainedResults[i]!.attachments as any[]; + if (originalAttachments.length !== retainedAttachments.length) fail(); + for (let j = 0; j < originalAttachments.length; j++) { + if (originalAttachments[j].path !== undefined) originalAttachments[j].path = retainedAttachments[j].path; + } + } + if (JSON.stringify(authoritative) !== JSON.stringify(value.report)) fail(); + const referenced = new Set(); + for (const result of results(value.report)) for (const attachment of result.attachments as any[]) { + if (attachment.path !== undefined) { if (typeof attachment.path !== "string") fail(); referenced.add(attachment.path); } + } + const seen = new Set(); let attachmentBytes = 0; + for (const row of value.attachments) { + if (!row || Object.keys(row).sort().join() !== "bytes,path,sha256" || !referenced.has(row.path) || seen.has(row.path) + || !Number.isSafeInteger(row.bytes) || row.bytes < 0 || !/^[0-9a-f]{64}$/.test(row.sha256)) fail(); + attachmentBytes += row.bytes; if (attachmentBytes > LIMIT) fail(); + const data = await regular(canonical, row.path); if (data.length !== row.bytes || hash(data) !== row.sha256) fail(); + seen.add(row.path); + } + if (seen.size !== referenced.size) fail(); + return value; +} +export async function referenceSealedFrameworkAttachments(directory: string, original: string, executionId: string) { + const value = await validatePrivateFrameworkReport(directory); + const sealed = await validatePrivateFrameworkReport(original); + if (JSON.stringify(value) !== JSON.stringify(sealed)) fail(); + for (const row of value.attachments.filter(r => r.path.startsWith("playwright-output/"))) { + const current = await regular(directory, row.path), canonical = await regular(original, row.path); + if (!current.equals(canonical)) fail(); + await rm(path.join(directory, row.path)); + } + value.canonicalExecutionId = executionId; + await writeFile(path.join(directory, PRIVATE_FRAMEWORK_REPORT), `${JSON.stringify(value)}\n`, { mode: 0o600 }); +} + +export async function materializePrivateFrameworkAttachments(directory: string, original: string, executionId: string) { + const value = await validatePrivateFrameworkReport(directory, original, executionId); + let bytes = 0; + for (const row of value.attachments) { + bytes += row.bytes; if (bytes > LIMIT) fail(); + if (!row.path.startsWith("playwright-output/")) continue; + const source = await regular(original, row.path); + if (source.length !== row.bytes || hash(source) !== row.sha256) fail(); + await mkdir(path.dirname(path.join(directory, row.path)), { recursive: true, mode: 0o700 }); + await copyFile(path.join(original, row.path), path.join(directory, row.path), constants.COPYFILE_EXCL); + const restored = await regular(directory, row.path); + if (hash(restored) !== row.sha256) fail(); + } +}