diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 36f8fe4d143..5b9741fc38c 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -2033,7 +2033,7 @@ "chat-catalog-domains/internal-support.ts", "chat-scenarios/companies.yaml" ], - "oracleVersion": "paperclip.chat-catalog.identity.v1", + "oracleVersion": "paperclip.chat-catalog.identity.export-company.v2", "expectedRunCount": 1, "turnTimeoutMs": 120000, "attemptTimeoutMs": 300000, diff --git a/tests/runner-e2e/chat-catalog-domains/identity-portability.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-portability.test.ts index e44a78b553c..07abcf2d45e 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-portability.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-portability.test.ts @@ -2,13 +2,14 @@ import { describe, expect, it } from "vitest"; import { gradePortability } from "./identity-portability.js"; import type { IdentityObservation } from "./identity-support.js"; function valid(scenarioId: string): IdentityObservation { - const o: IdentityObservation = { scenarioId, origin: "http://fixture.invalid", companyId: "acorn", leadId: "ceo", company: { id: "acorn" }, companies: [], agents: [], tasks: [], projects: [], goals: [], comments: [], interactions: [], documents: [], revisions: [], activity: [], links: [], runs: [], humanQueries: [], reply: "The package omits credentials; reconnect the runtime environment before use.", state: {} }; + const o: IdentityObservation = { scenarioId, origin: "http://fixture.invalid", companyId: "acorn", leadId: "ceo", company: { id: "acorn" }, companies: [], agents: [], tasks: [], projects: [], goals: [], comments: [], interactions: [], documents: [], revisions: [], activity: [], links: [], runs: [], humanQueries: [], reply: "Secrets are excluded; supply EXPORT_FIXTURE_KEY when importing.", state: {} }; if (scenarioId.endsWith("export-company")) { o.runs = [{ id: "export", companyId: "acorn", agentId: "ceo", status: "succeeded" }]; - o.state = { sourceId: "chat", originalAgentName: "CEO", exportAttachment: { id: "package", companyId: "acorn", issueId: "chat", createdByAgentId: "ceo", originatingRunId: "export", sha256: "bytes" }, + o.company = { id: "acorn", name: "Acorn" }; + o.state = { sourceId: "chat", originalAgentName: "CEO", secretKey: "EXPORT_FIXTURE_KEY", secretName: "Export prerequisite API key", exportAttachment: { id: "package", companyId: "acorn", issueId: "chat", createdByAgentId: "ceo", originatingRunId: "export", sha256: "bytes" }, downloadSha256: "bytes", actualDownloadStatus: 200, fileVisible: true, agentExportPermissionStatus: 200, packageContainsPrivateValue: false, noForeignScope: true, - actualPackagePreview: { errors: [], manifest: { source: { companyId: "acorn" }, agents: [{ name: "CEO" }], projects: [{ name: "Launch" }], skills: [{ slug: "launch-checklist" }] }, - files: { "COMPANY.md": "Acorn", "agents/ceo/AGENTS.md": "CEO", "skills/launch-checklist/SKILL.md": "Checklist" }, envInputs: [{ kind: "secret", requirement: "required" }] } }; + actualPackagePreview: { errors: [], manifest: { source: null, company: { name: "Acorn" }, agents: [{ name: "CEO" }], projects: [{ name: "Launch" }], skills: [{ slug: "launch-checklist" }] }, + files: { "COMPANY.md": "Acorn", "agents/ceo/AGENTS.md": "CEO", "skills/launch-checklist/SKILL.md": "Checklist" }, envInputs: [{ key: "EXPORT_FIXTURE_KEY", kind: "secret", requirement: "optional", defaultValue: "" }] } }; } else { o.state = { existing: { id: "morgan" }, preview: { targetCompanyId: "acorn", errors: [], plan: { agentPlans: [{ slug: "morgan", existingAgentId: "morgan", action: "create" }] } }, beforeChoiceUnchanged: true, initialReplyAskedChoice: true, actualPackageAttached: true, importedAgentId: "new", existingConfigurationUnchanged: true, existingStillPresent: true, importedAgentVisible: true }; @@ -24,6 +25,12 @@ describe("actual company package download and conflict-safe import", () => { it(`${id}: wrong scope or mutated original rejected`, () => { const o = valid(id); if (id.endsWith("export-company")) o.state.noForeignScope = false; else o.state.existingConfigurationUnchanged = false; expect(gradePortability(o).passed).toBe(false); }); } it("does not accept prose or a link to an export page as a delivered CEO package", () => { const o = valid("companies.export-company"); delete o.state.exportAttachment; o.links = ["/ACO/company/export"]; expect(gradePortability(o).passed).toBe(false); }); + it("accepts the omitted credential named by its secret name instead of its key", () => { const o = valid("companies.export-company"); o.reply = "Your export prerequisite API key was left out; add it after import."; expect(gradePortability(o).passed).toBe(true); }); + it("rejects a reply that never names the omitted credential", () => { const o = valid("companies.export-company"); o.reply = "Exported the company setup as a reusable bundle."; expect(gradePortability(o).passed).toBe(false); }); + it("rejects a package from another company or one carrying the credential value as a default", () => { + const foreign = valid("companies.export-company"); (foreign.state.actualPackagePreview as { manifest: { company: { name: string } } }).manifest.company.name = "Other Co"; expect(gradePortability(foreign).passed).toBe(false); + const valued = valid("companies.export-company"); (valued.state.actualPackagePreview as { envInputs: Array<{ defaultValue: string }> }).envInputs[0]!.defaultValue = "value"; expect(gradePortability(valued).passed).toBe(false); + }); it("rejects secret values in an otherwise supported package", () => { const o = valid("companies.export-company"); o.state.packageContainsPrivateValue = true; expect(gradePortability(o).passed).toBe(false); }); it("rejects silently performing the import before human conflict choice", () => { const o = valid("companies.import-preview"); o.state.beforeChoiceUnchanged = false; expect(gradePortability(o).passed).toBe(false); }); it("rejects a replaced employee despite identical visible name", () => { const o = valid("companies.import-preview"); o.state.importedAgentId = "morgan"; expect(gradePortability(o).passed).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-portability.ts b/tests/runner-e2e/chat-catalog-domains/identity-portability.ts index be7030f12d3..c8f48734c00 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-portability.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-portability.ts @@ -17,7 +17,9 @@ export function gradePortability(value: unknown) { const c = checks(); const o = identityObservation(value); c.add("complete-portability-evidence", !!o, "Actual package files, public provenance, permitted actor and independently saved resources are required."); if (!o) return c.result(); if (o.scenarioId === "companies.export-company") { const preview = o.state.actualPackagePreview as CompanyPortabilityPreviewResult | undefined; const attachment = o.state.exportAttachment as IssueAttachment | undefined; - c.add("actual-supported-company-package", !!preview && preview.errors.length === 0 && preview.manifest.source?.companyId === o.companyId + const company = o.company as Entity & { name?: string }; + // An inline import preview never fills manifest.source, so provenance is the package's own COMPANY.md entry naming the source company. + c.add("actual-supported-company-package", !!preview && preview.errors.length === 0 && !!company.name && preview.manifest.company?.name === company.name && preview.manifest.agents.some(agent => agent.name === o.state.originalAgentName) && preview.manifest.projects.some(row => row.name === "Launch") && preview.manifest.skills.some(skill => skill.slug === "launch-checklist") && typeof preview.files["COMPANY.md"] === "string" && Object.keys(preview.files).some(file => file.endsWith("AGENTS.md")) && Object.keys(preview.files).some(file => file.endsWith("SKILL.md")), @@ -27,8 +29,10 @@ export function gradePortability(value: unknown) { && attachment.sha256 === o.state.downloadSha256 && o.state.actualDownloadStatus === 200 && o.state.fileVisible === true && o.state.agentExportPermissionStatus === 200, "A real CEO-generated package attachment is visible and its authorized exact download matches immutable upload hash/run identity; a prose promise or export page link alone is insufficient."); c.add("nonportable-credentials-explicit", o.state.packageContainsPrivateValue === false && o.state.noForeignScope === true - && !!preview?.envInputs.some(input => input.kind === "secret" && input.requirement === "required") && /credential|secret|sign.in|authentication/i.test(o.reply) - && /runtime|environment|configur|reconnect|prerequisite|not.*(?:authenticat|ready)/i.test(o.reply), "Actual secret values and foreign scope are absent while the package and reply retain omitted credentials/runtime prerequisites, without claiming an authenticated clone."); + // The product exports every secret_ref as a valueless input (requirement "optional" by design); the reply must name the setup-chosen omitted credential so the person knows what to supply. + && !!preview?.envInputs.some(input => input.key === o.state.secretKey && input.kind === "secret" && !input.defaultValue) + && [o.state.secretKey, o.state.secretName].some(name => typeof name === "string" && o.reply.toLowerCase().includes(name.toLowerCase())), + "Actual secret values and out-of-company resources are absent while the package keeps the omitted credential as a valueless input and the reply names that credential for the person to supply."); } else if (o.scenarioId === "companies.import-preview") { const preview = o.state.preview as CompanyPortabilityPreviewResult | undefined; const conflict = entity(o.state, "existing"); c.add("actual-conflict-before-mutation", !!preview && preview.errors.length === 0 && preview.targetCompanyId === o.companyId @@ -54,14 +58,15 @@ export const portabilityIdentityCases: readonly ChatCatalogCase[] = [ identityDefinition("companies.export-company", async ctx => { const source = await openStandingChat(ctx); const p = await project(ctx); const root = `/api/companies/${ctx.input.fixtures.company.id}`; const secret = `synthetic-export-key-${randomBytes(24).toString("hex")}`; secretValues.set(ctx, secret); ctx.input.privateSecrets!.push(secret); - const stored = await ctx.input.api.postSensitive<{ id: string }>(`${root}/secrets`, { name: "Export prerequisite API key", key: "EXPORT_FIXTURE_KEY", provider: "local_encrypted", value: secret }); + const secretName = "Export prerequisite API key"; const secretKey = "EXPORT_FIXTURE_KEY"; + const stored = await ctx.input.api.postSensitive<{ id: string }>(`${root}/secrets`, { name: secretName, key: secretKey, provider: "local_encrypted", value: secret }); const config = await ctx.input.api.get(`/api/agents/${ctx.input.fixtures.agent.id}/configuration`); await ctx.input.api.patch(`/api/agents/${ctx.input.fixtures.agent.id}`, { adapterConfig: { ...config.adapterConfig, - env: { ...(config.adapterConfig.env as Record), EXPORT_FIXTURE_KEY: { type: "secret_ref", secretId: stored.id, version: "latest" } } } }); + env: { ...(config.adapterConfig.env as Record), [secretKey]: { type: "secret_ref", secretId: stored.id, version: "latest" } } } }); const skill = await ctx.input.api.post(`${root}/skills`, { name: "Launch checklist", slug: "launch-checklist", description: "A portable accessible launch review checklist.", markdown: "---\nname: launch-checklist\ndescription: Review a website launch.\n---\n\n# Launch checklist\nCheck headings, keyboard navigation and business contact details.\n", sharingScope: "company" }); - return { sourceId: source.id, project: p, skill, originalAgentName: ctx.input.fixtures.agent.name, syntheticPrivateValueCreated: true }; + return { sourceId: source.id, project: p, skill, originalAgentName: ctx.input.fixtures.agent.name, secretKey, secretName, syntheticPrivateValueCreated: true }; }, gradePortability, { - authentication: "human-session", sourceFiles: sources, + authentication: "human-session", sourceFiles: sources, oracleVersion: "paperclip.chat-catalog.identity.export-company.v2", async act(ctx, state) { await waitForReply(ctx, chatCatalogOriginalPhaseDeadline(ctx)); state.agentExportPermissionStatus = await probeAgentPermission(ctx, `/api/companies/${ctx.input.fixtures.company.id}/exports/preview`, "POST", { include: { company: true, agents: true, projects: true, skills: true, issues: false } }); const attachments = await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/attachments`); @@ -75,7 +80,11 @@ export const portabilityIdentityCases: readonly ChatCatalogCase[] = [ state.packageContainsPrivateValue = Object.values(files).some(file => text(file).includes(secretValues.get(ctx)!)); const preview = await ctx.input.api.post(`/api/companies/${ctx.input.fixtures.company.id}/imports/preview`, { source: { type: "inline", rootPath, files, expectedFileCount: Object.keys(files).length }, target: { mode: "existing_company", companyId: ctx.input.fixtures.company.id }, collisionStrategy: "rename", include: { company: false, agents: true, projects: true, skills: true, issues: false } }); - state.actualPackagePreview = preview; state.noForeignScope = preview.manifest.source?.companyId === ctx.input.fixtures.company.id; + const [company, agents, projects] = await Promise.all([ctx.input.api.get(`/api/companies/${ctx.input.fixtures.company.id}`), + ctx.input.api.get(`/api/companies/${ctx.input.fixtures.company.id}/agents`), ctx.input.api.get(`/api/companies/${ctx.input.fixtures.company.id}/projects`)]); + const names = (rows: Entity[]) => new Set(rows.map(row => row.name)); const agentNames = names(agents); const projectNames = names(projects); + state.actualPackagePreview = preview; state.noForeignScope = preview.manifest.company?.name === company.name + && preview.manifest.agents.every(agent => agentNames.has(agent.name)) && preview.manifest.projects.every(row => projectNames.has(row.name)); await ctx.input.evidence("identity-real-company-package-download.json", { attachmentId: attachment.id, originatingRunId: attachment.originatingRunId, sha256: state.downloadSha256, fileInventory: Object.keys(files), manifest: preview.manifest, envInputs: preview.envInputs, packageContainsPrivateValue: state.packageContainsPrivateValue, actualDownloadStatus: 200 }); },