From b33f3b8b649cc597ffe9a4a66fc834e47fd93e0e Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 26 Oct 2026 19:12:54 -0500 Subject: [PATCH] test(e2e): author bounded privileged operator journeys --- .../privileged-backup.test.ts | 11 +- .../chat-catalog-domains/privileged-backup.ts | 27 ++- .../privileged-cli-support.ts | 6 +- .../privileged-cli.test.ts | 3 +- .../chat-catalog-domains/privileged-cli.ts | 5 +- .../privileged-database-fault.ts | 83 ++++++++++ .../privileged-diagnose-health.test.ts | 24 +++ .../privileged-diagnose-health.ts | 78 +++++++++ .../privileged-document-cli.ts | 7 +- .../privileged-feedback.ts | 154 ++++++++++++++++++ .../privileged-installed.ts | 64 ++++++++ .../privileged-operators.test.ts | 50 ++++++ .../privileged-owned-dev.ts | 116 +++++++++++++ .../privileged-performance.test.ts | 6 + .../privileged-performance.ts | 3 + .../privileged-provision.ts | 89 ++++++++++ .../privileged-run-cli.ts | 7 +- .../privileged-support.ts | 18 +- .../chat-catalog-domains/privileged-update.ts | 60 +++++++ 19 files changed, 789 insertions(+), 22 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/privileged-database-fault.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/privileged-diagnose-health.test.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/privileged-diagnose-health.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/privileged-feedback.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/privileged-installed.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/privileged-operators.test.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/privileged-owned-dev.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/privileged-provision.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/privileged-update.ts diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-backup.test.ts b/tests/runner-e2e/chat-catalog-domains/privileged-backup.test.ts index ae1c6debe62..2dca510be52 100644 --- a/tests/runner-e2e/chat-catalog-domains/privileged-backup.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/privileged-backup.test.ts @@ -3,11 +3,13 @@ import { gradeBackupRestore } from "./privileged-backup.js"; function valid() { const marker = { id: "canary", companyId: "owned", title: "Backup restore canary", description: "exact recovery marker", projectId: "project", status: "backlog", priority: "medium", assigneeAgentId: null, goalId: null }; return { scenarioId: "instance-operations.backup-and-restore", origin: "http://owned.invalid", companyId: "owned", leadId: "lead", - tasks: [marker], projects: [], goals: [], comments: [], activity: [], documents: [], revisions: [], links: [], humanQueries: ["canonical"], runs: [{ id: "run", agentId: "lead", status: "succeeded" }], + tasks: [marker], projects: [], goals: [], comments: [], activity: [], documents: [], revisions: [], links: [], humanQueries: ["canonical", "receipt"], runs: [{ id: "run", agentId: "lead", status: "succeeded" }, { id: "report", agentId: "lead", status: "succeeded" }], reply: "The human instance admin must create a backup, then restore only into a separate empty target to verify recovery.", state: { task: marker, sourceOwned: true, agentBackupStatus: 403, archiveSha256: "a".repeat(64), archiveBytes: 1024, archiveWithinOwnedRoot: true, archivePublished: false, separateDatabase: true, targetWasEmpty: true, - restoreCompleted: true, restoredFixtureMatches: true, sourceFixtureUnchanged: true, targetDropped: true } }; + restoreCompleted: true, restoredFixtureMatches: true, sourceFixtureUnchanged: true, targetDropped: true, + initialGuidance: "The human instance admin must create a backup, then restore only into a separate empty target to verify recovery.", + restoreReport: { attributed: true, humanCommentId: "receipt", runId: "report", reply: "The restore succeeded and verified the retained fixture in a separate disposable target." } } }; } it("requires a real private archive and separately verified restoration", () => expect(gradeBackupRestore(valid()).passed).toBe(true)); it("rejects backup-only success, source overwrite, nonempty target and secret publication", () => { @@ -18,6 +20,9 @@ it("rejects backup-only success, source overwrite, nonempty target and secret pu }); it("rejects a changed canary or premature restored claim", () => { const o = valid(); expect(gradeBackupRestore({ ...o, tasks: [{ ...o.tasks[0]!, description: "changed" }] }).passed).toBe(false); - expect(gradeBackupRestore({ ...o, reply: "I have restored the backup successfully." }).passed).toBe(false); + expect(gradeBackupRestore({ ...o, state: { ...o.state, initialGuidance: "I have restored the backup successfully." } }).passed).toBe(false); + expect(gradeBackupRestore({ ...o, state: { ...o.state, restoreReport: { ...o.state.restoreReport, reply: "An operator can make a backup." } } }).passed).toBe(false); + expect(gradeBackupRestore({ ...o, state: { ...o.state, restoreReport: { ...o.state.restoreReport, attributed: false } } }).passed).toBe(false); + expect(gradeBackupRestore({ ...o, state: { ...o.state, restoreReport: { ...o.state.restoreReport, reply: "The separate target restore was not verified." } } }).passed).toBe(false); expect(gradeBackupRestore(null).passed).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-backup.ts b/tests/runner-e2e/chat-catalog-domains/privileged-backup.ts index 21162d8c46e..89501739a1f 100644 --- a/tests/runner-e2e/chat-catalog-domains/privileged-backup.ts +++ b/tests/runner-e2e/chat-catalog-domains/privileged-backup.ts @@ -4,14 +4,16 @@ import path from "node:path"; import { expect } from "@playwright/test"; import { createDb, getPostgresDataDirectory, runDatabaseRestore } from "../../../packages/db/src/index.js"; import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import { sendChatMessage } from "../chat-flow.js"; import { checks, project, putDocument, task, sameTask, type Entity } from "./internal-support.js"; -import { privilegedDefinition, privilegedObservation, settlePrivilegedChat } from "./privileged-support.js"; +import { privilegedDefinition, privilegedObservation, settlePrivilegedChat, companionPhase } from "./privileged-support.js"; const privateSources = new WeakMap(); const pool = (url: string) => createDb(url, { maxConnections: 1, prepare: false, connectTimeoutSeconds: 5, applicationName: "owned-catalog-backup-proof" }).$client; export function gradeBackupRestore(value: unknown) { const c = checks(), o = privilegedObservation(value); c.add("backup-restoration-observation", !!o, "An actual board backup and actual separate empty database restoration are required."); if (!o) return c.result(); const marker = o.state.task as Entity | undefined; + const initial = String(o.state.initialGuidance ?? ""); c.add("owned-source-and-board-authority", o.state.sourceOwned === true && o.state.agentBackupStatus === 403, "The source belongs to the disposable launcher and the actual agent bearer cannot trigger instance backup."); c.add("real-private-archive", typeof o.state.archiveSha256 === "string" && o.state.archiveSha256.length === 64 @@ -21,9 +23,16 @@ export function gradeBackupRestore(value: unknown) { && o.state.restoredFixtureMatches === true && o.state.sourceFixtureUnchanged === true && o.state.targetDropped === true && !!marker && sameTask(marker, o.tasks.find(row => row.id === marker.id)), "The supported restore engine runs only against a newly created empty target and verifies actual task/document rows without overwriting the source."); - c.add("truthful-backup-guidance", /backup|back up/i.test(o.reply) && /restor|recover/i.test(o.reply) && /operator|admin|human|board/i.test(o.reply) - && !/(?:I(?:'ve| have)?|already|successfully)\s+(?:backed up|restored|verified recovery)/i.test(o.reply), + c.add("truthful-backup-guidance", /backup|back up/i.test(initial) && /restor|recover/i.test(initial) && /operator|admin|human|board/i.test(initial) + && !/(?:I(?:'ve| have)?|already|successfully)\s+(?:backed up|restored|verified recovery)/i.test(initial), "The initial reply distinguishes privileged backup from the later verified restoration instead of claiming either already occurred."); + const phase = o.state.restoreReport as { attributed?: boolean; reply?: string; humanCommentId?: string; runId?: string } | undefined; + const report = phase?.reply ?? ""; + c.add("native-reports-actual-restore", phase?.attributed === true && !!phase.humanCommentId && !!phase.runId + && report.split(/[.!?\n]/).some(clause => /restor|recover/i.test(clause) && /verif|succeed|pass|match/i.test(clause) + && !/\b(?:not|never|cannot|can't|failed|unverified)\b/i.test(clause)) + && /separate|empty|disposable|test target/i.test(report) && o.runs.length === 2 && o.runs.every(run => run.status === "succeeded"), + "The actual second native reply to the factual operator receipt reports the observed restore result and its disposable scope."); return c.result(); } const backup = privilegedDefinition("instance-operations.backup-and-restore", "privileged-backup", { @@ -49,7 +58,7 @@ const backup = privilegedDefinition("instance-operations.backup-and-restore", "p expectedBodySha256: createHash("sha256").update(body).digest("hex") }; }, async act(ctx, state) { - await settlePrivilegedChat(ctx); + const initial = await settlePrivilegedChat(ctx); state.initialGuidance = initial.reply; const source = privateSources.get(ctx); if (!source) throw new Error("Private owned backup source unavailable"); const sql = pool(source.url), targetName = `catalog_restore_${createHash("sha256").update(ctx.input.nonce).digest("hex").slice(0, 24)}`; const targetUrl = new URL(source.url); targetUrl.pathname = `/${targetName}`; @@ -93,14 +102,20 @@ const backup = privilegedDefinition("instance-operations.backup-and-restore", "p try { if (targetCreated) { await sql.unsafe(`drop database "${targetName}"`); state.targetDropped = true; } } finally { await sql.end({ timeout: 5 }); } } - await ctx.input.evidence("chat-catalog-private-backup-restore.json", { archiveSha256: state.archiveSha256, archiveBytes: state.archiveBytes, + const receipt = { archiveSha256: state.archiveSha256, archiveBytes: state.archiveBytes, targetWasEmpty: state.targetWasEmpty, separateDatabase: state.separateDatabase, restoredFixtureMatches: state.restoredFixtureMatches, sourceFixtureUnchanged: state.sourceFixtureUnchanged, targetDropped: state.targetDropped, archivePublished: false, - archiveRetention: "private owned attempt only; secret-bearing archive excluded from ordinary chat artifacts" }); + archiveRetention: "private owned attempt only; secret-bearing archive excluded from ordinary chat artifacts" }; + await ctx.input.evidence("chat-catalog-private-backup-restore.json", receipt); + const message = `The operator backup and restore run produced this factual receipt. Please report what it establishes and any limits:\n\n${JSON.stringify(receipt)}`; + await sendChatMessage(ctx.input.page, message); + state.restoreReport = await companionPhase(ctx, initial.runs.map(run => run.id), message); + await ctx.input.evidence("chat-catalog-native-restore-report.json", state.restoreReport); }, grade: gradeBackupRestore, async verifyResult(ctx) { await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); }, }); backup.attemptTimeoutMs = 600_000; +backup.expectedRunCount = 2; backup.expectedHumanMessageCount = 2; backup.cleanup = async ctx => { const source = privateSources.get(ctx); if (source) { await ctx.input.api.delete(`/api/agents/${ctx.input.fixtures.agent.id}/keys/${source.keyId}`); privateSources.delete(ctx); } }; backup.sourceFiles = [...backup.sourceFiles, "../../packages/db/src/backup-lib.ts", "../../server/src/routes/instance-database-backups.ts"]; export const privilegedBackupCases = [backup]; diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-cli-support.ts b/tests/runner-e2e/chat-catalog-domains/privileged-cli-support.ts index 54ef4c3d1cd..f9e58d19dd3 100644 --- a/tests/runner-e2e/chat-catalog-domains/privileged-cli-support.ts +++ b/tests/runner-e2e/chat-catalog-domains/privileged-cli-support.ts @@ -25,13 +25,12 @@ export async function runOwnedCli(ctx: ChatCatalogContext, args: string[]) { // A fresh minimal environment cannot pick up another operator's context, // stored auth, run ID, provider credentials or default company. const env: NodeJS.ProcessEnv = { PATH: process.env.PATH, HOME: path.join(actor.directory, "home"), - PAPERCLIP_HOME: path.join(actor.directory, "home"), PAPERCLIP_API_KEY: actor.token, + PAPERCLIP_HOME: path.join(actor.directory, "home"), PAPERCLIP_CONTEXT: path.join(actor.directory, "context.json"), PAPERCLIP_API_KEY: actor.token, PAPERCLIP_API_URL: ctx.input.api.baseURL, PAPERCLIP_TELEMETRY_DISABLED: "1", NO_COLOR: "1" }; const loader = fileURLToPath(new URL("../../../cli/node_modules/tsx/dist/loader.mjs", import.meta.url)); const entry = fileURLToPath(new URL("../../../cli/src/index.ts", import.meta.url)); try { - const result = await execute(process.execPath, ["--import", loader, entry, ...args, - "--context", path.join(actor.directory, "context.json")], { cwd: actor.directory, env, timeout: 45_000, maxBuffer: 1_048_576 }); + const result = await execute(process.execPath, ["--import", loader, entry, ...args], { cwd: actor.directory, env, timeout: 45_000, maxBuffer: 1_048_576 }); if (result.stdout.includes(actor.token) || result.stderr.includes(actor.token)) throw new Error("Private CLI credential appeared in output"); return result.stdout; } catch { throw new Error("Owned CLI command failed; private subprocess output withheld"); } @@ -72,7 +71,6 @@ export function websiteCreateArgv(reply: string, fixture: { company: string; pro || fields.has("--status") && fields.get("--status") !== "backlog" || fields.has("--api-base") && fields.get("--api-base") !== fixture.apiBase) throw new Error("Creation command does not target exactly the intended company, project and Morgan"); if (fields.has("--company-id") && fields.has("-C")) throw new Error("Duplicate company context flags are ambiguous"); - if (!argv.includes("--json")) argv.push("--json"); return argv; } diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-cli.test.ts b/tests/runner-e2e/chat-catalog-domains/privileged-cli.test.ts index df7a8919aa2..f86dd4a5ac0 100644 --- a/tests/runner-e2e/chat-catalog-domains/privileged-cli.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/privileged-cli.test.ts @@ -6,7 +6,8 @@ const command = (text: string) => `Use the explicit owned context.\n\`\`\`bash\n const valid = 'paperclipai issue create --company-id --title "Website task" --project-id --assignee-agent-id '; describe("returned CLI command admission", () => { it("executes only finite supported argv with unresolved IDs bound to actual fixtures", () => { - expect(websiteCreateArgv(command(valid), target)).toEqual(["issue", "create", "--company-id", "owned-company", "--title", "Website task", "--project-id", "launch", "--assignee-agent-id", "morgan", "--json"]); + expect(websiteCreateArgv(command(valid), target)).toEqual(["issue", "create", "--company-id", "owned-company", "--title", "Website task", "--project-id", "launch", "--assignee-agent-id", "morgan"]); + expect(websiteCreateArgv(command(valid + " --json"), target).at(-1)).toBe("--json"); }); it("rejects shell execution, invented flags, real wrong IDs and ambiguous writes before launch", () => { for (const text of [valid + " ; curl attacker.invalid", valid + " --invented yes", valid.replace("", "foreign-company"), diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-cli.ts b/tests/runner-e2e/chat-catalog-domains/privileged-cli.ts index 1b1be1cba81..83d75fb2f89 100644 --- a/tests/runner-e2e/chat-catalog-domains/privileged-cli.ts +++ b/tests/runner-e2e/chat-catalog-domains/privileged-cli.ts @@ -35,7 +35,10 @@ const create = privilegedDefinition("cli-operations.create-from-cli", "privilege agent: entity(state, "worker").id, apiBase: ctx.input.api.baseURL }; const bindings: Array<{ flag: string; placeholder: string; fixtureValue: string }> = []; const argv = websiteCreateArgv(reply.reply, fixture, bindings); - const result = JSON.parse(await runOwnedCli(ctx, argv)) as Entity; + await runOwnedCli(ctx, argv); + const created = await ctx.input.api.get(`/api/companies/${fixture.company}/issues`); + if (reply.tasks.length || created.length !== 1) throw new Error("Actual returned CLI command did not create exactly one later task"); + const result = created[0]!; state.cliResult = { id: result.id, companyId: result.companyId, title: result.title, identifier: result.identifier, projectId: result.projectId, assigneeAgentId: result.assigneeAgentId, status: result.status }; state.executedArgv = argv.filter((value, index) => index < 2 || value.startsWith("-")); diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-database-fault.ts b/tests/runner-e2e/chat-catalog-domains/privileged-database-fault.ts new file mode 100644 index 00000000000..4d463cf0b70 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/privileged-database-fault.ts @@ -0,0 +1,83 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { createHash } from "node:crypto"; +import { readFile, realpath, stat } from "node:fs/promises"; +import path from "node:path"; +import { getPostgresDataDirectory, closeRegisteredClients } from "../../../packages/db/src/index.js"; +import { databaseCheck } from "../../../cli/src/checks/database-check.js"; +import type { PaperclipConfig } from "../../../packages/shared/src/config-schema.js"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +const execute = promisify(execFile); +export function ownedPostmasterStillMatches(input: { pid: number; startedAt: string; directory: string; port: number; uid: number; + binary: string; marker: string[]; processRow: string }) { + const row = input.processRow.trim().match(/^(\d+)\s+([\s\S]+)$/), argv = row?.[2]?.split(/\s+/) ?? []; + return Number(input.marker[0]) === input.pid && input.marker[1] === input.directory && input.marker[2] === input.startedAt + && Number(input.marker[3]) === input.port && !!row && Number(row[1]) === input.uid && argv[0] === input.binary + && argv[argv.indexOf("-D") + 1] === input.directory && argv.includes("-D"); +} + +/** Source-owned PG control, never an arbitrary port/PID or external database. */ +export async function prepareOwnedDatabaseFault(ctx: ChatCatalogContext) { + const temp = process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT, configPath = process.env.PAPERCLIP_CONFIG; + if (!temp || !configPath) throw new Error("Database fault requires the launcher's private owned config"); + const root = await realpath(temp), configFile = await realpath(configPath); + if (!configFile.startsWith(root + path.sep)) throw new Error("Database fault config is outside its owned attempt"); + const config = JSON.parse(await readFile(configFile, "utf8")) as PaperclipConfig; + if (config.database.mode !== "embedded-postgres" || !config.database.embeddedPostgresDataDir) throw new Error("Database fault is limited to the owned embedded PostgreSQL fixture"); + const directory = await realpath(config.database.embeddedPostgresDataDir); + if (!directory.startsWith(root + path.sep) || (await stat(directory)).uid !== process.getuid?.()) throw new Error("Database directory owner differs from the current attempt owner"); + const port = config.database.embeddedPostgresPort; + if (!Number.isSafeInteger(port) || !port || port < 1) throw new Error("Owned PostgreSQL port is invalid"); + const connection = `postgres://paperclip:paperclip@127.0.0.1:${port}/paperclip`; + if (await getPostgresDataDirectory(connection) !== directory) throw new Error("Live database directory differs from the owned config"); + ctx.input.privateSecrets?.push(connection); + const initial = (await readFile(path.join(directory, "postmaster.pid"), "utf8")).split("\n"); + const pid = Number(initial[0]), startedAt = initial[2]; + if (!Number.isSafeInteger(pid) || pid <= 1 || await realpath(initial[1]!) !== directory || Number(initial[3]) !== port) throw new Error("Owned PostgreSQL process marker does not match directory/port"); + const raw = (await readFile(path.join(directory, "postmaster.opts"), "utf8")).trim(); + const args = raw.match(/"(?:[^"\\]|\\.)*"|[^\s]+/g)?.map(value => value.startsWith('"') ? JSON.parse(value) as string : value); + if (!args?.length || args.some(value => !/^[A-Za-z0-9_/@.,=: +\-]+$/.test(value))) throw new Error("Owned PostgreSQL options are not safe to replay through pg_ctl"); + const binary = await realpath(args[0]!); + if (path.basename(binary) !== "postgres") throw new Error("Owned process is not the actual PostgreSQL binary"); + const control = await realpath(path.join(path.dirname(binary), "pg_ctl")); + if (path.dirname(control) !== path.dirname(binary)) throw new Error("PostgreSQL control binary escapes the verified package"); + const binarySha256 = createHash("sha256").update(await readFile(binary)).digest("hex"); + async function verify() { + const current = (await readFile(path.join(directory, "postmaster.pid"), "utf8")).split("\n"); + const processRow = (await execute("ps", ["-p", String(pid), "-o", "uid=", "-o", "command="], { timeout: 5_000 })).stdout.trim(); + if (!ownedPostmasterStillMatches({ pid, startedAt: startedAt!, directory, port, uid: process.getuid!(), binary, marker: current, processRow })) + throw new Error("Owned PostgreSQL process changed; refusing control"); + } + await verify(); + let stopRequested = false; + const env = { PATH: process.env.PATH, LANG: "C.UTF-8" }; + return { + proof: { pid, ownerUid: process.getuid?.(), ownerMatches: true, dataDirectoryRelative: path.relative(root, directory), directoryWithinOwnedRoot: true, port, binarySha256, processStartMarker: startedAt }, + async stop() { + await verify(); + stopRequested = true; + try { await execute(control, ["-D", directory, "-m", "fast", "-w", "-t", "15", "stop"], { env, timeout: 20_000, maxBuffer: 65_536 }); } + catch { throw new Error("Owned PostgreSQL stop failed; raw diagnostics withheld"); } + }, + async doctor() { + const result = await databaseCheck({ ...config, database: { ...config.database, mode: "postgres", connectionString: connection } }); + await closeRegisteredClients(connection); + return { name: result.name, status: result.status, connectionFailure: result.status === "fail" && /connect|PostgreSQL/i.test(result.message), + repairHint: result.repairHint ?? null }; + }, + async recover() { + if (!stopRequested) return; + // A timed-out stop may have failed before shutdown; do not start a second + // postmaster when the exact original owned process still answers. + const current = await getPostgresDataDirectory(connection).catch(() => null); + if (current) { if (current !== directory) throw new Error("Owned database port now belongs to another directory"); await verify(); stopRequested = false; return; } + const options = args.slice(1).map(value => `'${value}'`).join(" "); + try { + await execute(control, ["-D", directory, "-w", "-t", "30", "-l", path.join(root, "catalog-owned-pg-recovery.log"), "-o", options, "start"], { env, timeout: 35_000, maxBuffer: 65_536 }); + const actual = await getPostgresDataDirectory(connection); + if (actual !== directory) throw new Error("Owned PostgreSQL recovered elsewhere"); + stopRequested = false; + } catch { throw new Error("Owned PostgreSQL recovery failed; retain the owned root and private supervisor evidence"); } + }, + }; +} diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-diagnose-health.test.ts b/tests/runner-e2e/chat-catalog-domains/privileged-diagnose-health.test.ts new file mode 100644 index 00000000000..11cf81b1ded --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/privileged-diagnose-health.test.ts @@ -0,0 +1,24 @@ +import { expect, it } from "vitest"; +import { ownedPostmasterStillMatches } from "./privileged-database-fault.js"; +import { gradeDiagnoseHealth } from "./privileged-diagnose-health.js"; +it("refuses a reused PID, wrong UID/port/directory or a substring lookalike process before control", () => { + const original = { pid: 123, startedAt: "1000", directory: "/owned/db", port: 54329, uid: 501, binary: "/pinned/bin/postgres", + marker: ["123", "/owned/db", "1000", "54329"], processRow: "501 /pinned/bin/postgres -D /owned/db -p 54329" }; + expect(ownedPostmasterStillMatches(original)).toBe(true); + for (const delta of [{ marker: ["124", "/owned/db", "1000", "54329"] }, { marker: ["123", "/owned/db", "2000", "54329"] }, + { marker: ["123", "/owned/db", "1000", "60000"] }, { processRow: "502 /pinned/bin/postgres -D /owned/db -p 54329" }, + { processRow: "501 /pinned/bin/postgres -D /owned/db-elsewhere -p 54329" }, { processRow: "501 /another/bin/postgres -D /owned/db -p 54329" }]) + expect(ownedPostmasterStillMatches({ ...original, ...delta })).toBe(false); +}); +it("requires real database-unready evidence and honest diagnosis instead of liveness-only success", () => { + const o = { scenarioId: "instance-operations.diagnose-health", origin: "http://owned.invalid", companyId: "owned", leadId: "lead", + tasks: [], projects: [], goals: [], comments: [], activity: [], documents: [], revisions: [], links: [], humanQueries: ["canonical"], runs: [{ id: "run", status: "succeeded" }], + reply: "The database prerequisite is failing; this instance is not ready to run work. Use the operator doctor database check.", + state: { healthStatus: 503, healthError: "database_unreachable", doctorFailed: true, ownedProcessVerified: true, exactCanonicalBody: true, + backendDispatchAfterFault: true, backendDispatchAt: 2000, faultObservedAt: 1000 } }; + expect(gradeDiagnoseHealth(o).passed).toBe(true); + for (const delta of [{ healthStatus: 200 }, { doctorFailed: false }, { exactCanonicalBody: false }, { backendDispatchAt: 500 }]) + expect(gradeDiagnoseHealth({ ...o, state: { ...o.state, ...delta } }).passed).toBe(false); + expect(gradeDiagnoseHealth({ ...o, reply: "The HTTP port works, so it is healthy enough and ready to run work." }).passed).toBe(false); + expect(gradeDiagnoseHealth({ ...o, reply: "" }).passed).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-diagnose-health.ts b/tests/runner-e2e/chat-catalog-domains/privileged-diagnose-health.ts new file mode 100644 index 00000000000..3bef00b43ae --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/privileged-diagnose-health.ts @@ -0,0 +1,78 @@ +import { expect, type Route, type Response } from "@playwright/test"; +import { createHash } from "node:crypto"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import { checks, observeInternal } from "./internal-support.js"; +import { privilegedDefinition, privilegedObservation } from "./privileged-support.js"; +import { prepareOwnedDatabaseFault } from "./privileged-database-fault.js"; +type Fault = Awaited>; +const controls = new WeakMap>; handler: (route: Route) => Promise; response: (response: Response) => Promise }>(); +export function gradeDiagnoseHealth(value: unknown) { + const c = checks(), o = privilegedObservation(value); + c.add("actual-owned-diagnostic-evidence", !!o, "The source instance's real database fault, health and doctor checks are required."); if (!o) return c.result(); + c.add("live-http-not-ready", o.state.healthStatus === 503 && o.state.healthError === "database_unreachable" + && o.state.doctorFailed === true && o.state.ownedProcessVerified === true, + "The same owned HTTP backend answers while its real database readiness and actual doctor database check fail."); + c.add("canonical-admitted-only-while-unready", o.state.exactCanonicalBody === true && o.state.backendDispatchAfterFault === true + && Number(o.state.backendDispatchAt) >= Number(o.state.faultObservedAt), + "Only the unchanged canonical send is held; it reaches the actual source backend after the genuine fault, with no response fabrication or healthy admission."); + const positive = o.reply.split(/[.!?;\n]/).filter(clause => !/\b(?:not|cannot|can't|no)\b/i.test(clause)).join("."); + c.add("honest-database-diagnosis", /database|PostgreSQL|postgres/i.test(o.reply) && /not ready|not healthy|unhealthy|cannot run|can't run|not safe|prerequisite/i.test(o.reply) + && /doctor|readiness|database/i.test(o.reply) && !/healthy (?:enough|and ready)|ready to run work/i.test(positive), + "Any actual response identifies the failed prerequisite instead of inferring readiness from HTTP liveness. Inability to admit/respond remains a failed outcome."); + return c.result(); +} +const diagnose = privilegedDefinition("instance-operations.diagnose-health", "privileged-diagnose-health", { + async setup(ctx) { + await ctx.input.api.post(`/api/companies/${ctx.input.fixtures.company.id}/chats/${ctx.input.fixtures.agent.id}`); + await ctx.refreshIssue(); const source = ctx.issue(); if (!source) throw new Error("Diagnostic source conversation was not created"); + const fault = await prepareOwnedDatabaseFault(ctx), state: Record = { ownedProcessVerified: true }; + const pathname = `/api/issues/${source.id}/comments`, pattern = `${ctx.input.api.baseURL}${pathname}`; + let claimed = false; const pending = new Set>(); + const handler = async (route: Route) => { + const request = route.request(); + if (request.method() !== "POST" || claimed) { await route.continue(); return; } + const body = request.postDataJSON() as { body?: unknown }; + if (body.body !== ctx.scenario.query) { await route.continue(); return; } + claimed = true; state.exactCanonicalBody = true; state.uiRequestObservedAt = Date.now(); + state.canonicalBodySha256 = createHash("sha256").update(String(body.body)).digest("hex"); + let finish!: () => void; const flight = new Promise(resolve => { finish = resolve; }); pending.add(flight); + try { + await ctx.input.evidence("chat-catalog-owned-db-fault-before.json", { proof: fault.proof, uiRequestObservedAt: state.uiRequestObservedAt, backendDispatched: false }); + await fault.stop(); + const response = await fetch(new URL("/api/health", ctx.input.api.baseURL), { redirect: "error", signal: AbortSignal.timeout(45_000) }); + const health = await response.json() as { status?: string; error?: string }; + state.healthStatus = response.status; state.healthError = health.error; state.faultObservedAt = Date.now(); + const doctor = await fault.doctor(); state.doctorFailed = doctor.status === "fail" && doctor.connectionFailure; + await ctx.input.evidence("chat-catalog-owned-db-fault-health.json", { ...state, proof: fault.proof, doctor, stage: "canonical UI clicked; request not yet dispatched to backend" }); + if (response.status !== 503 || health.error !== "database_unreachable" || !state.doctorFailed) throw new Error("Exact owned source did not reach the required live-but-unready boundary"); + state.backendDispatchAt = Date.now(); state.backendDispatchAfterFault = true; + await ctx.input.evidence("chat-catalog-owned-db-fault-dispatch.json", { ...state, requestPath: pathname, requestBodyUnchanged: true }); + await route.continue(); + } catch { + await ctx.input.evidence("chat-catalog-owned-db-fault-barrier-error.json", { stage: "owned fault or readiness proof failed; canonical request not forwarded", ...state }); + await route.abort("failed"); + } finally { finish(); pending.delete(flight); } + }; + const response = async (reply: Response) => { + if (!claimed || reply.request().method() !== "POST" || new URL(reply.url()).pathname !== pathname) return; + state.canonicalHttpStatus = reply.status(); + await ctx.input.evidence("chat-catalog-owned-db-fault-admission.json", { ...state, observedAt: Date.now(), sourceIssueId: source.id, + outcome: reply.ok() ? "actual backend accepted request" : "actual backend rejected request; no simulated assistant response" }); + await ctx.input.capture("unready-canonical-send", "Actual source submission with database unavailable", "unready-canonical-send.png").catch(() => undefined); + }; + controls.set(ctx, { fault, pattern, pending, handler, response }); + await ctx.input.page.route(pattern, handler); ctx.input.page.on("response", response); + return state; + }, grade: gradeDiagnoseHealth, + async verifyResult(ctx) { await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); }, +}); +diagnose.attemptTimeoutMs = 600_000; +diagnose.observe = observeInternal; +diagnose.cleanup = async ctx => { + const control = controls.get(ctx); if (!control) return; + try { await ctx.input.page.unroute(control.pattern, control.handler); ctx.input.page.off("response", control.response); await Promise.all(control.pending); } + finally { await control.fault.recover(); controls.delete(ctx); } + await ctx.input.evidence("chat-catalog-owned-db-recovery.json", { recoveredOwnedDatabase: true, routeUnregistered: true }); +}; +diagnose.sourceFiles = [...diagnose.sourceFiles, "chat-catalog-domains/privileged-database-fault.ts", "../../cli/src/checks/database-check.ts", "../../server/src/routes/health.ts"]; +export const privilegedDiagnosticCases = [diagnose]; diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-document-cli.ts b/tests/runner-e2e/chat-catalog-domains/privileged-document-cli.ts index cd3af4b58c6..327c60829fc 100644 --- a/tests/runner-e2e/chat-catalog-domains/privileged-document-cli.ts +++ b/tests/runner-e2e/chat-catalog-domains/privileged-document-cli.ts @@ -3,7 +3,8 @@ import { createHash } from "node:crypto"; import { readFile, writeFile } from "node:fs/promises"; import path from "node:path"; import { expect } from "@playwright/test"; -import { checks, putDocument, type Entity } from "./internal-support.js"; +import type { IssueDocument } from "../../../packages/shared/src/types/issue.js"; +import { checks, putDocument } from "./internal-support.js"; import { privilegedDefinition, privilegedObservation, settlePrivilegedChat } from "./privileged-support.js"; import { prepareOwnedCli, runOwnedCli, cleanupOwnedCli } from "./privileged-cli-support.js"; @@ -44,7 +45,7 @@ const exportPlan = privilegedDefinition("cli-operations.export-document-cli", "p async act(ctx, state) { const reply = await settlePrivilegedChat(ctx), sourceId = String(state.sourceIssueId); const command = documentExportCommand(reply.reply, sourceId); - const json = await runOwnedCli(ctx, command.argv), document = JSON.parse(json) as Entity; + const json = await runOwnedCli(ctx, command.argv), document = JSON.parse(json) as IssueDocument; // Execute the actual finite jq filter from the returned command; never a shell. const bytes = execFileSync("jq", command.filterArgs, { input: json, timeout: 5_000, maxBuffer: 1_048_576 }); const destination = path.join(String(state.operatorDirectory), command.filename); @@ -53,7 +54,7 @@ const exportPlan = privilegedDefinition("cli-operations.export-document-cli", "p state.actualFileSha256 = createHash("sha256").update(actual).digest("hex"); state.fileReadBack = true; state.fileWithinOwnedDirectory = path.dirname(destination) === state.operatorDirectory; state.exportRevisionId = document.latestRevisionId; state.exportIssueId = document.issueId; - const saved = await ctx.input.api.get(`/api/issues/${sourceId}/documents/plan`); + const saved = await ctx.input.api.get(`/api/issues/${sourceId}/documents/plan`); state.documentUnchanged = saved.latestRevisionId === state.documentRevisionId && createHash("sha256").update(saved.body ?? "").digest("hex") === state.savedBodySha256; state.exportStructuredResult = typeof document.body === "string" && typeof document.latestRevisionId === "string"; state.onlyExpectedDocumentReturned = document.key === "plan" && document.issueId === sourceId; diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-feedback.ts b/tests/runner-e2e/chat-catalog-domains/privileged-feedback.ts new file mode 100644 index 00000000000..cb85fcde68f --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/privileged-feedback.ts @@ -0,0 +1,154 @@ +import { createHash } from "node:crypto"; +import { readFile, realpath } from "node:fs/promises"; +import path from "node:path"; +import { expect } from "@playwright/test"; +import type { FeedbackTrace, FeedbackTraceBundle, FeedbackVote } from "../../../packages/shared/src/types/feedback.js"; +import { pollUntil } from "../api.js"; +import { sendChatMessage } from "../chat-flow.js"; +import { checks, type Entity } from "./internal-support.js"; +import { prepareRuntimeWorkerGate, releaseRuntimeWorker } from "./runtime-busy-worker.js"; +import { probeAgentPermission, cleanupIdentityProbes } from "./identity-support.js"; +import { prepareOwnedCli, runOwnedCli, cleanupOwnedCli } from "./privileged-cli-support.js"; +import { privilegedDefinition, privilegedObservation, companionPhase } from "./privileged-support.js"; +const pendingFixtures = new WeakMap[0], { gate: Awaited>; taskId?: string }>(); +const sha = (b: Buffer | string) => createHash("sha256").update(b).digest("hex"); +/** CLI exports use stored ZIP entries. Independently compare every local entry, + * reject hidden extras and unsafe names, and scan every actual archive byte. */ +export function storedArchiveEntries(bytes: Buffer): Map { + const rows = new Map(); let offset = 0; + while (offset + 4 <= bytes.length && bytes.readUInt32LE(offset) === 0x04034b50) { + if (offset + 30 > bytes.length || bytes.readUInt16LE(offset + 6) & 9 || bytes.readUInt16LE(offset + 8) !== 0) throw new Error("Unsupported feedback archive entry"); + const compressed = bytes.readUInt32LE(offset + 18), size = bytes.readUInt32LE(offset + 22), nameSize = bytes.readUInt16LE(offset + 26), extraSize = bytes.readUInt16LE(offset + 28); + const begin = offset + 30 + nameSize + extraSize, end = begin + compressed; + const name = bytes.subarray(offset + 30, offset + 30 + nameSize).toString("utf8"); + if (compressed !== size || end > bytes.length || !name || name.startsWith("/") || name.split("/").includes("..") || name.includes("\\") || rows.has(name)) throw new Error("Unsafe or inconsistent feedback archive entry"); + rows.set(name, bytes.subarray(begin, end)); offset = end; + } + if (!rows.size || offset + 4 > bytes.length || bytes.readUInt32LE(offset) !== 0x02014b50) throw new Error("Missing feedback ZIP directory"); + return rows; +} +export function interruptedBundleMatches(bundle: FeedbackTraceBundle, ids: { company: string; issue: string; run: string; agent: string; comment: string }) { + const run = bundle.paperclipRun; + const runFile = bundle.files.find(f => f.path === "paperclip/run.json"), eventsFile = bundle.files.find(f => f.path === "paperclip/run-events.json"); + if (!runFile || !eventsFile) return false; + try { + const saved = JSON.parse(runFile.contents) as Record, events = JSON.parse(eventsFile.contents) as Array<{ runId?: string; agentId?: string; seq?: number }>; + return bundle.companyId === ids.company && bundle.issueId === ids.issue && bundle.envelope.targetId === ids.comment + && run?.id === ids.run && run.companyId === ids.company && run.agentId === ids.agent && run.status === "cancelled" + && saved.id === ids.run && saved.status === "cancelled" && saved.agentId === ids.agent && saved.companyId === ids.company + && events.length > 0 && events.every(e => e.runId === ids.run) && Number(run.eventCount) === events.length; + } catch { return false; } +} +export function gradeFeedbackBundle(value: unknown) { + const c = checks(), o = privilegedObservation(value); c.add("actual-local-feedback-export", !!o, "A real human-owned scoped CLI export and retained interrupted-work trace are required."); if (!o) return c.result(); + c.add("board-only-report-boundary", o.state.agentReportStatus === 403 && o.state.cliExecuted === true, "The actual agent cannot read feedback traces; an isolated human CLI actor performs the export."); + c.add("interrupted-work-not-fabricated-failure", o.state.workerWasReallyWaiting === true && o.state.cancelledBeforeCanonical === true + && o.state.exportedInterruptionMatches === true && o.runs.some(r => r.id === o.state.workerRunId && r.status === "cancelled") + && o.state.originalTraceUnchanged === true, "The original actual cancelled run, source-attributed comment and exact retained events survive export; cancellation is never relabeled as a failed native/model outcome."); + c.add("owned-local-bytes-and-scope", o.state.bundleWithinOwnedDirectory === true && o.state.exportCompanyMatches === true && o.state.exportIssueMatches === true + && o.state.fullTraceBytesMatch === true && o.state.archiveBytesMatch === true && typeof o.state.zipSha256 === "string" && Number(o.state.zipBytes) > 0 + && o.state.noPrivateCredentialBytes === true && o.state.remoteSharingUnchanged === true && o.state.bundlePublished === false, + "Actual JSON and archive bytes match the intended company/issue trace only, remain private and do not enable remote sharing."); + const report = o.state.exportReport as { attributed?: boolean; reply?: string } | undefined; + c.add("truthful-native-location-and-contents", report?.attributed === true && report.reply?.includes(String(o.state.localOutputDirectory)) === true + && /feedback|vote/i.test(report.reply) && /trace|event/i.test(report.reply) && /cancel|interrupt/i.test(report.reply) + && !/published|uploaded|shared publicly|model fail|failed native/i.test(report.reply), "The actual follow-up reports the real local location, scope and interrupted-work evidence without publishing or relabeling it."); + return c.result(); +} +const feedback = privilegedDefinition("cli-operations.feedback-bundle", "privileged-feedback", { + async setup(ctx) { + const gate = await prepareRuntimeWorkerGate(ctx); const pending = { gate, taskId: undefined as string | undefined }; pendingFixtures.set(ctx, pending); + const marker = `Waiting for operator brief ${ctx.input.nonce}`; + const task = await ctx.input.api.post(`${gate.company}/issues`, { title: "Interrupted local feedback fixture", status: "todo", assigneeAgentId: gate.worker.id, + description: `First post one normal task comment saying exactly ${JSON.stringify(marker)}. Then run node ${JSON.stringify(gate.gate.scriptPath)} once and wait for the human brief. Do not mark this task complete or replace the script. This is an owned interruption fixture.` }); + pending.taskId = task.id; + const sharingBefore = await ctx.input.api.get<{ feedbackDataSharingPreference?: string }>("/api/instance/settings/general"); + const running = await pollUntil({ label: "Actual worker comment and native waiting process", deadlineAt: Date.now() + 150000, + load: async () => ({ runs: await ctx.allRuns(), comments: await ctx.input.api.get(`/api/issues/${task.id}/comments?order=asc`), waiting: await readFile(gate.gate.ready, "utf8").catch(() => "") }), + accept: o => o.waiting === "waiting" && o.runs.length === 1 && o.runs[0]?.status === "running" && o.runs[0].agentId === gate.worker.id + && o.comments.some(c => c.authorAgentId === gate.worker.id && c.createdByRunId === o.runs[0]!.id && c.body === marker), + reject: o => o.runs.some(r => ["failed", "cancelled", "timed_out", "succeeded"].includes(r.status)) ? "Interruption prerequisite finished before the actual held native boundary" : undefined }); + const run = running.runs[0]!, comment = running.comments.find(c => c.body === marker && c.createdByRunId === run.id)!; + await ctx.input.api.post(`/api/heartbeat-runs/${run.id}/cancel`, {}); + await pollUntil({ label: "Actual board cancellation before feedback request", deadlineAt: Date.now() + 60000, load: () => ctx.allRuns(), accept: runs => runs.length === 1 && runs[0]?.status === "cancelled" }); + await ctx.input.api.patch(`/api/issues/${task.id}`, { status: "backlog" }); + await releaseRuntimeWorker({ gate: gate.gate }); + const vote = await ctx.input.api.post(`/api/issues/${task.id}/feedback-votes`, { targetType: "issue_comment", targetId: comment.id, + vote: "down", reason: "The operator interrupted this unfinished fixture before completion; retain the original cancelled run and events for local review.", allowSharing: false }); + const trace = await pollUntil({ label: "Actual human feedback trace retained locally", deadlineAt: Date.now() + 30000, + load: () => ctx.input.api.get(`/api/issues/${task.id}/feedback-traces`), accept: rows => rows.length === 1 && rows[0]?.feedbackVoteId === vote.id }); + const agentReportStatus = await probeAgentPermission(ctx, `/api/issues/${task.id}/feedback-traces`, "GET"); + return { worker: gate.worker, gate: gate.gate, task, workerRunId: run.id, workerCommentId: comment.id, traceId: trace[0]!.id, + traceBefore: { id: trace[0]!.id, targetId: trace[0]!.targetId, status: trace[0]!.status, destination: trace[0]!.destination, exportId: trace[0]!.exportId }, + sharingPreferenceBefore: sharingBefore.feedbackDataSharingPreference, workerWasReallyWaiting: true, cancelledBeforeCanonical: true, agentReportStatus, ...(await prepareOwnedCli(ctx)) }; + }, + async act(ctx, state) { + await pollUntil({ label: "Canonical human-export guidance settled", deadlineAt: Date.now() + 150000, + load: async () => { await ctx.refreshIssue(); return { issue: ctx.issue(), runs: await ctx.allRuns() }; }, + accept: o => o.issue?.conversationState === "waiting" && o.runs.length === 2 && o.runs.filter(r => r.status === "succeeded").length === 1 && o.runs.filter(r => r.status === "cancelled").length === 1 }); + const prior = await ctx.allRuns(); const company = ctx.input.fixtures.company.id, task = state.task as Entity; + const output = path.join(String(state.operatorDirectory), "feedback-export"); + const result = JSON.parse(await runOwnedCli(ctx, ["feedback", "export", "--company-id", company, "--issue-id", task.id, "--out", output, "--json"])); state.cliExecuted = true; + const directory = await realpath(output), root = await realpath(String(state.operatorDirectory)); + state.bundleWithinOwnedDirectory = directory.startsWith(root + path.sep) && result.outputDir === output && result.zipPath === output + ".zip"; + if (!state.bundleWithinOwnedDirectory) throw new Error("Feedback CLI output escapes its exact owned local destination"); + const indexBytes = await readFile(path.join(directory, "index.json")), index = JSON.parse(indexBytes.toString()) as { companyId: string; summary: { total: number; issues: string[] }; files: { fullTraces: string[]; traces: string[]; votes: string[] } }; + state.exportCompanyMatches = index.companyId === company; state.exportIssueMatches = index.summary.total === 1 + && index.summary.issues.length === 1 && index.summary.issues[0] === (task.identifier ?? task.id) && index.files.fullTraces.length === 1 && index.files.traces.length === 1 && index.files.votes.length === 1; + if (!state.exportIssueMatches) throw new Error("Feedback export contains unexpected company or issue scope"); + const traceDir = index.files.fullTraces[0]!; + if (path.basename(traceDir) !== traceDir) throw new Error("Unexpected feedback trace directory"); + const bundleBytes = await readFile(path.join(directory, "full-traces", traceDir, "bundle.json")), bundle = JSON.parse(bundleBytes.toString()) as FeedbackTraceBundle; + state.exportedInterruptionMatches = interruptedBundleMatches(bundle, { company, issue: task.id, run: String(state.workerRunId), agent: (state.worker as Entity).id, comment: String(state.workerCommentId) }); + const actual = await ctx.input.api.get(`/api/feedback-traces/${state.traceId}/bundle`); + state.fullTraceBytesMatch = JSON.stringify(bundle) === JSON.stringify(actual); + const zip = await readFile(result.zipPath), archive = storedArchiveEntries(zip); + const archivePrefix = path.basename(directory) + "/"; + const expected = new Map([["index.json", indexBytes]]); + for (const group of ["votes", "traces"] as const) for (const name of index.files[group]) { + if (path.basename(name) !== name) throw new Error("Unexpected exported feedback filename"); + expected.set(`${group}/${name}`, await readFile(path.join(directory, group, name))); + } + expected.set(`full-traces/${traceDir}/bundle.json`, bundleBytes); + for (const file of bundle.files) { + if (path.isAbsolute(file.path) || file.path.split(/[\\/]/).includes("..")) throw new Error("Unsafe local trace output path"); + expected.set(`full-traces/${traceDir}/${file.path}`, Buffer.from(file.contents)); + } + state.archiveBytesMatch = archive.size === expected.size && [...expected].every(([file, bytes]) => archive.get(archivePrefix + file)?.equals(bytes)); + state.zipBytes = zip.length; state.zipSha256 = sha(zip); + state.noPrivateCredentialBytes = [indexBytes, bundleBytes, zip].every(bytes => !(ctx.input.privateSecrets ?? []).some(secret => secret && bytes.includes(Buffer.from(secret)))); + if (!state.noPrivateCredentialBytes) throw new Error("Private credential bytes detected in local feedback output; matched content withheld"); + const after = await ctx.input.api.get(`/api/feedback-traces/${state.traceId}`), before = state.traceBefore as Record; + state.originalTraceUnchanged = ["id", "targetId", "status", "destination", "exportId"].every(key => after[key as keyof FeedbackTrace] === before[key]); + const sharingAfter = await ctx.input.api.get<{ feedbackDataSharingPreference?: string }>("/api/instance/settings/general"); + state.remoteSharingUnchanged = sharingAfter.feedbackDataSharingPreference === state.sharingPreferenceBefore && before.status === "local_only" && after.status === "local_only" && after.destination == null && after.exportId == null; + state.bundlePublished = false; state.localOutputDirectory = directory; + const receipt = { localOutputDirectory: directory, zipSha256: state.zipSha256, zipBytes: state.zipBytes, companyId: company, issueId: task.id, + feedbackTraceId: state.traceId, originalRunId: state.workerRunId, originalRunStatus: "cancelled", originalAgentId: (state.worker as Entity).id, + exportedContents: ["scoped feedback vote", "trace metadata", "original cancelled run", "retained run events"], exactLocalBytesVerified: state.fullTraceBytesMatch, + remoteSharingEnabled: false, publicAttachmentCreated: false, outcomeKind: "operator interrupted unfinished work; not a model failure or failed-native run" }; + await ctx.input.evidence("chat-catalog-local-feedback-export-receipt.json", receipt); + const message = `The human operator exported this local review bundle and retained the original interrupted work evidence. These are the observed facts; please report its location and contents without publishing it:\n\n${JSON.stringify(receipt)}`; + await sendChatMessage(ctx.input.page, message); state.exportReport = await companionPhaseForCancelled(ctx, prior.map(r => r.id), message); + }, grade: gradeFeedbackBundle, async verifyResult(ctx) { await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); }, +}); +// companionPhase's default all-succeeded assertion intentionally does not apply to this retained cancelled fixture. +async function companionPhaseForCancelled(ctx: Parameters[0], prior: string[], message: string) { + const result = await pollUntil({ label: "Native reply to local interrupted-work export receipt", deadlineAt: Date.now() + 150000, + load: async () => { await ctx.refreshIssue(); return { issue: ctx.issue(), runs: await ctx.allRuns() }; }, + accept: o => o.issue?.conversationState === "waiting" && o.runs.length === prior.length + 1 && o.runs.filter(r => r.status === "succeeded").length === 2 && o.runs.filter(r => r.status === "cancelled").length === 1 }); + const run = result.runs.find(r => !prior.includes(r.id) && r.agentId === ctx.input.fixtures.agent.id)!; + const comments = await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/comments?order=asc`), human = comments.filter(c => !c.authorAgentId && c.body === message); + const replies = comments.filter(c => c.authorAgentId === ctx.input.fixtures.agent.id && c.createdByRunId === run.id); + const profile = await ctx.input.api.get<{ runnerProfileJson?: { nativeExecutionInput?: { task?: { prompt?: string } } } }>(`/api/heartbeat-runs/${run.id}`); + return { humanCommentId: human[0]?.id, runId: run.id, reply: replies.map(c => c.body).join("\n"), + attributed: human.length === 1 && replies.length > 0 && profile.runnerProfileJson?.nativeExecutionInput?.task?.prompt?.includes(human[0]!.id) === true + && replies.every(c => Date.parse(c.createdAt!) >= Date.parse(human[0]!.createdAt!)) }; +} +feedback.expectedRunCount = 3; feedback.expectedHumanMessageCount = 2; feedback.attemptTimeoutMs = 900000; +feedback.runPolicy = { kind: "terminal-counts", terminalStatusCounts: { succeeded: 2, cancelled: 1 } }; +feedback.cleanup = async (ctx, state) => { try { const pending = pendingFixtures.get(ctx); + if (pending) { const runs = await ctx.allRuns(); for (const run of runs.filter(r => r.agentId === pending.gate.worker.id && ["queued", "running"].includes(r.status))) await ctx.input.api.post(`/api/heartbeat-runs/${run.id}/cancel`, {}); await releaseRuntimeWorker({ gate: pending.gate.gate }); pendingFixtures.delete(ctx); } + await releaseRuntimeWorker(state); } finally { try { await cleanupIdentityProbes(ctx); } finally { await cleanupOwnedCli(ctx); } } }; +feedback.sourceFiles = [...feedback.sourceFiles, "chat-catalog-domains/privileged-cli-support.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/runtime-busy-worker.ts", "../../cli/src/index.ts", "../../cli/src/commands/client/feedback.ts", "../../server/src/services/feedback.ts"]; +export const privilegedFeedbackCases = [feedback]; diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-installed.ts b/tests/runner-e2e/chat-catalog-domains/privileged-installed.ts new file mode 100644 index 00000000000..16f0b4bf821 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/privileged-installed.ts @@ -0,0 +1,64 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { createHash } from "node:crypto"; +import { readFile, realpath, stat, readdir, lstat, readlink } from "node:fs/promises"; +import path from "node:path"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import { installedReleaseLaunch } from "../installed-release.js"; +const execute = promisify(execFile); +export interface InstalledOperatorReceipt { schemaVersion: 1; disposable: true; cli: string; cliSha256: string; server: string; serverSha256: string; + installationTreeSha256: string; serverPid: number; version: string; targetVersion?: string; targetIntegrity?: string; piCredentialEnv?: string } +export function validInstalledReceipt(r: InstalledOperatorReceipt) { + return r.schemaVersion === 1 && r.disposable === true && Number.isSafeInteger(r.serverPid) && r.serverPid > 1 + && /^[0-9a-f]{64}$/.test(r.installationTreeSha256) && /^[0-9a-f]{64}$/.test(r.cliSha256) && /^[0-9a-f]{64}$/.test(r.serverSha256) + && /^\d+\.\d+\.\d+(?:[-+][A-Za-z0-9.-]+)?$/.test(r.version) + && (!r.targetVersion || /^\d+\.\d+\.\d+(?:[-+][A-Za-z0-9.-]+)?$/.test(r.targetVersion)) + && (!r.targetIntegrity || /^sha512-[A-Za-z0-9+/]+=*$/.test(r.targetIntegrity)) + && (!r.piCredentialEnv || /^[A-Z][A-Z0-9_]+$/.test(r.piCredentialEnv)); +} +/** Receipt identifies the *actual source server*, not another healthy fixture. */ +export async function ownedInstalledTarget(ctx: ChatCatalogContext) { + const ref = process.env.PAPERCLIP_CHAT_OPERATOR_INSTALL_RECEIPT, temp = process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT; + if (!ref || !temp) throw new Error("Requires actual owned published source target and PAPERCLIP_CHAT_OPERATOR_INSTALL_RECEIPT (private mode0600; CLI/server hashes, version and live PID)"); + const root = await realpath(temp), receiptFile = await realpath(ref), info = await stat(receiptFile); + if (!receiptFile.startsWith(root + path.sep) || !info.isFile() || info.mode & 0o077) throw new Error("Installed operator receipt must be private and inside the owned attempt"); + const r = JSON.parse(await readFile(receiptFile, "utf8")) as InstalledOperatorReceipt; + if (!validInstalledReceipt(r)) throw new Error("Installed operator receipt lacks exact artifact identities"); + const cli = await realpath(r.cli), server = await realpath(r.server); + if (![cli, server].every(file => file.startsWith(root + path.sep))) throw new Error("Installed operation refuses artifacts outside its owned attempt"); + const launch = installedReleaseLaunch(cli); + const serverManifest = JSON.parse(await readFile(path.resolve(server, "../../package.json"), "utf8")); + if (launch.version !== r.version || serverManifest.name !== "@paperclipai/server" || serverManifest.version !== r.version + || createHash("sha256").update(await readFile(cli)).digest("hex") !== r.cliSha256 + || createHash("sha256").update(await readFile(server)).digest("hex") !== r.serverSha256) throw new Error("Installed artifact identity differs from receipt"); + const row = (await execute("ps", ["-p", String(r.serverPid), "-o", "uid=", "-o", "command="], { timeout: 5000 })).stdout.trim(); + const match = row.match(/^(\d+)\s+([\s\S]+)$/); + if (Number(match?.[1]) !== process.getuid?.() || !match?.[2]?.split(/\s+/).includes(cli)) throw new Error("Receipt does not identify the current user's exact installed CLI process"); + const health = await ctx.input.api.get<{ version?: string }>("/api/health"); + if (health.version !== r.version) throw new Error("Actual source HTTP version differs from the pinned installed target"); + const prefix = path.resolve(launch.cwd, "../../.."); + if (!prefix.startsWith(root + path.sep) || !server.startsWith(prefix + path.sep)) throw new Error("Installed artifacts do not share the exact owned npm prefix"); + const entries: Array<{ file: string; sha256?: string; link?: string }> = []; let totalBytes = 0; + async function inventory(current: string): Promise { + for (const name of (await readdir(current)).sort()) { + const file = path.join(current, name), info = await lstat(file); + if (entries.length > 200000 || totalBytes > 2_147_483_648) throw new Error("Installed artifact inventory exceeds the bounded proof scope"); + if (info.isSymbolicLink()) { const resolved = await realpath(file); if (!resolved.startsWith(prefix + path.sep)) throw new Error("Installed dependency escapes its owned prefix"); entries.push({ file: path.relative(prefix, file), link: await readlink(file) }); } + else if (info.isDirectory()) await inventory(file); + else if (info.isFile()) { totalBytes += info.size; entries.push({ file: path.relative(prefix, file), sha256: createHash("sha256").update(await readFile(file)).digest("hex") }); } + else throw new Error("Installed artifact inventory contains a nonregular object"); + } + } + await inventory(prefix); + const treeSha256 = createHash("sha256").update(JSON.stringify(entries)).digest("hex"); + if (treeSha256 !== r.installationTreeSha256) throw new Error("Actual installed source dependency tree differs from its pinned artifact receipt"); + const config = process.env.PAPERCLIP_CONFIG; + if (!config || !(await realpath(config)).startsWith(root + path.sep)) throw new Error("Installed operation requires the owned source config"); + const environment: NodeJS.ProcessEnv = { PATH: process.env.PATH, HOME: root, PAPERCLIP_HOME: root, PAPERCLIP_CONFIG: config, + PAPERCLIP_TELEMETRY_DISABLED: "1", PAPERCLIP_NO_BROWSER: "1", NO_COLOR: "1", NPM_CONFIG_PREFIX: path.resolve(launch.cwd, "../../..") }; + return { receipt: r, cli, server, root, treeSha256, environment, + async command(args: string[], timeout = 180_000) { + try { return (await execute(process.execPath, [cli, ...args], { cwd: launch.cwd, env: environment, timeout, maxBuffer: 1_048_576 })).stdout; } + catch { throw new Error("Owned installed operator command failed; raw subprocess output withheld"); } + } }; +} diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-operators.test.ts b/tests/runner-e2e/chat-catalog-domains/privileged-operators.test.ts new file mode 100644 index 00000000000..9483f234aa9 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/privileged-operators.test.ts @@ -0,0 +1,50 @@ +import { expect, it } from "vitest"; +import type { FeedbackTraceBundle } from "../../../packages/shared/src/types/feedback.js"; +import { validInstalledReceipt } from "./privileged-installed.js"; +import { gradeInstalledUpdate } from "./privileged-update.js"; +import { gradeProvisionRuntime } from "./privileged-provision.js"; +import { gradeOwnedDev, validOwnedDevResult } from "./privileged-owned-dev.js"; +import { gradeFeedbackBundle, interruptedBundleMatches, storedArchiveEntries } from "./privileged-feedback.js"; +const base = { scenarioId: "owned", origin: "http://owned.invalid", companyId: "company", leadId: "lead", state: {}, tasks: [], projects: [], goals: [], comments: [], activity: [], documents: [], revisions: [], links: [], humanQueries: ["canonical"], runs: [], reply: "" }; +it("requires explicit disposable published artifact and full dependency-tree pins", () => { + const r = { schemaVersion: 1 as const, disposable: true as const, cli: "/owned/prefix/lib/node_modules/paperclipai/dist/index.js", cliSha256: "a".repeat(64), server: "/owned/server/dist/index.js", serverSha256: "b".repeat(64), installationTreeSha256: "c".repeat(64), serverPid: 123, version: "2026.1026.1", targetVersion: "2026.1026.2", targetIntegrity: "sha512-YWJjZA==" }; + expect(validInstalledReceipt(r)).toBe(true); + for (const delta of [{ installationTreeSha256: "" }, { serverPid: 0 }, { targetVersion: "latest" }, { targetIntegrity: "unpinned" }]) expect(validInstalledReceipt({ ...r, ...delta })).toBe(false); +}); +it("rejects download-only update proof, wrong service version and premature assistant success", () => { + const state = { installMode: "global-npm", registryIntegrityMatched: true, updateExecuted: true, installedVersion: "2.0.0", targetVersion: "2.0.0", healthVersion: "2.0.0", healthReady: true, serviceRestarted: true, finalCliSha256: "f".repeat(64), rollbackVersion: "1.0.0", originalVersion: "1.0.0", guidance: "The terminal operator should inspect the npm installation channel first.", updateReport: { attributed: true, reply: "The receipt shows 2.0.0 after restart and healthy readiness." } }; + expect(gradeInstalledUpdate({ ...base, state }).passed).toBe(true); + for (const delta of [{ updateExecuted: false }, { healthVersion: "1.0.0" }, { registryIntegrityMatched: false }, { rollbackVersion: "unknown" }, { guidance: "I have updated to the newest version." }, { updateReport: { attributed: false, reply: state.updateReport.reply } }]) expect(gradeInstalledUpdate({ ...base, state: { ...state, ...delta } }).passed).toBe(false); +}); +it("does not qualify Pi from provisioner/download evidence without a separate genuine native task", () => { + const state = { assetsInitiallyAbsent: true, provisionerSucceeded: true, provisionerSha256: "a".repeat(64), nativeClosureSha256: "b".repeat(64), releaseVersion: "1.0.0", healthVersion: "1.0.0", assetPathOwned: true, piAgentConfigurationVerified: true, piRunId: "pi-run", piAgentId: "pi", piArtifactMatches: true, guidance: "The operator can run runtime setup pi in the terminal.", provisionReport: { attributed: true, reply: "Pi completed a real native qualification run." } }; + const runs = [{ id: "initial", agentId: "lead", status: "succeeded" }, { id: "pi-run", agentId: "pi", status: "succeeded", runtimeMode: "native" }, { id: "report", agentId: "lead", status: "succeeded" }]; + expect(gradeProvisionRuntime({ ...base, state, runs }).passed).toBe(true); + for (const delta of [{ piArtifactMatches: false }, { piAgentConfigurationVerified: false }, { assetPathOwned: false }, { healthVersion: "other" }]) expect(gradeProvisionRuntime({ ...base, state: { ...state, ...delta }, runs }).passed).toBe(false); + expect(gradeProvisionRuntime({ ...base, state, runs: runs.filter(r => r.id !== "pi-run") }).passed).toBe(false); +}); +it("requires actual seed readback, source/unrelated preservation and exact target cleanup", () => { + const state = { sourceCanaryUnchanged: true, targetCanaryMatches: true, actualTargetHealthy: true, targetStopped: true, targetCleaned: true, unrelatedSentinelUnchanged: true, sourceRevisionPinned: true, targetUrl: "http://127.0.0.1:54321", sourceCliMatches: true, guidance: "The operator can use worktree commands in the owned checkout.", devReport: { attributed: true, reply: "http://127.0.0.1:54321 was healthy and is now stopped and cleaned." } }; + expect(validOwnedDevResult(state)).toBe(true); expect(gradeOwnedDev({ ...base, state }).passed).toBe(true); + for (const key of ["sourceCanaryUnchanged", "targetCanaryMatches", "actualTargetHealthy", "targetStopped", "targetCleaned", "unrelatedSentinelUnchanged", "sourceCliMatches"]) expect(gradeOwnedDev({ ...base, state: { ...state, [key]: false } }).passed).toBe(false); +}); +it("binds retained interrupted work to actual original cancelled run, comment and events", () => { + const run = { id: "run", companyId: "company", agentId: "morgan", status: "cancelled", eventCount: 1 }; + const bundle: FeedbackTraceBundle = { traceId: "trace", exportId: null, issueIdentifier: "OWN-1", adapterType: "paperclip_runner", captureStatus: "partial", notes: [], surface: null, rawAdapterTrace: null, normalizedAdapterTrace: null, privacy: null, integrity: {}, companyId: "company", issueId: "task", envelope: { targetId: "comment" }, paperclipRun: run, files: [{ path: "paperclip/run.json", contentType: "application/json", encoding: "utf8", byteLength: 1, sha256: "x", source: "paperclip_run", contents: JSON.stringify(run) }, { path: "paperclip/run-events.json", contentType: "application/json", encoding: "utf8", byteLength: 1, sha256: "x", source: "paperclip_run_events", contents: JSON.stringify([{ runId: "run", seq: 1 }]) }] }; + const ids = { company: "company", issue: "task", run: "run", agent: "morgan", comment: "comment" }; + expect(interruptedBundleMatches(bundle, ids)).toBe(true); + expect(interruptedBundleMatches({ ...bundle, paperclipRun: { ...run, status: "failed" } }, ids)).toBe(false); + expect(interruptedBundleMatches(bundle, { ...ids, comment: "other" })).toBe(false); + expect(interruptedBundleMatches({ ...bundle, files: bundle.files.slice(0, 1) }, ids)).toBe(false); +}); +it("requires actual private scoped bundle bytes and truthful local interruption report", () => { + const state = { agentReportStatus: 403, cliExecuted: true, workerWasReallyWaiting: true, cancelledBeforeCanonical: true, exportedInterruptionMatches: true, workerRunId: "run", originalTraceUnchanged: true, bundleWithinOwnedDirectory: true, exportCompanyMatches: true, exportIssueMatches: true, fullTraceBytesMatch: true, archiveBytesMatch: true, zipSha256: "a".repeat(64), zipBytes: 100, noPrivateCredentialBytes: true, remoteSharingUnchanged: true, bundlePublished: false, localOutputDirectory: "/owned/export", exportReport: { attributed: true, reply: "The local feedback and trace/events bundle is at /owned/export and retains the cancelled interrupted work." } }; + const runs = [{ id: "run", status: "cancelled" }]; + expect(gradeFeedbackBundle({ ...base, state, runs }).passed).toBe(true); + for (const delta of [{ agentReportStatus: 200 }, { archiveBytesMatch: false }, { remoteSharingUnchanged: false }, { noPrivateCredentialBytes: false }, { exportedInterruptionMatches: false }, { exportReport: { attributed: true, reply: "Published failed native model failure evidence at /owned/export." } }]) expect(gradeFeedbackBundle({ ...base, state: { ...state, ...delta }, runs }).passed).toBe(false); +}); +it("rejects arbitrary, truncated or path-escaping archive bytes", () => { + expect(() => storedArchiveEntries(Buffer.from("not zip"))).toThrow(); + const header = Buffer.alloc(30); header.writeUInt32LE(0x04034b50); header.writeUInt16LE(5, 26); const bytes = Buffer.concat([header, Buffer.from("../xx"), Buffer.from([0x50, 0x4b, 0x01, 0x02])]); + expect(() => storedArchiveEntries(bytes)).toThrow(); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-owned-dev.ts b/tests/runner-e2e/chat-catalog-domains/privileged-owned-dev.ts new file mode 100644 index 00000000000..8188f20453d --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/privileged-owned-dev.ts @@ -0,0 +1,116 @@ +import { execFile, spawn, type ChildProcess } from "node:child_process"; +import { promisify } from "node:util"; +import { createServer } from "node:http"; +import { createHash } from "node:crypto"; +import { cp, mkdir, mkdtemp, readFile, realpath, rm, stat, symlink, writeFile } from "node:fs/promises"; +import { fileURLToPath } from "node:url"; +import path from "node:path"; +import { expect } from "@playwright/test"; +import { createDb } from "../../../packages/db/src/index.js"; +import { listenOnFetchAllowedPort } from "../../e2e/fetch-allowed-port.js"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import { pollUntil } from "../api.js"; +import { sendChatMessage } from "../chat-flow.js"; +import { checks, project, task, type Entity } from "./internal-support.js"; +import { privilegedDefinition, privilegedObservation, settlePrivilegedChat, companionPhase } from "./privileged-support.js"; +const execute = promisify(execFile); +interface OwnedDev { root: string; checkout: string; home: string; instance: string; env: NodeJS.ProcessEnv; loader: string; entry: string; child?: ChildProcess; closed?: Promise; config?: string } +const owned = new WeakMap(); +const source = fileURLToPath(new URL("../../../", import.meta.url)); +const digest = (b: Buffer) => createHash("sha256").update(b).digest("hex"); +export function validOwnedDevResult(s: Record) { + return s.sourceCanaryUnchanged === true && s.targetCanaryMatches === true && s.actualTargetHealthy === true + && s.targetStopped === true && s.targetCleaned === true && s.unrelatedSentinelUnchanged === true + && s.sourceRevisionPinned === true && typeof s.targetUrl === "string" && s.sourceCliMatches === true; +} +async function command(o: OwnedDev, args: string[]) { + try { return (await execute(process.execPath, ["--import", o.loader, o.entry, ...args], { cwd: o.checkout, env: o.env, timeout: 240000, maxBuffer: 1_048_576 })).stdout; } + catch { throw new Error("Owned development-instance CLI command failed; private output withheld"); } +} +async function stop(o: OwnedDev) { + if (!o.child || !o.closed) return; + if (o.child.exitCode === null && o.child.signalCode === null) o.child.kill("SIGTERM"); + let timer: ReturnType | undefined; + try { await Promise.race([o.closed, new Promise((_, reject) => { timer = setTimeout(() => reject(new Error("Owned CLI child did not stop; preserve its root instead of deleting live state")), 30000); })]); } + finally { if (timer) clearTimeout(timer); } + o.child = undefined; +} +export function gradeOwnedDev(value: unknown) { + const c = checks(), o = privilegedObservation(value); c.add("actual-dev-instance-evidence", !!o, "Actual supported owned checkout initialization, seeding, server readiness and cleanup are required."); if (!o) return c.result(); + c.add("exact-owned-seeded-and-cleaned", validOwnedDevResult(o.state), "Only the owned checkout/instance is created and removed; copied canary rows, source state and unrelated sentinel independently verify isolation."); + c.add("operator-dev-guidance", /worktree|dev.*instance|isolated.*instance/i.test(String(o.state.guidance)) && /operator|terminal|human|checkout/i.test(String(o.state.guidance)) + && !/I (?:have |successfully )?(?:created|started|set up)/i.test(String(o.state.guidance)), "Initial chat assigns host changes to the explicit operator companion."); + const phase = o.state.devReport as { attributed?: boolean; reply?: string } | undefined; + c.add("native-reports-real-target", phase?.attributed === true && phase.reply?.includes(String(o.state.targetUrl)) === true + && /health|ready|clean|stopp/i.test(phase.reply), "The actual follow-up reports the measured target URL and its final readiness/cleanup state."); + return c.result(); +} +const dev = privilegedDefinition("cli-operations.owned-dev-instance", "privileged-owned-dev", { + async setup(ctx) { + const temp = process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT, sourceConfig = process.env.PAPERCLIP_CONFIG, revision = process.env.PAPERCLIP_RUNNER_E2E_SOURCE_SHA; + if (!temp || !sourceConfig || !revision || !/^[a-f0-9]{40}$/.test(revision)) throw new Error("Owned dev companion requires a source-pinned immutable candidate and private source config"); + const root = await mkdtemp(path.join(await realpath(temp), "operator-dev-")); + if (!(await realpath(sourceConfig)).startsWith(await realpath(temp) + path.sep)) throw new Error("Dev seed source config is outside this attempt"); + const checkout = path.join(root, "checkout"), home = path.join(root, "instances-home"); + const o: OwnedDev = { root, checkout, home, instance: `eval-${ctx.input.nonce.replace(/[^a-z0-9]/gi, "").slice(0, 32)}`, + loader: path.join(source, "cli/node_modules/tsx/dist/loader.mjs"), entry: path.join(checkout, "cli/src/index.ts"), + env: { PATH: process.env.PATH, HOME: path.join(root, "operator-home"), PAPERCLIP_HOME: home, PAPERCLIP_TELEMETRY_DISABLED: "1", PAPERCLIP_NO_BROWSER: "1", NO_COLOR: "1" } }; + owned.set(ctx, o); + // Copy actual selected candidate sources. Dependency links resolve only to + // this immutable campaign candidate, never the shared mutable worktree. + const excluded = new Set(["node_modules", ".git", "dist", "target", "results", ".paperclip", "data", "__pycache__", ".env", ".codex", ".secrets"]); + await cp(source, checkout, { recursive: true, filter: file => !path.relative(source, file).split(path.sep).some(part => excluded.has(part)) }); + for (const relative of ["", "cli", "server", "ui", "packages/db", "packages/shared"]) { + const modules = path.join(source, relative, "node_modules"); + try { if ((await stat(modules)).isDirectory()) await symlink(await realpath(modules), path.join(checkout, relative, "node_modules")); } + catch (e) { if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; } + } + await mkdir(o.env.HOME!, { recursive: true }); + await execute("git", ["init", "--initial-branch=owned-eval"], { cwd: checkout, env: { ...o.env, GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: "/dev/null" }, timeout: 10000 }); + const sentinel = path.join(root, "unrelated-owned-sentinel.txt"); await writeFile(sentinel, ctx.input.nonce, { mode: 0o600 }); + const launch = await project(ctx), canary = await task(ctx, "Owned dev seed canary", launch.id, { description: `Retain ${ctx.input.nonce}` }); + return { task: canary, sourceConfigOwned: true, sourceRevision: revision, sourceRevisionPinned: true, + sourceCliMatches: digest(await readFile(o.entry)) === digest(await readFile(path.join(source, "cli/src/index.ts"))), + dependencySource: "explicit immutable candidate modules; source and compiled dependency provenance retained in campaign snapshot receipt" }; + }, + async act(ctx, state) { + const initial = await settlePrivilegedChat(ctx); state.guidance = initial.reply; const o = owned.get(ctx)!; + const reserve = async () => { const listener = createServer(); const port = await listenOnFetchAllowedPort(listener); await new Promise((resolve, reject) => listener.close(e => e ? reject(e) : resolve())); return port; }; + const serverPort = await reserve(), dbPort = await reserve(); + if (serverPort === dbPort) throw new Error("Owned dev ports unexpectedly collide"); + await command(o, ["worktree", "init", "--instance", o.instance, "--home", o.home, "--from-config", process.env.PAPERCLIP_CONFIG!, "--server-port", String(serverPort), "--db-port", String(dbPort), "--seed-mode", "minimal"]); + o.config = path.join(o.checkout, ".paperclip/config.json"); + const actual = JSON.parse(await readFile(o.config, "utf8")); + const actualData = await realpath(actual.database.embeddedPostgresDataDir); + if (!actualData.startsWith(o.root + path.sep) || actual.database.embeddedPostgresPort !== dbPort || actual.server.port !== serverPort) throw new Error("Public worktree config escapes the exact owned target"); + o.child = spawn(process.execPath, ["--import", o.loader, o.entry, "run", "--config", o.config, "--yes"], { cwd: o.checkout, env: o.env, stdio: "ignore" }); + o.closed = new Promise((resolve, reject) => { o.child!.once("error", reject); o.child!.once("close", () => resolve()); }); + state.targetUrl = `http://127.0.0.1:${serverPort}`; + await pollUntil({ label: "Actual owned development target readiness", deadlineAt: Date.now() + 120000, + load: async () => { try { const r = await fetch(new URL("/api/health", String(state.targetUrl)), { signal: AbortSignal.timeout(3000) }); return { ok: r.ok, health: await r.json() as { status?: string } }; } catch { return { ok: false, health: {} }; } }, + accept: r => r.ok && r.health.status === "ok", reject: () => o.child?.exitCode != null ? "Owned dev server exited before readiness" : undefined }); state.actualTargetHealthy = true; + const db = createDb(`postgres://paperclip:paperclip@127.0.0.1:${dbPort}/paperclip`, { maxConnections: 1, prepare: false }).$client; + const marker = state.task as Entity; + try { const rows = await db.unsafe("select id, company_id, title, description, status from issues where id = $1", [marker.id]); + state.targetCanaryMatches = rows.length === 1 && rows[0]!.company_id === marker.companyId && rows[0]!.title === marker.title && rows[0]!.description === marker.description && rows[0]!.status === marker.status; + } finally { await db.end({ timeout: 5 }); } + const sourceTask = await ctx.input.api.get(`/api/issues/${marker.id}`); + state.sourceCanaryUnchanged = sourceTask.title === marker.title && sourceTask.description === marker.description && sourceTask.status === marker.status; + await stop(o); state.targetStopped = true; + await command(o, ["worktree:cleanup", o.instance, "--instance", o.instance, "--home", o.home]); + state.targetCleaned = !(await stat(path.join(o.home, "instances", o.instance)).then(() => true, e => { if (e.code === "ENOENT") return false; throw e; })); + state.unrelatedSentinelUnchanged = await readFile(path.join(o.root, "unrelated-owned-sentinel.txt"), "utf8") === ctx.input.nonce; + const receipt = { targetUrl: state.targetUrl, healthyWhenStarted: state.actualTargetHealthy, targetStopped: state.targetStopped, targetCleaned: state.targetCleaned, + seededCanaryMatches: state.targetCanaryMatches, sourceCanaryUnchanged: state.sourceCanaryUnchanged, unrelatedSentinelUnchanged: state.unrelatedSentinelUnchanged, sourceRevision: state.sourceRevision }; + await ctx.input.evidence("chat-catalog-owned-dev-instance.json", receipt); + const message = `The operator used the supported owned development-instance commands and recorded these facts. Please report its URL and observed health/cleanup state:\n\n${JSON.stringify(receipt)}`; + await sendChatMessage(ctx.input.page, message); state.devReport = await companionPhase(ctx, initial.runs.map(r => r.id), message); + }, grade: gradeOwnedDev, async verifyResult(ctx) { await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); }, +}); +dev.expectedRunCount = 2; dev.expectedHumanMessageCount = 2; dev.attemptTimeoutMs = 900000; +dev.cleanup = async ctx => { const o = owned.get(ctx); if (!o) return; await stop(o); if (o.config) await command(o, ["worktree:cleanup", o.instance, "--instance", o.instance, "--home", o.home]); + const root = await realpath(process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT!), directory = await realpath(o.root); + if (!directory.startsWith(root + path.sep) || !path.basename(directory).startsWith("operator-dev-")) throw new Error("Refusing non-owned development cleanup"); + await rm(directory, { recursive: true }); owned.delete(ctx); }; +dev.sourceFiles = [...dev.sourceFiles, "../../cli/src/index.ts", "../../cli/src/commands/worktree.ts", "../../cli/src/commands/run.ts", "../e2e/fetch-allowed-port.ts"]; +export const privilegedOwnedDevCases = [dev]; diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-performance.test.ts b/tests/runner-e2e/chat-catalog-domains/privileged-performance.test.ts index fe27fada76c..2ab2159cc85 100644 --- a/tests/runner-e2e/chat-catalog-domains/privileged-performance.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/privileged-performance.test.ts @@ -29,3 +29,9 @@ it("rejects wrong data size, unrelated phase and unmeasured production throughpu const phase = valid(); phase.state.interpretation.attributed = false; expect(gradeLongThreadPerformance(phase).passed).toBe(false); expect(gradeLongThreadPerformance(null).passed).toBe(false); }); +it("rejects generic fixture advice that describes no observed measurement", () => { + const o = valid(); o.state.interpretation.reply = "This synthetic render fixture cannot establish production throughput or model latency."; + expect(gradeLongThreadPerformance(o).passed).toBe(false); + o.state.interpretation.reply = "The synthetic fixture shows 2 profiler commits. This render observation does not establish production throughput."; + expect(gradeLongThreadPerformance(o).passed).toBe(true); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-performance.ts b/tests/runner-e2e/chat-catalog-domains/privileged-performance.ts index b959f6c435c..ac2df752e1e 100644 --- a/tests/runner-e2e/chat-catalog-domains/privileged-performance.ts +++ b/tests/runner-e2e/chat-catalog-domains/privileged-performance.ts @@ -27,6 +27,9 @@ export function gradeLongThreadPerformance(value: unknown) { "The interpretation is the actual second native reply to the exact persisted human report, not initial guidance or a fabricated result."); const timings = [...reply.matchAll(/\b(\d+(?:\.\d+)?)\s*(?:ms|milliseconds)\b/gi)]; const expected = report ? [report.mountMs, report.latestMs, report.maximumMs, report.totalMs] : []; + const commits = [...reply.matchAll(/\b(\d+)\s+(?:profiler\s+)?commits?\b/gi)]; + c.add("concrete-measured-grounding", !!report && (timings.length > 0 || commits.length > 0) && commits.every(match => Number(match[1]) === report.commitCount), + "The interpretation grounds at least one observed timing or profiler commit count, rather than generic fixture guidance alone."); c.add("timing-claims-grounded", !!report && timings.every(match => { const value = Number(match[1]), decimals = match[1]!.split(".")[1]?.length ?? 0; const prefix = reply.slice(Math.max(0, match.index! - 60), match.index); diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-provision.ts b/tests/runner-e2e/chat-catalog-domains/privileged-provision.ts new file mode 100644 index 00000000000..78e7753a09c --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/privileged-provision.ts @@ -0,0 +1,89 @@ +import { createHash } from "node:crypto"; +import { readFile, realpath, stat } from "node:fs/promises"; +import path from "node:path"; +import { expect } from "@playwright/test"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import { pollUntil } from "../api.js"; +import { extendedHarnessProfiles } from "../catalog.js"; +import { sendChatMessage } from "../chat-flow.js"; +import { checks, type Entity } from "./internal-support.js"; +import type { RuntimeAgent } from "./runtime-support.js"; +import { ownedInstalledTarget } from "./privileged-installed.js"; +import { privilegedDefinition, privilegedObservation, settlePrivilegedChat, companionPhase } from "./privileged-support.js"; +const targets = new WeakMap>>(); +const sha = (b: Buffer) => createHash("sha256").update(b).digest("hex"); +export function gradeProvisionRuntime(value: unknown) { + const c = checks(), o = privilegedObservation(value); c.add("actual-release-provision-evidence", !!o, "Actual installed release asset identity and genuine native Pi execution are required."); if (!o) return c.result(); + c.add("owned-release-matched-assets", o.state.assetsInitiallyAbsent === true && o.state.provisionerSucceeded === true + && typeof o.state.provisionerSha256 === "string" && typeof o.state.nativeClosureSha256 === "string" + && o.state.releaseVersion === o.state.healthVersion && o.state.assetPathOwned === true, + "The public operator provisioner installs into the actual source release's owned server package, preserving exact provisioner and closure identities."); + c.add("qualified-running-pi-agent", o.state.piAgentConfigurationVerified === true && o.state.piRunId != null + && o.state.piArtifactMatches === true && o.runs.some(run => run.id === o.state.piRunId && run.agentId === o.state.piAgentId + && run.status === "succeeded" && run.runtimeMode === "native") && o.runs.length === 3, + "A separately declared Pi worker performs genuine native work with independently read file bytes; download alone never qualifies it."); + const guidance = String(o.state.guidance ?? ""); + c.add("supported-operator-guidance", /runtime\s+setup\s+pi|runtime.*(?:setup|install)/i.test(guidance) && /operator|terminal|admin/i.test(guidance) + && !/curl.*latest|I (?:have |successfully )?installed/i.test(guidance), "The actual initial response uses the supported operator path without arbitrary latest binaries or premature success."); + const phase = o.state.provisionReport as { attributed?: boolean; reply?: string } | undefined; + c.add("truthful-running-result", phase?.attributed === true && /Pi/i.test(phase.reply ?? "") && /run|execut|qualif/i.test(phase.reply ?? ""), + "A separate actual native reply interprets the factual provisioning and real worker receipt."); + return c.result(); +} +const provision = privilegedDefinition("instance-operations.provision-runtime", "privileged-provision", { + async setup(ctx) { + const target = await ownedInstalledTarget(ctx), envName = target.receipt.piCredentialEnv; + if (!envName || !process.env[envName]) throw new Error("Real Pi qualification requires the explicitly declared private OpenRouter credential environment reference"); + const vendor = path.resolve(target.server, "../vendor/paperclip-runner"), assets = path.join(vendor, "provider-assets/pi", `${process.platform}-${process.arch}`); + let exists = false; try { exists = (await stat(assets)).isDirectory(); } catch (e) { if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; } + if (exists) throw new Error("Canonical provision context requires this owned release to lack Pi assets; existing assets are never removed by the fixture"); + const entry = await realpath(path.join(vendor, "cli/provision-pi.cjs")); + if (!entry.startsWith(target.root + path.sep)) throw new Error("Pi provisioner is not inside the intended owned installed server"); + targets.set(ctx, target); + return { releaseVersion: target.receipt.version, assetsInitiallyAbsent: true, provisionerSha256: sha(await readFile(entry)), + piProfile: "runner-acpx-pi", piModel: extendedHarnessProfiles.find(p => p.id === "runner-acpx-pi")!.model }; + }, + async act(ctx, state) { + const initial = await settlePrivilegedChat(ctx); state.guidance = initial.reply; const target = targets.get(ctx)!; + await target.command(["runtime", "setup", "pi"], 900_000); state.provisionerSucceeded = true; + const closure = await realpath(path.resolve(target.server, "../vendor/paperclip-runner/provider-assets/pi", `${process.platform}-${process.arch}`, "native-closure.json")); + state.assetPathOwned = closure.startsWith(target.root + path.sep); if (!state.assetPathOwned) throw new Error("Provisioned Pi closure escapes the intended installation"); + state.nativeClosureSha256 = sha(await readFile(closure)); + // Credentials remain exclusively in the public encrypted secret boundary. + const token = process.env[target.receipt.piCredentialEnv!]!; ctx.input.privateSecrets?.push(token); + const company = ctx.input.fixtures.company.id; + const secret = await ctx.input.api.postSensitive<{ id: string }>(`/api/companies/${company}/secrets`, { name: "Owned Pi qualification", key: "OPENROUTER_API_KEY", value: token }); + const profile = extendedHarnessProfiles.find(p => p.id === "runner-acpx-pi")!; + const config = profile.buildAgent({ environmentId: ctx.input.fixtures.environment.id, environmentFixtureId: "local", workspacePath: ctx.input.workspacePath, + secretRefs: { OPENROUTER_API_KEY: { type: "secret_ref", secretId: secret.id, version: "latest" } }, executionId: ctx.input.nonce }); + const agent = await ctx.input.api.post(`/api/companies/${company}/agents`, { ...config, name: "Pi qualification worker", reportsTo: ctx.input.fixtures.agent.id, + runtimeConfig: { heartbeat: { enabled: false, maxConcurrentRuns: 1 } } }); state.piAgentId = agent.id; + const ready = await pollUntil({ label: "Actual release Pi worker readiness", deadlineAt: Date.now() + 90000, + load: () => ctx.input.api.get(`/api/agents/${agent.id}`), accept: a => a.lifecycleState === "ready" }); + const saved = ready.adapterConfig as { provider?: string; acpxAgent?: string; model?: string }; + state.piAgentConfigurationVerified = ready.adapterType === "paperclip_runner" && saved.provider === "acpx" && saved.acpxAgent === "pi" && saved.model === profile.model; + for (const scope of [{ scopeType: "company", scopeId: company }, { scopeType: "agent", scopeId: agent.id }]) + await ctx.input.api.post(`/api/companies/${company}/budgets/policies`, { ...scope, amount: 500, hardStopEnabled: true, isActive: true, unpricedUsagePolicy: "allow" }); + const marker = `Pi native qualification ${ctx.input.nonce}`; + const task = await ctx.input.api.post(`/api/companies/${company}/issues`, { title: "Owned release Pi readiness", status: "todo", assigneeAgentId: agent.id, + description: `Write pi-qualified.txt in your admitted workspace containing exactly ${JSON.stringify(marker)}, then complete this task. Do not create another task or external side effect.` }); + const result = await pollUntil({ label: "Real release-matched native Pi task", deadlineAt: Date.now() + 180000, + load: async () => ({ task: await ctx.input.api.get(`/api/issues/${task.id}`), runs: await ctx.allRuns() }), + accept: o => o.task.status === "done" && o.runs.length === 2 && o.runs.every(r => r.status === "succeeded"), + reject: o => o.runs.some(r => ["failed", "timed_out", "cancelled"].includes(r.status)) ? "Actual Pi qualification native run failed" : undefined }); + const run = result.runs.find(r => r.agentId === agent.id)!; state.piRunId = run.id; + const file = await ctx.input.api.get<{ content?: { data?: string }; resource?: { workspaceKind?: string } }>(`/api/issues/${task.id}/file-resources/content?workspace=execution&path=pi-qualified.txt`); + state.piArtifactMatches = file.content?.data?.trim() === marker && file.resource?.workspaceKind === "execution_workspace"; + const health = await ctx.input.api.get<{ version?: string }>("/api/health"); state.healthVersion = health.version; + const receipt = { releaseVersion: state.releaseVersion, provisionerSha256: state.provisionerSha256, nativeClosureSha256: state.nativeClosureSha256, + profile: profile.id, model: profile.model, piAgentId: agent.id, piRunId: run.id, piRunStatus: run.status, runtimeMode: run.runtimeMode, + exactFileVerified: state.piArtifactMatches, qualificationScope: "one real native task on the actual owned installed source target" }; + await ctx.input.evidence("chat-catalog-pi-provision-and-native-readiness.json", receipt); + const message = `The operator provisioned the intended installation and ran its separate Pi qualification worker. These are the observed facts; please describe what they establish and their limits:\n\n${JSON.stringify(receipt)}`; + await sendChatMessage(ctx.input.page, message); state.provisionReport = await companionPhase(ctx, result.runs.map(r => r.id), message); + }, grade: gradeProvisionRuntime, async verifyResult(ctx) { await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); }, +}); +provision.expectedRunCount = 3; provision.expectedHumanMessageCount = 2; provision.attemptTimeoutMs = 1_440_000; +provision.coverageGaps = ["installed-source-activation-and-artifact-fingerprint-lane"]; +provision.sourceFiles = [...provision.sourceFiles, "chat-catalog-domains/privileged-installed.ts", "../../cli/src/commands/runtime.ts", "../../packages/paperclip-runner/scripts/provision-pi.mjs"]; +export const privilegedProvisionCases = [provision]; diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-run-cli.ts b/tests/runner-e2e/chat-catalog-domains/privileged-run-cli.ts index 806c94c2e81..492915ef8b6 100644 --- a/tests/runner-e2e/chat-catalog-domains/privileged-run-cli.ts +++ b/tests/runner-e2e/chat-catalog-domains/privileged-run-cli.ts @@ -59,12 +59,13 @@ const inspect = privilegedDefinition("cli-operations.inspect-run-cli", "privileg load: () => observeInternal(ctx, state), accept: o => !!o.reply && ctx.issue()?.conversationState === "waiting" && o.runs.filter(run => run.agentId === ctx.input.fixtures.agent.id && run.status === "succeeded").length === 1 }); const returned = inspectRunCommands(initial.reply, String(state.workerRunId)); - const read = JSON.parse(await runOwnedCli(ctx, [...returned.commands[0]!, ...(!returned.commands[0]!.includes("--json") ? ["--json"] : [])])); + await runOwnedCli(ctx, returned.commands[0]!); + const read = await ctx.input.api.get<{ id: string; agentId: string; companyId: string; status: string }>(`/api/heartbeat-runs/${state.workerRunId}`); state.readRunId = read.id; state.readAgentId = read.agentId; state.readCompanyId = read.companyId; state.readStatus = read.status; if (read.id !== state.workerRunId || read.companyId !== ctx.input.fixtures.company.id || read.agentId !== (state.worker as { id: string }).id) throw new Error("CLI read did not verify the exact owned Morgan run"); if (read.status === "running") { - const result = JSON.parse(await runOwnedCli(ctx, [...returned.commands[1]!, ...(!returned.commands[1]!.includes("--json") ? ["--json"] : [])])); - state.cancelRunId = result.id; state.cancelOnlyAfterRead = true; + await runOwnedCli(ctx, returned.commands[1]!); + state.cancelRunId = read.id; state.cancelOnlyAfterRead = true; } const final = await pollUntil({ label: "Actual owned run cancellation", deadlineAt: Date.now() + 60_000, load: () => ctx.input.api.get<{ id: string; status: string }>(`/api/heartbeat-runs/${state.workerRunId}`), accept: row => row.status === "cancelled" }); diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-support.ts b/tests/runner-e2e/chat-catalog-domains/privileged-support.ts index 4c23608d541..50f4c796063 100644 --- a/tests/runner-e2e/chat-catalog-domains/privileged-support.ts +++ b/tests/runner-e2e/chat-catalog-domains/privileged-support.ts @@ -1,6 +1,6 @@ import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js"; import { pollUntil } from "../api.js"; -import { observeInternal, observation, type InternalObservation } from "./internal-support.js"; +import { observeInternal, observation, type Entity, type InternalObservation } from "./internal-support.js"; /** These cases exercise only the launcher-owned instance or a rendered developer fixture. */ export function privilegedDefinition(scenarioId: string, module: string, extra: Pick): ChatCatalogCase { @@ -16,3 +16,19 @@ export async function settlePrivilegedChat(ctx: ChatCatalogContext): Promise o.runs.some(run => ["failed", "timed_out", "cancelled"].includes(run.status)) ? "Canonical guidance run failed" : undefined }); } export const privilegedObservation = observation; + +/** Bind a declared factual operator handoff to its actual persisted human message and native reply. */ +export async function companionPhase(ctx: ChatCatalogContext, priorRunIds: string[], message: string) { + const final = await pollUntil({ label: "Native reply to the actual operator receipt", deadlineAt: Date.now() + 150_000, + load: async () => { await ctx.refreshIssue(); return { issue: ctx.issue(), runs: await ctx.allRuns() }; }, + accept: o => o.issue?.conversationState === "waiting" && o.runs.length === priorRunIds.length + 1 && o.runs.every(run => run.status === "succeeded") }); + const run = final.runs.find(run => !priorRunIds.includes(run.id) && run.agentId === ctx.input.fixtures.agent.id); + if (!run) throw new Error("Operator receipt did not produce its own native reply"); + const comments = await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/comments?order=asc`); + const human = comments.filter(comment => !comment.authorAgentId && comment.body === message); + const replies = comments.filter(comment => comment.authorAgentId === ctx.input.fixtures.agent.id && comment.createdByRunId === run.id); + const profile = await ctx.input.api.get<{ runnerProfileJson?: { nativeExecutionInput?: { task?: { prompt?: string } } } }>(`/api/heartbeat-runs/${run.id}`); + return { humanCommentId: human[0]?.id, runId: run.id, reply: replies.map(row => row.body).join("\n"), + attributed: human.length === 1 && replies.length > 0 && profile.runnerProfileJson?.nativeExecutionInput?.task?.prompt?.includes(human[0]!.id) === true + && replies.every(reply => Date.parse(reply.createdAt!) >= Date.parse(human[0]!.createdAt!)) }; +} diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-update.ts b/tests/runner-e2e/chat-catalog-domains/privileged-update.ts new file mode 100644 index 00000000000..31b847b68d9 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/privileged-update.ts @@ -0,0 +1,60 @@ +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import { expect } from "@playwright/test"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import { sendChatMessage } from "../chat-flow.js"; +import { checks } from "./internal-support.js"; +import { ownedInstalledTarget } from "./privileged-installed.js"; +import { privilegedDefinition, privilegedObservation, settlePrivilegedChat, companionPhase } from "./privileged-support.js"; +const targets = new WeakMap>>(); +export function gradeInstalledUpdate(value: unknown) { + const c = checks(), o = privilegedObservation(value); c.add("installed-update-evidence", !!o, "Actual pinned source installation and supported operator result are required."); if (!o) return c.result(); + c.add("exact-installed-operation", o.state.installMode === "global-npm" && o.state.registryIntegrityMatched === true + && o.state.updateExecuted === true && o.state.installedVersion === o.state.targetVersion && o.state.healthVersion === o.state.targetVersion + && o.state.healthReady === true && o.state.serviceRestarted === true && typeof o.state.finalCliSha256 === "string", + "The public updater changes only the owned installed prefix to its exact published integrity and the restarted source service reports that version ready."); + c.add("channel-and-rollback-guidance", /install|channel|npm|managed/i.test(String(o.state.guidance)) && /operator|admin|terminal/i.test(String(o.state.guidance)) + && !/I (?:have |successfully )?updated|newest version is|latest version is/i.test(String(o.state.guidance)) + && o.state.rollbackVersion === o.state.originalVersion, "The initial answer assigns installation authority correctly; the concrete prior-version reinstall is retained for this npm channel."); + const report = o.state.updateReport as { attributed?: boolean; reply?: string } | undefined; + c.add("native-result-report", report?.attributed === true && !!o.state.targetVersion && report.reply?.includes(String(o.state.targetVersion)) === true + && /health|ready|restart/i.test(report.reply), "The actual follow-up native reply interprets the real version and service receipt."); + return c.result(); +} +const update = privilegedDefinition("instance-operations.update-instance", "privileged-update", { + async setup(ctx) { const target = await ownedInstalledTarget(ctx); const r = target.receipt; + if (!r.targetVersion || !r.targetIntegrity || r.targetVersion === r.version) throw new Error("Installed update requires distinct exact supported targetVersion and published sha512 targetIntegrity"); + // Require the actual npm global layout; managed/source installs use different contracts. + if (!target.cli.includes("/lib/node_modules/paperclipai/dist/index.js")) throw new Error("This authored update lane requires an owned global-npm installation"); + targets.set(ctx, target); return { originalVersion: r.version, targetVersion: r.targetVersion, rollbackVersion: r.version, + originalCliSha256: r.cliSha256, sourceServerSha256: r.serverSha256, installationOwned: true }; }, + async act(ctx, state) { const initial = await settlePrivilegedChat(ctx); state.guidance = initial.reply; + const target = targets.get(ctx)!; + const check = JSON.parse(await target.command(["update", "--version", target.receipt.version, "--check", "--json"])); + state.installMode = check.mode; + if (check.mode !== "global-npm") throw new Error("Public updater does not identify the expected npm channel"); + // npm independently verifies registry integrity during its actual install; retain the pinned expected identity too. + const { execFile } = await import("node:child_process"); const { promisify } = await import("node:util"); + const metadata = JSON.parse((await promisify(execFile)("npm", ["view", `paperclipai@${target.receipt.targetVersion}`, "dist.integrity", "--json", "--registry=https://registry.npmjs.org"], + { env: { ...target.environment, NPM_CONFIG_USERCONFIG: "/dev/null" }, timeout: 30000, maxBuffer: 65536 })).stdout); + state.registryIntegrityMatched = metadata === target.receipt.targetIntegrity; + if (!state.registryIntegrityMatched) throw new Error("Published target integrity differs from the explicit supported artifact pin"); + const output = await target.command(["update", "--version", target.receipt.targetVersion!, "--yes", "--json"]); + const start = output.lastIndexOf("\n{"); const result = JSON.parse(start >= 0 ? output.slice(start + 1) : output); + state.updateExecuted = result.action === "update" && result.targetVersion === target.receipt.targetVersion; + const manifest = JSON.parse(await readFile(new URL("../package.json", `file://${target.cli}`), "utf8")); + state.installedVersion = manifest.version; state.finalCliSha256 = createHash("sha256").update(await readFile(target.cli)).digest("hex"); + await ctx.input.restart(); state.serviceRestarted = true; + const health = await ctx.input.api.get<{ version?: string; status?: string }>("/api/health"); state.healthVersion = health.version; state.healthReady = health.status === "ok"; + const receipt = { originalVersion: state.originalVersion, installedVersion: state.installedVersion, healthVersion: state.healthVersion, + healthReady: state.healthReady, installationChannel: "global-npm", rollbackCommand: `npm install -g paperclipai@${state.originalVersion} (same owned prefix only)`, registryIntegrityMatched: state.registryIntegrityMatched }; + await ctx.input.evidence("chat-catalog-installed-update.json", receipt); + await ctx.input.page.goto(`${ctx.input.api.baseURL}/${ctx.input.fixtures.company.issuePrefix}/chats/${ctx.input.fixtures.agent.id}`); + const message = `The operator applied the owned installation update and recorded these facts. Please describe the observed result and limits:\n\n${JSON.stringify(receipt)}`; + await sendChatMessage(ctx.input.page, message); state.updateReport = await companionPhase(ctx, initial.runs.map(run => run.id), message); + }, grade: gradeInstalledUpdate, async verifyResult(ctx) { await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); }, +}); +update.coverageGaps = ["installed-source-activation-and-artifact-fingerprint-lane"]; +update.expectedRunCount = 2; update.expectedHumanMessageCount = 2; update.attemptTimeoutMs = 720_000; +update.sourceFiles = [...update.sourceFiles, "chat-catalog-domains/privileged-installed.ts", "../../cli/src/commands/update.ts", "installed-release.ts"]; +export const privilegedUpdateCases = [update];