From f6b2d50c845e5582d29c2757cd26d2cc0a035b19 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 27 Oct 2026 01:58:20 -0500 Subject: [PATCH] Author owned lost-sandbox reconstruction chat evaluation --- ...untime-daytona-destroy-canary-receipt.json | 26 +++++ .../runtime-daytona-destroy-privacy.spec.ts | 16 +++ .../runtime-daytona-destroy-worker.ts | 40 +++++++ .../runtime-daytona-destroy.test.ts | 12 ++ .../runtime-daytona-destroy.ts | 36 ++++++ .../runtime-lost-sandbox.test.ts | 13 +++ .../runtime-lost-sandbox.ts | 105 ++++++++++++++++++ 7 files changed, 248 insertions(+) create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-daytona-destroy-canary-receipt.json create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-daytona-destroy-privacy.spec.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-daytona-destroy-worker.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-daytona-destroy.test.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-daytona-destroy.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-lost-sandbox.test.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-lost-sandbox.ts diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-daytona-destroy-canary-receipt.json b/tests/runner-e2e/chat-catalog-domains/runtime-daytona-destroy-canary-receipt.json new file mode 100644 index 00000000000..b840046a977 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-daytona-destroy-canary-receipt.json @@ -0,0 +1,26 @@ +{ + "syntheticOnly": true, + "tests": 2, + "childFailureAndExpectedParentFailure": true, + "invalidOwnedBindingRejectedBeforeHttp": true, + "realProviderCalls": 0, + "canonicalAttempts": 0, + "browserLaunches": 0, + "qualificationClaim": "private child/reporting failure boundary only; actual provider deletion and positive provider receipt unqualified", + "checkedSinks": [ + "stdout", + "JUnit", + "blob ZIP entries", + "HTML embedded report ZIP", + "error-context" + ], + "scannedItems": 12, + "privateValueCount": 4, + "findings": [], + "sourceSha256": { + "runtime-daytona-destroy.ts": "59a90160ced0df51e8afdfaea8c5e6203c36f71acf0f46b2a3fd45227a913943", + "runtime-daytona-destroy-worker.ts": "7fb18291c9c7969bad3642938a323854eaa62fe35c33e029c77441eecefd543a", + "runtime-daytona-destroy-privacy.spec.ts": "2c68d01edb27a99bffd84d165bbfbff26d40345b32d16af3d6565113eb8309f4" + }, + "configSha256": "b4de9cfed9d4cf51b035988d5177ceab1b02cac518809cef869cc7e3cda0b071" +} diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-daytona-destroy-privacy.spec.ts b/tests/runner-e2e/chat-catalog-domains/runtime-daytona-destroy-privacy.spec.ts new file mode 100644 index 00000000000..db492ae7fd1 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-daytona-destroy-privacy.spec.ts @@ -0,0 +1,16 @@ +/** Opt-in synthetic retention calibration only. Invalid ownership stops before + * HTTP/provider access; no real key, resource, DB or canonical admission. */ +import { randomUUID } from "node:crypto"; +import { appendFile } from "node:fs/promises"; +import { test, expect } from "@playwright/test"; +import { destroyOwnedFailedSandbox } from "./runtime-daytona-destroy.js"; +for (const branch of ["child-failure", "parent-failure"] as const) test(`private owned sandbox fault reporting ${branch}`, async () => { + test.skip(!process.env.PAPERCLIP_DAYTONA_FAULT_CANARY_PRIVATE_FILE, "Explicit synthetic-only private provider child calibration"); + const credential = `synthetic-provider-key-${randomUUID()}`, cookie = `synthetic-session-${randomUUID()}`; + await appendFile(process.env.PAPERCLIP_DAYTONA_FAULT_CANARY_PRIVATE_FILE!, `${credential}\n${cookie}\n`, { mode: 0o600 }); + const secrets: string[] = []; + const result = destroyOwnedFailedSandbox({ scenarioId: "workspaces.recover-lost-sandbox", executionId: "chat-features.runner-codex-subscription.local.workspaces.recover-lost-sandbox", binding: { companyId: "intentionally-invalid-owned-context", environmentId: "invalid", taskId: "invalid", runId: "invalid", leaseId: "invalid", sandboxId: "invalid", actionId: "invalid", attempt: "synthetic" }, baseURL: "http://127.0.0.1:1", credential, cookies: [{ name: "session", value: cookie, domain: "127.0.0.1", path: "/", expires: -1, httpOnly: true, secure: false, sameSite: "Lax" }] }, secrets); + await expect(result).rejects.toThrow("private provider request/diagnostics withheld"); + expect(secrets.length).toBe(2); + if (branch === "parent-failure") { test.fail(); throw new Error("Deliberate safe parent failure after isolated synthetic provider child rejection"); } +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-daytona-destroy-worker.ts b/tests/runner-e2e/chat-catalog-domains/runtime-daytona-destroy-worker.ts new file mode 100644 index 00000000000..397fce08b1d --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-daytona-destroy-worker.ts @@ -0,0 +1,40 @@ +import { createRequire } from "node:module"; +import { pathToFileURL } from "node:url"; +import { readFile, lstat } from "node:fs/promises"; +import { test, type Cookie } from "@playwright/test"; +import type { Environment, EnvironmentLease } from "../../../packages/shared/src/types/environment.js"; +import type { IssueRecoveryAction } from "../../../packages/shared/src/types/issue.js"; +import type { NativeFaultRun } from "./runtime-native-fault.js"; +import { createRemoteFixtureClient } from "../remote-native-bootstrap.js"; +import { providerSandboxIsOwned, type DestroySandboxBinding, type DestroySandboxReceipt } from "./runtime-daytona-destroy.js"; +const record = (v: unknown): Record => v && typeof v === "object" ? v as Record : {}; +async function main(): Promise { + let attached = false; try { test.info(); attached = true; } catch { /* standalone child only */ } if (attached) throw new Error("Private provider fault must not execute in Test"); + const file = process.argv[2]!; const info = await lstat(file); if (!info.isFile() || info.mode & 0o077 || info.uid !== process.getuid?.() || info.size > 65536) throw new Error("Invalid private provider fault request"); + const input = JSON.parse(await readFile(file, "utf8")) as { scenarioId: string; executionId: string; binding: DestroySandboxBinding; baseURL: string; cookies: Cookie[]; credential: string }; + const base = new URL(input.baseURL), b = input.binding; + if (input.scenarioId !== "workspaces.recover-lost-sandbox" || !/^chat-features\.[a-z0-9-]+\.local\.workspaces\.recover-lost-sandbox$/.test(input.executionId) || base.protocol !== "http:" || base.hostname !== "127.0.0.1" || base.origin !== input.baseURL || ![b.companyId, b.environmentId, b.taskId, b.runId, b.leaseId, b.sandboxId, b.actionId].every(id => /^[a-f0-9-]{36}$/i.test(id)) || !/^[a-zA-Z0-9_-]{1,160}$/.test(b.attempt)) throw new Error("Provider fault requires exact owned loopback fixture identities"); + async function get(pathname: string): Promise { const response = await fetch(base.origin + pathname, { headers: { Cookie: input.cookies.map(row => `${row.name}=${row.value}`).join("; "), Origin: base.origin }, redirect: "error", signal: AbortSignal.timeout(15000) }); if (!response.ok) { await response.body?.cancel(); throw new Error("Exact provider fault public ownership check failed"); } return response.json() as Promise; } + const check = async () => { + const [environment, listed, lease, leases, run, recovery] = await Promise.all([get(`/api/environments/${b.environmentId}`), get(`/api/companies/${b.companyId}/environments`), get(`/api/environment-leases/${b.leaseId}`), get(`/api/environments/${b.environmentId}/leases`), get(`/api/heartbeat-runs/${b.runId}`), get<{ actions: IssueRecoveryAction[] }>(`/api/issues/${b.taskId}/recovery-actions`)]); + const ref = record(record(run.runnerProfileJson).nativeWorkspaceSync), action = recovery.actions.find(row => row.id === b.actionId); + if (environment.config.provider !== "daytona" || environment.metadata?.attempt !== b.attempt || environment.metadata.companyId !== b.companyId || !listed.some(row => row.id === environment.id) + || lease.companyId !== b.companyId || lease.environmentId !== b.environmentId || lease.issueId !== b.taskId || lease.heartbeatRunId !== b.runId || lease.providerLeaseId !== b.sandboxId || lease.provider !== "daytona" || lease.metadata?.pendingCleanupInFlight === true + || leases.some(row => row.id !== lease.id && row.providerLeaseId === b.sandboxId && ["active", "retained", "pending_cleanup"].includes(row.status)) + || run.companyId !== b.companyId || run.nativeIssueId !== b.taskId || run.runtimeMode !== "native" || run.status !== "failed" || run.nativePhase !== "terminal_failure" || ref.state !== "prepared" || ref.leaseId !== b.leaseId || ref.providerLeaseId !== b.sandboxId + || !action || action.ownerType !== "board" || !["active", "escalated"].includes(action.status) || action.evidence.runId !== b.runId || !["native_workspace_sync_out_unrecoverable", "native_workspace_sync_out_retry_exhausted"].includes(action.cause)) throw new Error("Refusing sandbox mutation without exact quiescent failed native source/board-owned scope"); + }; + await check(); + const client = await createRemoteFixtureClient(input.credential) as unknown as { get(id: string): Promise<{ id: string; name: string; labels: Record; refreshData(): Promise; delete(timeout: number, force: boolean): Promise }> }; + const sandbox = await client.get(b.sandboxId); await sandbox.refreshData(); if (!providerSandboxIsOwned(sandbox, b)) throw new Error("Actual provider labels/name do not bind the owned failed sandbox"); + const ownership = sandbox.labels["paperclip-create-attempt"]; + await check(); await sandbox.refreshData(); if (!providerSandboxIsOwned(sandbox, b) || sandbox.labels["paperclip-create-attempt"] !== ownership) throw new Error("Provider sandbox ownership changed before exact deletion"); + await sandbox.delete(60, true); + const require = createRequire(new URL("../../../packages/plugins/sandbox-providers/daytona/package.json", import.meta.url)); + const { DaytonaNotFoundError } = await import(pathToFileURL(require.resolve("@daytonaio/sdk")).href); + let absent = false; + try { await client.get(b.sandboxId); } catch (error) { absent = error instanceof DaytonaNotFoundError; } + if (!absent) throw new Error("Actual provider did not independently confirm the exact destroyed sandbox absent"); + return { ...b, processIsolation: true, testRunnerAttached: false, apiStepRecorder: false, actualOwnedProviderLabels: true, actualDeleteAccepted: true, independentlyAbsent: true }; +} +main().then(value => process.stdout.write(JSON.stringify(value))).catch(() => { process.stderr.write("Exact owned sandbox fault failed; private diagnostics withheld"); process.exitCode = 1; }); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-daytona-destroy.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-daytona-destroy.test.ts new file mode 100644 index 00000000000..9995241c381 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-daytona-destroy.test.ts @@ -0,0 +1,12 @@ +import { expect, it } from "vitest"; +import { providerSandboxIsOwned, validateDestroyedSandboxReceipt, type DestroySandboxBinding } from "./runtime-daytona-destroy.js"; +const binding: DestroySandboxBinding = { companyId: "company", environmentId: "environment", taskId: "website", runId: "run", leaseId: "lease", sandboxId: "sandbox", actionId: "action", attempt: "attempt" }; +const attempt = "12345678-1234-1234-1234-123456789012"; +const sandbox = { id: "sandbox", name: `paperclip-create-${attempt}`, labels: { "paperclip-provider": "daytona", "paperclip-company-id": "company", "paperclip-environment-id": "environment", "paperclip-run-id": "run", "paperclip-create-attempt": attempt, "paperclip-reuse-lease": "false" } }; +it("binds only the actual newly created failed-run provider allocation", () => expect(providerSandboxIsOwned(sandbox, binding)).toBe(true)); +it("never adopts an unowned, reused or renamed sandbox", () => { for (const key of Object.keys(sandbox.labels)) expect(providerSandboxIsOwned({ ...sandbox, labels: { ...sandbox.labels, [key]: "foreign" } }, binding)).toBe(false); expect(providerSandboxIsOwned({ ...sandbox, id: "another" }, binding)).toBe(false); expect(providerSandboxIsOwned({ ...sandbox, name: "preexisting-resource" }, binding)).toBe(false); }); +it("requires exact destruction and independent actual absence receipts without secret fields", () => { + const receipt = { ...binding, processIsolation: true, testRunnerAttached: false, apiStepRecorder: false, actualOwnedProviderLabels: true, actualDeleteAccepted: true, independentlyAbsent: true }; + expect(validateDestroyedSandboxReceipt(receipt, binding)).toEqual(receipt); + for (const mutation of [{ independentlyAbsent: false }, { actualDeleteAccepted: false }, { runId: "other" }, { testRunnerAttached: true }, { token: "synthetic-private-token" }]) expect(() => validateDestroyedSandboxReceipt({ ...receipt, ...mutation }, binding)).toThrow("closed independent receipt"); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-daytona-destroy.ts b/tests/runner-e2e/chat-catalog-domains/runtime-daytona-destroy.ts new file mode 100644 index 00000000000..781aff95a45 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-daytona-destroy.ts @@ -0,0 +1,36 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { mkdtemp, chmod, writeFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import type { Cookie } from "@playwright/test"; +export interface DestroySandboxBinding { companyId: string; environmentId: string; taskId: string; runId: string; leaseId: string; sandboxId: string; actionId: string; attempt: string } +export interface DestroySandboxReceipt extends DestroySandboxBinding { processIsolation: true; testRunnerAttached: false; apiStepRecorder: false; actualOwnedProviderLabels: true; actualDeleteAccepted: true; independentlyAbsent: true } +export function providerSandboxIsOwned(sandbox: { id?: string; name?: string; labels?: Record }, binding: DestroySandboxBinding) { + const labels = sandbox.labels, attempt = labels?.["paperclip-create-attempt"]; + return sandbox.id === binding.sandboxId && !!attempt && /^[a-f0-9-]{36}$/i.test(attempt) && sandbox.name === `paperclip-create-${attempt}` && labels?.["paperclip-provider"] === "daytona" + && labels["paperclip-company-id"] === binding.companyId && labels["paperclip-environment-id"] === binding.environmentId && labels["paperclip-run-id"] === binding.runId && labels["paperclip-reuse-lease"] === "false"; +} +export function validateDestroyedSandboxReceipt(value: unknown, expected: DestroySandboxBinding): DestroySandboxReceipt { + const r = value as DestroySandboxReceipt; + const keys = [...Object.keys(expected), "processIsolation", "testRunnerAttached", "apiStepRecorder", "actualOwnedProviderLabels", "actualDeleteAccepted", "independentlyAbsent"]; + if (!r || Object.keys(r).sort().join() !== keys.sort().join() || Object.entries(expected).some(([key, value]) => r[key as keyof DestroySandboxReceipt] !== value) + || r.processIsolation !== true || r.testRunnerAttached !== false || r.apiStepRecorder !== false || r.actualOwnedProviderLabels !== true || r.actualDeleteAccepted !== true || r.independentlyAbsent !== true) throw new Error("Exact owned sandbox destruction did not return its closed independent receipt"); + return r; +} +export async function destroyOwnedFailedSandbox(input: { scenarioId: "workspaces.recover-lost-sandbox"; executionId: string; binding: DestroySandboxBinding; baseURL: string; cookies: Cookie[]; credential: string }, privateSecrets: string[]): Promise { + if (input.credential.length < 12) throw new Error("The exact selected Daytona key is required for the owned sandbox fault"); + privateSecrets.push(input.credential, ...input.cookies.map(row => row.value)); + const directory = await mkdtemp(path.join(tmpdir(), "paperclip-owned-sandbox-fault-")); await chmod(directory, 0o700); + try { + const request = path.join(directory, "request.json"); await writeFile(request, JSON.stringify(input), { flag: "wx", mode: 0o600 }); + const loader = fileURLToPath(new URL("../../../server/node_modules/tsx/dist/loader.mjs", import.meta.url)), worker = fileURLToPath(new URL("./runtime-daytona-destroy-worker.ts", import.meta.url)); + let output: string; + try { output = (await promisify(execFile)(process.execPath, ["--import", loader, worker, request], { timeout: 120000, maxBuffer: 16384, env: { PATH: process.env.PATH ?? "", HOME: directory, TMPDIR: directory, NODE_ENV: "test", DO_NOT_TRACK: "1" } })).stdout; } + catch { throw new Error("Owned sandbox fault failed; private provider request/diagnostics withheld"); } + if (privateSecrets.some(secret => secret && [secret, encodeURIComponent(secret), Buffer.from(secret).toString("base64")].some(value => output.includes(value)))) throw new Error("Owned sandbox fault exceeded its private receipt boundary; output withheld"); + let receipt: unknown; try { receipt = JSON.parse(output); } catch { throw new Error("Owned sandbox fault returned no closed receipt"); } + return validateDestroyedSandboxReceipt(receipt, input.binding); + } finally { await rm(directory, { recursive: true, force: true }); } +} diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-lost-sandbox.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-lost-sandbox.test.ts new file mode 100644 index 00000000000..f59d260989c --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-lost-sandbox.test.ts @@ -0,0 +1,13 @@ +import { expect, it } from "vitest"; +import type { EnvironmentLease } from "../../../packages/shared/src/types/environment.js"; +import { destroyedLeaseBound, gradeLostSandbox, lostSandboxGuidance } from "./runtime-lost-sandbox.js"; +function valid() { return { companyId: "company", leadId: "ceo", scenarioId: "workspaces.recover-lost-sandbox", origin: "http://127.0.0.1:42123", reply: "", links: [], agents: [], humanQueries: [], activity: [], tasks: [{ id: "website" }], runs: [{ id: "old", status: "failed" }, { id: "canonical", status: "succeeded" }, { id: "reconstruction", status: "succeeded" }, { id: "report", status: "succeeded" }], state: { task: { id: "website" }, exactProviderDestroyed: true, platformDestructionBound: true, survivingAttachmentVerified: true, noReconstructionBeforeHuman: true, guidanceReply: "The original sandbox was destroyed. A board member can use Continue from saved sources with the surviving attachments to reconstruct the page; missing unregistered files are not recovered.", actualReconstructionUi: true, actualReconstructionStatus: 202, humanDecisionBound: true, missingWorkAcknowledged: true, freshSessionBound: true, decisionSourcesBound: true, reconstructedArtifactVerified: true, newerWorkPreserved: true, originalFailureRetained: true, originalEventsRetained: true, confirmationBound: true, confirmationReply: "The website was reconstructed from the surviving attachment and the newer edits were preserved. The original run remains failed; missing unregistered state from the destroyed workspace is still unavailable." } }; } +it("accepts real authorized reconstruction while preserving the old failure", () => expect(gradeLostSandbox(valid()).passed).toBe(true)); +it("accepts a natural saved-source guidance and factual result paraphrase", () => { const o = valid(); o.state.guidanceReply = "You need to choose the saved attachments and acknowledge the lost workspace state before rebuilding. An operator owns that decision."; o.state.confirmationReply = "Only the attached source was used to rebuild the page; your newer files were preserved separately. Unattached work cannot be restored from the destroyed allocation."; expect(gradeLostSandbox(o).passed).toBe(true); }); +it("rejects SDK absence alone without independent platform evidence", () => { const o = valid(); o.state.platformDestructionBound = false; expect(gradeLostSandbox(o).passed).toBe(false); }); +it("rejects an API-only substitute for the real routed human decision", () => { const o = valid(); o.state.actualReconstructionUi = false; expect(gradeLostSandbox(o).passed).toBe(false); }); +it("rejects wrong decision source, stale session, missing newer edits and wrong reply phase", () => { for (const key of ["decisionSourcesBound", "freshSessionBound", "newerWorkPreserved", "confirmationBound", "humanDecisionBound"] as const) { const o = valid(); o.state[key] = false; expect(gradeLostSandbox(o).passed).toBe(false); } }); +it("rejects erasing the failed source or claiming all lost files recovered", () => { const o = valid(); o.runs[0]!.status = "succeeded"; expect(gradeLostSandbox(o).passed).toBe(false); const p = valid(); p.state.confirmationReply += " All lost work was recovered."; expect(gradeLostSandbox(p).passed).toBe(false); expect(lostSandboxGuidance("I already reconstructed all lost files from that destroyed sandbox.")).toBe(false); }); +it("requires the exact persisted destroyed termination receipt and no ongoing cleanup", () => { const binding = { companyId: "company", taskId: "website", runId: "old", leaseId: "lease", sandboxId: "sandbox", environmentId: "environment" }; const lease = { id: "lease", companyId: "company", issueId: "website", heartbeatRunId: "old", environmentId: "environment", provider: "daytona", providerLeaseId: "sandbox", metadata: { remoteExecutionTermination: { state: "destroyed" }, pendingCleanupInFlight: false } } as unknown as EnvironmentLease; expect(destroyedLeaseBound(lease, binding)).toBe(true); expect(destroyedLeaseBound({ ...lease, providerLeaseId: "another" }, binding)).toBe(false); expect(destroyedLeaseBound({ ...lease, metadata: { remoteExecutionTermination: { state: "stopped" } } }, binding)).toBe(false); expect(destroyedLeaseBound({ ...lease, metadata: { ...lease.metadata, pendingCleanupInFlight: true } }, binding)).toBe(false); }); + +it("rejects guidance that waives the actual human reconstruction decision", () => { expect(lostSandboxGuidance("The sandbox was destroyed; you can reconstruct from saved attachments without any board approval.")).toBe(false); expect(lostSandboxGuidance("You cannot rebuild the lost workspace without human approval of the saved attachments.")).toBe(true); }); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-lost-sandbox.ts b/tests/runner-e2e/chat-catalog-domains/runtime-lost-sandbox.ts new file mode 100644 index 00000000000..1a139d69b24 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-lost-sandbox.ts @@ -0,0 +1,105 @@ +import { randomUUID, createHash } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { expect } from "@playwright/test"; +import type { IssueRecoveryAction } from "../../../packages/shared/src/types/issue.js"; +import type { EnvironmentLease } from "../../../packages/shared/src/types/environment.js"; +import type { NativeWorkspaceSyncReference } from "../../../server/src/services/native-runtime/native-workspace-sync.js"; +import { pollUntil } from "../api.js"; +import { sendChatMessage } from "../chat-flow.js"; +import { readRegisteredArtifacts } from "../registered-artifact.js"; +import { collectRunEvents } from "../run-observations.js"; +import { checks, type Entity } from "./internal-support.js"; +import { runtimeDefinition, runtimeObservation, observeRuntime } from "./runtime-support.js"; +import { prepareFailedWebsiteExport, cleanupFailedWebsiteExport } from "./runtime-workspace-repair.js"; +import { destroyOwnedFailedSandbox } from "./runtime-daytona-destroy.js"; +import { selectedRuntimeInputs, runtimePrivateSpec } from "./runtime-private.js"; +import type { NativeFaultRun } from "./runtime-native-fault.js"; +import type { OwnedDaytonaTarget } from "./runtime-daytona-target.js"; +const record = (value: unknown): Record => value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; +const digest = (value: unknown) => createHash("sha256").update(JSON.stringify(value)).digest("hex"); +type RecoveryList = { active?: IssueRecoveryAction; actions: IssueRecoveryAction[] }; +export function destroyedLeaseBound(lease: EnvironmentLease, input: { companyId: string; taskId: string; runId: string; leaseId: string; sandboxId: string; environmentId: string }) { + const termination = record(lease.metadata?.remoteExecutionTermination); + return lease.id === input.leaseId && lease.companyId === input.companyId && lease.issueId === input.taskId && lease.heartbeatRunId === input.runId && lease.environmentId === input.environmentId && lease.provider === "daytona" && lease.providerLeaseId === input.sandboxId && termination.state === "destroyed" && lease.metadata?.pendingCleanupInFlight !== true; +} +export function lostSandboxGuidance(reply: string) { return /saved (?:sources?|attachments?)|surviv(?:ing|ed).*(?:files?|attachments?)|attached.*(?:files?|source)/i.test(reply) && /human|board|operator|you (?:can|must|need)/i.test(reply) && /reconstruct|rebuild|continue/i.test(reply) && /destroy|lost|missing|unrecover|cannot|can't|not recover/i.test(reply) && !/(?:human|board|operator) (?:approval|decision|action) (?:is |was )?(?:not (?:needed|required)|unnecessary)|\b(?:can|will)\b[^.!?\n]{0,100}\b(?:rebuild|reconstruct|continue)\b[^.!?\n]{0,60}\bwithout (?:any )?(?:human|board|operator) (?:approval|decision|action)/i.test(reply) && !/(?:I(?:'ve| have)?|already|successfully)\s+(?:recovered|restored|reconstructed)|all.*lost.*(?:recovered|restored)/i.test(reply); } +export function gradeLostSandbox(value: unknown) { + const c = checks(), o = runtimeObservation(value); c.add("real-historical-failure-destroyed-source-and-survivors", !!o, "A real failed export, actual exact provider destruction, independent platform proof and durable attributed survivors are required."); if (!o) return c.result(); const s = o.state; + c.add("exact-destroyed-platform-allocation-and-surviving-source", s.exactProviderDestroyed === true && s.platformDestructionBound === true && s.survivingAttachmentVerified === true && s.noReconstructionBeforeHuman === true && lostSandboxGuidance(String(s.guidanceReply)), "Chat identifies the actual saved sources and destroyed original; no agent or fixture may reconstruct before the authorized human decision."); + c.add("actual-routed-human-decision-with-explicit-known-losses", s.actualReconstructionUi === true && s.actualReconstructionStatus === 202 && s.humanDecisionBound === true && s.missingWorkAcknowledged === true, "The actual routed Continue from saved sources control selects the actual attachment, records losses/previous external effects and explicit human acknowledgement."); + c.add("fresh-supported-admission-consumes-actual-decision", s.freshSessionBound === true && s.decisionSourcesBound === true && s.reconstructedArtifactVerified === true && s.newerWorkPreserved === true, "Exactly one fresh native worker session is admitted through normal gates with real source receipts; its actual reconstructed page is grounded in the selected surviving bytes while newer destination work remains intact."); + c.add("old-failure-events-and-no-silent-loss-recovery", s.originalFailureRetained === true && s.originalEventsRetained === true && s.confirmationBound === true && /saved|surviv|attach|source/i.test(String(s.confirmationReply)) && /reconstruct|rebuilt|recover|preserv/i.test(String(s.confirmationReply)) && /lost|missing|unattached|cannot|can't|only.*(?:source|attachment)/i.test(String(s.confirmationReply)) && !/all.*lost.*(?:recovered|restored)|nothing (?:was |is )?lost/i.test(String(s.confirmationReply)), "The final own-phase reply distinguishes reconstruction from unsupported recovery; the old failed native run and immutable original events survive."); + c.add("exact-native-and-task-scope", o.tasks.length === 1 && o.tasks[0]?.id === (s.task as Entity).id && o.runs.length === 4 && o.runs.filter(row => row.status === "failed").length === 1 && o.runs.filter(row => row.status === "succeeded").length === 3, "One original failed source, one reconstruction worker and two explicit controller turns are the entire native history."); + return c.result(); +} +export const runtimeLostSandboxCases = [runtimeDefinition("workspaces.recover-lost-sandbox", async ctx => { + try { + const s = await prepareFailedWebsiteExport(ctx), selected = selectedRuntimeInputs(ctx, "workspaces.recover-lost-sandbox"), ref = s.reference, companyId = ctx.input.fixtures.company.id; + if (!ctx.input.privateSecrets) throw new Error("Exact provider fault requires the retained private secret sink"); + const destruction = await destroyOwnedFailedSandbox({ scenarioId: "workspaces.recover-lost-sandbox", executionId: ctx.input.execution.id, binding: { companyId, environmentId: s.target.environmentId, taskId: s.task.id, runId: s.originalRun.id, leaseId: ref.leaseId, sandboxId: ref.providerLeaseId, actionId: s.action.id, attempt: ctx.input.nonce }, baseURL: ctx.input.api.baseURL, cookies: await ctx.input.page.context().cookies(ctx.input.api.baseURL), credential: selected.credentials.DAYTONA_API_KEY! }, ctx.input.privateSecrets); + // The product must independently prove absence/termination itself. The + // standalone SDK delete receipt never substitutes for platform evidence. + await ctx.input.api.post(`/api/issues/${s.task.id}/recovery-actions/recover-workspace`, { actionId: s.action.id, runId: s.originalRun.id, idempotencyKey: randomUUID() }); + const gone = await pollUntil({ label: "Actual platform verifies historical sandbox destruction and holds reconstruction", deadlineAt: Date.now() + 240000, + load: async () => ({ lease: await ctx.input.api.get(`/api/environment-leases/${ref.leaseId}`), run: await ctx.input.api.get(`/api/heartbeat-runs/${s.originalRun.id}`), recovery: await ctx.input.api.get(`/api/issues/${s.task.id}/recovery-actions`) }), + accept: v => destroyedLeaseBound(v.lease, { companyId, taskId: s.task.id, runId: s.originalRun.id, leaseId: ref.leaseId, sandboxId: ref.providerLeaseId, environmentId: s.target.environmentId }) && v.run.status === "failed" && record(v.recovery.actions.find(row => row.id === s.action.id)?.evidence.workspaceRecovery).status === "blocked" }); + const [artifact] = await readRegisteredArtifacts(ctx.input.api, s.task.id, s.originalRun.id, ["website.html"]); + return { ...s, destruction, exactProviderDestroyed: destruction.actualDeleteAccepted && destruction.independentlyAbsent, platformDestructionBound: true, providerFixtureOperations: [{ phase: "historical-source-loss", operation: "exact-owned-sandbox-delete-and-confirm-absence", status: "confirmed", usage: "unavailable", cost: "unavailable" }], destroyedLease: gone.lease, survivingAttachmentVerified: artifact?.sha256 === s.artifact.sha256 && artifact.content === s.incoming }; + } catch (error) { await cleanupFailedWebsiteExport(ctx); throw error; } +}, gradeLostSandbox, { + ...runtimePrivateSpec("workspaces.recover-lost-sandbox"), authentication: "human-session", expectedHumanMessageCount: 2, expectedRunCount: 4, attemptTimeoutMs: 1800000, + runPolicy: { kind: "terminal-counts", terminalStatusCounts: { failed: 1, succeeded: 3 } }, setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["real-failed-destroyed-sandbox-and-attributed-surviving-attachment"] }, + sourceFiles: ["chat-catalog-domains/runtime-lost-sandbox.ts", "chat-catalog-domains/runtime-workspace-repair.ts", "chat-catalog-domains/runtime-daytona-target.ts", "chat-catalog-domains/runtime-daytona-destroy.ts", "chat-catalog-domains/runtime-daytona-destroy-worker.ts", "remote-native-bootstrap.ts", "registered-artifact.ts", "../../server/src/services/native-runtime/native-workspace-reconstruction.ts", "../../server/src/services/workspace-recovery.ts", "../../ui/src/components/WorkspaceReconstructionRecovery.tsx", "chat-scenarios/workspaces.yaml"], + async verifySetup(ctx, s) { + const old = s.originalRun as NativeFaultRun, ref = s.reference as NativeWorkspaceSyncReference; + const [run, lease, artifacts] = await Promise.all([ctx.input.api.get(`/api/heartbeat-runs/${old.id}`), ctx.input.api.get(`/api/environment-leases/${ref.leaseId}`), readRegisteredArtifacts(ctx.input.api, (s.task as Entity).id, old.id, ["website.html"])]); + const c = checks(); c.add("real-failed-destroyed-sandbox-and-attributed-surviving-attachment", run.status === "failed" && run.nativePhase === "terminal_failure" && destroyedLeaseBound(lease, { companyId: ctx.input.fixtures.company.id, taskId: (s.task as Entity).id, runId: old.id, leaseId: ref.leaseId, sandboxId: ref.providerLeaseId, environmentId: (s.target as OwnedDaytonaTarget).environmentId }) && artifacts[0]?.content === s.incoming && await readFile(path.join(String(s.cwd), "website.html"), "utf8") === s.newer, "The exact old failed native source is independently destroyed; actual surviving source and newer human destination remain present before canonical guidance."); return c.result(); + }, + async act(ctx, s) { + const api = ctx.input.api, task = s.task as Entity, old = s.originalRun as NativeFaultRun, source = ctx.input.page.url(); + await pollUntil({ label: "Canonical lost-source guidance before board decision", deadlineAt: Date.now() + 180000, load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; }, accept: v => v.source?.conversationState === "waiting" && v.runs.length === 2 && v.runs.filter(row => row.status === "succeeded").length === 1 }); + const first = await observeRuntime(ctx, s); s.guidanceReply = first.reply; s.noReconstructionBeforeHuman = first.runs.length === 2 && first.runs.filter(row => row.agentId === (s.worker as Entity).id).length === 1; + if (!s.noReconstructionBeforeHuman || await readFile(path.join(String(s.cwd), "website.html"), "utf8") !== s.newer) throw new Error("Canonical guidance created premature work or overwrote the newer human destination"); + await writeFile(path.join(String(s.cwd), "newer-human-website-preserved.html"), String(s.newer), { flag: "wx" }); + await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" }); + const summary = ctx.input.page.getByText("Continue from saved sources", { exact: true }); await expect(summary).toBeVisible(); await summary.click(); s.actualReconstructionUi = true; + await ctx.input.page.getByLabel(/^website\.html \(/).check(); + const plan = "Copy the surviving website.html attachment byte-for-byte into recovered-website.html and register that actual copy as the downloadable recovered-website.html artifact. Keep the newer destination website.html and newer-human-website-preserved.html byte-for-byte unchanged. The destroyed sandbox's other unregistered state is unavailable; do not claim it recovered. Complete this same task without creating another task, redoing external actions, or regenerating missing sources."; + const evidence = "The previous native result saved the attached website source. No email, deployment or other external action was authorized or performed. Do not repeat any external action; preserve the original failed run."; + await ctx.input.page.getByLabel("Missing work and reconstruction plan", { exact: true }).fill(plan); + await ctx.input.page.getByLabel("Previously completed external actions", { exact: true }).selectOption("not_performed"); + await ctx.input.page.getByLabel("Evidence and actions that must not be repeated", { exact: true }).fill(evidence); + await ctx.input.page.getByLabel("Continue from these sources. The missing edits have not been recovered.", { exact: true }).check(); s.missingWorkAcknowledged = true; + const responsePromise = ctx.input.page.waitForResponse(response => response.request().method() === "POST" && new URL(response.url()).pathname === `/api/issues/${task.id}/recovery-actions/reconstruct-workspace`); + await ctx.input.page.getByRole("button", { name: "Rebuild and continue", exact: true }).click(); s.actualReconstructionStatus = (await responsePromise).status(); + const rebuilt = await pollUntil({ label: "Fresh supported worker reconstruction after actual routed human decision", deadlineAt: Date.now() + 360000, + load: async () => ({ task: await api.get(`/api/issues/${task.id}`), runs: await ctx.allRuns(), recovery: await api.get(`/api/issues/${task.id}/recovery-actions`) }), + accept: v => v.task.status === "done" && v.runs.length === 3 && v.runs.filter(row => row.status === "succeeded").length === 2, + reject: v => v.runs.length > 3 || v.runs.some(row => row.id !== old.id && ["failed", "cancelled", "timed_out"].includes(row.status)) ? "Actual reconstruction failed or admitted extra work" : undefined }); + const next = rebuilt.runs.find(row => row.agentId === (s.worker as Entity).id && row.id !== old.id)!; + const actual = await api.get(`/api/heartbeat-runs/${next.id}`), action = rebuilt.recovery.actions.find(row => row.id === (s.action as IssueRecoveryAction).id)!, decision = record(action.evidence.executionReconciliation), reconstruction = record(decision.workspaceReconstruction); + const context = record(actual.contextSnapshot), selectedSources = reconstruction.sources as Array<{ attachmentId: string; sha256: string }> | undefined; + s.freshSessionBound = actual.runtimeMode === "native" && actual.nativeIssueId === task.id && actual.companyId === first.companyId && actual.agentId === (s.worker as Entity).id && context.forceFreshSession === true && context.previousRunId === old.id && context.workspaceReconstructionActionId === action.id && !actual.sessionIdBefore && !!actual.sessionIdAfter && actual.nativeSessionId !== old.nativeSessionId && next.status === "succeeded"; + s.decisionSourcesBound = reconstruction.schema === "paperclip.workspace-reconstruction.v1" && reconstruction.missingWork === plan && selectedSources?.length === 1 && selectedSources[0]?.attachmentId === (s.artifact as { attachmentId: string }).attachmentId && selectedSources[0].sha256 === (s.artifact as { sha256: string }).sha256 && decision.outcomeEvidence === evidence && decision.actionOutcome === "not_performed"; + const person = await api.get<{ user: { id: string } }>("/api/auth/get-session"), activity = await api.get>(`/api/issues/${task.id}/activity`); + s.humanDecisionBound = s.decisionSourcesBound === true && activity.some(row => row.action === "issue.workspace_reconstruction_authorized" && row.actorType === "user" && row.actorId === person.user.id && row.entityId === task.id); + const [artifact] = await readRegisteredArtifacts(api, task.id, actual.id, ["recovered-website.html"]); + s.reconstructedArtifactVerified = artifact?.content === s.incoming; + s.newerWorkPreserved = await readFile(path.join(String(s.cwd), "website.html"), "utf8") === s.newer && await readFile(path.join(String(s.cwd), "newer-human-website-preserved.html"), "utf8") === s.newer; + const original = await api.get(`/api/heartbeat-runs/${old.id}`); s.originalFailureRetained = original.status === "failed" && original.errorCode === old.errorCode && original.nativeSessionId === old.nativeSessionId; + const events = await collectRunEvents<{ seq?: number; payload?: unknown }>((afterSeq, limit) => api.get(`/api/heartbeat-runs/${old.id}/events?afterSeq=${afterSeq}&limit=${limit}`)); + s.originalEventsRetained = (s.originalEventReceipts as Array<{ seq: number; sha256: string }>).every(before => events.some(after => after.seq === before.seq && digest(after.payload) === before.sha256)); + await ctx.input.capture("lost-sandbox-reconstructed", "The original routed Website task rebuilt from selected surviving sources", "runtime-lost-sandbox-reconstructed.png"); + await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); + const before = (await ctx.allRuns()).map(row => row.id), followup = "The human used Continue from saved sources and the reconstruction run has finished. Inspect the actual surviving attachment, reconstructed website and original failed run. Report what was reconstructed, whether the newer human edits survived, and which destroyed or unattached work is still unavailable. Do not claim that missing work was recovered."; + await sendChatMessage(ctx.input.page, followup); + const final = await pollUntil({ label: "Phase-bound factual reconstruction report", deadlineAt: Date.now() + 180000, load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; }, accept: v => v.source?.conversationState === "waiting" && v.runs.length === 4 && v.runs.filter(row => row.status === "succeeded").length === 3 }); + const replyRun = final.runs.find(row => !before.includes(row.id))!, comments = await api.get(`/api/issues/${ctx.issue()!.id}/comments?order=asc`), human = comments.filter(row => !row.authorAgentId && row.body === followup); + s.confirmationBound = human.length === 1 && Array.isArray(replyRun.contextSnapshot?.wakeCommentIds) && replyRun.contextSnapshot.wakeCommentIds.length === 1 && replyRun.contextSnapshot.wakeCommentIds[0] === human[0]!.id; + s.confirmationReply = comments.filter(row => row.createdByRunId === replyRun.id && row.authorAgentId === first.leadId).map(row => row.body).join("\n"); + }, + observe: observeRuntime, + async verifyResult(ctx, s) { const source = ctx.input.page.url(), task = s.task as Entity; await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" }); await expect(ctx.input.page.getByRole("heading", { name: "Website task", exact: true }).first()).toBeVisible(); await expect(ctx.input.page.getByText(/recovered-website\.html/).first()).toBeVisible(); await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); }, + async cleanup(ctx) { await cleanupFailedWebsiteExport(ctx); }, +})];