From 7b8aa60b6d79b6cd9e4d46ca1369515c33e2d5d4 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Tue, 3 Nov 2026 00:46:13 -0600 Subject: [PATCH] test(e2e): add repository continuation journey (#18729) ## Thinking Path > - Paperclip keeps agent work in project workspaces. > - A later user request must continue from the saved repository state. > - A successful reply alone cannot prove file or Git history preservation. > - The repository journey in #18652 passed against QA staging with independent byte and commit checks. > - Its separate completion journey remains unpassed and must stay open. > - This PR delivers only repository continuation and reuses the merged shared support from #18728. ## Linked Issues or Issue Description Refs: #18652. Builds on merged #18728. This is the repository-only delivery from #18652. The original completion implementation and all failed attempts remain on that open PR. ## What Changed - Add `repository-continuation` with the exact measured source fixture, two native turns, and eight required checks. - Start from measured source `ac4efb7c6f4782ea806661683c656fd2b4cc3189`. Keep the repository flow and Git/byte oracle unchanged. Clarify the two file-content instructions to require the word first or verified followed by exactly one LF newline; retain the exact expected bytes and the original random-reference contract. - Document the two-turn proof boundary: final saved hidden Git metadata is not independently read. Keep all eight success assertions. - Use the original-deadline and routine failure-reporting repairs from merged #18728. Integrate its actual master squash `e5713d95c6d9a258882649789ea5eb071cc17dc0` through one normal merge. - Use a fixed public read-only repository commit. Verify downloaded Git objects, parent commits, changed paths, saved workspace identity, and exact source bytes across both turns. Reopen the registered outputs through the browser. - Keep the shared artifact reader behavior unchanged and narrow its type to the API capabilities it uses. - Add only this case to the catalog, feature association, dispatcher, source hash, types, run/check contract, strict catalog expectation, and journey documentation. ## Verification - Current head `86808b7ec17e1727bd55a3fd46bc7e0409ca0e31` passed fresh [QA staging run 41589522430](https://github.com/paperclipai/paperclip-cloud/actions/runs/41589522430) and its [full report](https://pages.paperclip.ing/evals-qa-smoke-41589522430-1/). All eight checks and five phases passed. Both native turns succeeded and committed in the same workspace. The independent same-buffer Git audit verified that the second commit has the first commit as its parent. The saved original files and browser downloads matched their expected bytes. This does not add a final hidden `.git`/HEAD assertion. - The fresh run used OpenCode with `openrouter/deepseek/deepseek-v4.1-flash-1017` on QA staging/Daytona. It used Core `09162fe893b464dd2ccdafd14bd3ef1611202a86`, Cloud `cloud-41570439002-1`, and environment configuration version 9. Actual checkouts matched public source `86808b7ec17e1727bd55a3fd46bc7e0409ca0e31`, adapter `7367cf6fd0ee1cf52c2651a8a6671e087728a9c1`, and protected workflow bootstrap `a9ad6877de62fe3f65aacadc8a9dfba7d90729cf`. The bootstrap matched the accepted 117-file QA contract. - Normal cleanup completed. The original manifest is complete, evidence cleanup is complete, no operations remain active, and the owned agent is terminal. The coordinator verified ACTIVE TestOps case 685 / result 3695 and closed [launch 1921](https://paperclip.testops.cloud/launch/1921). This records normal cleanup closure; no separate physical provider-stop acknowledgement was collected. - Measured source `ac4efb7c6f4782ea806661683c656fd2b4cc3189` passed [QA staging run 41573331189](https://pages.paperclip.ing/evals-qa-smoke-41573331189-1/) against Core `09162fe893b464dd2ccdafd14bd3ef1611202a86`. TestOps launch 1723 / result 3501 maps to case 685. All eight checks and five phases passed. The two run bindings, independent saved-byte/Git audit, and completed cleanup were verified. - Initial extraction `54103639850d08d4d125ebe0493d8d3d49b58211` preserved the measured case files. The current head also clarifies the two LF instructions, documents the final Git-metadata limit, and inherits the reviewed helper repairs. This changed the fixture conditions. The current head now has its own live pass, listed below. The original ac4 result remains a pass under its original conditions; it was not used to qualify this head. - The emitted-prompt control failed on the old quoted backslash+n instructions, while 14 sibling controls passed. All 47 focused repository/artifact controls then passed after the two-sentence repair. Local: 891 related support tests, runner E2E typecheck, recursive typecheck, build, maps, token gates, and `git diff --check` passed. The inherited routine source has 62 causal read/cleanup controls and 26 unchanged real component controls. - The initial repository head's normal full CI run 41578686494 passed. Its 4/5 review and all original findings remain recorded. The later stacked head `037b979477161881a4877230500e6f3de8faea5c` also passed full CI run 41581342503 and a fresh 5/5 review. Current integration head `86808b7ec17e1727bd55a3fd46bc7e0409ca0e31` passed normal full [CI run 41584981651](https://github.com/paperclipai/paperclip/actions/runs/41584981651), attempt 1. Its fresh exact-head Greptile review is 5/5, with zero unresolved threads and no open findings. The broad local suite was not repeated. The earlier full local run recorded in #18643 had four failures and did not reach 164 serialized groups; this PR does not claim an all-local pass. - Earlier repository failures remain recorded: provider-pack transfer timed out in one attempt; a later attempt could not read saved workspace files. The later pass does not regrade those failures. The product repairs are separate from this unchanged case delivery. - The fixed actual #18728 squash is now integrated. The PR targets master. Its aggregate diff has exactly 14 repository-only paths, each identical to `037b979477161881a4877230500e6f3de8faea5c`; all other master changes are preserved. No later master head was chased. The current CI, review, and fresh staging qualification all passed for this integration head. ## Risks - Git bundles and workspace bytes are private evidence. Public reports retain closed checks; the fixture never pushes to the public source repository. - The second committed output must retain the first commit as its parent, the same workspace binding, and original saved file bytes. Final hidden `.git`/HEAD after the second save is not independently exposed by the supported remote workspace API; this test does not claim third-turn Git metadata durability. - Only JSON metadata observations retry transient transport failures. Binary attachment and browser downloads remain single attempts. Transport failures and digest mismatches remain visible failures. - The existing artifact reader is shared. Its runtime code is unchanged and its regression coverage remains included. - Daily source pins and plan selection stay with the coordinator. This PR does not register or bypass the unpassed completion case. ## Model Used OpenAI GPT-6 through Codex, with tool use and code execution. The exact deployment ID and context window are not exposed by this session. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- tests/runner-e2e/PRODUCT-JOURNEYS.md | 7 + tests/runner-e2e/catalog.test.ts | 2 +- tests/runner-e2e/journey-features.ts | 1 + .../product-journeys/agent-flows.ts | 2 +- tests/runner-e2e/product-journeys/catalog.ts | 1 + tests/runner-e2e/product-journeys/launch.ts | 7 +- .../repository-continuation-contract.ts | 30 ++++ .../repository-continuation-evidence.test.ts | 105 +++++++++++++ .../repository-continuation-evidence.ts | 82 ++++++++++ .../repository-continuation-flow.ts | 146 ++++++++++++++++++ .../repository-continuation.md | 51 ++++++ .../product-journeys/success-contract.ts | 3 +- tests/runner-e2e/product-journeys/types.ts | 2 +- tests/runner-e2e/registered-artifact.ts | 4 +- 14 files changed, 435 insertions(+), 8 deletions(-) create mode 100644 tests/runner-e2e/product-journeys/repository-continuation-contract.ts create mode 100644 tests/runner-e2e/product-journeys/repository-continuation-evidence.test.ts create mode 100644 tests/runner-e2e/product-journeys/repository-continuation-evidence.ts create mode 100644 tests/runner-e2e/product-journeys/repository-continuation-flow.ts create mode 100644 tests/runner-e2e/product-journeys/repository-continuation.md diff --git a/tests/runner-e2e/PRODUCT-JOURNEYS.md b/tests/runner-e2e/PRODUCT-JOURNEYS.md index 40e9ddfad12..74463320bd9 100644 --- a/tests/runner-e2e/PRODUCT-JOURNEYS.md +++ b/tests/runner-e2e/PRODUCT-JOURNEYS.md @@ -23,6 +23,7 @@ never start a server or depend on a tenant filesystem. | Journey | Required starting state | Observed outcome | | --- | --- | --- | +| `repository-continuation` | Native agent; two work runs; a pinned public read-only repository | Verify independent Git bundle history and workspace bytes across two turns, preserve the original random file, and reopen registered outputs after reload. See [repository-continuation.md](product-journeys/repository-continuation.md) | | `routine-scheduled-run` | Native agent; one work run; an owned project and routine | Browser-create a routine, enable its minute schedule, verify one scheduled invocation and run-attributed saved output, then pause and observe the next schedule boundary with no new work | | `file-input-answer` | Native agent; one work run; synthetic CSV uploaded through the composer | Verify original uploaded bytes, independently calculated accepted-row count and units, a run-attributed task document after reload, and preservation of the input attachment. The task prompt contains neither the random file reference nor the answers | | `inbox-triage` | Signed-in board user; a separately authenticated QA human author; a fresh self-assigned task; zero model runs | Mark an unread task read, archive it through Mine, reload both views, verify unchanged work status, then observe a real Blocked transition resurface it through the normal archive policy | @@ -480,3 +481,9 @@ uses the server's timestamp; client and server wall clocks are never compared. The journey retries only transient read transport failures within the existing phase and journey deadlines. Private read-failure receipts retain those observations. Mutations, user messages and agent work are never replayed. + +Repository JSON metadata observations retry only transient transport failures +within the original journey deadline. Binary attachment and browser downloads +remain single attempts; transport failures and digest mismatches fail visibly. Git inspection consumes the exact digest-verified download +buffer; browser downloads remain an independent result-access check. The +repository journey shares the observation reader and browser helpers above. diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index c040b57595a..13989a0efe5 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -126,7 +126,7 @@ describe("runner E2E catalog", () => { }); it("exports deployment journeys separately from the model matrix and default selection", () => { - expect(productJourneys.map(journey => journey.id)).toEqual(["routine-scheduled-run", "inbox-triage", "file-input-answer", "task-create", "project-task-context", "ask-question", "question-answer-resume", "task-properties", "task-document-persistence", "task-search-filter", "plan-revise-accept", "build-revise", "protected-actions", "create-skill-studio", "stop-redirect", "followup-while-running", "hire-delegate-reuse", "cloud-onboarding"]); + expect(productJourneys.map(journey => journey.id)).toEqual(["repository-continuation", "routine-scheduled-run", "inbox-triage", "file-input-answer", "task-create", "project-task-context", "ask-question", "question-answer-resume", "task-properties", "task-document-persistence", "task-search-filter", "plan-revise-accept", "build-revise", "protected-actions", "create-skill-studio", "stop-redirect", "followup-while-running", "hire-delegate-reuse", "cloud-onboarding"]); expect(runnerSuites.some(suite => suite.id === "product-journeys")).toBe(false); expect(runnerMatrix.some(execution => execution.suite.id === "product-journeys")).toBe(false); expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(execution => execution.suite.id === "product-journeys")).toBe(false); diff --git a/tests/runner-e2e/journey-features.ts b/tests/runner-e2e/journey-features.ts index 9abc7caf528..7da187c149e 100644 --- a/tests/runner-e2e/journey-features.ts +++ b/tests/runner-e2e/journey-features.ts @@ -19,6 +19,7 @@ function associate(suites: readonly string[], cases: Record; -type AgentJourney = Exclude; +type AgentJourney = Exclude; interface TaskState { currentIssue: Row & { id: string; status: string }; taskRuns: Array; comments: Row[]; interactions: Array; attachments: StoryDelivery[] } interface Options { deadlineAt: number; directory: string; capture(id: string, label: string, file: string): Promise } const active = (run: Row) => ["queued", "running"].includes(run.status); diff --git a/tests/runner-e2e/product-journeys/catalog.ts b/tests/runner-e2e/product-journeys/catalog.ts index 156220ce89e..2b47d1d842e 100644 --- a/tests/runner-e2e/product-journeys/catalog.ts +++ b/tests/runner-e2e/product-journeys/catalog.ts @@ -2,6 +2,7 @@ import { agentJourneyRuns } from "./success-contract.js"; export { agentJourneyRuns, agentJourneyRunCountValid } from "./success-contract.js"; import type { JourneyId, TargetConfig } from "./types.js"; export const productJourneys = [ + { id: "repository-continuation", label: "Continue repository work with the same saved workspace", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["projects/task-context", "documents-artifacts/attachments"], paid: true }, { id: "routine-scheduled-run", label: "Run a scheduled routine once and verify pause and saved history", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["routines/definition", "routines/triggers", "routines/history", "routines/management"], paid: true }, { id: "inbox-triage", label: "Read, archive and resurface an owned inbox task", requires: ["company", "board-user", "task-write", "inbox-author"], capabilities: ["inbox-search/views", "inbox-search/triage"], paid: false }, { id: "file-input-answer", label: "Upload a reference file and verify the agent uses its contents", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["documents-artifacts/attachments", "documents-artifacts/documents"], paid: true }, diff --git a/tests/runner-e2e/product-journeys/launch.ts b/tests/runner-e2e/product-journeys/launch.ts index 9ca26c76c51..edfe84bbe75 100644 --- a/tests/runner-e2e/product-journeys/launch.ts +++ b/tests/runner-e2e/product-journeys/launch.ts @@ -17,6 +17,7 @@ import { projectTaskContext } from "./project-context-flow.js"; import { runAgentJourney } from "./agent-flows.js"; import { runActiveFollowup } from "./active-followup-flow.js"; import { runClarificationJourney } from "./clarification-flow.js"; +import { runRepositoryContinuation } from "./repository-continuation-flow.js"; import { runScheduledRoutine } from "./routine-schedule-flow.js"; import { runInputFile } from "./input-file-flow.js"; import { inboxTriage } from "./inbox-triage-flow.js"; @@ -111,7 +112,7 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri try { sha = execFileSync("git", ["rev-parse", "HEAD"], { cwd: repositoryRoot, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim(); } catch { /* Packaged harnesses may not include Git metadata. */ } const source = resolveRunnerE2ESource({ sha, ref: null, workflowRunUrl: null }); const definitionHash = createHash("sha256").update(JSON.stringify(productJourneys)); - for (const name of ["observation-read.ts", "../connection-config.ts", "catalog.ts", "../user-actions.ts", "../connection-target.ts", "routine-schedule-flow.ts", "routine-schedule-evidence.ts", "input-file-flow.ts", "input-file-evidence.ts", "inbox-triage-flow.ts", "inbox-triage-evidence.ts", "inbox-triage-fixture.ts", "inbox-author-contract.ts", "task-properties.ts", "task-search-filter.ts", "task-search-evidence.ts", "basic-flows.ts", "basic-evidence.ts", "task-document-flow.ts", "task-document-evidence.ts", "project-context-flow.ts", "project-context-readback.ts", "project-context-evidence.ts", "agent-flows.ts", "hire-delegate-flow.ts", "hire-delegate-composer.ts", "hire-delegate-evidence.ts", "active-followup-flow.ts", "active-followup-evidence.ts", "stop-redirect-evidence.ts", "clarification-flow.ts", "clarification-evidence.ts", "../question-resume-ui.ts", "../question-resume-scoring.ts", "../runtime-question-readiness.ts", "../chat-flow.ts", "../continuation-readiness.ts", "../continuation-cases.ts", "success-contract.ts", "product-screenshot.ts", "cloud-onboarding.ts", "onboarding-connect.ts", "onboarding-first-task.ts", "workspace-deferral.ts", "onboarding-pre-acceptance.ts", "artifact-evidence.ts", "fixtures.ts", "launch.ts", "types.ts", "collection.ts", "local-target.ts", "../../../ui/src/lib/markdown-reply-content.ts", "../../../ui/src/lib/issue-reference.ts", "../../../ui/src/lib/company-routes.ts", "../../../ui/package.json"]) definitionHash.update(await readFile(path.join(import.meta.dirname, name))); + for (const name of ["observation-read.ts", "../connection-config.ts", "catalog.ts", "repository-continuation-flow.ts", "repository-continuation-contract.ts", "repository-continuation-evidence.ts", "../registered-artifact.ts", "../user-actions.ts", "../connection-target.ts", "../../../packages/shared/src/project-mentions.ts", "routine-schedule-flow.ts", "routine-schedule-evidence.ts", "input-file-flow.ts", "input-file-evidence.ts", "inbox-triage-flow.ts", "inbox-triage-evidence.ts", "inbox-triage-fixture.ts", "inbox-author-contract.ts", "task-properties.ts", "task-search-filter.ts", "task-search-evidence.ts", "basic-flows.ts", "basic-evidence.ts", "task-document-flow.ts", "task-document-evidence.ts", "project-context-flow.ts", "project-context-readback.ts", "project-context-evidence.ts", "agent-flows.ts", "hire-delegate-flow.ts", "hire-delegate-composer.ts", "hire-delegate-evidence.ts", "active-followup-flow.ts", "active-followup-evidence.ts", "stop-redirect-evidence.ts", "clarification-flow.ts", "clarification-evidence.ts", "../question-resume-ui.ts", "../question-resume-scoring.ts", "../runtime-question-readiness.ts", "../chat-flow.ts", "../continuation-readiness.ts", "../continuation-cases.ts", "success-contract.ts", "product-screenshot.ts", "cloud-onboarding.ts", "onboarding-connect.ts", "onboarding-first-task.ts", "workspace-deferral.ts", "onboarding-pre-acceptance.ts", "artifact-evidence.ts", "fixtures.ts", "launch.ts", "types.ts", "collection.ts", "local-target.ts", "../../../ui/src/lib/markdown-reply-content.ts", "../../../ui/src/lib/issue-reference.ts", "../../../ui/src/lib/company-routes.ts", "../../../ui/package.json"]) definitionHash.update(await readFile(path.join(import.meta.dirname, name))); for (const file of ["catalog.ts", "api.ts", "run-observations.ts", "native-task-reply.ts", "native-event-envelope.ts", "user-actions.ts", "plan-revision-flow.ts", "skill-studio-flow.ts", "protected-action-flow.ts", "story-artifact.ts", "stop-redirect-flow.ts", "everyday-cases.ts", "everyday-delivery.ts", "chat-hardening.ts", "everyday-decisions.ts", "connection-guidance-evidence.ts", "everyday-artifact.py"]) definitionHash.update(await readFile(path.join(import.meta.dirname, "..", file))); definitionHash.update(runnerSuites.flatMap(suite => suite.tasks).find(task => task.id === "ask-question")!.buildPrompt("definition")); const suiteDefinitionHash = definitionHash.digest("hex"); @@ -207,7 +208,9 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri result.provider = ready.agent.provider; result.model = ready.agent.model; result.runtimeMode = ready.agent.runtimeMode; - if (id === "routine-scheduled-run") await runScheduledRoutine(ready, session!, campaignId, evidence, + if (id === "repository-continuation") await runRepositoryContinuation(ready, session!, campaignId, evidence, + { deadlineAt, directory: path.join(outputDir, "private-stories", id) }); + else if (id === "routine-scheduled-run") await runScheduledRoutine(ready, session!, campaignId, evidence, { deadlineAt, directory: path.join(outputDir, "private-stories", id) }); else if (id === "file-input-answer") await runInputFile(ready, session!, campaignId, evidence, { deadlineAt, directory: path.join(outputDir, "private-stories", id) }); diff --git a/tests/runner-e2e/product-journeys/repository-continuation-contract.ts b/tests/runner-e2e/product-journeys/repository-continuation-contract.ts new file mode 100644 index 00000000000..4d301919f18 --- /dev/null +++ b/tests/runner-e2e/product-journeys/repository-continuation-contract.ts @@ -0,0 +1,30 @@ +/** Public, read-only fixture verified through GitHub's commit and content APIs. */ +export const repositoryContinuationSource = { + url: "https://github.com/octocat/Hello-World", + commit: "7fd1a60b01f91b314f59955a4e4d4e80d8edf11d", + readme: "Hello World!\n", +} as const; + +export function repositoryContinuationStory(campaignId: string) { + if (!/^journey-[0-9]+-[a-f0-9]{8}$/.test(campaignId)) throw new Error("invalid_repository_journey_campaign"); + const suffix = campaignId.replaceAll("-", ""); + const files = { context: `qa-context-${suffix}.txt`, first: `qa-first-${suffix}.txt`, + followup: `qa-followup-${suffix}.txt`, firstBundle: `qa-first-${suffix}.bundle`, finalBundle: `qa-final-${suffix}.bundle` }; + return { + files, + first: [ + "Use this task's configured project repository. Work only in its existing checkout; do not create another repository, project, or task. Do not push anything to a remote.", + `Verify that its starting HEAD is ${repositoryContinuationSource.commit}, then read the committed README. Stop and report a mismatch if this is not the configured repository.`, + `Generate a fresh unpredictable 32-byte hexadecimal reference from the operating system random source and save it followed by one LF in ${files.context}. Do not print the reference in comments or your final answer.`, + `Create ${files.first} containing only the text first followed by exactly one LF newline. Make one local Git commit containing only these two new files. Preserve every original repository file.`, + `After the commit, create a self-contained Git bundle of HEAD at ${files.firstBundle}. Do not add the bundle to Git.`, + `Register both ${files.context} and ${files.firstBundle} as downloadable artifacts with those exact filenames and titles. Deliver their accessible links and finish the task. I will send a later follow-up.`, + ].join("\n"), + followup: [ + "Continue in the same project checkout. Keep the earlier commit and original files.", + `Read ${files.context} and ${files.first} from the current workspace. If either is missing or invalid, stop and report the failure. Never recreate either file or recover it from conversation, attachments, or another checkout.`, + `Keep their original bytes. Add ${files.followup} containing only the text verified followed by exactly one LF newline. Make one more local Git commit containing only that new file. Do not amend or rewrite the earlier commit, and do not push.`, + `Create a new self-contained Git bundle of HEAD at ${files.finalBundle}. Register it as a downloadable artifact with that exact filename and title. Deliver its accessible link and finish the task.`, + ].join("\n"), + }; +} diff --git a/tests/runner-e2e/product-journeys/repository-continuation-evidence.test.ts b/tests/runner-e2e/product-journeys/repository-continuation-evidence.test.ts new file mode 100644 index 00000000000..b201da677f5 --- /dev/null +++ b/tests/runner-e2e/product-journeys/repository-continuation-evidence.test.ts @@ -0,0 +1,105 @@ +import { execFile as execFileCallback } from "node:child_process"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { repositoryContinuationSource, repositoryContinuationStory } from "./repository-continuation-contract.js"; +import { inspectRepositoryBundle, repositoryCommitMatches, repositoryWorkspaceBindingMatches, repositoryWorkspaceFileMatches, + type RepositoryBundleEvidence } from "./repository-continuation-evidence.js"; +import type { WorkspaceFileContent } from "../../../packages/shared/src/types/workspace-file-resource.js"; + +const execFile = promisify(execFileCallback); +describe("repository continuation Git evidence", () => { + let root: string, sourceCommit: string, first: RepositoryBundleEvidence, final: RepositoryBundleEvidence, firstBytes: Buffer; + const reference = `${"c".repeat(64)}\n`, firstFile = "first.txt", contextFile = "context.txt", finalFile = "followup.txt"; + beforeAll(async () => { + root = await mkdtemp(path.join(os.tmpdir(), "paperclip-qa-repo-control-")); + const git = async (...args: string[]) => (await execFile("git", ["-c", "core.hooksPath=/dev/null", "-C", root, ...args], { + env: { PATH: process.env.PATH, GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: "/dev/null", + GIT_AUTHOR_NAME: "Fixture", GIT_AUTHOR_EMAIL: "fixture@example.test", GIT_COMMITTER_NAME: "Fixture", GIT_COMMITTER_EMAIL: "fixture@example.test" }, + encoding: "utf8", timeout: 10_000, + })).stdout; + await git("init"); + await writeFile(path.join(root, "README"), repositoryContinuationSource.readme); + await git("add", "README"); await git("commit", "-m", "source"); + sourceCommit = (await git("rev-parse", "HEAD")).trim(); + await writeFile(path.join(root, contextFile), reference); await writeFile(path.join(root, firstFile), "first\n"); + await git("add", contextFile, firstFile); await git("commit", "-m", "first"); + await git("bundle", "create", path.join(root, "first.bundle"), "HEAD"); + firstBytes = await readFile(path.join(root, "first.bundle")); + first = await inspectRepositoryBundle(firstBytes, { sourceCommit, files: [contextFile, firstFile, "README"] }); + await writeFile(path.join(root, finalFile), "verified\n"); await git("add", finalFile); await git("commit", "-m", "second"); + await git("bundle", "create", "--version=3", path.join(root, "final.bundle"), "HEAD"); + final = await inspectRepositoryBundle(await readFile(path.join(root, "final.bundle")), { sourceCommit, files: [contextFile, firstFile, finalFile, "README"] }); + }); + afterAll(async () => { if (root) await rm(root, { recursive: true, force: true }); }); + const expectedFirst = () => ({ parent: sourceCommit, sourceReadme: repositoryContinuationSource.readme, + files: { [contextFile]: reference, [firstFile]: "first\n", README: repositoryContinuationSource.readme }, added: [contextFile, firstFile] }); + it("reads real Git commits, parent links and file objects from both downloaded bundles", () => { + expect(repositoryCommitMatches(first, expectedFirst())).toBe(true); + expect(repositoryCommitMatches(final, { parent: first.head, sourceReadme: repositoryContinuationSource.readme, + files: { [contextFile]: reference, [firstFile]: "first\n", [finalFile]: "verified\n", README: repositoryContinuationSource.readme }, added: [finalFile] })).toBe(true); + }); + const invalidEvidence: Array<[string, () => RepositoryBundleEvidence | null]> = [ + ["missing", () => null], + ["rewritten history", () => ({ ...first, parent: "a".repeat(40) })], + ["no new commit", () => ({ ...first, head: sourceCommit })], + ["wrong repository", () => ({ ...first, sourceReadme: "Other repository\n" })], + ["original changed", () => ({ ...first, files: { ...first.files, [contextFile]: "replacement\n" } })], + ["extra changes", () => ({ ...first, changes: [...first.changes, { status: "M", path: "README" }] })], + ["missing saved file", () => ({ ...first, files: {} })], + ["duplicate tree path", () => ({ ...first, changes: [first.changes[0]!, first.changes[0]!] })], + ["modified original", () => ({ ...first, changes: first.changes.map(change => ({ ...change, status: "M" })) })], + ]; + it.each(invalidEvidence)("rejects %s evidence", (_name, change) => { expect(repositoryCommitMatches(change(), expectedFirst())).toBe(false); }); + it("rejects malformed, corrupt, incomplete and unrelated Git bundles", async () => { + await expect(inspectRepositoryBundle(Buffer.from("claimed git commit"), { sourceCommit, files: [] })).rejects.toThrow(); + await expect(inspectRepositoryBundle(firstBytes.subarray(0, 150), { sourceCommit, files: [] })).rejects.toThrow(); + await expect(inspectRepositoryBundle(firstBytes, { sourceCommit: "f".repeat(40), files: [] })).rejects.toThrow(); + await expect(inspectRepositoryBundle(firstBytes, { sourceCommit, files: ["../escape"] })).rejects.toThrow(); + }); +}); + +describe("repository continuation target binding", () => { + const issue = { id: "issue", companyId: "company", projectId: "project", executionWorkspaceId: "workspace" }; + const run = { companyId: "company", runnerProfileJson: { nativeExecutionInput: { binding: { executionWorkspaceId: "workspace" } } } }; + const workspace = { id: "workspace", companyId: "company", projectId: "project", projectWorkspaceId: "project-workspace", repoUrl: repositoryContinuationSource.url }; + const input = { companyId: "company", issueId: "issue", projectId: "project", projectWorkspaceId: "project-workspace", workspaceId: "workspace", + repoUrl: repositoryContinuationSource.url, issue, view: { issueId: "issue", workspace }, runs: [run] }; + it("requires the configured project, actual task workspace and every run binding", () => { + expect(repositoryWorkspaceBindingMatches(input)).toBe(true); + expect(repositoryWorkspaceBindingMatches({ ...input, runs: [] })).toBe(false); + expect(repositoryWorkspaceBindingMatches({ ...input, view: {} })).toBe(false); + for (const change of [{ id: "foreign" }, { companyId: "foreign" }, { projectId: "foreign" }, + { projectWorkspaceId: "foreign" }, { repoUrl: "https://example.test/other" }]) { + expect(repositoryWorkspaceBindingMatches({ ...input, view: { ...input.view, workspace: { ...workspace, ...change } } })).toBe(false); + } + expect(repositoryWorkspaceBindingMatches({ ...input, runs: [run, { ...run, companyId: "foreign" }] })).toBe(false); + expect(repositoryWorkspaceBindingMatches({ ...input, runs: [run, { companyId: "company" }] })).toBe(false); + expect(repositoryWorkspaceBindingMatches({ ...input, issue: { ...issue, executionWorkspaceId: "replacement" } })).toBe(false); + }); + it("compares exact workspace API bytes and rejects another workspace or file", () => { + const file = { resource: { kind: "file", workspaceKind: "execution_workspace", workspaceId: "workspace", title: "context.txt", byteSize: 4 }, + content: { encoding: "utf8", data: "abc\n" } } as WorkspaceFileContent; + expect(repositoryWorkspaceFileMatches(file, "workspace", "context.txt", "abc\n")).toBe(true); + expect(repositoryWorkspaceFileMatches(null, "workspace", "context.txt", "abc\n")).toBe(false); + expect(repositoryWorkspaceFileMatches(file, "foreign", "context.txt", "abc\n")).toBe(false); + expect(repositoryWorkspaceFileMatches(file, "workspace", "wrong.txt", "abc\n")).toBe(false); + expect(repositoryWorkspaceFileMatches(file, "workspace", "context.txt", "xyz\n")).toBe(false); + }); + it("keeps the later user request free from the unpredictable earlier reference", () => { + const story = repositoryContinuationStory("journey-1793642415731-3c34246f"); + expect(story.first).toContain(repositoryContinuationSource.commit); + expect(story.followup).toContain("Never recreate"); + expect(story.followup).not.toContain("c".repeat(64)); + expect(() => repositoryContinuationStory("../unsafe")).toThrow(); + }); + it("emits unambiguous LF instructions for the independently verified file bytes", () => { + const story = repositoryContinuationStory("journey-1793642415731-3c34246f"); + expect(story.first).toContain("containing only the text first followed by exactly one LF newline."); + expect(story.followup).toContain("containing only the text verified followed by exactly one LF newline."); + expect(story.first).not.toContain("first\\n"); + expect(story.followup).not.toContain("verified\\n"); + }); +}); diff --git a/tests/runner-e2e/product-journeys/repository-continuation-evidence.ts b/tests/runner-e2e/product-journeys/repository-continuation-evidence.ts new file mode 100644 index 00000000000..25f35f269a9 --- /dev/null +++ b/tests/runner-e2e/product-journeys/repository-continuation-evidence.ts @@ -0,0 +1,82 @@ +import { execFile as execFileCallback } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import type { WorkspaceFileContent } from "../../../packages/shared/src/types/workspace-file-resource.js"; + +const execFile = promisify(execFileCallback); +const sha = (bytes: Buffer) => createHash("sha256").update(bytes).digest("hex"); +export interface RepositoryBundleEvidence { + bundleSha256: string; + head: string; + parent: string; + changes: Array<{ status: string; path: string }>; + files: Record; + sourceReadme: string; +} + +/** Inspect actual Git objects. No checkout, remote fetch, hook, or repo script runs. */ +export async function inspectRepositoryBundle(bytes: Buffer, input: { sourceCommit: string; files: string[] }): Promise { + if (!/^[a-f0-9]{40}$/.test(input.sourceCommit) || bytes.length < 32 || bytes.length > 262_144 + || !/^# v[23] git bundle\n/.test(bytes.subarray(0, 32).toString("ascii")) + || input.files.some(file => !/^[A-Za-z0-9][A-Za-z0-9.-]{0,120}$/.test(file))) throw new Error("invalid_repository_bundle"); + const root = await mkdtemp(path.join(os.tmpdir(), "paperclip-qa-repository-inspection-")); + const gitDir = path.join(root, "objects.git"), bundle = path.join(root, "proof.bundle"); + const git = async (...args: string[]) => (await execFile("git", ["--no-replace-objects", "-c", "core.hooksPath=/dev/null", + "-c", "core.fsmonitor=false", "-c", "protocol.allow=never", "-c", "protocol.file.allow=always", "--git-dir", gitDir, ...args], { + cwd: root, env: { PATH: process.env.PATH, GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: "/dev/null", GIT_TERMINAL_PROMPT: "0" }, + timeout: 20_000, maxBuffer: 262_144, encoding: "utf8", + })).stdout; + try { + await mkdir(gitDir); + await writeFile(bundle, bytes, { mode: 0o600, flag: "wx" }); + await git("init", "--bare"); + await git("bundle", "verify", bundle); + await git("fetch", "--no-tags", bundle, "HEAD:refs/heads/observed"); + await git("fsck", "--full", "--strict"); + const head = (await git("rev-parse", "refs/heads/observed")).trim(); + await git("merge-base", "--is-ancestor", input.sourceCommit, head); + const parent = (await git("show", "-s", "--format=%P", head)).trim(); + // A merge or omitted parent cannot satisfy either single-commit turn. + if (!/^[a-f0-9]{40}$/.test(head) || !/^[a-f0-9]{40}$/.test(parent)) throw new Error("repository_commit_parent_mismatch"); + const rawChanges = (await git("diff-tree", "--no-commit-id", "--name-status", "-r", "-z", parent, head)).split("\0"); + if (rawChanges.pop() !== "" || rawChanges.length % 2) throw new Error("repository_tree_changes_invalid"); + const changes = Array.from({ length: rawChanges.length / 2 }, (_, i) => ({ status: rawChanges[i * 2]!, path: rawChanges[i * 2 + 1]! })); + const files: Record = {}; + for (const file of input.files) files[file] = await git("show", `${head}:${file}`); + return { bundleSha256: sha(bytes), head, parent, changes, files, sourceReadme: await git("show", `${input.sourceCommit}:README`) }; + } finally { await rm(root, { recursive: true, force: true }); } +} + +export function repositoryWorkspaceFileMatches(value: WorkspaceFileContent | null, workspaceId: string, filename: string, bytes: string) { + return Boolean(value && workspaceId && value.resource.kind === "file" && value.resource.workspaceKind === "execution_workspace" + && value.resource.workspaceId === workspaceId && value.resource.title === filename + && value.resource.byteSize === Buffer.byteLength(bytes) && value.content.encoding === "utf8" && value.content.data === bytes); +} + +export function repositoryWorkspaceBindingMatches(input: { + companyId: string; issueId: string; projectId: string; projectWorkspaceId: string; workspaceId: string; repoUrl: string; + issue: Record; view: Record; runs: Array>; +}) { + const { workspace } = input.view; + return Boolean(input.workspaceId && input.issue.id === input.issueId && input.issue.companyId === input.companyId + && input.issue.projectId === input.projectId && input.issue.executionWorkspaceId === input.workspaceId + && input.view.issueId === input.issueId && workspace?.id === input.workspaceId && workspace.companyId === input.companyId + && workspace.projectId === input.projectId && workspace.projectWorkspaceId === input.projectWorkspaceId + && workspace.repoUrl === input.repoUrl && input.runs.length > 0 + && input.runs.every(run => run.companyId === input.companyId + && run.runnerProfileJson?.nativeExecutionInput?.binding?.executionWorkspaceId === input.workspaceId)); +} + +export function repositoryCommitMatches(value: RepositoryBundleEvidence | null, expected: { + parent: string; sourceReadme: string; files: Record; added: string[]; +}) { + return Boolean(value && /^[a-f0-9]{40}$/.test(value.head) && value.head !== expected.parent + && value.parent === expected.parent && value.sourceReadme === expected.sourceReadme + && value.changes.length === expected.added.length && new Set(value.changes.map(change => change.path)).size === expected.added.length + && value.changes.every(change => change.status === "A" && expected.added.includes(change.path)) + && Object.keys(value.files).length === Object.keys(expected.files).length + && Object.entries(expected.files).every(([file, bytes]) => value.files[file] === bytes)); +} diff --git a/tests/runner-e2e/product-journeys/repository-continuation-flow.ts b/tests/runner-e2e/product-journeys/repository-continuation-flow.ts new file mode 100644 index 00000000000..ebeb43c4fc2 --- /dev/null +++ b/tests/runner-e2e/product-journeys/repository-continuation-flow.ts @@ -0,0 +1,146 @@ +import { createObservedReader } from "./observation-read.js"; +import { createHash } from "node:crypto"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { expect } from "@playwright/test"; +import type { WorkspaceFileContent } from "../../../packages/shared/src/types/workspace-file-resource.js"; +import { readRegisteredArtifacts } from "../registered-artifact.js"; +import { createTaskThroughUi, submitTaskReply } from "../user-actions.js"; +import { fixtureReplyStoredOnce, validateAgentTaskState, waitFor } from "./agent-flows.js"; +import { repositoryContinuationSource, repositoryContinuationStory } from "./repository-continuation-contract.js"; +import { inspectRepositoryBundle, repositoryCommitMatches, repositoryWorkspaceBindingMatches, repositoryWorkspaceFileMatches } from "./repository-continuation-evidence.js"; +import { openDocumentAuthoring } from "./task-document-flow.js"; +import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSession } from "./types.js"; + +type Row = Record; +const id = "repository-continuation"; + +/** Attached and managed targets share the browser story and byte/Git oracle. */ +export async function runRepositoryContinuation(context: JourneyContext, session: TargetSession, campaignId: string, + evidence: JourneyEvidence, options: { deadlineAt: number; directory: string }) { + const { api, page, company, agent } = context; + const observations = createObservedReader(api, options.deadlineAt); + const readObserved = observations.get; + try { + if (!agent || agent.runtimeMode !== "native" || !context.environmentId) + throw new JourneyError("unsupported", "native_agent_and_environment_required"); + const story = repositoryContinuationStory(campaignId), { files } = story, source = repositoryContinuationSource; + await mkdir(options.directory, { mode: 0o700, recursive: true }); + const retain = (filename: string, value: unknown) => writeFile(path.join(options.directory, filename), JSON.stringify(value, null, 2), { mode: 0o600 }); + const check = (name: string, passed: boolean) => { + evidence.checks.push({ id: name, passed }); + if (!passed) throw new JourneyError("failed", name.replaceAll("-", "_")); + }; + const projectName = `Journey QA repository ${campaignId}`; + await session.own({ kind: "project", name: projectName, journeyId: id }); + const project = await api.post(`/api/companies/${company.id}/projects`, { + name: projectName, description: "Disposable read-only public repository continuation fixture.", + executionWorkspacePolicy: { enabled: true, defaultMode: "shared_workspace", sharedWorkspaceConcurrency: "serialize", + allowIssueOverride: false, environmentId: context.environmentId, workspaceStrategy: { type: "project_primary" } }, + workspace: { name: "Primary", sourceType: "git_repo", repoUrl: source.url, repoRef: source.commit, isPrimary: true }, + }); + await session.own({ kind: "project", name: projectName, id: project.id, journeyId: id }); + const configured = await readObserved(`/api/projects/${project.id}`); + const projectWorkspaces = configured.workspaces?.filter((workspace: Row) => workspace.repoUrl === source.url && workspace.repoRef === source.commit && workspace.isPrimary); + if (configured.companyId !== company.id || !Array.isArray(projectWorkspaces) || projectWorkspaces.length !== 1) + throw new JourneyError("failed", "repository_project_source_mismatch"); + const projectWorkspaceId = projectWorkspaces[0]!.id; + const title = `Journey QA repository continuation ${campaignId}`; + const created = await createTaskThroughUi({ page, origin: api.origin, issuePrefix: company.issuePrefix, + agentName: agent.name, projectName, title, prompt: story.first, workMode: "standard", requireExplicitTitle: true, + beforeDispatch: () => session.own({ kind: "issue", name: title, journeyId: id }) }); + await session.own({ kind: "issue", name: title, id: created.issueId, journeyId: id }); + const route = `/api/issues/${created.issueId}`; + const read = async () => { + const [currentIssue, summaries, comments, interactions] = await Promise.all([ + readObserved(route), readObserved(`${route}/runs`), + readObserved(`${route}/comments?order=asc`), readObserved>(`${route}/interactions`), + ]); + if (!Array.isArray(summaries) || summaries.length > 2) throw new JourneyError("failed", "repository_run_limit_exceeded"); + const taskRuns = (await Promise.all(summaries.map(summary => readObserved(`/api/heartbeat-runs/${summary.runId}`)))) + .sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt)) || String(a.id).localeCompare(String(b.id))); + const state = { currentIssue, taskRuns, comments, interactions, attachments: [] }; + await retain("repository-latest-task.json", state); + validateAgentTaskState(context, created.issueId, state, 2); + if (currentIssue.projectId !== project.id) throw new JourneyError("failed", "repository_project_binding_mismatch"); + return state; + }; + const settled = (count: number) => waitFor(options.deadlineAt, read, state => state.currentIssue.status === "done" + && state.taskRuns.length === count && state.taskRuns.every(run => run.status === "succeeded" + && run.nativePhase === "committed" && run.resultJson?.finalizationPhase === "committed") + && state.interactions.every(row => row.status !== "pending"), "repository_task_not_completed"); + // Use the attached target's origin for the common registration/download oracle. + const artifactApi = { get: (route: string) => readObserved(route), request: { get: (route: string) => api.response(route) } }; + const bundle = async (name: string, runId: string) => { + const [artifact] = await readRegisteredArtifacts(artifactApi, created.issueId, runId, [name]); + const bytes = await api.bytes(`/api/attachments/${artifact!.attachmentId}/content?download=1`); + if (createHash("sha256").update(bytes).digest("hex") !== artifact!.sha256) throw new JourneyError("failed", "repository_bundle_download_changed"); + await writeFile(path.join(options.directory, name), bytes, { mode: 0o600, flag: "wx" }); + return { artifact: artifact!, bytes }; + }; + // Task-chat layouts keep generic output controls in Artifacts. Use the + // production download link for this exact attachment, not a coincidental + // URL in a model reply or an API-only readback. + const browserDownload = async (artifact: { attachmentId: string; sha256: string }, phase: string) => { + await openDocumentAuthoring(context); + const link = page.getByRole("link", { name: /^Download/ }).and( + page.locator(`a[href*="/api/attachments/${artifact.attachmentId}/content"]`)).first(); + await expect(link).toBeVisible(); + const [download] = await Promise.all([page.waitForEvent("download", { timeout: Math.max(1, Math.min(30_000, options.deadlineAt - Date.now())) }), link.click()]); + const filename = path.join(options.directory, `browser-${phase}`); + await download.saveAs(filename); + const bytes = await readFile(filename); + if (createHash("sha256").update(bytes).digest("hex") !== artifact.sha256) throw new JourneyError("failed", "repository_browser_download_changed"); + }; + const file = (name: string) => readObserved(`${route}/file-resources/content?workspace=execution&path=${encodeURIComponent(name)}`); + const first = await settled(1), firstRun = first.taskRuns[0]!; + const initialView = await readObserved(`${route}/workspace`), workspaceId = initialView.workspace?.id; + const binding = (issue: Row, view: Row, runs: Row[]) => repositoryWorkspaceBindingMatches({ companyId: company.id, + issueId: created.issueId, projectId: project.id, projectWorkspaceId, workspaceId, repoUrl: source.url, issue, view, runs }); + check("repository-configured-source-bound", binding(first.currentIssue, initialView, first.taskRuns)); + const [referenceArtifact] = await readRegisteredArtifacts(artifactApi, created.issueId, firstRun.id, [files.context]); + const original = referenceArtifact!.content; + if (!/^[a-f0-9]{64}\n$/.test(original)) throw new JourneyError("failed", "repository_original_reference_invalid"); + const { artifact: firstBundle, bytes: originalBytes } = await bundle(files.firstBundle, firstRun.id); + const initialGit = await inspectRepositoryBundle(originalBytes, { sourceCommit: source.commit, files: [files.context, files.first, "README"] }); + check("repository-first-commit-verified", repositoryCommitMatches(initialGit, { parent: source.commit, sourceReadme: source.readme, + files: { [files.context]: original, [files.first]: "first\n", README: source.readme }, added: [files.context, files.first] })); + check("repository-original-files-saved", repositoryWorkspaceFileMatches(await file(files.context), workspaceId, files.context, original) + && repositoryWorkspaceFileMatches(await file(files.first), workspaceId, files.first, "first\n") + && repositoryWorkspaceFileMatches(await file("README"), workspaceId, "README", source.readme)); + await retain("repository-first-observation.json", { source, projectId: project.id, workspaceId, issueId: created.issueId, + runId: firstRun.id, referenceArtifact, initialGit, bundle: { attachmentId: firstBundle.attachmentId, sha256: firstBundle.sha256 } }); + const url = `${api.origin}/${company.issuePrefix}/issues/${first.currentIssue.identifier ?? created.issueId}`; + await page.goto(url, { waitUntil: "domcontentloaded" }); + await page.reload(); + await browserDownload(firstBundle, "first-bundle"); + // The original random reference and commit are not supplied by this follow-up. + await submitTaskReply(page, story.followup); + const withReply = await waitFor(Math.min(options.deadlineAt, Date.now() + 30_000), read, + state => state.comments.some(comment => comment.authorUserId === context.user.id && !first.comments.some(old => old.id === comment.id)), + "repository_followup_not_stored"); + check("repository-followup-stored-once", fixtureReplyStoredOnce(first.comments, withReply.comments, context.user.id, story.followup)); + const final = await settled(2), lastRun = final.taskRuns[1]!; + const finalView = await readObserved(`${route}/workspace`); + check("repository-same-workspace-retained", binding(final.currentIssue, finalView, final.taskRuns) + && Date.parse(lastRun.startedAt) >= Date.parse(firstRun.finishedAt)); + const { artifact: finalBundle, bytes: finalBytes } = await bundle(files.finalBundle, lastRun.id); + const finalGit = await inspectRepositoryBundle(finalBytes, + { sourceCommit: source.commit, files: [files.context, files.first, files.followup, "README"] }); + check("repository-second-commit-verified", repositoryCommitMatches(finalGit, { parent: initialGit.head, sourceReadme: source.readme, + files: { [files.context]: original, [files.first]: "first\n", [files.followup]: "verified\n", README: source.readme }, added: [files.followup] })); + await page.goto(url, { waitUntil: "domcontentloaded" }); + await page.reload(); + for (const [index, artifact] of [referenceArtifact!, firstBundle, finalBundle].entries()) await browserDownload(artifact, `reopened-${index}`); + const [retained, retainedBundle] = await readRegisteredArtifacts(artifactApi, created.issueId, firstRun.id, [files.context, files.firstBundle]); + check("repository-original-and-result-reopened", retained!.attachmentId === referenceArtifact!.attachmentId && retained!.content === original + && retainedBundle!.attachmentId === firstBundle.attachmentId && retainedBundle!.sha256 === firstBundle.sha256 + && repositoryWorkspaceFileMatches(await file(files.context), workspaceId, files.context, original) + && repositoryWorkspaceFileMatches(await file(files.first), workspaceId, files.first, "first\n") + && repositoryWorkspaceFileMatches(await file(files.followup), workspaceId, files.followup, "verified\n") + && repositoryWorkspaceFileMatches(await file("README"), workspaceId, "README", source.readme)); + check("configured-model-and-environment", true); // Both actual native runs passed validateAgentTaskState. + await retain("repository-continuation-checks.json", { source, projectId: project.id, workspaceId, issueId: created.issueId, + initialGit, finalGit, referenceArtifact, firstBundle, finalBundle, runIds: final.taskRuns.map(run => run.id) }); + } finally { await observations.preserve(options.directory); } +} diff --git a/tests/runner-e2e/product-journeys/repository-continuation.md b/tests/runner-e2e/product-journeys/repository-continuation.md new file mode 100644 index 00000000000..981a27d5ae7 --- /dev/null +++ b/tests/runner-e2e/product-journeys/repository-continuation.md @@ -0,0 +1,51 @@ +# Repository continuation journey + +`repository-continuation` uses the target's configured native agent and execution +environment. It creates one owned project and one owned task. The project uses +the public `octocat/Hello-World` repository at commit +`7fd1a60b01f91b314f59955a4e4d4e80d8edf11d`. The fixture never pushes or changes +the remote repository. + +The first browser request creates a random reference file and a short note in +the project checkout. The agent makes one local commit and registers the +reference and a self-contained Git bundle as downloads. The observer downloads +and verifies them before it sends the second request. That request supplies no +reference value or commit ID. It requires the original files to remain intact, +adds one follow-up file, and delivers a second committed Git bundle. + +The oracle inspects actual Git objects in disposable bare repositories. It runs +no checkout, hook, project script, or remote fetch. Each commit must have the +expected parent and only the declared file additions. The repository's original +files must stay unchanged. The observer compares the committed file bytes with +the ordinary task workspace file API, checks the configured project source and +both native workspace bindings, and reopens all output links after reload. +Missing input, recreated history, extra native runs, wrong bytes, or a replaced +workspace fails the journey. + +There are exactly two native runs, one per user request. The target supplies the +unchanged model, billing policy and overall deadline. This case does not test +automatic repository preparation, automatic continuation, remote pushes or pull +requests. Those are separate journeys. + +The target provider owns authentication, source identity, usage collection and +cleanup. Intent and receipts are saved for the owned project and task. Git +bundles, raw API receipts, source bytes and workspace identifiers remain in the +private Product journey evidence directory. They must not enter a public report. +The normal final screenshot is restricted to the authenticated owned task page. + +Registration and local assertion calibration do not establish a deployed pass. +Daily admission requires a fresh pass against the intended QA target with the +same source and assertion set, completed collection, and successful cleanup. + +Browser result access opens the task's ordinary Artifacts panel. Each exact +registered attachment is downloaded through its visible production control, and +the downloaded digest must match the independent registered artifact receipt. +A URL present only in a chat message cannot satisfy this check. + +The two-turn proof has a boundary: the final saved checkout's hidden `.git` +metadata and `HEAD` are not independently read after turn two. The ordinary +workspace file API protects `.git`, and remote workspace close-readiness does +not expose Git metadata. The second verified commit must still have the first +verified commit as its parent, the original saved files must remain intact, and +both runs must retain the same workspace binding. This case does not qualify +post-completion Git metadata durability for a third turn. diff --git a/tests/runner-e2e/product-journeys/success-contract.ts b/tests/runner-e2e/product-journeys/success-contract.ts index 45620f41532..7141438d29c 100644 --- a/tests/runner-e2e/product-journeys/success-contract.ts +++ b/tests/runner-e2e/product-journeys/success-contract.ts @@ -1,5 +1,5 @@ /** Closed, target-independent success requirements used by dispatch and retained evidence. */ -export const agentJourneyRuns = { "routine-scheduled-run": 1, "file-input-answer": 1, "ask-question": 1, "plan-revise-accept": 3, "build-revise": 2, +export const agentJourneyRuns = { "repository-continuation": 2, "routine-scheduled-run": 1, "file-input-answer": 1, "ask-question": 1, "plan-revise-accept": 3, "build-revise": 2, "followup-while-running": 2, "protected-actions": 4, "create-skill-studio": 1, "stop-redirect": 3, "question-answer-resume": 3, "hire-delegate-reuse": 7 } as const; export function agentJourneyRunCountValid(id: string, count: unknown): boolean { @@ -9,6 +9,7 @@ export function agentJourneyRunCountValid(id: string, count: unknown): boolean { return (count as number) >= minimum && (count as number) <= maximum; } export const journeyRequiredChecks: Record = { + "repository-continuation": ["repository-configured-source-bound", "repository-first-commit-verified", "repository-original-files-saved", "repository-followup-stored-once", "repository-same-workspace-retained", "repository-second-commit-verified", "repository-original-and-result-reopened", "configured-model-and-environment"], "routine-scheduled-run": ["routine-browser-created", "routine-scheduler-created-execution", "routine-result-attributed", "routine-history-preserved", "routine-pause-prevented-next-run", "routine-result-reopened", "configured-model-and-environment"], "inbox-triage": ["inbox-external-human-activity", "inbox-read-persisted", "inbox-archive-persisted", "inbox-archive-preserved-task-status", "inbox-relevant-activity-resurfaced", "no-agent-work"], "file-input-answer": ["input-upload-bytes-preserved", "input-content-used-correctly", "input-result-attributed-to-run", "input-result-reopened", "original-input-retained", "configured-model-and-environment"], diff --git a/tests/runner-e2e/product-journeys/types.ts b/tests/runner-e2e/product-journeys/types.ts index f753c1873d2..0de951ff574 100644 --- a/tests/runner-e2e/product-journeys/types.ts +++ b/tests/runner-e2e/product-journeys/types.ts @@ -3,7 +3,7 @@ import type { ConnectionApi } from "../connection-target.js"; export type JourneyOutcome = "passed" | "failed" | "blocked" | "unsupported"; export type JourneyId = "task-properties" | "task-create" | "project-task-context" | "task-document-persistence" | "ask-question" | "cloud-onboarding" - | "question-answer-resume" | "task-search-filter" | "followup-while-running" | "plan-revise-accept" | "build-revise" | "protected-actions" | "create-skill-studio" | "stop-redirect" | "hire-delegate-reuse" | "file-input-answer" | "inbox-triage" | "routine-scheduled-run"; + | "question-answer-resume" | "task-search-filter" | "followup-while-running" | "plan-revise-accept" | "build-revise" | "protected-actions" | "create-skill-studio" | "stop-redirect" | "hire-delegate-reuse" | "file-input-answer" | "inbox-triage" | "routine-scheduled-run" | "repository-continuation"; export type TargetMode = "managed-self-hosted" | "existing-cloud" | "new-cloud"; export interface DeploymentIdentity { kind: "self-hosted" | "cloud"; diff --git a/tests/runner-e2e/registered-artifact.ts b/tests/runner-e2e/registered-artifact.ts index 9531119fe36..3d55332d074 100644 --- a/tests/runner-e2e/registered-artifact.ts +++ b/tests/runner-e2e/registered-artifact.ts @@ -1,7 +1,7 @@ import { createHash } from "node:crypto"; import type { IssueAttachment } from "../../packages/shared/src/types/issue.js"; import type { IssueWorkProduct } from "../../packages/shared/src/types/work-product.js"; -import type { RunnerApi } from "./api.js"; +import type { APIRequestContext } from "@playwright/test"; export class RegisteredArtifactNotFoundError extends Error { constructor(artifactName: string) { @@ -13,7 +13,7 @@ export class RegisteredArtifactNotFoundError extends Error { const REGISTERED_ARTIFACT_STATUSES = new Set(["active", "ready_for_review", "approved"]); /** Registration, attribution and downloaded bytes must agree independently. */ -export async function readRegisteredArtifacts(api: RunnerApi, issueId: string, runId: string, names: readonly string[]) { +export async function readRegisteredArtifacts(api: { get(route: string): Promise; request: Pick }, issueId: string, runId: string, names: readonly string[]) { if (names.length === 0) return []; const [products, attachments] = await Promise.all([ api.get(`/api/issues/${encodeURIComponent(issueId)}/work-products`),