diff --git a/tests/runner-e2e/PRODUCT-JOURNEYS.md b/tests/runner-e2e/PRODUCT-JOURNEYS.md index 40e9ddfad12..74463320bd9 100644 --- a/tests/runner-e2e/PRODUCT-JOURNEYS.md +++ b/tests/runner-e2e/PRODUCT-JOURNEYS.md @@ -23,6 +23,7 @@ never start a server or depend on a tenant filesystem. | Journey | Required starting state | Observed outcome | | --- | --- | --- | +| `repository-continuation` | Native agent; two work runs; a pinned public read-only repository | Verify independent Git bundle history and workspace bytes across two turns, preserve the original random file, and reopen registered outputs after reload. See [repository-continuation.md](product-journeys/repository-continuation.md) | | `routine-scheduled-run` | Native agent; one work run; an owned project and routine | Browser-create a routine, enable its minute schedule, verify one scheduled invocation and run-attributed saved output, then pause and observe the next schedule boundary with no new work | | `file-input-answer` | Native agent; one work run; synthetic CSV uploaded through the composer | Verify original uploaded bytes, independently calculated accepted-row count and units, a run-attributed task document after reload, and preservation of the input attachment. The task prompt contains neither the random file reference nor the answers | | `inbox-triage` | Signed-in board user; a separately authenticated QA human author; a fresh self-assigned task; zero model runs | Mark an unread task read, archive it through Mine, reload both views, verify unchanged work status, then observe a real Blocked transition resurface it through the normal archive policy | @@ -480,3 +481,9 @@ uses the server's timestamp; client and server wall clocks are never compared. The journey retries only transient read transport failures within the existing phase and journey deadlines. Private read-failure receipts retain those observations. Mutations, user messages and agent work are never replayed. + +Repository JSON metadata observations retry only transient transport failures +within the original journey deadline. Binary attachment and browser downloads +remain single attempts; transport failures and digest mismatches fail visibly. Git inspection consumes the exact digest-verified download +buffer; browser downloads remain an independent result-access check. The +repository journey shares the observation reader and browser helpers above. diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index c040b57595a..13989a0efe5 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -126,7 +126,7 @@ describe("runner E2E catalog", () => { }); it("exports deployment journeys separately from the model matrix and default selection", () => { - expect(productJourneys.map(journey => journey.id)).toEqual(["routine-scheduled-run", "inbox-triage", "file-input-answer", "task-create", "project-task-context", "ask-question", "question-answer-resume", "task-properties", "task-document-persistence", "task-search-filter", "plan-revise-accept", "build-revise", "protected-actions", "create-skill-studio", "stop-redirect", "followup-while-running", "hire-delegate-reuse", "cloud-onboarding"]); + expect(productJourneys.map(journey => journey.id)).toEqual(["repository-continuation", "routine-scheduled-run", "inbox-triage", "file-input-answer", "task-create", "project-task-context", "ask-question", "question-answer-resume", "task-properties", "task-document-persistence", "task-search-filter", "plan-revise-accept", "build-revise", "protected-actions", "create-skill-studio", "stop-redirect", "followup-while-running", "hire-delegate-reuse", "cloud-onboarding"]); expect(runnerSuites.some(suite => suite.id === "product-journeys")).toBe(false); expect(runnerMatrix.some(execution => execution.suite.id === "product-journeys")).toBe(false); expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(execution => execution.suite.id === "product-journeys")).toBe(false); diff --git a/tests/runner-e2e/journey-features.ts b/tests/runner-e2e/journey-features.ts index 9abc7caf528..7da187c149e 100644 --- a/tests/runner-e2e/journey-features.ts +++ b/tests/runner-e2e/journey-features.ts @@ -19,6 +19,7 @@ function associate(suites: readonly string[], cases: Record; -type AgentJourney = Exclude; +type AgentJourney = Exclude; interface TaskState { currentIssue: Row & { id: string; status: string }; taskRuns: Array; comments: Row[]; interactions: Array; attachments: StoryDelivery[] } interface Options { deadlineAt: number; directory: string; capture(id: string, label: string, file: string): Promise } const active = (run: Row) => ["queued", "running"].includes(run.status); diff --git a/tests/runner-e2e/product-journeys/catalog.ts b/tests/runner-e2e/product-journeys/catalog.ts index 156220ce89e..2b47d1d842e 100644 --- a/tests/runner-e2e/product-journeys/catalog.ts +++ b/tests/runner-e2e/product-journeys/catalog.ts @@ -2,6 +2,7 @@ import { agentJourneyRuns } from "./success-contract.js"; export { agentJourneyRuns, agentJourneyRunCountValid } from "./success-contract.js"; import type { JourneyId, TargetConfig } from "./types.js"; export const productJourneys = [ + { id: "repository-continuation", label: "Continue repository work with the same saved workspace", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["projects/task-context", "documents-artifacts/attachments"], paid: true }, { id: "routine-scheduled-run", label: "Run a scheduled routine once and verify pause and saved history", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["routines/definition", "routines/triggers", "routines/history", "routines/management"], paid: true }, { id: "inbox-triage", label: "Read, archive and resurface an owned inbox task", requires: ["company", "board-user", "task-write", "inbox-author"], capabilities: ["inbox-search/views", "inbox-search/triage"], paid: false }, { id: "file-input-answer", label: "Upload a reference file and verify the agent uses its contents", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["documents-artifacts/attachments", "documents-artifacts/documents"], paid: true }, diff --git a/tests/runner-e2e/product-journeys/launch.ts b/tests/runner-e2e/product-journeys/launch.ts index 9ca26c76c51..edfe84bbe75 100644 --- a/tests/runner-e2e/product-journeys/launch.ts +++ b/tests/runner-e2e/product-journeys/launch.ts @@ -17,6 +17,7 @@ import { projectTaskContext } from "./project-context-flow.js"; import { runAgentJourney } from "./agent-flows.js"; import { runActiveFollowup } from "./active-followup-flow.js"; import { runClarificationJourney } from "./clarification-flow.js"; +import { runRepositoryContinuation } from "./repository-continuation-flow.js"; import { runScheduledRoutine } from "./routine-schedule-flow.js"; import { runInputFile } from "./input-file-flow.js"; import { inboxTriage } from "./inbox-triage-flow.js"; @@ -111,7 +112,7 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri try { sha = execFileSync("git", ["rev-parse", "HEAD"], { cwd: repositoryRoot, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim(); } catch { /* Packaged harnesses may not include Git metadata. */ } const source = resolveRunnerE2ESource({ sha, ref: null, workflowRunUrl: null }); const definitionHash = createHash("sha256").update(JSON.stringify(productJourneys)); - for (const name of ["observation-read.ts", "../connection-config.ts", "catalog.ts", "../user-actions.ts", "../connection-target.ts", "routine-schedule-flow.ts", "routine-schedule-evidence.ts", "input-file-flow.ts", "input-file-evidence.ts", "inbox-triage-flow.ts", "inbox-triage-evidence.ts", "inbox-triage-fixture.ts", "inbox-author-contract.ts", "task-properties.ts", "task-search-filter.ts", "task-search-evidence.ts", "basic-flows.ts", "basic-evidence.ts", "task-document-flow.ts", "task-document-evidence.ts", "project-context-flow.ts", "project-context-readback.ts", "project-context-evidence.ts", "agent-flows.ts", "hire-delegate-flow.ts", "hire-delegate-composer.ts", "hire-delegate-evidence.ts", "active-followup-flow.ts", "active-followup-evidence.ts", "stop-redirect-evidence.ts", "clarification-flow.ts", "clarification-evidence.ts", "../question-resume-ui.ts", "../question-resume-scoring.ts", "../runtime-question-readiness.ts", "../chat-flow.ts", "../continuation-readiness.ts", "../continuation-cases.ts", "success-contract.ts", "product-screenshot.ts", "cloud-onboarding.ts", "onboarding-connect.ts", "onboarding-first-task.ts", "workspace-deferral.ts", "onboarding-pre-acceptance.ts", "artifact-evidence.ts", "fixtures.ts", "launch.ts", "types.ts", "collection.ts", "local-target.ts", "../../../ui/src/lib/markdown-reply-content.ts", "../../../ui/src/lib/issue-reference.ts", "../../../ui/src/lib/company-routes.ts", "../../../ui/package.json"]) definitionHash.update(await readFile(path.join(import.meta.dirname, name))); + for (const name of ["observation-read.ts", "../connection-config.ts", "catalog.ts", "repository-continuation-flow.ts", "repository-continuation-contract.ts", "repository-continuation-evidence.ts", "../registered-artifact.ts", "../user-actions.ts", "../connection-target.ts", "../../../packages/shared/src/project-mentions.ts", "routine-schedule-flow.ts", "routine-schedule-evidence.ts", "input-file-flow.ts", "input-file-evidence.ts", "inbox-triage-flow.ts", "inbox-triage-evidence.ts", "inbox-triage-fixture.ts", "inbox-author-contract.ts", "task-properties.ts", "task-search-filter.ts", "task-search-evidence.ts", "basic-flows.ts", "basic-evidence.ts", "task-document-flow.ts", "task-document-evidence.ts", "project-context-flow.ts", "project-context-readback.ts", "project-context-evidence.ts", "agent-flows.ts", "hire-delegate-flow.ts", "hire-delegate-composer.ts", "hire-delegate-evidence.ts", "active-followup-flow.ts", "active-followup-evidence.ts", "stop-redirect-evidence.ts", "clarification-flow.ts", "clarification-evidence.ts", "../question-resume-ui.ts", "../question-resume-scoring.ts", "../runtime-question-readiness.ts", "../chat-flow.ts", "../continuation-readiness.ts", "../continuation-cases.ts", "success-contract.ts", "product-screenshot.ts", "cloud-onboarding.ts", "onboarding-connect.ts", "onboarding-first-task.ts", "workspace-deferral.ts", "onboarding-pre-acceptance.ts", "artifact-evidence.ts", "fixtures.ts", "launch.ts", "types.ts", "collection.ts", "local-target.ts", "../../../ui/src/lib/markdown-reply-content.ts", "../../../ui/src/lib/issue-reference.ts", "../../../ui/src/lib/company-routes.ts", "../../../ui/package.json"]) definitionHash.update(await readFile(path.join(import.meta.dirname, name))); for (const file of ["catalog.ts", "api.ts", "run-observations.ts", "native-task-reply.ts", "native-event-envelope.ts", "user-actions.ts", "plan-revision-flow.ts", "skill-studio-flow.ts", "protected-action-flow.ts", "story-artifact.ts", "stop-redirect-flow.ts", "everyday-cases.ts", "everyday-delivery.ts", "chat-hardening.ts", "everyday-decisions.ts", "connection-guidance-evidence.ts", "everyday-artifact.py"]) definitionHash.update(await readFile(path.join(import.meta.dirname, "..", file))); definitionHash.update(runnerSuites.flatMap(suite => suite.tasks).find(task => task.id === "ask-question")!.buildPrompt("definition")); const suiteDefinitionHash = definitionHash.digest("hex"); @@ -207,7 +208,9 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri result.provider = ready.agent.provider; result.model = ready.agent.model; result.runtimeMode = ready.agent.runtimeMode; - if (id === "routine-scheduled-run") await runScheduledRoutine(ready, session!, campaignId, evidence, + if (id === "repository-continuation") await runRepositoryContinuation(ready, session!, campaignId, evidence, + { deadlineAt, directory: path.join(outputDir, "private-stories", id) }); + else if (id === "routine-scheduled-run") await runScheduledRoutine(ready, session!, campaignId, evidence, { deadlineAt, directory: path.join(outputDir, "private-stories", id) }); else if (id === "file-input-answer") await runInputFile(ready, session!, campaignId, evidence, { deadlineAt, directory: path.join(outputDir, "private-stories", id) }); diff --git a/tests/runner-e2e/product-journeys/repository-continuation-contract.ts b/tests/runner-e2e/product-journeys/repository-continuation-contract.ts new file mode 100644 index 00000000000..4d301919f18 --- /dev/null +++ b/tests/runner-e2e/product-journeys/repository-continuation-contract.ts @@ -0,0 +1,30 @@ +/** Public, read-only fixture verified through GitHub's commit and content APIs. */ +export const repositoryContinuationSource = { + url: "https://github.com/octocat/Hello-World", + commit: "7fd1a60b01f91b314f59955a4e4d4e80d8edf11d", + readme: "Hello World!\n", +} as const; + +export function repositoryContinuationStory(campaignId: string) { + if (!/^journey-[0-9]+-[a-f0-9]{8}$/.test(campaignId)) throw new Error("invalid_repository_journey_campaign"); + const suffix = campaignId.replaceAll("-", ""); + const files = { context: `qa-context-${suffix}.txt`, first: `qa-first-${suffix}.txt`, + followup: `qa-followup-${suffix}.txt`, firstBundle: `qa-first-${suffix}.bundle`, finalBundle: `qa-final-${suffix}.bundle` }; + return { + files, + first: [ + "Use this task's configured project repository. Work only in its existing checkout; do not create another repository, project, or task. Do not push anything to a remote.", + `Verify that its starting HEAD is ${repositoryContinuationSource.commit}, then read the committed README. Stop and report a mismatch if this is not the configured repository.`, + `Generate a fresh unpredictable 32-byte hexadecimal reference from the operating system random source and save it followed by one LF in ${files.context}. Do not print the reference in comments or your final answer.`, + `Create ${files.first} containing only the text first followed by exactly one LF newline. Make one local Git commit containing only these two new files. Preserve every original repository file.`, + `After the commit, create a self-contained Git bundle of HEAD at ${files.firstBundle}. Do not add the bundle to Git.`, + `Register both ${files.context} and ${files.firstBundle} as downloadable artifacts with those exact filenames and titles. Deliver their accessible links and finish the task. I will send a later follow-up.`, + ].join("\n"), + followup: [ + "Continue in the same project checkout. Keep the earlier commit and original files.", + `Read ${files.context} and ${files.first} from the current workspace. If either is missing or invalid, stop and report the failure. Never recreate either file or recover it from conversation, attachments, or another checkout.`, + `Keep their original bytes. Add ${files.followup} containing only the text verified followed by exactly one LF newline. Make one more local Git commit containing only that new file. Do not amend or rewrite the earlier commit, and do not push.`, + `Create a new self-contained Git bundle of HEAD at ${files.finalBundle}. Register it as a downloadable artifact with that exact filename and title. Deliver its accessible link and finish the task.`, + ].join("\n"), + }; +} diff --git a/tests/runner-e2e/product-journeys/repository-continuation-evidence.test.ts b/tests/runner-e2e/product-journeys/repository-continuation-evidence.test.ts new file mode 100644 index 00000000000..b201da677f5 --- /dev/null +++ b/tests/runner-e2e/product-journeys/repository-continuation-evidence.test.ts @@ -0,0 +1,105 @@ +import { execFile as execFileCallback } from "node:child_process"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { repositoryContinuationSource, repositoryContinuationStory } from "./repository-continuation-contract.js"; +import { inspectRepositoryBundle, repositoryCommitMatches, repositoryWorkspaceBindingMatches, repositoryWorkspaceFileMatches, + type RepositoryBundleEvidence } from "./repository-continuation-evidence.js"; +import type { WorkspaceFileContent } from "../../../packages/shared/src/types/workspace-file-resource.js"; + +const execFile = promisify(execFileCallback); +describe("repository continuation Git evidence", () => { + let root: string, sourceCommit: string, first: RepositoryBundleEvidence, final: RepositoryBundleEvidence, firstBytes: Buffer; + const reference = `${"c".repeat(64)}\n`, firstFile = "first.txt", contextFile = "context.txt", finalFile = "followup.txt"; + beforeAll(async () => { + root = await mkdtemp(path.join(os.tmpdir(), "paperclip-qa-repo-control-")); + const git = async (...args: string[]) => (await execFile("git", ["-c", "core.hooksPath=/dev/null", "-C", root, ...args], { + env: { PATH: process.env.PATH, GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: "/dev/null", + GIT_AUTHOR_NAME: "Fixture", GIT_AUTHOR_EMAIL: "fixture@example.test", GIT_COMMITTER_NAME: "Fixture", GIT_COMMITTER_EMAIL: "fixture@example.test" }, + encoding: "utf8", timeout: 10_000, + })).stdout; + await git("init"); + await writeFile(path.join(root, "README"), repositoryContinuationSource.readme); + await git("add", "README"); await git("commit", "-m", "source"); + sourceCommit = (await git("rev-parse", "HEAD")).trim(); + await writeFile(path.join(root, contextFile), reference); await writeFile(path.join(root, firstFile), "first\n"); + await git("add", contextFile, firstFile); await git("commit", "-m", "first"); + await git("bundle", "create", path.join(root, "first.bundle"), "HEAD"); + firstBytes = await readFile(path.join(root, "first.bundle")); + first = await inspectRepositoryBundle(firstBytes, { sourceCommit, files: [contextFile, firstFile, "README"] }); + await writeFile(path.join(root, finalFile), "verified\n"); await git("add", finalFile); await git("commit", "-m", "second"); + await git("bundle", "create", "--version=3", path.join(root, "final.bundle"), "HEAD"); + final = await inspectRepositoryBundle(await readFile(path.join(root, "final.bundle")), { sourceCommit, files: [contextFile, firstFile, finalFile, "README"] }); + }); + afterAll(async () => { if (root) await rm(root, { recursive: true, force: true }); }); + const expectedFirst = () => ({ parent: sourceCommit, sourceReadme: repositoryContinuationSource.readme, + files: { [contextFile]: reference, [firstFile]: "first\n", README: repositoryContinuationSource.readme }, added: [contextFile, firstFile] }); + it("reads real Git commits, parent links and file objects from both downloaded bundles", () => { + expect(repositoryCommitMatches(first, expectedFirst())).toBe(true); + expect(repositoryCommitMatches(final, { parent: first.head, sourceReadme: repositoryContinuationSource.readme, + files: { [contextFile]: reference, [firstFile]: "first\n", [finalFile]: "verified\n", README: repositoryContinuationSource.readme }, added: [finalFile] })).toBe(true); + }); + const invalidEvidence: Array<[string, () => RepositoryBundleEvidence | null]> = [ + ["missing", () => null], + ["rewritten history", () => ({ ...first, parent: "a".repeat(40) })], + ["no new commit", () => ({ ...first, head: sourceCommit })], + ["wrong repository", () => ({ ...first, sourceReadme: "Other repository\n" })], + ["original changed", () => ({ ...first, files: { ...first.files, [contextFile]: "replacement\n" } })], + ["extra changes", () => ({ ...first, changes: [...first.changes, { status: "M", path: "README" }] })], + ["missing saved file", () => ({ ...first, files: {} })], + ["duplicate tree path", () => ({ ...first, changes: [first.changes[0]!, first.changes[0]!] })], + ["modified original", () => ({ ...first, changes: first.changes.map(change => ({ ...change, status: "M" })) })], + ]; + it.each(invalidEvidence)("rejects %s evidence", (_name, change) => { expect(repositoryCommitMatches(change(), expectedFirst())).toBe(false); }); + it("rejects malformed, corrupt, incomplete and unrelated Git bundles", async () => { + await expect(inspectRepositoryBundle(Buffer.from("claimed git commit"), { sourceCommit, files: [] })).rejects.toThrow(); + await expect(inspectRepositoryBundle(firstBytes.subarray(0, 150), { sourceCommit, files: [] })).rejects.toThrow(); + await expect(inspectRepositoryBundle(firstBytes, { sourceCommit: "f".repeat(40), files: [] })).rejects.toThrow(); + await expect(inspectRepositoryBundle(firstBytes, { sourceCommit, files: ["../escape"] })).rejects.toThrow(); + }); +}); + +describe("repository continuation target binding", () => { + const issue = { id: "issue", companyId: "company", projectId: "project", executionWorkspaceId: "workspace" }; + const run = { companyId: "company", runnerProfileJson: { nativeExecutionInput: { binding: { executionWorkspaceId: "workspace" } } } }; + const workspace = { id: "workspace", companyId: "company", projectId: "project", projectWorkspaceId: "project-workspace", repoUrl: repositoryContinuationSource.url }; + const input = { companyId: "company", issueId: "issue", projectId: "project", projectWorkspaceId: "project-workspace", workspaceId: "workspace", + repoUrl: repositoryContinuationSource.url, issue, view: { issueId: "issue", workspace }, runs: [run] }; + it("requires the configured project, actual task workspace and every run binding", () => { + expect(repositoryWorkspaceBindingMatches(input)).toBe(true); + expect(repositoryWorkspaceBindingMatches({ ...input, runs: [] })).toBe(false); + expect(repositoryWorkspaceBindingMatches({ ...input, view: {} })).toBe(false); + for (const change of [{ id: "foreign" }, { companyId: "foreign" }, { projectId: "foreign" }, + { projectWorkspaceId: "foreign" }, { repoUrl: "https://example.test/other" }]) { + expect(repositoryWorkspaceBindingMatches({ ...input, view: { ...input.view, workspace: { ...workspace, ...change } } })).toBe(false); + } + expect(repositoryWorkspaceBindingMatches({ ...input, runs: [run, { ...run, companyId: "foreign" }] })).toBe(false); + expect(repositoryWorkspaceBindingMatches({ ...input, runs: [run, { companyId: "company" }] })).toBe(false); + expect(repositoryWorkspaceBindingMatches({ ...input, issue: { ...issue, executionWorkspaceId: "replacement" } })).toBe(false); + }); + it("compares exact workspace API bytes and rejects another workspace or file", () => { + const file = { resource: { kind: "file", workspaceKind: "execution_workspace", workspaceId: "workspace", title: "context.txt", byteSize: 4 }, + content: { encoding: "utf8", data: "abc\n" } } as WorkspaceFileContent; + expect(repositoryWorkspaceFileMatches(file, "workspace", "context.txt", "abc\n")).toBe(true); + expect(repositoryWorkspaceFileMatches(null, "workspace", "context.txt", "abc\n")).toBe(false); + expect(repositoryWorkspaceFileMatches(file, "foreign", "context.txt", "abc\n")).toBe(false); + expect(repositoryWorkspaceFileMatches(file, "workspace", "wrong.txt", "abc\n")).toBe(false); + expect(repositoryWorkspaceFileMatches(file, "workspace", "context.txt", "xyz\n")).toBe(false); + }); + it("keeps the later user request free from the unpredictable earlier reference", () => { + const story = repositoryContinuationStory("journey-1793642415731-3c34246f"); + expect(story.first).toContain(repositoryContinuationSource.commit); + expect(story.followup).toContain("Never recreate"); + expect(story.followup).not.toContain("c".repeat(64)); + expect(() => repositoryContinuationStory("../unsafe")).toThrow(); + }); + it("emits unambiguous LF instructions for the independently verified file bytes", () => { + const story = repositoryContinuationStory("journey-1793642415731-3c34246f"); + expect(story.first).toContain("containing only the text first followed by exactly one LF newline."); + expect(story.followup).toContain("containing only the text verified followed by exactly one LF newline."); + expect(story.first).not.toContain("first\\n"); + expect(story.followup).not.toContain("verified\\n"); + }); +}); diff --git a/tests/runner-e2e/product-journeys/repository-continuation-evidence.ts b/tests/runner-e2e/product-journeys/repository-continuation-evidence.ts new file mode 100644 index 00000000000..25f35f269a9 --- /dev/null +++ b/tests/runner-e2e/product-journeys/repository-continuation-evidence.ts @@ -0,0 +1,82 @@ +import { execFile as execFileCallback } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import type { WorkspaceFileContent } from "../../../packages/shared/src/types/workspace-file-resource.js"; + +const execFile = promisify(execFileCallback); +const sha = (bytes: Buffer) => createHash("sha256").update(bytes).digest("hex"); +export interface RepositoryBundleEvidence { + bundleSha256: string; + head: string; + parent: string; + changes: Array<{ status: string; path: string }>; + files: Record; + sourceReadme: string; +} + +/** Inspect actual Git objects. No checkout, remote fetch, hook, or repo script runs. */ +export async function inspectRepositoryBundle(bytes: Buffer, input: { sourceCommit: string; files: string[] }): Promise { + if (!/^[a-f0-9]{40}$/.test(input.sourceCommit) || bytes.length < 32 || bytes.length > 262_144 + || !/^# v[23] git bundle\n/.test(bytes.subarray(0, 32).toString("ascii")) + || input.files.some(file => !/^[A-Za-z0-9][A-Za-z0-9.-]{0,120}$/.test(file))) throw new Error("invalid_repository_bundle"); + const root = await mkdtemp(path.join(os.tmpdir(), "paperclip-qa-repository-inspection-")); + const gitDir = path.join(root, "objects.git"), bundle = path.join(root, "proof.bundle"); + const git = async (...args: string[]) => (await execFile("git", ["--no-replace-objects", "-c", "core.hooksPath=/dev/null", + "-c", "core.fsmonitor=false", "-c", "protocol.allow=never", "-c", "protocol.file.allow=always", "--git-dir", gitDir, ...args], { + cwd: root, env: { PATH: process.env.PATH, GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: "/dev/null", GIT_TERMINAL_PROMPT: "0" }, + timeout: 20_000, maxBuffer: 262_144, encoding: "utf8", + })).stdout; + try { + await mkdir(gitDir); + await writeFile(bundle, bytes, { mode: 0o600, flag: "wx" }); + await git("init", "--bare"); + await git("bundle", "verify", bundle); + await git("fetch", "--no-tags", bundle, "HEAD:refs/heads/observed"); + await git("fsck", "--full", "--strict"); + const head = (await git("rev-parse", "refs/heads/observed")).trim(); + await git("merge-base", "--is-ancestor", input.sourceCommit, head); + const parent = (await git("show", "-s", "--format=%P", head)).trim(); + // A merge or omitted parent cannot satisfy either single-commit turn. + if (!/^[a-f0-9]{40}$/.test(head) || !/^[a-f0-9]{40}$/.test(parent)) throw new Error("repository_commit_parent_mismatch"); + const rawChanges = (await git("diff-tree", "--no-commit-id", "--name-status", "-r", "-z", parent, head)).split("\0"); + if (rawChanges.pop() !== "" || rawChanges.length % 2) throw new Error("repository_tree_changes_invalid"); + const changes = Array.from({ length: rawChanges.length / 2 }, (_, i) => ({ status: rawChanges[i * 2]!, path: rawChanges[i * 2 + 1]! })); + const files: Record = {}; + for (const file of input.files) files[file] = await git("show", `${head}:${file}`); + return { bundleSha256: sha(bytes), head, parent, changes, files, sourceReadme: await git("show", `${input.sourceCommit}:README`) }; + } finally { await rm(root, { recursive: true, force: true }); } +} + +export function repositoryWorkspaceFileMatches(value: WorkspaceFileContent | null, workspaceId: string, filename: string, bytes: string) { + return Boolean(value && workspaceId && value.resource.kind === "file" && value.resource.workspaceKind === "execution_workspace" + && value.resource.workspaceId === workspaceId && value.resource.title === filename + && value.resource.byteSize === Buffer.byteLength(bytes) && value.content.encoding === "utf8" && value.content.data === bytes); +} + +export function repositoryWorkspaceBindingMatches(input: { + companyId: string; issueId: string; projectId: string; projectWorkspaceId: string; workspaceId: string; repoUrl: string; + issue: Record; view: Record; runs: Array>; +}) { + const { workspace } = input.view; + return Boolean(input.workspaceId && input.issue.id === input.issueId && input.issue.companyId === input.companyId + && input.issue.projectId === input.projectId && input.issue.executionWorkspaceId === input.workspaceId + && input.view.issueId === input.issueId && workspace?.id === input.workspaceId && workspace.companyId === input.companyId + && workspace.projectId === input.projectId && workspace.projectWorkspaceId === input.projectWorkspaceId + && workspace.repoUrl === input.repoUrl && input.runs.length > 0 + && input.runs.every(run => run.companyId === input.companyId + && run.runnerProfileJson?.nativeExecutionInput?.binding?.executionWorkspaceId === input.workspaceId)); +} + +export function repositoryCommitMatches(value: RepositoryBundleEvidence | null, expected: { + parent: string; sourceReadme: string; files: Record; added: string[]; +}) { + return Boolean(value && /^[a-f0-9]{40}$/.test(value.head) && value.head !== expected.parent + && value.parent === expected.parent && value.sourceReadme === expected.sourceReadme + && value.changes.length === expected.added.length && new Set(value.changes.map(change => change.path)).size === expected.added.length + && value.changes.every(change => change.status === "A" && expected.added.includes(change.path)) + && Object.keys(value.files).length === Object.keys(expected.files).length + && Object.entries(expected.files).every(([file, bytes]) => value.files[file] === bytes)); +} diff --git a/tests/runner-e2e/product-journeys/repository-continuation-flow.ts b/tests/runner-e2e/product-journeys/repository-continuation-flow.ts new file mode 100644 index 00000000000..ebeb43c4fc2 --- /dev/null +++ b/tests/runner-e2e/product-journeys/repository-continuation-flow.ts @@ -0,0 +1,146 @@ +import { createObservedReader } from "./observation-read.js"; +import { createHash } from "node:crypto"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { expect } from "@playwright/test"; +import type { WorkspaceFileContent } from "../../../packages/shared/src/types/workspace-file-resource.js"; +import { readRegisteredArtifacts } from "../registered-artifact.js"; +import { createTaskThroughUi, submitTaskReply } from "../user-actions.js"; +import { fixtureReplyStoredOnce, validateAgentTaskState, waitFor } from "./agent-flows.js"; +import { repositoryContinuationSource, repositoryContinuationStory } from "./repository-continuation-contract.js"; +import { inspectRepositoryBundle, repositoryCommitMatches, repositoryWorkspaceBindingMatches, repositoryWorkspaceFileMatches } from "./repository-continuation-evidence.js"; +import { openDocumentAuthoring } from "./task-document-flow.js"; +import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSession } from "./types.js"; + +type Row = Record; +const id = "repository-continuation"; + +/** Attached and managed targets share the browser story and byte/Git oracle. */ +export async function runRepositoryContinuation(context: JourneyContext, session: TargetSession, campaignId: string, + evidence: JourneyEvidence, options: { deadlineAt: number; directory: string }) { + const { api, page, company, agent } = context; + const observations = createObservedReader(api, options.deadlineAt); + const readObserved = observations.get; + try { + if (!agent || agent.runtimeMode !== "native" || !context.environmentId) + throw new JourneyError("unsupported", "native_agent_and_environment_required"); + const story = repositoryContinuationStory(campaignId), { files } = story, source = repositoryContinuationSource; + await mkdir(options.directory, { mode: 0o700, recursive: true }); + const retain = (filename: string, value: unknown) => writeFile(path.join(options.directory, filename), JSON.stringify(value, null, 2), { mode: 0o600 }); + const check = (name: string, passed: boolean) => { + evidence.checks.push({ id: name, passed }); + if (!passed) throw new JourneyError("failed", name.replaceAll("-", "_")); + }; + const projectName = `Journey QA repository ${campaignId}`; + await session.own({ kind: "project", name: projectName, journeyId: id }); + const project = await api.post(`/api/companies/${company.id}/projects`, { + name: projectName, description: "Disposable read-only public repository continuation fixture.", + executionWorkspacePolicy: { enabled: true, defaultMode: "shared_workspace", sharedWorkspaceConcurrency: "serialize", + allowIssueOverride: false, environmentId: context.environmentId, workspaceStrategy: { type: "project_primary" } }, + workspace: { name: "Primary", sourceType: "git_repo", repoUrl: source.url, repoRef: source.commit, isPrimary: true }, + }); + await session.own({ kind: "project", name: projectName, id: project.id, journeyId: id }); + const configured = await readObserved(`/api/projects/${project.id}`); + const projectWorkspaces = configured.workspaces?.filter((workspace: Row) => workspace.repoUrl === source.url && workspace.repoRef === source.commit && workspace.isPrimary); + if (configured.companyId !== company.id || !Array.isArray(projectWorkspaces) || projectWorkspaces.length !== 1) + throw new JourneyError("failed", "repository_project_source_mismatch"); + const projectWorkspaceId = projectWorkspaces[0]!.id; + const title = `Journey QA repository continuation ${campaignId}`; + const created = await createTaskThroughUi({ page, origin: api.origin, issuePrefix: company.issuePrefix, + agentName: agent.name, projectName, title, prompt: story.first, workMode: "standard", requireExplicitTitle: true, + beforeDispatch: () => session.own({ kind: "issue", name: title, journeyId: id }) }); + await session.own({ kind: "issue", name: title, id: created.issueId, journeyId: id }); + const route = `/api/issues/${created.issueId}`; + const read = async () => { + const [currentIssue, summaries, comments, interactions] = await Promise.all([ + readObserved(route), readObserved(`${route}/runs`), + readObserved(`${route}/comments?order=asc`), readObserved>(`${route}/interactions`), + ]); + if (!Array.isArray(summaries) || summaries.length > 2) throw new JourneyError("failed", "repository_run_limit_exceeded"); + const taskRuns = (await Promise.all(summaries.map(summary => readObserved(`/api/heartbeat-runs/${summary.runId}`)))) + .sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt)) || String(a.id).localeCompare(String(b.id))); + const state = { currentIssue, taskRuns, comments, interactions, attachments: [] }; + await retain("repository-latest-task.json", state); + validateAgentTaskState(context, created.issueId, state, 2); + if (currentIssue.projectId !== project.id) throw new JourneyError("failed", "repository_project_binding_mismatch"); + return state; + }; + const settled = (count: number) => waitFor(options.deadlineAt, read, state => state.currentIssue.status === "done" + && state.taskRuns.length === count && state.taskRuns.every(run => run.status === "succeeded" + && run.nativePhase === "committed" && run.resultJson?.finalizationPhase === "committed") + && state.interactions.every(row => row.status !== "pending"), "repository_task_not_completed"); + // Use the attached target's origin for the common registration/download oracle. + const artifactApi = { get: (route: string) => readObserved(route), request: { get: (route: string) => api.response(route) } }; + const bundle = async (name: string, runId: string) => { + const [artifact] = await readRegisteredArtifacts(artifactApi, created.issueId, runId, [name]); + const bytes = await api.bytes(`/api/attachments/${artifact!.attachmentId}/content?download=1`); + if (createHash("sha256").update(bytes).digest("hex") !== artifact!.sha256) throw new JourneyError("failed", "repository_bundle_download_changed"); + await writeFile(path.join(options.directory, name), bytes, { mode: 0o600, flag: "wx" }); + return { artifact: artifact!, bytes }; + }; + // Task-chat layouts keep generic output controls in Artifacts. Use the + // production download link for this exact attachment, not a coincidental + // URL in a model reply or an API-only readback. + const browserDownload = async (artifact: { attachmentId: string; sha256: string }, phase: string) => { + await openDocumentAuthoring(context); + const link = page.getByRole("link", { name: /^Download/ }).and( + page.locator(`a[href*="/api/attachments/${artifact.attachmentId}/content"]`)).first(); + await expect(link).toBeVisible(); + const [download] = await Promise.all([page.waitForEvent("download", { timeout: Math.max(1, Math.min(30_000, options.deadlineAt - Date.now())) }), link.click()]); + const filename = path.join(options.directory, `browser-${phase}`); + await download.saveAs(filename); + const bytes = await readFile(filename); + if (createHash("sha256").update(bytes).digest("hex") !== artifact.sha256) throw new JourneyError("failed", "repository_browser_download_changed"); + }; + const file = (name: string) => readObserved(`${route}/file-resources/content?workspace=execution&path=${encodeURIComponent(name)}`); + const first = await settled(1), firstRun = first.taskRuns[0]!; + const initialView = await readObserved(`${route}/workspace`), workspaceId = initialView.workspace?.id; + const binding = (issue: Row, view: Row, runs: Row[]) => repositoryWorkspaceBindingMatches({ companyId: company.id, + issueId: created.issueId, projectId: project.id, projectWorkspaceId, workspaceId, repoUrl: source.url, issue, view, runs }); + check("repository-configured-source-bound", binding(first.currentIssue, initialView, first.taskRuns)); + const [referenceArtifact] = await readRegisteredArtifacts(artifactApi, created.issueId, firstRun.id, [files.context]); + const original = referenceArtifact!.content; + if (!/^[a-f0-9]{64}\n$/.test(original)) throw new JourneyError("failed", "repository_original_reference_invalid"); + const { artifact: firstBundle, bytes: originalBytes } = await bundle(files.firstBundle, firstRun.id); + const initialGit = await inspectRepositoryBundle(originalBytes, { sourceCommit: source.commit, files: [files.context, files.first, "README"] }); + check("repository-first-commit-verified", repositoryCommitMatches(initialGit, { parent: source.commit, sourceReadme: source.readme, + files: { [files.context]: original, [files.first]: "first\n", README: source.readme }, added: [files.context, files.first] })); + check("repository-original-files-saved", repositoryWorkspaceFileMatches(await file(files.context), workspaceId, files.context, original) + && repositoryWorkspaceFileMatches(await file(files.first), workspaceId, files.first, "first\n") + && repositoryWorkspaceFileMatches(await file("README"), workspaceId, "README", source.readme)); + await retain("repository-first-observation.json", { source, projectId: project.id, workspaceId, issueId: created.issueId, + runId: firstRun.id, referenceArtifact, initialGit, bundle: { attachmentId: firstBundle.attachmentId, sha256: firstBundle.sha256 } }); + const url = `${api.origin}/${company.issuePrefix}/issues/${first.currentIssue.identifier ?? created.issueId}`; + await page.goto(url, { waitUntil: "domcontentloaded" }); + await page.reload(); + await browserDownload(firstBundle, "first-bundle"); + // The original random reference and commit are not supplied by this follow-up. + await submitTaskReply(page, story.followup); + const withReply = await waitFor(Math.min(options.deadlineAt, Date.now() + 30_000), read, + state => state.comments.some(comment => comment.authorUserId === context.user.id && !first.comments.some(old => old.id === comment.id)), + "repository_followup_not_stored"); + check("repository-followup-stored-once", fixtureReplyStoredOnce(first.comments, withReply.comments, context.user.id, story.followup)); + const final = await settled(2), lastRun = final.taskRuns[1]!; + const finalView = await readObserved(`${route}/workspace`); + check("repository-same-workspace-retained", binding(final.currentIssue, finalView, final.taskRuns) + && Date.parse(lastRun.startedAt) >= Date.parse(firstRun.finishedAt)); + const { artifact: finalBundle, bytes: finalBytes } = await bundle(files.finalBundle, lastRun.id); + const finalGit = await inspectRepositoryBundle(finalBytes, + { sourceCommit: source.commit, files: [files.context, files.first, files.followup, "README"] }); + check("repository-second-commit-verified", repositoryCommitMatches(finalGit, { parent: initialGit.head, sourceReadme: source.readme, + files: { [files.context]: original, [files.first]: "first\n", [files.followup]: "verified\n", README: source.readme }, added: [files.followup] })); + await page.goto(url, { waitUntil: "domcontentloaded" }); + await page.reload(); + for (const [index, artifact] of [referenceArtifact!, firstBundle, finalBundle].entries()) await browserDownload(artifact, `reopened-${index}`); + const [retained, retainedBundle] = await readRegisteredArtifacts(artifactApi, created.issueId, firstRun.id, [files.context, files.firstBundle]); + check("repository-original-and-result-reopened", retained!.attachmentId === referenceArtifact!.attachmentId && retained!.content === original + && retainedBundle!.attachmentId === firstBundle.attachmentId && retainedBundle!.sha256 === firstBundle.sha256 + && repositoryWorkspaceFileMatches(await file(files.context), workspaceId, files.context, original) + && repositoryWorkspaceFileMatches(await file(files.first), workspaceId, files.first, "first\n") + && repositoryWorkspaceFileMatches(await file(files.followup), workspaceId, files.followup, "verified\n") + && repositoryWorkspaceFileMatches(await file("README"), workspaceId, "README", source.readme)); + check("configured-model-and-environment", true); // Both actual native runs passed validateAgentTaskState. + await retain("repository-continuation-checks.json", { source, projectId: project.id, workspaceId, issueId: created.issueId, + initialGit, finalGit, referenceArtifact, firstBundle, finalBundle, runIds: final.taskRuns.map(run => run.id) }); + } finally { await observations.preserve(options.directory); } +} diff --git a/tests/runner-e2e/product-journeys/repository-continuation.md b/tests/runner-e2e/product-journeys/repository-continuation.md new file mode 100644 index 00000000000..981a27d5ae7 --- /dev/null +++ b/tests/runner-e2e/product-journeys/repository-continuation.md @@ -0,0 +1,51 @@ +# Repository continuation journey + +`repository-continuation` uses the target's configured native agent and execution +environment. It creates one owned project and one owned task. The project uses +the public `octocat/Hello-World` repository at commit +`7fd1a60b01f91b314f59955a4e4d4e80d8edf11d`. The fixture never pushes or changes +the remote repository. + +The first browser request creates a random reference file and a short note in +the project checkout. The agent makes one local commit and registers the +reference and a self-contained Git bundle as downloads. The observer downloads +and verifies them before it sends the second request. That request supplies no +reference value or commit ID. It requires the original files to remain intact, +adds one follow-up file, and delivers a second committed Git bundle. + +The oracle inspects actual Git objects in disposable bare repositories. It runs +no checkout, hook, project script, or remote fetch. Each commit must have the +expected parent and only the declared file additions. The repository's original +files must stay unchanged. The observer compares the committed file bytes with +the ordinary task workspace file API, checks the configured project source and +both native workspace bindings, and reopens all output links after reload. +Missing input, recreated history, extra native runs, wrong bytes, or a replaced +workspace fails the journey. + +There are exactly two native runs, one per user request. The target supplies the +unchanged model, billing policy and overall deadline. This case does not test +automatic repository preparation, automatic continuation, remote pushes or pull +requests. Those are separate journeys. + +The target provider owns authentication, source identity, usage collection and +cleanup. Intent and receipts are saved for the owned project and task. Git +bundles, raw API receipts, source bytes and workspace identifiers remain in the +private Product journey evidence directory. They must not enter a public report. +The normal final screenshot is restricted to the authenticated owned task page. + +Registration and local assertion calibration do not establish a deployed pass. +Daily admission requires a fresh pass against the intended QA target with the +same source and assertion set, completed collection, and successful cleanup. + +Browser result access opens the task's ordinary Artifacts panel. Each exact +registered attachment is downloaded through its visible production control, and +the downloaded digest must match the independent registered artifact receipt. +A URL present only in a chat message cannot satisfy this check. + +The two-turn proof has a boundary: the final saved checkout's hidden `.git` +metadata and `HEAD` are not independently read after turn two. The ordinary +workspace file API protects `.git`, and remote workspace close-readiness does +not expose Git metadata. The second verified commit must still have the first +verified commit as its parent, the original saved files must remain intact, and +both runs must retain the same workspace binding. This case does not qualify +post-completion Git metadata durability for a third turn. diff --git a/tests/runner-e2e/product-journeys/success-contract.ts b/tests/runner-e2e/product-journeys/success-contract.ts index 45620f41532..7141438d29c 100644 --- a/tests/runner-e2e/product-journeys/success-contract.ts +++ b/tests/runner-e2e/product-journeys/success-contract.ts @@ -1,5 +1,5 @@ /** Closed, target-independent success requirements used by dispatch and retained evidence. */ -export const agentJourneyRuns = { "routine-scheduled-run": 1, "file-input-answer": 1, "ask-question": 1, "plan-revise-accept": 3, "build-revise": 2, +export const agentJourneyRuns = { "repository-continuation": 2, "routine-scheduled-run": 1, "file-input-answer": 1, "ask-question": 1, "plan-revise-accept": 3, "build-revise": 2, "followup-while-running": 2, "protected-actions": 4, "create-skill-studio": 1, "stop-redirect": 3, "question-answer-resume": 3, "hire-delegate-reuse": 7 } as const; export function agentJourneyRunCountValid(id: string, count: unknown): boolean { @@ -9,6 +9,7 @@ export function agentJourneyRunCountValid(id: string, count: unknown): boolean { return (count as number) >= minimum && (count as number) <= maximum; } export const journeyRequiredChecks: Record = { + "repository-continuation": ["repository-configured-source-bound", "repository-first-commit-verified", "repository-original-files-saved", "repository-followup-stored-once", "repository-same-workspace-retained", "repository-second-commit-verified", "repository-original-and-result-reopened", "configured-model-and-environment"], "routine-scheduled-run": ["routine-browser-created", "routine-scheduler-created-execution", "routine-result-attributed", "routine-history-preserved", "routine-pause-prevented-next-run", "routine-result-reopened", "configured-model-and-environment"], "inbox-triage": ["inbox-external-human-activity", "inbox-read-persisted", "inbox-archive-persisted", "inbox-archive-preserved-task-status", "inbox-relevant-activity-resurfaced", "no-agent-work"], "file-input-answer": ["input-upload-bytes-preserved", "input-content-used-correctly", "input-result-attributed-to-run", "input-result-reopened", "original-input-retained", "configured-model-and-environment"], diff --git a/tests/runner-e2e/product-journeys/types.ts b/tests/runner-e2e/product-journeys/types.ts index f753c1873d2..0de951ff574 100644 --- a/tests/runner-e2e/product-journeys/types.ts +++ b/tests/runner-e2e/product-journeys/types.ts @@ -3,7 +3,7 @@ import type { ConnectionApi } from "../connection-target.js"; export type JourneyOutcome = "passed" | "failed" | "blocked" | "unsupported"; export type JourneyId = "task-properties" | "task-create" | "project-task-context" | "task-document-persistence" | "ask-question" | "cloud-onboarding" - | "question-answer-resume" | "task-search-filter" | "followup-while-running" | "plan-revise-accept" | "build-revise" | "protected-actions" | "create-skill-studio" | "stop-redirect" | "hire-delegate-reuse" | "file-input-answer" | "inbox-triage" | "routine-scheduled-run"; + | "question-answer-resume" | "task-search-filter" | "followup-while-running" | "plan-revise-accept" | "build-revise" | "protected-actions" | "create-skill-studio" | "stop-redirect" | "hire-delegate-reuse" | "file-input-answer" | "inbox-triage" | "routine-scheduled-run" | "repository-continuation"; export type TargetMode = "managed-self-hosted" | "existing-cloud" | "new-cloud"; export interface DeploymentIdentity { kind: "self-hosted" | "cloud"; diff --git a/tests/runner-e2e/registered-artifact.ts b/tests/runner-e2e/registered-artifact.ts index 9531119fe36..3d55332d074 100644 --- a/tests/runner-e2e/registered-artifact.ts +++ b/tests/runner-e2e/registered-artifact.ts @@ -1,7 +1,7 @@ import { createHash } from "node:crypto"; import type { IssueAttachment } from "../../packages/shared/src/types/issue.js"; import type { IssueWorkProduct } from "../../packages/shared/src/types/work-product.js"; -import type { RunnerApi } from "./api.js"; +import type { APIRequestContext } from "@playwright/test"; export class RegisteredArtifactNotFoundError extends Error { constructor(artifactName: string) { @@ -13,7 +13,7 @@ export class RegisteredArtifactNotFoundError extends Error { const REGISTERED_ARTIFACT_STATUSES = new Set(["active", "ready_for_review", "approved"]); /** Registration, attribution and downloaded bytes must agree independently. */ -export async function readRegisteredArtifacts(api: RunnerApi, issueId: string, runId: string, names: readonly string[]) { +export async function readRegisteredArtifacts(api: { get(route: string): Promise; request: Pick }, issueId: string, runId: string, names: readonly string[]) { if (names.length === 0) return []; const [products, attachments] = await Promise.all([ api.get(`/api/issues/${encodeURIComponent(issueId)}/work-products`),