diff --git a/doc/evals.md b/doc/evals.md index e2db0f49f9e..8a57f31089e 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -113,6 +113,8 @@ reply with the late requirement, and persistence after reload. Its attached and managed targets share the same story and permit one steered run or one normal successor. Registration and calibration are separate from a live staging pass. +The explicit [dependency continuation journey](../tests/runner-e2e/product-journeys/task-dependency-resume.md) checks a finite blocked wait and one automatic native continuation after a browser-completed blocker. Local evidence controls are separate from a deployed QA pass. + [Reusable Product journeys](../tests/runner-e2e/PRODUCT-JOURNEYS.md) add explicit deployment targets and starting-state recipes. The shared task-properties journey and task-document create/edit/reopen journey use no model; Cloud diff --git a/tests/runner-e2e/PRODUCT-JOURNEYS.md b/tests/runner-e2e/PRODUCT-JOURNEYS.md index 6fb291d0ac7..1b73f1e566a 100644 --- a/tests/runner-e2e/PRODUCT-JOURNEYS.md +++ b/tests/runner-e2e/PRODUCT-JOURNEYS.md @@ -30,6 +30,7 @@ never start a server or depend on a tenant filesystem. | `inbox-triage` | Signed-in board user; a separately authenticated QA human author; a fresh self-assigned task; zero model runs | Mark an unread task read, archive it through Mine, reload both views, verify unchanged work status, then observe a real Blocked transition resurface it through the normal archive policy | | `task-create` | Company and signed-in board user | Create a task through the shared production composer, assign it to yourself, reopen and reload it, verify the saved prompt and no agent work | | `project-task-context` | Company, signed-in board user, task-write capability; no model runs | Create a project and a self-assigned task with the project selected through the production dialogs. Reopen the project, open its task, reload both, and verify exact saved company, project, prompt, assignment, standard work mode and zero runs | +| `task-dependency-resume` | Native agent; one automatic work run; two browser-created tasks | Keep an assigned dependent blocked for a measured ten seconds; complete its human-owned blocker through the UI; require one automatic native wake, run-authored saved document, and reloaded result. See [contract](product-journeys/task-dependency-resume.md). | | `ask-question` | Company, configured agent capability, positive model allowance and exactly one permitted work run | Reuse the existing cross-model Ask question prompt; require one successful run with matching company/task/agent/model/environment, one saved exact answer, and visible readback after reload | | `chat-continuity` | Fresh owned native agent with no prior work runs; Agent Chat enabled; fresh owned user chat; three provider turns and one synthetic reset | Recall prompt-hidden context after reopening, then use `/new` to advance the same saved conversation to a fresh native session, preserve history and verify the final reply after reload. See [CHAT-CONTINUITY.md](product-journeys/CHAT-CONTINUITY.md) | | `question-answer-resume` | Native agent; up to three work runs | Reuse the classic two-question story: answer a choice and a free-text question through the browser, verify each native continuation, then read the exact saved answers in an agent-authored task document after reload | diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 4ab07867b7f..e5e598a5e78 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -135,6 +135,7 @@ describe("runner E2E catalog", () => { "file-input-answer", "task-create", "project-task-context", + "task-dependency-resume", "ask-question", "chat-continuity", "question-answer-resume", diff --git a/tests/runner-e2e/journey-features.ts b/tests/runner-e2e/journey-features.ts index b964103c45d..1816d62bd96 100644 --- a/tests/runner-e2e/journey-features.ts +++ b/tests/runner-e2e/journey-features.ts @@ -21,6 +21,7 @@ function associate(suites: readonly string[], cases: Record = { "task-create": ["browser-created-task-persisted", "no-agent-work"], "task-document-persistence": ["browser-document-created", "document-revision-history-preserved", "document-reopened-after-reload", "no-agent-work"], "project-task-context": ["browser-created-project-persisted", "browser-created-project-task-persisted", "project-task-association-reloaded", "project-task-context-visible", "no-agent-work"], + "task-dependency-resume": ["dependency-browser-created-and-linked", "dependency-unresolved-prevented-execution", "dependency-blocker-completed-by-board", "dependency-one-automatic-resume", "dependency-output-authored-by-run", "dependency-result-reopened", "dependency-history-preserved", "configured-model-and-environment"], "task-search-filter": ["task-search-results-exact", "task-status-filter-excludes-controls", "task-search-filter-reloaded", "task-search-result-reopened", "no-agent-work"], "ask-question": ["question-answer-persisted", "one-bound-successful-run", "configured-model-and-environment"], "chat-continuity": ["chat-fresh-owned-conversation", "chat-user-messages-stored-once", "chat-reload-preserved-context", "chat-new-session-boundary", "chat-fresh-context-answer", "chat-history-preserved-after-reload", "chat-no-unrequested-work", "chat-exact-turn-accounting", "configured-model-and-environment"], diff --git a/tests/runner-e2e/product-journeys/task-dependency-resume-contract.test.ts b/tests/runner-e2e/product-journeys/task-dependency-resume-contract.test.ts new file mode 100644 index 00000000000..5c997d832be --- /dev/null +++ b/tests/runner-e2e/product-journeys/task-dependency-resume-contract.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "vitest"; +import { parseTargetConfig, productJourneys } from "./catalog.js"; +import { agentJourneyRunCountValid, journeyRequiredChecks } from "./success-contract.js"; +import { TASK_DEPENDENCY_RESUME_CHECKS } from "./task-dependency-resume-evidence.js"; +import { validateRetainedRunnerResult } from "../result-validation.js"; +import type { RunnerE2EResult } from "../types.js"; + +const caseId = "task-dependency-resume"; +const configuration = () => ({ version: 1, id: "qa-fixture", mode: "existing-cloud", executionEnvironment: "daytona", + adapterModule: "/private/qa-provider.mjs", journeys: [caseId], baseline: { features: {}, connections: [] }, + limits: { durationMs: 600_000, modelBudgetCents: 500, maxRuns: 1 } }); +function retained(): RunnerE2EResult { + return { schema: "paperclip.runner-e2e.result/v2", suiteId: "product-journeys", suiteDefinitionHash: "fixture", caseId, + executionId: `product-journeys.qa-fixture.configured-agent.daytona.${caseId}`, profileId: "configured-agent", environmentId: "daytona", + provider: "opencode", model: "fixture-model", runtimeMode: "native", status: "passed", cleanup: "passed", attempt: 1, + startedAt: "2020-01-01T00:00:00Z", finishedAt: "2020-01-01T00:01:00Z", durationMs: 60_000, runIds: ["run"], + productJourney: { outcome: "passed", startingState: "recipe", privacy: "private", resourceDisposition: "retained", + phases: (["authentication", "setup", "journey", "collection", "cleanup"] as const).map(phase => ({ phase, outcome: "passed" })), + checks: TASK_DEPENDENCY_RESUME_CHECKS.map(id => ({ id, passed: true })), deployment: { kind: "cloud", targetId: "qa-fixture", + origin: "https://qa.example.test", commit: "a".repeat(40), cloudRelease: "b".repeat(40), runtimeImage: null, configurationHash: "fixture" } } }; +} +describe("dependency journey registration and retained acceptance", () => { + it("is explicit and requires exactly one bounded configured native run", () => { + expect(productJourneys.find(row => row.id === caseId)).toMatchObject({ paid: true, requires: ["company", "board-user", "task-write", "agent"] }); + expect(parseTargetConfig(configuration()).journeys).toEqual([caseId]); + expect(journeyRequiredChecks[caseId]).toEqual(TASK_DEPENDENCY_RESUME_CHECKS); + expect(agentJourneyRunCountValid(caseId, 1)).toBe(true); + for (const count of [0, 2]) { + const value = configuration(); value.limits.maxRuns = count; + expect(() => parseTargetConfig(value)).toThrow("agent_journeys_require_bounded_runs"); + expect(agentJourneyRunCountValid(caseId, count)).toBe(false); + } + expect(() => parseTargetConfig({ ...configuration(), limits: { ...configuration().limits, modelBudgetCents: 0 } })).toThrow(); + }); + it("accepts a complete retained measurement and rejects partial checks, runtime or cleanup", () => { + expect(() => validateRetainedRunnerResult(retained())).not.toThrow(); + for (const change of [ + (r: RunnerE2EResult) => { r.productJourney!.checks.pop(); }, + (r: RunnerE2EResult) => { r.productJourney!.checks[0]!.passed = false; }, + (r: RunnerE2EResult) => { r.runIds = []; }, + (r: RunnerE2EResult) => { r.runIds = ["run", "extra"]; }, + (r: RunnerE2EResult) => { r.runtimeMode = "none"; }, + (r: RunnerE2EResult) => { r.cleanup = "failed"; }, + ]) { const row = retained(); change(row); expect(() => validateRetainedRunnerResult(row)).toThrow(); } + }); +}); diff --git a/tests/runner-e2e/product-journeys/task-dependency-resume-evidence.test.ts b/tests/runner-e2e/product-journeys/task-dependency-resume-evidence.test.ts new file mode 100644 index 00000000000..c10c27f6896 --- /dev/null +++ b/tests/runner-e2e/product-journeys/task-dependency-resume-evidence.test.ts @@ -0,0 +1,123 @@ +import { describe, expect, it } from "vitest"; +import { dependencyOutputMatches, gradeTaskDependencyResume, TASK_DEPENDENCY_RESUME_CHECKS, type DependencyEvidence } from "./task-dependency-resume-evidence.js"; +import { dependencyEvidenceFixture } from "./task-dependency-resume-fixture.js"; + +describe("task dependency resume evidence", () => { + it("accepts a finite blocked observation followed by one automatic, attributed native result", () => { + const checks = gradeTaskDependencyResume(dependencyEvidenceFixture()); + expect(checks).toEqual(TASK_DEPENDENCY_RESUME_CHECKS.map(id => ({ id, passed: true }))); + }); + const changeRuns = (f: DependencyEvidence, mutate: (run: Record) => void) => { + for (const point of [f.finished, f.reopened]) for (const run of [...point.dependentRuns, ...point.agentRuns]) mutate(run); + }; + it.each([ + ["provider", (f: DependencyEvidence) => changeRuns(f, run => { run.runnerProfileJson.nativeExecutionInput.provider.kind = "other"; }), "configured-model-and-environment"], + ["model", (f: DependencyEvidence) => changeRuns(f, run => { run.runnerProfileJson.nativeExecutionInput.provider.model = "other"; }), "configured-model-and-environment"], + ["environment", (f: DependencyEvidence) => changeRuns(f, run => { run.contextSnapshot.paperclipEnvironment.id = "other"; }), "configured-model-and-environment"], + ["runtime mode", (f: DependencyEvidence) => changeRuns(f, run => { run.runtimeMode = "legacy"; }), "configured-model-and-environment"], + ["uncommitted phase", (f: DependencyEvidence) => changeRuns(f, run => { run.nativePhase = "saving"; }), "configured-model-and-environment"], + ["document author", (f: DependencyEvidence) => { f.finished.document!.createdByUserId = "user"; }, "dependency-output-authored-by-run"], + ["revision run", (f: DependencyEvidence) => { f.finished.revision!.createdByRunId = "foreign"; }, "dependency-output-authored-by-run"], + ["reload visibility", (f: DependencyEvidence) => { f.reopened.resultVisible = false; }, "dependency-result-reopened"], + ["activity coverage", (f: DependencyEvidence) => { f.reopened.activityCoverage.agent = false; }, "dependency-history-preserved"], + ["extra child work", (f: DependencyEvidence) => { f.reopened.activity.push({ runId: "run", action: "issue.created" }); }, "dependency-history-preserved"], + ["retained completion", (f: DependencyEvidence) => { f.reopened.blocker.completedAt = "2020-01-01T00:00:22.000Z"; }, "dependency-history-preserved"], + ] as const)("attributes the %s defect to its own evidence check", (_name, mutate, failedId) => { + const f = dependencyEvidenceFixture(); mutate(f); + expect(gradeTaskDependencyResume(f).filter(check => !check.passed).map(check => check.id)).toEqual([failedId]); + }); + const mutants: Array<[string, (value: DependencyEvidence) => void]> = [ + ["missing observation", f => { f.waiting = []; }], + ["short quiet window", f => { f.waiting[1]!.observedAt--; }], + ["non-monotonic observation", f => { f.waiting.reverse(); }], + ["quiet window before creation", f => { f.created.observedAt = f.waiting[0]!.observedAt + 1; }], + ["release before final quiet read", f => { f.releaseRequestedAt = f.waiting[1]!.observedAt - 1; }], + ["blocker not completed", f => { f.released.blocker.status = "todo"; }], + ["wrong board actor", f => { f.reopened.activity[0]!.actorId = "foreign"; }], + ["agent claimed board completion", f => { f.reopened.activity[0]!.actorType = "agent"; }], + ["missing board completion audit", f => { f.reopened.activity = []; }], + ["duplicate board completion", f => { f.reopened.activity.push({ ...f.reopened.activity[0] }); }], + ["wrong blocker relation", f => { f.waiting[0]!.dependent.blockedByIssueIds = ["foreign"]; }], + ["removed relation after completion", f => { f.reopened.dependent.blockedByIssueIds = []; }], + ["parked backlog instead of blocked", f => { f.waiting[0]!.dependent.status = "backlog"; }], + ["premature execution", f => { f.waiting[0]!.dependentRuns = [f.finished.dependentRuns[0]!]; }], + ["unknown work on owned agent", f => { f.waiting[0]!.agentRuns = [f.finished.agentRuns[0]!]; }], + ["blocker ran an agent", f => { f.reopened.blockerRuns = [f.finished.dependentRuns[0]!]; }], + ["dependency never resumed", f => { f.finished.dependent.status = "blocked"; }], + ["duplicate native run", f => { f.reopened.dependentRuns.push({ ...f.reopened.dependentRuns[0], id: "extra" }); }], + ["native run preceded blocker completion", f => { f.finished.dependentRuns[0]!.startedAt = "2020-01-01T00:00:19.000Z"; }], + ["manual run instead of automatic resume", f => { f.finished.dependentRuns[0]!.invocationSource = "on_demand"; }], + ["wrong wake reason", f => { f.finished.dependentRuns[0]!.contextSnapshot.wakeReason = "issue_assigned"; }], + ["mutation name used as wake source", f => { for (const p of [f.finished, f.reopened]) for (const r of [...p.dependentRuns, ...p.agentRuns]) r.contextSnapshot.source = "issue.blocker_done"; }], + ["missing saved wake source", f => { for (const p of [f.finished, f.reopened]) for (const r of [...p.dependentRuns, ...p.agentRuns]) delete r.contextSnapshot.source; }], + ["wrong resolved blocker", f => { f.finished.dependentRuns[0]!.contextSnapshot.resolvedBlockerIssueId = "foreign"; }], + ["conflicting native issue binding", f => { f.finished.dependentRuns[0]!.issueId = "foreign"; }], + ["reopened run changed status", f => { f.reopened.dependentRuns[0]!.status = "running"; }], + ["reopened native work still owns checkout", f => { f.reopened.dependent.executionRunId = "run"; }], + ["wrong provider", f => { f.finished.dependentRuns[0]!.runnerProfileJson.nativeExecutionInput.provider.kind = "other"; }], + ["wrong model", f => { f.finished.dependentRuns[0]!.runnerProfileJson.nativeExecutionInput.provider.model = "other"; }], + ["wrong environment", f => { f.finished.dependentRuns[0]!.contextSnapshot.paperclipEnvironment.id = "other"; }], + ["uncommitted native phase", f => { for (const p of [f.finished, f.reopened]) for (const r of [...p.dependentRuns, ...p.agentRuns]) r.nativePhase = "saving"; }], + ["missing native phase", f => { for (const p of [f.finished, f.reopened]) for (const r of [...p.dependentRuns, ...p.agentRuns]) delete r.nativePhase; }], + ["incomplete activity coverage", f => { f.reopened.activityCoverage.agent = false; }], + ["wrong saved reference", f => { f.finished.document!.body = '{"reference":"wrong","blockerIdentifier":"QA-1"}'; }], + ["wrong document task", f => { f.finished.document!.issueId = "foreign"; }], + ["board-authored document", f => { f.finished.document!.createdByUserId = "user"; }], + ["revision from another run", f => { f.finished.revision!.createdByRunId = "foreign"; }], + ["missing saved result", f => { delete f.finished.document; }], + ["missing revision", f => { delete f.finished.revision; }], + ["result not visible after reload", f => { f.reopened.resultVisible = false; }], + ["replaced document after reload", f => { f.reopened.document!.id = "replacement"; }], + ["reference leaked in task prompt", f => { f.scope.prompt += f.scope.reference; }], + ["foreign creator", f => { f.created.dependent.createdByUserId = "foreign"; }], + ["unrequested child work", f => { f.reopened.activity.push({ runId: "run", action: "issue.created" }); }], + ]; + it.each(mutants)("rejects %s", (_name, mutate) => { + const f = dependencyEvidenceFixture(); mutate(f); + expect(gradeTaskDependencyResume(f).some(check => !check.passed)).toBe(true); + }); + it("missing input never becomes a pass", () => { expect(gradeTaskDependencyResume(undefined).every(c => !c.passed)).toBe(true); }); + it.each([null, {}, { summary: "Compacted result" }])("accepts the public run projection without duplicate finalization metadata: %j", resultJson => { + const f = dependencyEvidenceFixture(); + for (const point of [f.finished, f.reopened]) for (const run of [...point.dependentRuns, ...point.agentRuns]) run.resultJson = resultJson; + expect(gradeTaskDependencyResume(f).every(check => check.passed)).toBe(true); + }); + const withHistory = () => { + const f = dependencyEvidenceFixture(); + const prior = { ...structuredClone(f.finished.agentRuns[0]!), id: "prior", nativeIssueId: "prior-task", + contextSnapshot: { issueId: "prior-task" }, status: "failed", startedAt: "2019-12-31T23:00:00Z" }; + f.startingAgentRuns = [prior]; + for (const p of [f.created, ...f.waiting, f.released, f.finished, f.reopened]) p.agentRuns.push(structuredClone(prior)); + return f; + }; + it("retains completed earlier campaign work while measuring only new runs", () => { + const f = withHistory(); + expect(gradeTaskDependencyResume(f).every(check => check.passed)).toBe(true); + expect(f.reopened.agentRuns.map(run => run.id)).toEqual(["run", "prior"]); + }); + it.each([ + ["missing baseline", (f: DependencyEvidence) => { delete (f as Partial).startingAgentRuns; }], + ["foreign baseline", (f: DependencyEvidence) => { f.startingAgentRuns[0]!.agentId = "foreign"; }], + ["active baseline", (f: DependencyEvidence) => { f.startingAgentRuns[0]!.status = "running"; }], + ["baseline run disappeared", (f: DependencyEvidence) => { f.waiting[0]!.agentRuns = []; }], + ["duplicate baseline ID", (f: DependencyEvidence) => { f.startingAgentRuns.push(f.startingAgentRuns[0]!); }], + ["premature run hidden as prior work", (f: DependencyEvidence) => { f.startingAgentRuns.push(structuredClone(f.finished.dependentRuns[0]!)); }], + ["new unrelated run while blocked", (f: DependencyEvidence) => { f.waiting[0]!.agentRuns.push({ ...structuredClone(f.startingAgentRuns[0]!), id: "extra" }); }], + ] as const)("rejects %s", (_name, mutate) => { + const f = withHistory(); mutate(f); + expect(gradeTaskDependencyResume(f).some(check => !check.passed)).toBe(true); + }); + it("has no global activity length cap after complete scoped observation", () => { + const f = dependencyEvidenceFixture(); + f.reopened.activity.push(...Array.from({ length: 600 }, (_, i) => ({ id: `prior-${i}`, companyId: f.scope.companyId, + entityType: "issue", entityId: f.scope.dependentId, action: "issue.observed" }))); + expect(gradeTaskDependencyResume(f).every(check => check.passed)).toBe(true); + }); + it("accepts plain or fenced JSON but rejects extra keys and prose", () => { + const s = dependencyEvidenceFixture().scope, body = JSON.stringify({ reference: s.reference, blockerIdentifier: s.blockerIdentifier }); + expect(dependencyOutputMatches(body, s)).toBe(true); + expect(dependencyOutputMatches(`\`\`\`json\n${body}\n\`\`\``, s)).toBe(true); + expect(dependencyOutputMatches(JSON.stringify({ reference: s.reference, blockerIdentifier: s.blockerIdentifier, extra: true }), s)).toBe(false); + expect(dependencyOutputMatches(`Completed ${body}`, s)).toBe(false); + }); +}); diff --git a/tests/runner-e2e/product-journeys/task-dependency-resume-evidence.ts b/tests/runner-e2e/product-journeys/task-dependency-resume-evidence.ts new file mode 100644 index 00000000000..d212a1f3ce2 --- /dev/null +++ b/tests/runner-e2e/product-journeys/task-dependency-resume-evidence.ts @@ -0,0 +1,131 @@ +type Row = Record; +export const DEPENDENCY_IDLE_MS = 10_000; +export const TASK_DEPENDENCY_RESUME_CHECKS = [ + "dependency-browser-created-and-linked", "dependency-unresolved-prevented-execution", + "dependency-blocker-completed-by-board", "dependency-one-automatic-resume", + "dependency-output-authored-by-run", "dependency-result-reopened", + "dependency-history-preserved", "configured-model-and-environment", +] as const; + +export interface DependencyScope { + companyId: string; userId: string; agentId: string; environmentId: string; provider: string; model: string; + blockerId: string; blockerIdentifier: string; blockerTitle: string; blockerDescription: string; + dependentId: string; dependentTitle: string; prompt: string; reference: string; documentKey: string; +} +export interface DependencyPoint { + observedAt: number; blocker: Row; dependent: Row; + blockerRuns: Row[]; dependentRuns: Row[]; agentRuns: Row[]; activity: Row[]; + activityCoverage: { blocker: boolean; dependent: boolean; agent: boolean }; + document?: Row; revision?: Row; resultVisible?: boolean; +} +export interface DependencyEvidence { + scope: DependencyScope; startingAgentRuns: Row[]; created: DependencyPoint; waiting: DependencyPoint[]; + releaseRequestedAt: number; released: DependencyPoint; finished: DependencyPoint; reopened: DependencyPoint; +} +const nonempty = (value: unknown): value is string => typeof value === "string" && value.length > 0; +const date = (value: unknown) => typeof value === "string" ? Date.parse(value) : NaN; +const exactId = (rows: unknown, id: string) => Array.isArray(rows) && rows.length === 1 && rows[0] === id; +const runFacts = (run: Row) => JSON.stringify([run.id, run.companyId, run.agentId, dependencyRunIssueId(run), run.status, + run.runtimeMode, run.nativePhase, run.invocationSource, run.triggerDetail, run.startedAt, + run.contextSnapshot?.wakeReason, run.contextSnapshot?.source, run.contextSnapshot?.resolvedBlockerIssueId, + run.contextSnapshot?.blockerIssueIds, run.contextSnapshot?.paperclipEnvironment?.id, + run.runnerProfileJson?.nativeExecutionInput?.provider?.kind, run.runnerProfileJson?.nativeExecutionInput?.provider?.model]); +export function dependencyNewAgentRuns(runs: Row[], startingRuns: Row[]) { + const priorIds = new Set(startingRuns.map(run => run.id)); + return runs.filter(run => !priorIds.has(run.id)); +} +export function dependencyRunIssueId(run: Row) { + const ids = [run.issueId, run.nativeIssueId, run.contextSnapshot?.issueId, run.contextSnapshot?.taskId].filter(value => value != null); + return ids.length && ids.every(nonempty) && new Set(ids).size === 1 ? ids[0] as string : undefined; +} +export function dependencyOutputMatches(body: unknown, scope: Pick) { + if (typeof body !== "string") return false; + const text = body.trim(), json = /^```(?:json)?\s*\n([\s\S]*?)\n```$/.exec(text)?.[1] ?? text; + try { + const value = JSON.parse(json); + return Boolean(value && !Array.isArray(value) && Object.keys(value).sort().join(",") === "blockerIdentifier,reference" + && value.reference === scope.reference && value.blockerIdentifier === scope.blockerIdentifier); + } catch { return false; } +} + +export function gradeTaskDependencyResume(value: DependencyEvidence | undefined) { + const fail = () => TASK_DEPENDENCY_RESUME_CHECKS.map(id => ({ id, passed: false })); + if (!value?.scope || !Array.isArray(value.startingAgentRuns) || !value.created || !Array.isArray(value.waiting) || value.waiting.length < 2 + || !value.released || !value.finished || !value.reopened) return fail(); + const { scope: s, startingAgentRuns, created, waiting, released, finished, reopened } = value; + const points = [created, ...waiting, released, finished, reopened]; + if (Object.values(s).some(item => !nonempty(item)) || s.blockerId === s.dependentId || s.prompt.includes(s.reference) + || !s.blockerDescription.includes(s.reference) + || points.some(p => !p?.blocker || !p.dependent || !Number.isFinite(p.observedAt) + || [p.blockerRuns, p.dependentRuns, p.agentRuns, p.activity].some(rows => !Array.isArray(rows)))) return fail(); + const startingIds = startingAgentRuns.map(run => run.id); + const historyComplete = startingIds.every(nonempty) && new Set(startingIds).size === startingIds.length + && startingAgentRuns.every(run => run.companyId === s.companyId && run.agentId === s.agentId + && ["succeeded", "failed", "cancelled", "timed_out"].includes(run.status) + && ![s.blockerId, s.dependentId].includes(dependencyRunIssueId(run) ?? "")) + && points.every(p => new Set(p.agentRuns.map(run => run.id)).size === p.agentRuns.length + && p.agentRuns.every(run => nonempty(run.id) && run.companyId === s.companyId && run.agentId === s.agentId) + && startingAgentRuns.every(prior => p.agentRuns.some(run => runFacts(run) === runFacts(prior)))); + const newRuns = (point: DependencyPoint) => dependencyNewAgentRuns(point.agentRuns, startingAgentRuns); + const scoped = historyComplete && points.every(p => p.blocker.id === s.blockerId && p.dependent.id === s.dependentId + && p.blocker.companyId === s.companyId && p.dependent.companyId === s.companyId + && p.blocker.title === s.blockerTitle && p.dependent.title === s.dependentTitle + && p.blocker.description === s.blockerDescription && p.dependent.description === s.prompt + && p.blocker.createdByUserId === s.userId && p.dependent.createdByUserId === s.userId + && !p.blocker.createdByAgentId && !p.dependent.createdByAgentId + && p.blocker.assigneeUserId === s.userId && p.blocker.assigneeAgentId === null); + const linked = scoped && [created.blocker, created.dependent].every(task => task.status === "todo") + && created.dependent.assigneeUserId === s.userId && created.dependent.assigneeAgentId === null + && [created.blockerRuns, created.dependentRuns, newRuns(created)].every(rows => rows.length === 0) + && points.slice(1).every(p => p.dependent.assigneeAgentId === s.agentId && p.dependent.assigneeUserId === null + && exactId(p.dependent.blockedByIssueIds, s.blockerId)); + const held = linked && waiting.every(p => p.blocker.status === "todo" && p.dependent.status === "blocked" + && !p.dependent.executionRunId && !p.dependent.checkoutRunId + && [p.blockerRuns, p.dependentRuns, newRuns(p)].every(rows => rows.length === 0)) + && waiting.every((p, index) => index === 0 || p.observedAt >= waiting[index - 1]!.observedAt) + && waiting[waiting.length - 1]!.observedAt - waiting[0]!.observedAt >= DEPENDENCY_IDLE_MS + && waiting[0]!.observedAt >= created.observedAt + && Number.isFinite(value.releaseRequestedAt) && value.releaseRequestedAt >= waiting[waiting.length - 1]!.observedAt + && released.observedAt >= value.releaseRequestedAt && finished.observedAt >= released.observedAt && reopened.observedAt >= finished.observedAt; + const completedByBoard = held && [released, finished, reopened].every(p => p.blocker.status === "done" + && Number.isFinite(date(p.blocker.completedAt))) + && reopened.activity.filter(row => row.companyId === s.companyId && row.entityType === "issue" + && row.entityId === s.blockerId && row.action === "issue.updated" && row.actorType === "user" + && row.actorId === s.userId && row.details?.status === "done").length === 1; + const run = finished.dependentRuns[0]; + const oneRun = completedByBoard && finished.dependent.status === "done" && reopened.dependent.status === "done" + && !finished.dependent.executionRunId && !finished.dependent.checkoutRunId + && !reopened.dependent.executionRunId && !reopened.dependent.checkoutRunId + && finished.dependentRuns.length === 1 && reopened.dependentRuns.length === 1 + && newRuns(finished).length === 1 && newRuns(reopened).length === 1 + && nonempty(run?.id) && [reopened.dependentRuns[0], newRuns(finished)[0], newRuns(reopened)[0]].every(row => runFacts(row) === runFacts(run)) + && run.companyId === s.companyId && run.agentId === s.agentId && dependencyRunIssueId(run) === s.dependentId + && run.status === "succeeded" && run.invocationSource === "automation" && run.triggerDetail === "system" + && run.contextSnapshot?.wakeReason === "issue_blockers_resolved" && run.contextSnapshot?.source === "issue.blockers_resolved" + && run.contextSnapshot?.resolvedBlockerIssueId === s.blockerId && exactId(run.contextSnapshot?.blockerIssueIds, s.blockerId) + && Number.isFinite(date(run.startedAt)) && date(run.startedAt) >= date(released.blocker.completedAt); + const boundRun = scoped && [run, reopened.dependentRuns[0], newRuns(finished)[0], newRuns(reopened)[0]].every(row => + nonempty(row?.id) && row.id === run?.id && row.companyId === s.companyId && row.agentId === s.agentId + && dependencyRunIssueId(row) === s.dependentId); + const runtime = boundRun && run?.runtimeMode === "native" && run.nativePhase === "committed" + && run.runnerProfileJson?.nativeExecutionInput?.provider?.kind === s.provider + && run.runnerProfileJson?.nativeExecutionInput?.provider?.model === s.model + && run.contextSnapshot?.paperclipEnvironment?.id === s.environmentId; + const documentMatches = (p: DependencyPoint) => p.document && p.revision + && nonempty(p.document.id) && p.document.companyId === s.companyId && p.document.issueId === s.dependentId + && p.document.key === s.documentKey && p.document.createdByAgentId === s.agentId && p.document.createdByUserId === null + && p.document.latestRevisionId === p.revision.id && p.revision.documentId === p.document.id + && p.revision.createdByAgentId === s.agentId && p.revision.createdByUserId === null && p.revision.createdByRunId === run?.id + && dependencyOutputMatches(p.document.body, s) && dependencyOutputMatches(p.revision.body, s); + const authored = boundRun && documentMatches(finished); + const resultReopened = boundRun && documentMatches(reopened) && finished.document && reopened.resultVisible === true + && reopened.document!.id === finished.document.id && reopened.document!.latestRevisionId === finished.document.latestRevisionId + && reopened.document!.body === finished.document.body; + const history = boundRun && points.every(p => p.blockerRuns.length === 0) + && points.every(p => p.activityCoverage?.blocker === true && p.activityCoverage?.dependent === true && p.activityCoverage?.agent === true) + && !reopened.activity.some(row => row.runId === run?.id + && ["issue.created", "issue.child_created", "project.created", "agent.created", "agent.hire_created"].includes(row.action)) + && reopened.blocker.completedAt === released.blocker.completedAt; + const values = [linked, held, completedByBoard, oneRun, authored, resultReopened, history, runtime]; + return TASK_DEPENDENCY_RESUME_CHECKS.map((id, index) => ({ id, passed: Boolean(values[index]) })); +} diff --git a/tests/runner-e2e/product-journeys/task-dependency-resume-fixture.ts b/tests/runner-e2e/product-journeys/task-dependency-resume-fixture.ts new file mode 100644 index 00000000000..984459110b0 --- /dev/null +++ b/tests/runner-e2e/product-journeys/task-dependency-resume-fixture.ts @@ -0,0 +1,38 @@ +import type { DependencyEvidence, DependencyPoint } from "./task-dependency-resume-evidence.js"; + +export function dependencyEvidenceFixture(): DependencyEvidence { + const start = Date.parse("2020-01-01T00:00:00.000Z"); + const scope = { companyId: "company", userId: "user", agentId: "agent", environmentId: "environment", provider: "opencode", model: "fixture-model", + blockerId: "blocker", blockerIdentifier: "QA-1", blockerTitle: "Review gate", blockerDescription: "The review reference is secret-reference.", + dependentId: "dependent", dependentTitle: "Continue after review", prompt: "Read QA-1 after its review and write dependency-proof.", reference: "secret-reference", documentKey: "dependency-proof" }; + const point = (observedAt: number): DependencyPoint => ({ observedAt, + blocker: { id: scope.blockerId, identifier: scope.blockerIdentifier, companyId: scope.companyId, title: scope.blockerTitle, + description: scope.blockerDescription, status: "todo", createdByUserId: scope.userId, createdByAgentId: null, + assigneeUserId: scope.userId, assigneeAgentId: null }, + dependent: { id: scope.dependentId, identifier: "QA-2", companyId: scope.companyId, title: scope.dependentTitle, + description: scope.prompt, status: "blocked", createdByUserId: scope.userId, createdByAgentId: null, + assigneeUserId: null, assigneeAgentId: scope.agentId, blockedByIssueIds: [scope.blockerId], executionRunId: null, checkoutRunId: null }, + blockerRuns: [], dependentRuns: [], agentRuns: [], activity: [], activityCoverage: { blocker: true, dependent: true, agent: true } }); + const created = point(start); + Object.assign(created.dependent, { status: "todo", assigneeAgentId: null, assigneeUserId: scope.userId, blockedByIssueIds: [] }); + const released = point(start + 20_000); Object.assign(released.blocker, { status: "done", completedAt: "2020-01-01T00:00:20.000Z" }); + const run = { id: "run", companyId: scope.companyId, agentId: scope.agentId, nativeIssueId: scope.dependentId, + status: "succeeded", runtimeMode: "native", nativePhase: "committed", invocationSource: "automation", triggerDetail: "system", + startedAt: "2020-01-01T00:00:21.000Z", + contextSnapshot: { issueId: scope.dependentId, taskId: scope.dependentId, wakeReason: "issue_blockers_resolved", source: "issue.blockers_resolved", + resolvedBlockerIssueId: scope.blockerId, blockerIssueIds: [scope.blockerId], paperclipEnvironment: { id: scope.environmentId } }, + runnerProfileJson: { nativeExecutionInput: { provider: { kind: scope.provider, model: scope.model } } } }; + const document = { id: "document", companyId: scope.companyId, issueId: scope.dependentId, key: scope.documentKey, + body: JSON.stringify({ reference: scope.reference, blockerIdentifier: scope.blockerIdentifier }), latestRevisionId: "revision", + createdByAgentId: scope.agentId, createdByUserId: null }; + const revision = { id: "revision", documentId: document.id, body: document.body, createdByAgentId: scope.agentId, + createdByUserId: null, createdByRunId: run.id }; + const finished = structuredClone(released); finished.observedAt = start + 30_000; finished.dependent.status = "done"; + finished.dependentRuns = [structuredClone(run)]; finished.agentRuns = [structuredClone(run)]; + finished.document = document; finished.revision = revision; + const reopened = structuredClone(finished); reopened.observedAt = start + 31_000; reopened.resultVisible = true; + reopened.activity = [{ id: "completion", companyId: scope.companyId, actorType: "user", actorId: scope.userId, + action: "issue.updated", entityType: "issue", entityId: scope.blockerId, details: { status: "done" } }]; + return { scope, startingAgentRuns: [], created, waiting: [point(start + 1_000), point(start + 11_000)], releaseRequestedAt: start + 19_000, + released, finished, reopened }; +} diff --git a/tests/runner-e2e/product-journeys/task-dependency-resume.md b/tests/runner-e2e/product-journeys/task-dependency-resume.md new file mode 100644 index 00000000000..f8f18dee135 --- /dev/null +++ b/tests/runner-e2e/product-journeys/task-dependency-resume.md @@ -0,0 +1,64 @@ +# Blocked task resumes after its human-owned blocker + +`task-dependency-resume` is an explicit Product E2E journey for an authenticated +existing Cloud target or a target provider with a qualified native agent. It uses +the ordinary task UI and public observation APIs. Its registration is not evidence +of a local or deployed pass. + +The browser creates two tasks assigned to the signed-in person. The blocker holds +a random review reference absent from the dependent's prompt. On the dependent, +the person adds the blocker, selects Blocked, and assigns the disposable agent. +Closed properties panes are opened through their visible toolbar control; mounted +inert or accessibility-hidden rows cannot substitute for an interactive pane. +Status choices are bound to the matching labeled glyph inside the open picker. +The saved dependency, blocked status, and empty task/agent run histories must +remain observable for at least ten seconds. This is a finite measured wait, +not an unlimited claim that no execution can ever occur. + +The person completes the blocker through the task's status control. Paperclip must +then start exactly one native run automatically. Its durable wake must name the +same resolved blocker, reason `issue_blockers_resolved`, and saved source +`issue.blockers_resolved`; `blocker_done` is the producer's separate mutation +payload. Manual +Run or a follow-up message cannot substitute. The run starts after the saved +blocker completion, uses the configured provider/model/environment, commits its +result through the durable committed native phase, and completes the dependent. +The public API may omit or compact `resultJson`; duplicate finalization metadata +in that optional object is not required. A task document named `dependency-proof` +contains the review reference and blocker identifier. Its current revision must +be authored by this agent and run. Reloading the task must show that same saved +revision and preserve the two tasks' descriptions, dependency, and board-owned +blocker completion audit. No agent work is permitted on the human-owned blocker. + +The case requires `company`, `board-user`, `task-write`, and `agent`. It declares +one work run, uses the existing bounded case budget, and shares the original +campaign deadline. The native agent must have nonblank provider and model +identifiers before any observation read or task creation. An incomplete execution +identity returns `unsupported` without task or agent work. +It saves the agent's starting run history and retains those +records while checking only newly appearing runs for this case. Earlier terminal +campaign work stays in the accounting and does not count as new execution. +Owned-task activity reads are complete and uncapped; the separate agent-filtered +activity read fails closed if its 500-row bound is reached. The agent run list +likewise fails closed at the public API's 1000-row cap. Unrelated company history +does not enter these completeness checks. +Agent setup usage is outside task-run accounting; missing +provider prices remain unknown. The owning target provider controls normal +cancellation, termination, retained history, and evidence cleanup. It never +changes a shared source agent, company budget, or execution environment. + +The private evidence folder contains `task-dependency-observations.json` and +`observation-read-failures.json`, with allowlisted owned task, run, audit, and +document facts. The normal launcher captures the final owned task screenshot. +Both evidence writes are attempted. A write failure retains an existing journey +error and its outcome; a previously successful journey fails if required evidence +cannot be saved. Diagnostics name only the affected evidence file. +Runtime identity, document authorship, reload visibility, and retained history +use their own observations bound to the same company, task, agent, and run. One +failed control does not mark unrelated controls failed. All eight checks remain +required for acceptance. +There are no direct database writes, private runner hooks, manually dispatched +resumption wakes, retry loops for failed model work, or fixture-authored outputs. +Independent controls reject premature or duplicate execution, a never-resumed +or manually started task, foreign/changed runtime identity, missing completion +audit, wrong output or revision authorship, and missing or shortened observations. diff --git a/tests/runner-e2e/product-journeys/task-dependency-resume.test.ts b/tests/runner-e2e/product-journeys/task-dependency-resume.test.ts new file mode 100644 index 00000000000..fa694af7d1d --- /dev/null +++ b/tests/runner-e2e/product-journeys/task-dependency-resume.test.ts @@ -0,0 +1,285 @@ +import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSession } from "./types.js"; +import { dependencyEvidenceFixture } from "./task-dependency-resume-fixture.js"; +import * as observedReader from "./observation-read.js"; + +const controls = vi.hoisted(() => ({ create: vi.fn() })); +vi.mock("../user-actions.js", () => ({ createTaskThroughUi: (...args: unknown[]) => controls.create(...args) })); +vi.mock("./task-document-flow.js", () => ({ openDocumentAuthoring: vi.fn(async () => {}) })); +vi.mock("@playwright/test", () => ({ expect: Object.assign((locator: any) => ({ + async toBeVisible() { expect(await locator.isVisible()).toBe(true); }, + async toBeInViewport() { expect(await locator.isVisible()).toBe(true); }, + async toContainText(text: string) { expect(await locator.textContent()).toContain(text); }, +}), { poll: (read: () => Promise) => ({ + async toBe(value: unknown) { expect(await read()).toBe(value); }, + async toEqual(value: unknown) { expect(await read()).toEqual(value); }, +}) }) })); +import { taskDependencyResume } from "./task-dependency-resume.js"; + +interface FixtureOptions { priorRuns?: number; unrelatedActivity?: number; ownedActivity?: number; agentActivity?: number; missingActivity?: boolean; activePrior?: boolean; + closedPanel?: "properties" | "toggle"; missingProperty?: boolean; missingStatusChoice?: boolean } +async function fixture(premature = false, options: FixtureOptions = {}) { + const directory = await mkdtemp(path.join(os.tmpdir(), "dependency-flow-")); + const f = dependencyEvidenceFixture(), s = f.scope; + const blocker = { ...f.created.blocker }, dependent = { ...f.created.dependent }; + const actions: string[] = [], owned: any[] = [], writes: unknown[] = [], reads: string[] = []; + let completed = false, assigned = false, panelOpen = !options.closedPanel; + const reference = () => /review reference is ([a-f0-9]{32})/.exec(blocker.description)?.[1]; + const output = () => JSON.stringify({ reference: reference(), blockerIdentifier: s.blockerIdentifier }); + const runs = () => assigned && (completed || premature) ? [structuredClone(f.finished.dependentRuns[0]!)] : []; + const priorRuns = Array.from({ length: options.priorRuns ?? 0 }, (_, i) => ({ ...structuredClone(f.finished.agentRuns[0]!), + id: `prior-${i}`, nativeIssueId: `prior-task-${i}`, contextSnapshot: { issueId: `prior-task-${i}` }, + startedAt: "2019-12-31T23:00:00Z", status: options.activePrior ? "running" : i % 2 ? "failed" : "succeeded" })); + const activity = (count: number, entityId: string, agentId?: string) => Array.from({ length: count }, (_, i) => ({ + id: `${entityId}-activity-${i}`, companyId: s.companyId, entityType: "issue", entityId, agentId, + actorType: agentId ? "agent" : "user", actorId: agentId ?? s.userId, action: "issue.observed" })); + const taskDocument = () => ({ ...f.finished.document, body: output() }); + const revision = () => ({ ...f.finished.revision, body: output() }); + controls.create.mockImplementation(async (input: any) => { + await input.beforeDispatch?.(); + const row = input.title.includes(" blocker ") ? blocker : dependent; + Object.assign(row, { title: input.title, description: input.prompt }); + actions.push(`create:${row.id}`); + return { issueId: row.id, submittedAtMs: Date.now() }; + }); + const locator = (key: string): any => ({ + key, + filter(value: any) { + if (key === '[data-slot="popover-content"]' && value?.has) { + return locator(options.missingStatusChoice ? "missing-status-choice" : value.has.key); + } + return this; + }, first() { return this; }, last() { return this; }, + locator() { return this; }, + getByRole(_role: string, options?: any) { return locator(options?.name ? String(options.name) : key); }, + async isVisible() { + if (key.includes("data-property-row")) return panelOpen && !options.missingProperty; + if (key === "Toggle side panel") return !panelOpen && options.closedPanel === "toggle"; + if (key.includes("Show properties")) return !panelOpen && options.closedPanel === "properties"; + return true; + }, + async scrollIntoViewIfNeeded() { actions.push(`scroll:${key}`); }, + async fill(value: string) { actions.push(`fill:${key}:${value}`); }, + async click() { + actions.push(`click:${key}`); + if (key === "Toggle side panel" || key.includes("Show properties")) panelOpen = true; + if (key === "missing-status-choice") throw new Error("missing_status_choice"); + if (key === "blocker-choice") dependent.blockedByIssueIds = [blocker.id]; + if (key === "Blocked") dependent.status = "blocked"; + if (key === "agent-choice") { assigned = true; dependent.assigneeAgentId = s.agentId; dependent.assigneeUserId = null; } + if (key === "Done") { + completed = true; blocker.status = "done"; blocker.completedAt = "2020-01-01T00:00:20.000Z"; + dependent.status = "done"; + } + }, + async textContent() { return output(); }, + }); + const page: any = { + locator(selector: string) { return locator(selector); }, + getByRole(_role: string, options?: any) { + if (_role === "button" && options?.exact && ["Blocked", "Done"].includes(options.name)) return locator("missing-status-choice"); + const l = locator(options?.name ? String(options.name) : "buttons"); + l.filter = (options: any) => options?.hasText === blocker.title ? locator("blocker-choice") + : options?.hasText === "Fixture agent" ? locator("agent-choice") : l; + return l; + }, + getByPlaceholder(name: string) { return locator(name); }, + getByTestId(name: string) { return locator(name); }, + async goto(url: string) { actions.push(`goto:${url}`); }, async reload() { actions.push("reload"); }, + }; + const api: any = { origin: "https://qa.example.test", async get(route: string) { + reads.push(route); + if (route === `/api/issues/${s.blockerId}`) return structuredClone(blocker); + if (route === `/api/issues/${s.dependentId}`) return structuredClone(dependent); + if (route === `/api/issues/${s.blockerId}/runs`) return []; + if (route === `/api/issues/${s.dependentId}/runs`) return runs().map(r => ({ runId: r.id })); + if (route.includes("/heartbeat-runs?")) return [...runs(), ...structuredClone(priorRuns)]; + if (route === "/api/heartbeat-runs/run") return runs()[0]; + if (route === `/api/issues/${s.blockerId}/activity`) return completed ? structuredClone(f.reopened.activity) : []; + if (route === `/api/issues/${s.dependentId}/activity`) return activity(options.ownedActivity ?? 0, s.dependentId); + if (route === `/api/companies/${s.companyId}/activity?agentId=${s.agentId}&limit=500`) { + return options.missingActivity ? undefined : activity(options.agentActivity ?? 0, "prior-task", s.agentId); + } + if (route === `/api/companies/${s.companyId}/activity?limit=500`) return activity(options.unrelatedActivity ?? 0, "unrelated-task").slice(0, 500); + if (route === `/api/issues/${s.dependentId}/documents`) return completed ? [taskDocument()] : []; + if (route.endsWith("/documents/dependency-proof")) return taskDocument(); + if (route.endsWith("/revisions/revision")) return revision(); + throw new Error(`unexpected_read:${route}`); + }, async post(...args: unknown[]) { writes.push(args); throw new Error("unexpected_mutation"); }, + async patch(...args: unknown[]) { writes.push(args); throw new Error("unexpected_mutation"); } }; + const context = { page, api, company: { id: s.companyId, issuePrefix: "QA" }, user: { id: s.userId }, environmentId: s.environmentId, + capabilities: ["company", "board-user", "task-write", "agent"], agent: { id: s.agentId, name: "Fixture agent", provider: s.provider, model: s.model, runtimeMode: "native" } } as JourneyContext; + const session = { own: vi.fn(async resource => { owned.push(resource); }) } as unknown as TargetSession; + const evidence = { checks: [] } as unknown as JourneyEvidence; + return { directory, context, session, evidence, actions, owned, writes, reads }; +} +afterEach(() => { vi.useRealTimers(); controls.create.mockReset(); }); +describe("dependency browser flow orchestration", () => { + const evidenceFiles = ["task-dependency-observations.json", "observation-read-failures.json"] as const; + const writerFaults = ["observations", "reads", "both"] as const; + const obstructWriters = async (directory: string, fault: typeof writerFaults[number]) => { + const names = fault === "both" ? [...evidenceFiles] : [evidenceFiles[fault === "observations" ? 0 : 1]]; + await Promise.all(names.map(name => mkdir(path.join(directory, name)))); + return names; + }; + it.each(writerFaults.flatMap(fault => (["unsupported", "failed"] as const).map(outcome => [fault, outcome] as const)))( + "preserves the original %s writer failure outcome %s and attempts both evidence files", async (fault, outcome) => { + const f = await fixture(), blocked = await obstructWriters(f.directory, fault); + const logging = vi.spyOn(console, "error").mockImplementation(() => {}); + const original = Object.freeze(new JourneyError("failed", "dependency_observation_control_failed")); + if (outcome === "unsupported") f.context.agent!.provider = ""; + else vi.spyOn(f.context.api, "get").mockRejectedValue(original); + try { + const operation = taskDependencyResume(f.context, f.session, "journey-123-abcdef01", f.evidence, + { deadlineAt: Date.now() + 60_000, directory: f.directory }); + if (outcome === "unsupported") await expect(operation).rejects.toMatchObject({ outcome, code: "dependency_execution_identity_unavailable" }); + else await expect(operation).rejects.toBe(original); + for (const name of evidenceFiles.filter(name => !blocked.includes(name))) { + expect(JSON.parse(await readFile(path.join(f.directory, name), "utf8"))).toBeTypeOf("object"); + } + expect(logging).toHaveBeenCalledWith("dependency_evidence_preservation_failed", blocked); + expect(JSON.stringify(logging.mock.calls)).not.toContain(f.directory); + expect(f.actions).toEqual([]); expect(f.owned).toEqual([]); expect(f.writes).toEqual([]); + } finally { logging.mockRestore(); await rm(f.directory, { recursive: true, force: true }); } + }); + it.each(["unsupported", "failed"] as const)("awaits the other evidence file when the read writer throws synchronously after %s", async outcome => { + const f = await fixture(), createReader = observedReader.createObservedReader; + const original = Object.freeze(new JourneyError("failed", "dependency_observation_control_failed")); + const failingWrite = vi.fn(() => { throw new Error("private writer diagnostic"); }); + const reader = vi.spyOn(observedReader, "createObservedReader").mockImplementation((...args) => ({ + ...createReader(...args), preserve: failingWrite, + })); + const logging = vi.spyOn(console, "error").mockImplementation(() => {}); + if (outcome === "unsupported") f.context.agent!.model = ""; + else vi.spyOn(f.context.api, "get").mockRejectedValue(original); + try { + const operation = taskDependencyResume(f.context, f.session, "journey-123-abcdef01", f.evidence, + { deadlineAt: Date.now() + 60_000, directory: f.directory }); + if (outcome === "unsupported") await expect(operation).rejects.toMatchObject({ outcome, code: "dependency_execution_identity_unavailable" }); + else await expect(operation).rejects.toBe(original); + expect(failingWrite).toHaveBeenCalledOnce(); + expect(JSON.parse(await readFile(path.join(f.directory, "task-dependency-observations.json"), "utf8"))).toHaveProperty("stage", + outcome === "unsupported" ? "prerequisites" : "observe_starting_runs"); + expect(logging).toHaveBeenCalledWith("dependency_evidence_preservation_failed", ["observation-read-failures.json"]); + expect(JSON.stringify(logging.mock.calls)).not.toContain("private writer diagnostic"); + } finally { logging.mockRestore(); reader.mockRestore(); await rm(f.directory, { recursive: true, force: true }); } + }); + it.each(writerFaults)("fails a successful journey when its %s evidence writer fails", async fault => { + const f = await fixture(), blocked = await obstructWriters(f.directory, fault); + const logging = vi.spyOn(console, "error").mockImplementation(() => {}); + vi.useFakeTimers(); vi.setSystemTime(new Date("2020-01-01T00:00:00Z")); + try { + const rejection = expect(taskDependencyResume(f.context, f.session, "journey-123-abcdef01", f.evidence, + { deadlineAt: Date.now() + 60_000, directory: f.directory })).rejects.toMatchObject({ + outcome: "failed", code: "dependency_evidence_preservation_failed", + }); + await vi.advanceTimersByTimeAsync(11_000); await rejection; + expect(f.evidence.checks).toHaveLength(8); expect(f.evidence.checks.every(check => check.passed)).toBe(true); + expect(logging).toHaveBeenCalledWith("dependency_evidence_preservation_failed", blocked); + for (const name of evidenceFiles.filter(name => !blocked.includes(name))) { + expect(JSON.parse(await readFile(path.join(f.directory, name), "utf8"))).toBeTypeOf("object"); + } + expect(f.writes).toEqual([]); + } finally { logging.mockRestore(); await rm(f.directory, { recursive: true, force: true }); } + }); + it.each((["provider", "model"] as const).flatMap(field => [ + [field, "missing", undefined], [field, "null", null], [field, "empty", ""], [field, "blank", " \t "], + ] as const))("rejects a %s identity that is %s before observation or task work", async (field, _kind, value) => { + const f = await fixture(); + Object.assign(f.context.agent!, { [field]: value }); + const get = vi.spyOn(f.context.api, "get").mockRejectedValue(new Error("preflight_read_attempted")); + try { + await expect(taskDependencyResume(f.context, f.session, "journey-123-abcdef01", f.evidence, + { deadlineAt: Date.now() + 60_000, directory: f.directory })).rejects.toMatchObject({ + outcome: "unsupported", code: "dependency_execution_identity_unavailable", + }); + expect(get).not.toHaveBeenCalled(); expect(controls.create).not.toHaveBeenCalled(); + expect(f.actions).toEqual([]); expect(f.owned).toEqual([]); expect(f.writes).toEqual([]); + expect(f.evidence.checks).toEqual([]); + expect(JSON.parse(await readFile(path.join(f.directory, "task-dependency-observations.json"), "utf8"))).toHaveProperty("stage", "prerequisites"); + expect(JSON.parse(await readFile(path.join(f.directory, "observation-read-failures.json"), "utf8"))).toEqual({ failures: [] }); + } finally { await rm(f.directory, { recursive: true, force: true }); } + }); + it.each([ + ["empty history", {}], + ["completed earlier campaign runs", { priorRuns: 3 }], + ["more than 100 earlier campaign runs", { priorRuns: 150 }], + ["more than 500 unrelated company events", { unrelatedActivity: 800 }], + ["more than 500 owned task events", { ownedActivity: 600 }], + ["prior agent events", { priorRuns: 2, agentActivity: 300 }], + ] as const)("uses browser actions with %s and never writes output through API observations", async (_name, options) => { + const f = await fixture(false, options); vi.useFakeTimers(); vi.setSystemTime(new Date("2020-01-01T00:00:00Z")); + try { + const operation = taskDependencyResume(f.context, f.session, "journey-123-abcdef01", f.evidence, + { deadlineAt: Date.now() + 60_000, directory: f.directory }); + await vi.advanceTimersByTimeAsync(11_000); await operation; + expect(f.evidence.checks).toHaveLength(8); expect(f.evidence.checks.every(check => check.passed)).toBe(true); + expect(f.writes).toEqual([]); + expect(f.actions.indexOf("click:Blocked")).toBeLessThan(f.actions.indexOf("click:agent-choice")); + expect(f.actions.indexOf("click:agent-choice")).toBeLessThan(f.actions.indexOf("click:Done")); + expect(f.owned).toEqual(expect.arrayContaining([ + expect.objectContaining({ kind: "issue", id: "blocker", journeyId: "task-dependency-resume" }), + expect.objectContaining({ kind: "issue", id: "dependent", journeyId: "task-dependency-resume" }), + ])); + const saved = JSON.parse(await readFile(path.join(f.directory, "task-dependency-observations.json"), "utf8")); + expect(saved.stage).toBe("complete"); expect(saved.scope.prompt).not.toContain(saved.scope.reference); + expect(saved.waiting.at(-1).observedAt - saved.waiting[0].observedAt).toBeGreaterThanOrEqual(10_000); + expect(saved.startingAgentRuns).toHaveLength("priorRuns" in options ? options.priorRuns : 0); + expect(saved.reopened.agentRuns).toHaveLength(saved.startingAgentRuns.length + 1); + expect(f.reads.some(route => route.startsWith("/api/heartbeat-runs/prior-"))).toBe(false); + expect(f.reads).toContain("/api/issues/blocker/activity"); expect(f.reads).toContain("/api/issues/dependent/activity"); + expect(f.reads).not.toContain("/api/companies/company/activity?limit=500"); + } finally { await rm(f.directory, { recursive: true, force: true }); } + }); + it.each(["properties", "toggle"] as const)("opens a closed properties pane through its %s control before linking", async closedPanel => { + const f = await fixture(false, { closedPanel }); vi.useFakeTimers(); vi.setSystemTime(new Date("2020-01-01T00:00:00Z")); + try { + const operation = taskDependencyResume(f.context, f.session, "journey-123-abcdef01", f.evidence, + { deadlineAt: Date.now() + 60_000, directory: f.directory }); + await vi.advanceTimersByTimeAsync(11_000); await operation; + expect(f.evidence.checks).toHaveLength(8); expect(f.evidence.checks.every(check => check.passed)).toBe(true); + const opener = f.actions.findIndex(action => action.startsWith("click:") && (action.includes("Show properties") || action === "click:Toggle side panel")); + expect(opener).toBeGreaterThanOrEqual(0); expect(opener).toBeLessThan(f.actions.indexOf("click:blocker-choice")); + expect(f.actions.some(action => action.startsWith("scroll:[data-property-row]:not([inert] *):not([aria-hidden=\"true\"] *)"))).toBe(true); + expect(f.writes).toEqual([]); + } finally { await rm(f.directory, { recursive: true, force: true }); } + }); + it.each([ + ["missing property", { missingProperty: true }, "link_blocker"], + ["missing matching status glyph", { missingStatusChoice: true }, "set_dependent_blocked"], + ] as const)("fails closed on %s before agent assignment or blocker completion", async (_name, options, stage) => { + const f = await fixture(false, options); + try { + await expect(taskDependencyResume(f.context, f.session, "journey-123-abcdef01", f.evidence, + { deadlineAt: Date.now() + 60_000, directory: f.directory })).rejects.toThrow(); + expect(f.actions).not.toContain("click:agent-choice"); expect(f.actions).not.toContain("click:Done"); expect(f.writes).toEqual([]); + expect(JSON.parse(await readFile(path.join(f.directory, "task-dependency-observations.json"), "utf8"))).toHaveProperty("stage", stage); + } finally { await rm(f.directory, { recursive: true, force: true }); } + }); + it.each([ + ["truncated agent activity", { agentActivity: 500 }, "dependency_activity_coverage_unavailable"], + ["missing agent activity", { missingActivity: true }, "dependency_activity_coverage_unavailable"], + ["active earlier work", { priorRuns: 1, activePrior: true }, "dependency_starting_agent_not_idle"], + ["saturated run history", { priorRuns: 1000 }, "dependency_run_coverage_unavailable"], + ] as const)("rejects %s before completing the blocker", async (_name, options, code) => { + const f = await fixture(false, options); + try { + await expect(taskDependencyResume(f.context, f.session, "journey-123-abcdef01", f.evidence, + { deadlineAt: Date.now() + 60_000, directory: f.directory })).rejects.toThrow(code); + expect(f.actions).not.toContain("click:Done"); expect(f.writes).toEqual([]); + } finally { await rm(f.directory, { recursive: true, force: true }); } + }); + it("premature work fails before the browser completes the blocker and preserves its observations", async () => { + const f = await fixture(true); + try { + await expect(taskDependencyResume(f.context, f.session, "journey-123-abcdef01", f.evidence, + { deadlineAt: Date.now() + 60_000, directory: f.directory })).rejects.toThrow("dependency_ran_before_blocker_completed"); + expect(f.actions).not.toContain("click:Done"); expect(f.writes).toEqual([]); + expect(JSON.parse(await readFile(path.join(f.directory, "task-dependency-observations.json"), "utf8"))).toHaveProperty("stage", "observe_blocked_wait"); + expect(JSON.parse(await readFile(path.join(f.directory, "observation-read-failures.json"), "utf8"))).toEqual({ failures: [] }); + } finally { await rm(f.directory, { recursive: true, force: true }); } + }); +}); diff --git a/tests/runner-e2e/product-journeys/task-dependency-resume.ts b/tests/runner-e2e/product-journeys/task-dependency-resume.ts new file mode 100644 index 00000000000..ed13e309878 --- /dev/null +++ b/tests/runner-e2e/product-journeys/task-dependency-resume.ts @@ -0,0 +1,243 @@ +import { randomBytes } from "node:crypto"; +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { expect } from "@playwright/test"; +import { HEARTBEAT_RUN_LIST_MAX_LIMIT } from "../../../packages/shared/src/constants.js"; +import { createTaskThroughUi } from "../user-actions.js"; +import { createObservedReader } from "./observation-read.js"; +import { openDocumentAuthoring } from "./task-document-flow.js"; +import { DEPENDENCY_IDLE_MS, dependencyNewAgentRuns, gradeTaskDependencyResume, type DependencyEvidence, type DependencyPoint, type DependencyScope } from "./task-dependency-resume-evidence.js"; +import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSession } from "./types.js"; + +type Row = Record; +const id = "task-dependency-resume" as const; +const taskSnapshot = (task: Row) => ({ id: task.id, companyId: task.companyId, identifier: task.identifier, + title: task.title, description: task.description, status: task.status, assigneeAgentId: task.assigneeAgentId, + assigneeUserId: task.assigneeUserId, createdByUserId: task.createdByUserId, createdByAgentId: task.createdByAgentId, + completedAt: task.completedAt, executionRunId: task.executionRunId, checkoutRunId: task.checkoutRunId, + blockedByIssueIds: task.blockedByIssueIds ?? task.blockedBy?.map((row: Row) => row.id) ?? [] }); +const runSnapshot = (run: Row) => ({ id: run.id, companyId: run.companyId, agentId: run.agentId, + issueId: run.issueId, nativeIssueId: run.nativeIssueId, status: run.status, runtimeMode: run.runtimeMode, + nativePhase: run.nativePhase, invocationSource: run.invocationSource, triggerDetail: run.triggerDetail, startedAt: run.startedAt, + contextSnapshot: { issueId: run.contextSnapshot?.issueId, taskId: run.contextSnapshot?.taskId, + source: run.contextSnapshot?.source, wakeReason: run.contextSnapshot?.wakeReason, + resolvedBlockerIssueId: run.contextSnapshot?.resolvedBlockerIssueId, blockerIssueIds: run.contextSnapshot?.blockerIssueIds, + paperclipEnvironment: { id: run.contextSnapshot?.paperclipEnvironment?.id } }, + runnerProfileJson: { nativeExecutionInput: { provider: { kind: run.runnerProfileJson?.nativeExecutionInput?.provider?.kind, + model: run.runnerProfileJson?.nativeExecutionInput?.provider?.model } } } }); +const documentSnapshot = (row: Row) => Object.fromEntries(["id", "companyId", "issueId", "key", "body", "latestRevisionId", + "documentId", "createdByAgentId", "createdByUserId", "createdByRunId"].map(key => [key, row[key]])); + +async function properties(context: JourneyContext, label: string) { + const { page } = context; + const row = page.locator('[data-property-row]:not([inert] *):not([aria-hidden="true"] *)') + .filter({ has: page.locator(`[data-property-label="${label}"]`) }).filter({ visible: true }); + const toggle = page.getByRole("button", { name: "Toggle side panel", exact: true, pressed: false }).filter({ visible: true }); + const propertiesButton = page.getByRole("button", { name: /^(Show properties|Properties)$/ }).filter({ visible: true }); + await expect.poll(async () => await row.isVisible() || await toggle.isVisible() || await propertiesButton.isVisible()).toBe(true); + if (!await row.isVisible()) { + if (await toggle.isVisible()) await toggle.click(); + else await propertiesButton.click(); + } + await expect(row).toBeVisible(); + await row.scrollIntoViewIfNeeded(); + await expect(row).toBeInViewport(); + return row; +} +async function setStatus(context: JourneyContext, label: string) { + const { page } = context; + await page.getByTestId("issue-detail-header").filter({ visible: true }) + .getByRole("button", { name: /^Change status \(current:/ }).click(); + await page.locator('[data-slot="popover-content"]').filter({ visible: true }) + .getByRole("button").filter({ has: page.getByRole("img", { name: label, exact: true }) }).click(); +} + +export async function taskDependencyResume(context: JourneyContext, session: TargetSession, campaignId: string, evidence: JourneyEvidence, + options: { deadlineAt: number; directory: string }) { + const { api, page, company, user, agent } = context; + const read = createObservedReader(api, options.deadlineAt); + const reference = randomBytes(16).toString("hex"); + let scope: DependencyScope | undefined, stage = "prerequisites", journeyFailed = false; + let latest: DependencyPoint | undefined; + const observations: Partial = { waiting: [] }; + const preserve = async () => { + await mkdir(options.directory, { recursive: true, mode: 0o700 }); + await writeFile(path.join(options.directory, "task-dependency-observations.json"), + JSON.stringify({ schema: "paperclip.task-dependency-observations/v1", stage, ...observations, latest, scope }, null, 2), { mode: 0o600 }); + }; + const url = (identifier: string) => `${api.origin}/${company.issuePrefix}/issues/${identifier}`; + try { + for (const capability of ["company", "board-user", "task-write", "agent"]) { + if (!context.capabilities.includes(capability)) throw new JourneyError("unsupported", `missing_${capability}`); + } + if (!agent || agent.runtimeMode !== "native" || !context.environmentId) throw new JourneyError("unsupported", "native_agent_and_environment_required"); + if ([agent.provider, agent.model].some(value => typeof value !== "string" || !value.trim())) { + throw new JourneyError("unsupported", "dependency_execution_identity_unavailable"); + } + const summaries = async (route: string, limit?: number) => { + const rows = await read.get(route); + if (!Array.isArray(rows) || limit !== undefined && rows.length >= limit) throw new JourneyError("failed", "dependency_run_coverage_unavailable"); + const ids = rows.map(row => row.runId ?? row.id); + if (ids.some(value => typeof value !== "string" || !value) || new Set(ids).size !== ids.length) throw new JourneyError("failed", "dependency_run_identity_invalid"); + return rows.map((row, index) => ({ ...row, id: ids[index] })); + }; + const runDetail = async (runId: string) => { + const run = await read.get(`/api/heartbeat-runs/${runId}`); + if (run.id !== runId) throw new JourneyError("failed", "dependency_run_identity_invalid"); + return runSnapshot(run); + }; + const snapshots = async (route: string) => Promise.all((await summaries(route)).map(row => runDetail(row.id))); + const agentRoute = `/api/companies/${company.id}/heartbeat-runs?agentId=${agent.id}&limit=${HEARTBEAT_RUN_LIST_MAX_LIMIT}`; + stage = "observe_starting_runs"; + const startingRuns = observations.startingAgentRuns = (await summaries(agentRoute, HEARTBEAT_RUN_LIST_MAX_LIMIT)).map(runSnapshot); + if (startingRuns.some(run => run.companyId !== company.id || run.agentId !== agent.id + || !["succeeded", "failed", "cancelled", "timed_out"].includes(run.status))) { + throw new JourneyError("failed", "dependency_starting_agent_not_idle"); + } + const startingIds = new Set(startingRuns.map(run => run.id)); + const agentSnapshots = async () => Promise.all((await summaries(agentRoute, HEARTBEAT_RUN_LIST_MAX_LIMIT)) + .map(row => startingIds.has(row.id) ? runSnapshot(row) : runDetail(row.id))); + stage = "create_blocker"; + const blockerTitle = `Journey QA dependency blocker ${campaignId}`; + const blockerDescription = `Human review gate. The review reference is ${reference}. Keep this task for my review; no agent work is required.`; + const blocker = await createTaskThroughUi({ page, origin: api.origin, issuePrefix: company.issuePrefix, + agentName: "Me", title: blockerTitle, prompt: blockerDescription, workMode: "standard", requireExplicitTitle: true, + beforeDispatch: () => session.own({ kind: "issue", name: blockerTitle, journeyId: id }) }); + await session.own({ kind: "issue", name: blockerTitle, id: blocker.issueId, journeyId: id }); + const savedBlocker = await read.get(`/api/issues/${blocker.issueId}`); + if (typeof savedBlocker.identifier !== "string" || !savedBlocker.identifier) throw new JourneyError("failed", "dependency_blocker_receipt_invalid"); + const dependentTitle = `Journey QA dependency dependent ${campaignId}`, documentKey = "dependency-proof"; + const prompt = `After the blocker ${savedBlocker.identifier} is completed, read its description and save a task document with key ${documentKey}. ` + + `Its body must be JSON with exactly reference (the review reference in the blocker description) and blockerIdentifier ("${savedBlocker.identifier}"). ` + + "Then finish this task. No other tasks, agents or projects are needed."; + stage = "create_dependent"; + const dependent = await createTaskThroughUi({ page, origin: api.origin, issuePrefix: company.issuePrefix, + agentName: "Me", title: dependentTitle, prompt, workMode: "standard", requireExplicitTitle: true, + beforeDispatch: () => session.own({ kind: "issue", name: dependentTitle, journeyId: id }) }); + await session.own({ kind: "issue", name: dependentTitle, id: dependent.issueId, journeyId: id }); + scope = { companyId: company.id, userId: user.id, agentId: agent.id, environmentId: context.environmentId, + provider: agent.provider, model: agent.model, blockerId: blocker.issueId, blockerIdentifier: savedBlocker.identifier, + blockerTitle, blockerDescription, dependentId: dependent.issueId, dependentTitle, prompt, reference, documentKey }; + const observe = async (): Promise => { + const [currentBlocker, currentDependent, blockerRuns, dependentRuns, agentRuns, blockerActivity, dependentActivity, agentActivity] = await Promise.all([ + read.get(`/api/issues/${blocker.issueId}`), read.get(`/api/issues/${dependent.issueId}`), + snapshots(`/api/issues/${blocker.issueId}/runs`), snapshots(`/api/issues/${dependent.issueId}/runs`), + agentSnapshots(), read.get(`/api/issues/${blocker.issueId}/activity`), read.get(`/api/issues/${dependent.issueId}/activity`), + read.get(`/api/companies/${company.id}/activity?agentId=${agent.id}&limit=500`), + ]); + if ([blockerActivity, dependentActivity, agentActivity].some(rows => !Array.isArray(rows)) || agentActivity.length >= 500) { + throw new JourneyError("failed", "dependency_activity_coverage_unavailable"); + } + const newAgentRuns = dependencyNewAgentRuns(agentRuns, startingRuns); + const activity = [...new Map([...blockerActivity, ...dependentActivity, ...agentActivity].map(row => [row.id, row])).values()]; + if (activity.some(row => typeof row.id !== "string" || !row.id || row.companyId !== company.id) + || agentActivity.some(row => row.agentId !== agent.id)) throw new JourneyError("failed", "dependency_activity_identity_invalid"); + const point: DependencyPoint = { observedAt: Date.now(), blocker: taskSnapshot(currentBlocker), dependent: taskSnapshot(currentDependent), + blockerRuns, dependentRuns, agentRuns, activityCoverage: { blocker: true, dependent: true, agent: true }, + activity: activity.filter(row => [blocker.issueId, dependent.issueId].includes(row.entityId) + || newAgentRuns.some(run => run.id === row.runId)).map(row => ({ id: row.id, companyId: row.companyId, actorType: row.actorType, + actorId: row.actorId, runId: row.runId, action: row.action, entityType: row.entityType, entityId: row.entityId, + createdAt: row.createdAt, details: { status: row.details?.status } })) }; + latest = point; + if (dependentRuns.length > 1 || newAgentRuns.length > 1 || blockerRuns.length + || [...dependentRuns, ...newAgentRuns].some(run => ["failed", "cancelled", "timed_out"].includes(run.status))) { + throw new JourneyError("failed", "dependency_unexpected_or_failed_work"); + } + if (currentDependent.status === "done") { + const documents = await read.get(`/api/issues/${dependent.issueId}/documents`); + const candidate = documents.find(row => row.key === documentKey); + if (candidate) { + const document = await read.get(`/api/issues/${dependent.issueId}/documents/${documentKey}`); + point.document = documentSnapshot(document); + point.revision = documentSnapshot(await read.get(`/api/issues/${dependent.issueId}/documents/${documentKey}/revisions/${document.latestRevisionId}`)); + } + } + return point; + }; + observations.created = await observe(); + if ([observations.created.blocker, observations.created.dependent].some(task => task.companyId !== company.id + || task.assigneeUserId !== user.id || task.assigneeAgentId !== null || task.createdByUserId !== user.id + || task.createdByAgentId || task.status !== "todo") || observations.created.blocker.id !== blocker.issueId + || observations.created.dependent.id !== dependent.issueId || observations.created.blockerRuns.length + || observations.created.dependentRuns.length || dependencyNewAgentRuns(observations.created.agentRuns, startingRuns).length) { + throw new JourneyError("failed", "dependency_created_task_binding_mismatch"); + } + stage = "link_blocker"; + const dependentSaved = observations.created.dependent; + await page.goto(url(dependentSaved.identifier ?? dependent.issueId), { waitUntil: "domcontentloaded" }); + const blockedBy = await properties(context, "Blocked by"); + await blockedBy.locator("[data-property-value]").getByRole("button").first().click(); + await page.getByRole("textbox", { name: "Search tasks to add as blockers", exact: true }).fill(blockerTitle); + await page.getByRole("button").filter({ hasText: blockerTitle }).last().click(); + await expect.poll(async () => (taskSnapshot(await read.get(`/api/issues/${dependent.issueId}`)).blockedByIssueIds)).toEqual([blocker.issueId]); + stage = "set_dependent_blocked"; + await setStatus(context, "Blocked"); + await expect.poll(async () => (await read.get(`/api/issues/${dependent.issueId}`)).status).toBe("blocked"); + stage = "assign_blocked_task"; + const assignee = await properties(context, "Assignee"); + await assignee.locator("[data-property-value]").getByRole("button").first().click(); + await page.getByPlaceholder("Search assignees...", { exact: true }).fill(agent.name); + await page.getByRole("button").filter({ hasText: agent.name }).last().click(); + await expect.poll(async () => (await read.get(`/api/issues/${dependent.issueId}`)).assigneeAgentId).toBe(agent.id); + await page.reload(); + stage = "observe_blocked_wait"; + const waiting = observations.waiting!; + waiting.push(await observe()); + if (waiting[0]!.dependentRuns.length || dependencyNewAgentRuns(waiting[0]!.agentRuns, startingRuns).length || waiting[0]!.dependent.status !== "blocked" + || waiting[0]!.blocker.status !== "todo") throw new JourneyError("failed", "dependency_ran_before_blocker_completed"); + const until = waiting[0]!.observedAt + DEPENDENCY_IDLE_MS; + while (Date.now() < until) { + if (Date.now() + 1000 >= options.deadlineAt) throw new JourneyError("failed", "dependency_wait_deadline"); + await new Promise(resolve => setTimeout(resolve, Math.min(1000, until - Date.now()))); + const point = await observe(); waiting.push(point); + if (point.dependentRuns.length || dependencyNewAgentRuns(point.agentRuns, startingRuns).length || point.dependent.status !== "blocked" || point.blocker.status !== "todo") { + throw new JourneyError("failed", "dependency_ran_before_blocker_completed"); + } + } + if (waiting[waiting.length - 1]!.observedAt < until) waiting.push(await observe()); + stage = "complete_blocker"; + await page.goto(url(savedBlocker.identifier), { waitUntil: "domcontentloaded" }); + observations.releaseRequestedAt = Date.now(); + await setStatus(context, "Done"); + await expect.poll(async () => (await read.get(`/api/issues/${blocker.issueId}`)).status).toBe("done"); + observations.released = await observe(); + stage = "await_automatic_resume"; + while (Date.now() < options.deadlineAt) { + const point = await observe(); observations.finished = point; + if (point.dependent.status === "done" && point.dependentRuns.length === 1 && point.dependentRuns[0]?.status === "succeeded" + && point.dependentRuns[0]?.nativePhase === "committed" && point.document && point.revision + && !point.dependent.executionRunId && !point.dependent.checkoutRunId) break; + await new Promise(resolve => setTimeout(resolve, 1000)); + } + if (!observations.finished || observations.finished.dependent.status !== "done" || observations.finished.dependentRuns[0]?.status !== "succeeded" + || observations.finished.dependentRuns[0]?.nativePhase !== "committed" || !observations.finished.document || !observations.finished.revision) { + throw new JourneyError("failed", "dependency_resume_deadline"); + } + stage = "reopen_result"; + await page.goto(url(dependentSaved.identifier ?? dependent.issueId), { waitUntil: "domcontentloaded" }); + await page.reload(); await openDocumentAuthoring(context); + const document = page.locator(`#document-${documentKey}`); + await expect(document).toContainText(reference); + observations.reopened = await observe(); + observations.reopened.resultVisible = await document.isVisible() && (await document.textContent())?.includes(reference) === true; + const checks = gradeTaskDependencyResume({ ...observations, scope } as DependencyEvidence); + evidence.checks.push(...checks); + const failed = checks.find(check => !check.passed); + if (failed) throw new JourneyError("failed", failed.id.replaceAll("-", "_")); + stage = "complete"; + } catch (error) { + journeyFailed = true; + throw error; + } finally { + const writers = [ + { label: "task-dependency-observations.json", write: preserve }, + { label: "observation-read-failures.json", write: () => read.preserve(options.directory) }, + ]; + const results = await Promise.allSettled(writers.map(writer => Promise.resolve().then(writer.write))); + const failedWriters = results.flatMap((result, index) => result.status === "rejected" ? [writers[index]!.label] : []); + if (failedWriters.length) { + console.error("dependency_evidence_preservation_failed", failedWriters); + if (!journeyFailed) throw new JourneyError("failed", "dependency_evidence_preservation_failed"); + } + } +} diff --git a/tests/runner-e2e/product-journeys/types.ts b/tests/runner-e2e/product-journeys/types.ts index cc92860f48b..2a776353237 100644 --- a/tests/runner-e2e/product-journeys/types.ts +++ b/tests/runner-e2e/product-journeys/types.ts @@ -7,6 +7,7 @@ export type JourneyId = | "task-properties" | "task-create" | "project-task-context" + | "task-dependency-resume" | "task-document-persistence" | "ask-question" | "chat-continuity"