mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-27 19:35:13 +01:00
fix(runner): persist warm Daytona workspaces
This commit is contained in:
1 parent
3da58b185e
commit
ee11cd65d9
44 files changed
+4019
-359
No files matched your search
@@ -58,9 +58,13 @@ registry therefore discovers that row through the public environments API.
|
||||
This still provides full isolation because every cell starts a new Paperclip
|
||||
instance and database.
|
||||
|
||||
Daytona creates a sandbox environment through the public API. Keep
|
||||
`reuseLease:false`, `runnerLifecycleMode:"per_turn"`, short provider cleanup
|
||||
backstops, a Daytona secret reference, and an immutable image digest. Teardown
|
||||
Daytona creates sandbox environments through the public API. The core fixture
|
||||
keeps `reuseLease:false` and `runnerLifecycleMode:"per_turn"`. The dedicated
|
||||
warm-continuity fixture uses `reuseLease:true` and
|
||||
`runnerLifecycleMode:"warm"`; its distinct `configurationKey` is part of the
|
||||
suite fingerprint even though both fixtures report `environmentId:"daytona"`.
|
||||
Keep short provider cleanup backstops, a Daytona secret reference, and an
|
||||
immutable image digest. Teardown
|
||||
must delete the environment with reusable-lease destruction and must fail the
|
||||
cell if cleanup cannot be confirmed. Keep CPU, memory, and disk explicit: lease
|
||||
metadata and the per-test public-list-price runtime estimate depend on that
|
||||
@@ -93,7 +97,10 @@ marker factories. `question_resume_completion` must define the deterministic
|
||||
browser answer and prove exactly two successful runs with no pending
|
||||
interaction. `plan_approval_completion` must target the exact two-step
|
||||
canonical Plan revision, capture its pending UI, approve in the browser, and
|
||||
prove exactly two successful runs.
|
||||
prove exactly two successful runs. `warm_three_turn` provides exactly two
|
||||
browser follow-up messages, preserves one project/execution-workspace scope,
|
||||
verifies host file contents after every turn, and finishes within three
|
||||
ten-minute turn deadlines.
|
||||
|
||||
Every selected case runs in its own isolated Paperclip process, and independent
|
||||
cases may run concurrently. Follow-up turns inside one case retain their shared
|
||||
|
||||
@@ -68,10 +68,11 @@ pnpm test:e2e:runner -- --group native --environment local
|
||||
pnpm test:e2e:runner -- --profile runner-codex --case message-marker
|
||||
pnpm test:e2e:runner -- --case plan-revise-accept --group local
|
||||
pnpm test:e2e:runner -- --case ask-question --group native
|
||||
pnpm test:e2e:runner -- --suite daytona-warm-continuity
|
||||
pnpm test:e2e:runner -- --all
|
||||
```
|
||||
|
||||
The catalog contains three suites. `core-compatibility` (**Core Runner
|
||||
The catalog contains four suites. `core-compatibility` (**Core Runner
|
||||
Compatibility**) is seven major runner profiles × local/Daytona × three
|
||||
workflows: 42 cells. Its cases are:
|
||||
|
||||
@@ -107,7 +108,19 @@ duplicating the final response. The second workflow restarts the isolated
|
||||
Paperclip server while the interaction is waiting, reloads that state, and
|
||||
then resumes it. The suite has no Daytona cells.
|
||||
|
||||
The complete catalog is 66 cells (45 local and 21 Daytona) and 114 expected
|
||||
`daytona-warm-continuity` (**Daytona Warm Continuity**) is exactly two paid
|
||||
cells: legacy Codex and Runner Codex against one reusable warm Daytona
|
||||
configuration. Each cell creates a real project with a primary local-path
|
||||
workspace through the API, selects it in the browser task dialog, and performs
|
||||
three browser-driven turns on one issue. Every turn reads and extends the same
|
||||
nonce file, verifies host copy-back, records scheduler/run/end-to-end timing,
|
||||
and asserts `created`, `resumed`, `resumed` lease acquisition on one sandbox.
|
||||
Runner Codex additionally proves stable native session, provider session,
|
||||
runner instance, PID, and process-start identity. Each turn is bounded to ten
|
||||
minutes, the cell to thirty minutes, and cleanup explicitly deletes the
|
||||
sandbox rather than waiting for Daytona's idle timeout.
|
||||
|
||||
The complete catalog is 68 cells (45 local and 23 Daytona) and 120 expected
|
||||
paid agent turns. Follow-up steps remain ordered within their cell; all other
|
||||
cells are independent. Narrow selectors are strongly recommended while
|
||||
developing fixtures.
|
||||
@@ -351,7 +364,7 @@ Set `RUNNER_E2E_AWS_ENABLED=true` to route paid cells to the repository-scoped
|
||||
ephemeral AWS RunsOn fleet selected by
|
||||
`runs-on/fleet=paperclip-public-pr-x64/env=public-ci`. Any other value uses the
|
||||
proven GitHub-hosted `ubuntu-latest` target. Set `RUNNER_E2E_MAX_PARALLEL` to an
|
||||
integer from 1–100 on AWS (default 100); use at least 66 to run the current
|
||||
integer from 1–100 on AWS (default 100); use at least 68 to run the current
|
||||
complete catalog in one wave. The fallback runner retains its 1–57 limit and
|
||||
default of 32. Multi-turn steps are sequential inside their cell while
|
||||
independent cells overlap. Artifacts and merged HTML/JUnit/normalized reports
|
||||
|
||||
@@ -10,7 +10,10 @@ import {
|
||||
runnerProfiles,
|
||||
runnerSuites,
|
||||
runnerTasks,
|
||||
daytonaWarmContinuityTask,
|
||||
daytonaWarmEnvironment,
|
||||
isImmutableDaytonaImage,
|
||||
suiteDefinitionHash,
|
||||
validateRunnerCatalog,
|
||||
} from "./catalog.js";
|
||||
import {
|
||||
@@ -21,7 +24,7 @@ import {
|
||||
} from "./selectors.js";
|
||||
|
||||
describe("runner E2E catalog", () => {
|
||||
it("validates the core, local-integrity, and breadth suites", () => {
|
||||
it("validates the core, local-integrity, breadth, and warm suites", () => {
|
||||
expect(runnerProfiles).toHaveLength(7);
|
||||
expect(openRouterBreadthProfiles).toHaveLength(4);
|
||||
expect(runnerEnvironments).toHaveLength(2);
|
||||
@@ -29,10 +32,10 @@ describe("runner E2E catalog", () => {
|
||||
expect(localIntegrityTasks).toHaveLength(2);
|
||||
expect(openRouterBreadthTasks).toHaveLength(3);
|
||||
expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([
|
||||
42, 14, 10,
|
||||
42, 14, 10, 2,
|
||||
]);
|
||||
expect(validateRunnerCatalog()).toHaveLength(66);
|
||||
expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(66);
|
||||
expect(validateRunnerCatalog()).toHaveLength(68);
|
||||
expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(68);
|
||||
expect(
|
||||
runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"),
|
||||
).toHaveLength(42);
|
||||
@@ -46,18 +49,81 @@ describe("runner E2E catalog", () => {
|
||||
(entry) => entry.suite.id === "openrouter-model-breadth",
|
||||
),
|
||||
).toHaveLength(10);
|
||||
expect(
|
||||
runnerMatrix.filter(
|
||||
(entry) => entry.suite.id === "daytona-warm-continuity",
|
||||
),
|
||||
).toHaveLength(2);
|
||||
expect(
|
||||
runnerMatrix.reduce(
|
||||
(total, execution) => total + execution.task.expectedRunCount,
|
||||
0,
|
||||
),
|
||||
).toBe(114);
|
||||
).toBe(120);
|
||||
expect(
|
||||
runnerTasks.find((task) => task.id === "plan-revise-accept")
|
||||
?.attemptTimeoutMs,
|
||||
).toEqual({ local: 8 * 60_000, daytona: 12 * 60_000 });
|
||||
});
|
||||
|
||||
it("defines the warm Daytona continuity fixture as exactly two Codex cells", () => {
|
||||
expect(daytonaWarmEnvironment).toMatchObject({
|
||||
id: "daytona",
|
||||
configurationKey: "warm-reuse-v1",
|
||||
groups: ["daytona", "warm"],
|
||||
});
|
||||
expect(
|
||||
daytonaWarmEnvironment.buildEnvironment({
|
||||
secretRefs: {
|
||||
DAYTONA_API_KEY: {
|
||||
type: "secret_ref",
|
||||
secretId: "22222222-2222-4222-8222-222222222222",
|
||||
version: "latest",
|
||||
},
|
||||
},
|
||||
daytonaImage: `runner@sha256:${"a".repeat(64)}`,
|
||||
executionId: "warm",
|
||||
}),
|
||||
).toMatchObject({
|
||||
config: {
|
||||
reuseLease: true,
|
||||
runnerLifecycleMode: "warm",
|
||||
autoStopInterval: 5,
|
||||
autoArchiveInterval: 15,
|
||||
autoDeleteInterval: 60,
|
||||
},
|
||||
});
|
||||
expect(daytonaWarmContinuityTask).toMatchObject({
|
||||
flow: "warm_three_turn",
|
||||
expectedRunCount: 3,
|
||||
turnTimeoutMs: 600_000,
|
||||
});
|
||||
expect(
|
||||
daytonaWarmContinuityTask.buildFollowupMessages?.("nonce"),
|
||||
).toHaveLength(2);
|
||||
const cells = runnerMatrix.filter(
|
||||
(entry) => entry.suite.id === "daytona-warm-continuity",
|
||||
);
|
||||
expect(cells.map((entry) => entry.profile.id)).toEqual([
|
||||
"legacy-codex",
|
||||
"runner-codex",
|
||||
]);
|
||||
expect(cells.every((entry) => entry.environment.id === "daytona")).toBe(
|
||||
true,
|
||||
);
|
||||
const suite = runnerSuites.find(
|
||||
(candidate) => candidate.id === "daytona-warm-continuity",
|
||||
)!;
|
||||
expect(
|
||||
suiteDefinitionHash({
|
||||
...suite,
|
||||
environments: [
|
||||
{ ...daytonaWarmEnvironment, configurationKey: "changed" },
|
||||
],
|
||||
}),
|
||||
).not.toBe(suiteDefinitionHash(suite));
|
||||
});
|
||||
|
||||
it("derives the qualified local native OpenCode profiles from the ranked snapshot", () => {
|
||||
expect(openRouterBreadthExcludedModelIds).toEqual(["xiaomi/mimo-v2.5"]);
|
||||
expect(openRouterBreadthExcludedExecutionIds).toEqual([
|
||||
@@ -181,9 +247,10 @@ describe("runner E2E catalog", () => {
|
||||
question?.buildPrompt("nonce"),
|
||||
...breadthTasks,
|
||||
]) {
|
||||
expect(prompt).toContain("then emit exactly");
|
||||
const terminalTextInstruction = prompt?.match(/then emit (?:exactly|only)/)?.[0];
|
||||
expect(terminalTextInstruction).toBeDefined();
|
||||
expect(prompt!.indexOf("paperclip_finish exactly once")).toBeLessThan(
|
||||
prompt!.indexOf("then emit exactly"),
|
||||
prompt!.indexOf(terminalTextInstruction!),
|
||||
);
|
||||
expect(prompt).toContain("Wait for that tool call to succeed");
|
||||
}
|
||||
@@ -336,7 +403,7 @@ describe("runner E2E catalog", () => {
|
||||
"those two tool calls form one indivisible response sequence",
|
||||
);
|
||||
expect(task!.buildPrompt("nonce")).toContain(
|
||||
"Do not emit assistant text, end the heartbeat, or stop after write_document alone",
|
||||
"Do not emit assistant text, end the response or heartbeat, or stop after write_document alone",
|
||||
);
|
||||
expect(task!.buildPrompt("nonce")).toContain(
|
||||
"one atomic issue PATCH with status `done` and that exact comment",
|
||||
@@ -433,7 +500,7 @@ describe("runner E2E selectors", () => {
|
||||
"daytona",
|
||||
]);
|
||||
const selected = selectRunnerExecutions(options);
|
||||
expect(selected).toHaveLength(12);
|
||||
expect(selected).toHaveLength(13);
|
||||
expect(
|
||||
selected.every(
|
||||
(entry) =>
|
||||
@@ -443,7 +510,7 @@ describe("runner E2E selectors", () => {
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects groups outside the advertised four", () => {
|
||||
it("rejects unknown groups", () => {
|
||||
const options = parseRunnerSelectors(["--group", "codex"]);
|
||||
expect(() => selectRunnerExecutions(options)).toThrow("Unknown group");
|
||||
});
|
||||
@@ -452,10 +519,10 @@ describe("runner E2E selectors", () => {
|
||||
const jobs = buildMatrixJobs(
|
||||
selectRunnerExecutions(parseRunnerSelectors(["--all"])),
|
||||
);
|
||||
expect(jobs).toHaveLength(66);
|
||||
expect(jobs.filter((job) => job.needsDaytona)).toHaveLength(21);
|
||||
expect(jobs).toHaveLength(68);
|
||||
expect(jobs.filter((job) => job.needsDaytona)).toHaveLength(23);
|
||||
expect(jobs.filter((job) => !job.needsDaytona)).toHaveLength(45);
|
||||
expect(new Set(jobs.map((job) => job.executionId)).size).toBe(66);
|
||||
expect(new Set(jobs.map((job) => job.executionId)).size).toBe(68);
|
||||
expect(
|
||||
jobs.find(
|
||||
(job) =>
|
||||
|
||||
+139
-4
@@ -27,6 +27,7 @@ const SELECTABLE_GROUPS = [
|
||||
"native",
|
||||
"local",
|
||||
"daytona",
|
||||
"warm",
|
||||
"core",
|
||||
"breadth",
|
||||
] as const;
|
||||
@@ -374,6 +375,50 @@ export const runnerEnvironments: readonly EnvironmentFixture[] = [
|
||||
},
|
||||
] as const;
|
||||
|
||||
export const daytonaWarmEnvironment: EnvironmentFixture = {
|
||||
id: "daytona",
|
||||
configurationKey: "warm-reuse-v1",
|
||||
label: "Daytona warm reusable sandbox",
|
||||
groups: ["daytona", "warm"],
|
||||
driver: "sandbox",
|
||||
provider: "daytona",
|
||||
credential: "DAYTONA_API_KEY",
|
||||
lifecycle: {
|
||||
setup: "create_via_api",
|
||||
probe: "run_context_via_api",
|
||||
cleanup: "delete_via_api_and_destroy_leases",
|
||||
},
|
||||
expectedExecutionTarget: { kind: "remote", transport: "sandbox" },
|
||||
buildEnvironment(input) {
|
||||
if (!isImmutableDaytonaImage(input.daytonaImage)) {
|
||||
throw new Error(
|
||||
"PAPERCLIP_E2E_DAYTONA_IMAGE must be an immutable image digest",
|
||||
);
|
||||
}
|
||||
return {
|
||||
name: `Runner E2E Daytona warm ${input.executionId}`,
|
||||
description: "Ephemeral reusable Daytona runner E2E environment",
|
||||
driver: "sandbox",
|
||||
config: {
|
||||
provider: "daytona",
|
||||
apiKey: requiredDaytonaSecret(input),
|
||||
image: input.daytonaImage,
|
||||
cpu: 4,
|
||||
memory: 4,
|
||||
disk: 10,
|
||||
reuseLease: true,
|
||||
runnerLifecycleMode: "warm",
|
||||
autoStopInterval: 5,
|
||||
autoArchiveInterval: 15,
|
||||
autoDeleteInterval: 60,
|
||||
timeoutMs: 300_000,
|
||||
livenessTimeoutMs: 30_000,
|
||||
},
|
||||
envVars: {},
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
export const runnerTasks: readonly RunnerTaskFixture[] = [
|
||||
{
|
||||
id: "message-marker",
|
||||
@@ -719,6 +764,80 @@ const localEnvironment = runnerEnvironments.find(
|
||||
(environment) => environment.id === "local",
|
||||
)!;
|
||||
|
||||
function warmTurnMarker(turn: 1 | 2 | 3, nonce: string) {
|
||||
return `PAPERCLIP_E2E_WARM_T${turn}_${nonce}`;
|
||||
}
|
||||
|
||||
function warmWorkspaceLine(turn: 1 | 2 | 3, nonce: string) {
|
||||
return `T${turn}_${nonce}`;
|
||||
}
|
||||
|
||||
function warmTurnInstructions(turn: 1 | 2 | 3, nonce: string) {
|
||||
const file = `daytona-warm-${nonce}.txt`;
|
||||
const lines = Array.from({ length: turn }, (_, index) =>
|
||||
warmWorkspaceLine((index + 1) as 1 | 2 | 3, nonce),
|
||||
);
|
||||
const marker = warmTurnMarker(turn, nonce);
|
||||
const finalTurn = turn === 3;
|
||||
return [
|
||||
`This is warm Daytona continuity turn ${turn} of 3. Work only in the current execution workspace.`,
|
||||
turn === 1
|
||||
? `Create ${file} with exactly this one line followed by a newline: ${lines[0]}`
|
||||
: `Before changing anything, read ${file} and verify its content is exactly ${lines.slice(0, -1).join("\\n")} followed by a newline. Then append exactly ${lines.at(-1)} followed by a newline.`,
|
||||
`After the write, verify ${file} contains exactly these lines, once each and in order: ${lines.join(" | ")}.`,
|
||||
`In a native runner, call paperclip_finish exactly once with {reportedWorkDisposition:"${finalTurn ? "done" : "needs_review"}",summary:"${marker}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[{commandOrCheck:"read ${file}",status:"passed"}]}. Wait for that tool call to succeed, then emit exactly ${marker} once as the complete user-facing final response.`,
|
||||
`In a legacy runner, make exactly one public-API completion write after verification: PATCH /api/issues/$PAPERCLIP_TASK_ID with {"status":"${finalTurn ? "done" : "in_review"}","comment":"${marker}"}. Include Authorization and X-Paperclip-Run-Id. Do not POST a separate comment.`,
|
||||
`Do not include ${marker} in any other visible response or write. Do not recreate, truncate, reorder, or duplicate prior lines.`,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export const daytonaWarmContinuityTask: RunnerTaskFixture = {
|
||||
id: "warm-three-turn",
|
||||
label: "Warm three-turn workspace continuity",
|
||||
groups: ["warm"],
|
||||
workMode: "standard",
|
||||
flow: "warm_three_turn",
|
||||
expectedRunCount: 3,
|
||||
attemptTimeoutMs: { local: 30 * 60_000, daytona: 30 * 60_000 },
|
||||
turnTimeoutMs: 10 * 60_000,
|
||||
expectedTerminalState: { issue: "done", run: "succeeded" },
|
||||
buildTitle: (nonce) => `Runner E2E warm Daytona continuity ${nonce}`,
|
||||
buildVisibleMarker: (nonce) => warmTurnMarker(3, nonce),
|
||||
buildPrompt: (nonce) => warmTurnInstructions(1, nonce),
|
||||
buildFollowupMessages: (nonce) => [
|
||||
warmTurnInstructions(2, nonce),
|
||||
warmTurnInstructions(3, nonce),
|
||||
],
|
||||
buildMatchers(nonce, execution) {
|
||||
const markers = ([1, 2, 3] as const).map((turn) =>
|
||||
warmTurnMarker(turn, nonce),
|
||||
);
|
||||
return [
|
||||
{ kind: "message_exact", expected: markers[2] },
|
||||
...markers.map(
|
||||
(expected) =>
|
||||
({ kind: "message_occurrences", expected, count: 1 }) as const,
|
||||
),
|
||||
{ kind: "message_ordered", expected: markers },
|
||||
{
|
||||
kind: "file_exact",
|
||||
path: `daytona-warm-${nonce}.txt`,
|
||||
expected: `${([1, 2, 3] as const)
|
||||
.map((turn) => warmWorkspaceLine(turn, nonce))
|
||||
.join("\n")}\n`,
|
||||
},
|
||||
{ kind: "issue_status", expected: "done" },
|
||||
{ kind: "run_status", expected: "succeeded" },
|
||||
{ kind: "runtime_mode", expected: execution.profile.expectedRuntimeMode },
|
||||
{ kind: "environment", expected: "daytona" },
|
||||
];
|
||||
},
|
||||
};
|
||||
|
||||
const codexContinuityProfiles = runnerProfiles.filter((profile) =>
|
||||
["legacy-codex", "runner-codex"].includes(profile.id),
|
||||
);
|
||||
|
||||
export const runnerSuites: readonly RunnerSuiteFixture[] = [
|
||||
{
|
||||
id: "core-compatibility",
|
||||
@@ -762,6 +881,17 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
|
||||
excludedExecutionIds: openRouterBreadthExcludedExecutionIds,
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "daytona-warm-continuity",
|
||||
label: "Daytona Warm Continuity",
|
||||
description:
|
||||
"Three browser-driven turns on one reusable Daytona sandbox for legacy and native Codex.",
|
||||
groups: ["daytona", "warm"],
|
||||
profiles: codexContinuityProfiles,
|
||||
environments: [daytonaWarmEnvironment],
|
||||
tasks: [daytonaWarmContinuityTask],
|
||||
expectedMatrixSize: 2,
|
||||
},
|
||||
] as const;
|
||||
|
||||
export function suiteDefinitionHash(suite: RunnerSuiteFixture) {
|
||||
@@ -774,7 +904,10 @@ export function suiteDefinitionHash(suite: RunnerSuiteFixture) {
|
||||
model: profile.model,
|
||||
qualification: profile.modelQualification,
|
||||
})),
|
||||
environments: suite.environments.map((environment) => environment.id),
|
||||
environments: suite.environments.map((environment) => ({
|
||||
id: environment.id,
|
||||
configurationKey: environment.configurationKey ?? "default",
|
||||
})),
|
||||
tasks: suite.tasks.map((task) => ({
|
||||
id: task.id,
|
||||
flow: task.flow,
|
||||
@@ -869,6 +1002,7 @@ export function validateRunnerCatalog(): MatrixExecution[] {
|
||||
...runnerTasks,
|
||||
...localIntegrityTasks,
|
||||
...openRouterBreadthTasks,
|
||||
daytonaWarmContinuityTask,
|
||||
];
|
||||
for (const [label, values] of [
|
||||
["suite", runnerSuites],
|
||||
@@ -888,6 +1022,7 @@ export function validateRunnerCatalog(): MatrixExecution[] {
|
||||
...runnerSuites,
|
||||
...allProfiles,
|
||||
...runnerEnvironments,
|
||||
daytonaWarmEnvironment,
|
||||
...allTasks,
|
||||
]) {
|
||||
const unknownGroups = fixture.groups.filter(
|
||||
@@ -916,7 +1051,7 @@ export function validateRunnerCatalog(): MatrixExecution[] {
|
||||
]),
|
||||
);
|
||||
|
||||
for (const environment of runnerEnvironments) {
|
||||
for (const environment of [...runnerEnvironments, daytonaWarmEnvironment]) {
|
||||
const payload = environment.buildEnvironment({
|
||||
secretRefs: sampleRefs,
|
||||
daytonaImage:
|
||||
@@ -968,8 +1103,8 @@ export function validateRunnerCatalog(): MatrixExecution[] {
|
||||
);
|
||||
}
|
||||
}
|
||||
if (matrix.length !== 66)
|
||||
throw new Error(`Expected 66 runner executions; received ${matrix.length}`);
|
||||
if (matrix.length !== 68)
|
||||
throw new Error(`Expected 68 runner executions; received ${matrix.length}`);
|
||||
return matrix;
|
||||
}
|
||||
|
||||
|
||||
@@ -196,6 +196,18 @@ function renderCase(
|
||||
</tr>`,
|
||||
)
|
||||
.join("");
|
||||
const turnTimingRows = (entry?.result.turnTimings ?? [])
|
||||
.map(
|
||||
(timing) => `<tr>
|
||||
<td>${timing.turn}</td>
|
||||
<td><code>${html(timing.runId)}</code></td>
|
||||
<td>${html(timing.leaseAcquisitionOutcome)}</td>
|
||||
<td>${html(timing.schedulerLatencyMs === null ? "unavailable" : durationLabel(timing.schedulerLatencyMs))}</td>
|
||||
<td>${html(timing.runDurationMs === null ? "unavailable" : durationLabel(timing.runDurationMs))}</td>
|
||||
<td>${html(timing.responseLatencyMs === null ? "unavailable" : durationLabel(timing.responseLatencyMs))}</td>
|
||||
</tr>`,
|
||||
)
|
||||
.join("");
|
||||
const gallery = screenshots.length
|
||||
? `<div class="gallery" aria-label="Screenshots for ${html(execution.id)}">${screenshots
|
||||
.map(
|
||||
@@ -268,6 +280,7 @@ function renderCase(
|
||||
: ""
|
||||
}
|
||||
<p class="detail">${html(detail)}</p>
|
||||
${turnTimingRows ? `<div class="matcher-wrap"><table class="matchers"><thead><tr><th>Turn</th><th>Run</th><th>Lease</th><th>Scheduler</th><th>Run duration</th><th>Response</th></tr></thead><tbody>${turnTimingRows}</tbody></table></div>` : ""}
|
||||
${matcherRows ? `<div class="matcher-wrap"><table class="matchers"><thead><tr><th>Result</th><th>Matcher</th><th>Expectation</th><th>Detail</th></tr></thead><tbody>${matcherRows}</tbody></table></div>` : `<p class="detail">No matcher result was recorded.</p>`}
|
||||
${entry ? `<details class="usage"><summary>Usage and billing metadata</summary><pre>${html(JSON.stringify({ billing, rawUsage: entry.result.usage ?? null }, null, 2))}</pre></details>` : ""}
|
||||
${links}
|
||||
|
||||
@@ -195,8 +195,8 @@ describe("runner E2E campaign history", () => {
|
||||
expect(index).toContain("Runner E2E campaigns");
|
||||
expect(index).toContain("complete-green");
|
||||
expect(index).toContain("complete-red");
|
||||
expect(index).toContain("66/66 passed");
|
||||
expect(index).toContain("65/66 passed");
|
||||
expect(index).toContain("68/68 passed");
|
||||
expect(index).toContain("67/68 passed");
|
||||
expect(index).toContain("Open report →");
|
||||
expect(index).toContain(
|
||||
"campaigns/complete-red/public-images/campaign-summary.png",
|
||||
|
||||
@@ -70,4 +70,80 @@ describe("live runner fixtures", () => {
|
||||
"DELETE /api/environments/environment-1?destroyReusableSandboxLeases=true",
|
||||
);
|
||||
});
|
||||
|
||||
it("creates a primary project workspace for reusable Daytona scope", async () => {
|
||||
const calls: string[] = [];
|
||||
const api = {
|
||||
async post(path: string, data?: Record<string, unknown>) {
|
||||
calls.push(`POST ${path}`);
|
||||
if (path === "/api/plugins/install") {
|
||||
return {
|
||||
id: "plugin-daytona",
|
||||
pluginKey: "paperclip.daytona-sandbox-provider",
|
||||
status: "ready",
|
||||
};
|
||||
}
|
||||
if (path === "/api/companies") {
|
||||
return { id: "company-1", name: "Runner E2E" };
|
||||
}
|
||||
if (path.endsWith("/environments")) {
|
||||
return { id: "environment-1", driver: "sandbox" };
|
||||
}
|
||||
if (path.endsWith("/agents")) {
|
||||
return { id: "agent-1", name: "Agent", companyId: "company-1" };
|
||||
}
|
||||
if (path.endsWith("/projects")) {
|
||||
expect(data).toMatchObject({
|
||||
executionWorkspacePolicy: {
|
||||
enabled: true,
|
||||
defaultMode: "shared_workspace",
|
||||
environmentId: "environment-1",
|
||||
},
|
||||
workspace: {
|
||||
sourceType: "local_path",
|
||||
cwd: "/tmp/workspace",
|
||||
isPrimary: true,
|
||||
},
|
||||
});
|
||||
return {
|
||||
id: "project-1",
|
||||
name: data?.name,
|
||||
primaryWorkspace: {
|
||||
id: "project-workspace-1",
|
||||
cwd: "/tmp/workspace",
|
||||
},
|
||||
};
|
||||
}
|
||||
throw new Error(`Unexpected POST ${path}`);
|
||||
},
|
||||
async postSensitive(_path: string, data?: Record<string, unknown>) {
|
||||
return { id: `secret-${String(data?.key).toLowerCase()}` };
|
||||
},
|
||||
async delete() {},
|
||||
} as unknown as RunnerApi;
|
||||
const execution = runnerMatrix.find(
|
||||
(candidate) =>
|
||||
candidate.id ===
|
||||
"daytona-warm-continuity.runner-codex.daytona.warm-three-turn",
|
||||
)!;
|
||||
|
||||
const fixtures = await setupLiveFixtures({
|
||||
api,
|
||||
execution,
|
||||
executionNonce: "nonce",
|
||||
workspacePath: "/tmp/workspace",
|
||||
credentials: {
|
||||
OPENAI_API_KEY: "openai-test-value",
|
||||
DAYTONA_API_KEY: "daytona-test-value",
|
||||
},
|
||||
daytonaImage:
|
||||
"ghcr.io/paperclip/image@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
});
|
||||
|
||||
expect(fixtures.project?.primaryWorkspace?.id).toBe("project-workspace-1");
|
||||
expect(
|
||||
calls.indexOf("POST /api/companies/company-1/environments"),
|
||||
).toBeLessThan(calls.indexOf("POST /api/companies/company-1/projects"));
|
||||
await fixtures.teardown();
|
||||
});
|
||||
});
|
||||
@@ -32,12 +32,21 @@ interface AgentRecord {
|
||||
name: string;
|
||||
companyId: string;
|
||||
}
|
||||
interface ProjectRecord {
|
||||
id: string;
|
||||
name: string;
|
||||
primaryWorkspace?: {
|
||||
id: string;
|
||||
cwd?: string | null;
|
||||
} | null;
|
||||
}
|
||||
|
||||
export interface LiveFixtureValues {
|
||||
company: CompanyRecord;
|
||||
secretRefs: SecretReferenceMap;
|
||||
environment: EnvironmentRecord;
|
||||
agent: AgentRecord;
|
||||
project?: ProjectRecord;
|
||||
teardown(): Promise<void>;
|
||||
}
|
||||
|
||||
@@ -210,12 +219,51 @@ export async function setupLiveFixtures(input: {
|
||||
},
|
||||
});
|
||||
|
||||
if (execution.environment.configurationKey === "warm-reuse-v1") {
|
||||
registry.register<ProjectRecord>({
|
||||
id: "project",
|
||||
dependencies: ["company", "environment"],
|
||||
async setup(resolved) {
|
||||
const company = value<CompanyRecord>(resolved, "company");
|
||||
const environment = value<EnvironmentRecord>(resolved, "environment");
|
||||
return api.post<ProjectRecord>(
|
||||
`/api/companies/${company.id}/projects`,
|
||||
{
|
||||
name: `Runner E2E warm project ${input.executionNonce}`,
|
||||
description:
|
||||
"Ephemeral project anchoring a reusable Daytona execution workspace",
|
||||
executionWorkspacePolicy: {
|
||||
enabled: true,
|
||||
defaultMode: "shared_workspace",
|
||||
sharedWorkspaceConcurrency: "serialize",
|
||||
allowIssueOverride: false,
|
||||
environmentId: environment.id,
|
||||
workspaceStrategy: { type: "project_primary" },
|
||||
},
|
||||
workspace: {
|
||||
name: "Primary",
|
||||
sourceType: "local_path",
|
||||
cwd: input.workspacePath,
|
||||
isPrimary: true,
|
||||
},
|
||||
},
|
||||
);
|
||||
},
|
||||
async teardown() {
|
||||
// The isolated instance is deleted after provider resources are gone.
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const setup = await registry.setupAll();
|
||||
return {
|
||||
company: value<CompanyRecord>(setup.values, "company"),
|
||||
secretRefs: value<SecretReferenceMap>(setup.values, "secrets"),
|
||||
environment: value<EnvironmentRecord>(setup.values, "environment"),
|
||||
agent: value<AgentRecord>(setup.values, "agent"),
|
||||
...(setup.values.has("project")
|
||||
? { project: value<ProjectRecord>(setup.values, "project") }
|
||||
: {}),
|
||||
teardown: setup.teardown,
|
||||
};
|
||||
}
|
||||
@@ -116,6 +116,7 @@ export async function evaluateMatcher(
|
||||
passed = actual === matcher.expected;
|
||||
} else if (
|
||||
matcher.kind === "file_exists" ||
|
||||
matcher.kind === "file_exact" ||
|
||||
matcher.kind === "file_contains"
|
||||
) {
|
||||
try {
|
||||
@@ -124,7 +125,9 @@ export async function evaluateMatcher(
|
||||
(await readFile(matcher.path, "utf8"));
|
||||
passed =
|
||||
matcher.kind === "file_exists" ||
|
||||
String(actual).includes(matcher.expected);
|
||||
(matcher.kind === "file_exact"
|
||||
? String(actual) === matcher.expected
|
||||
: String(actual).includes(matcher.expected));
|
||||
} catch {
|
||||
actual = undefined;
|
||||
passed = false;
|
||||
|
||||
@@ -44,6 +44,19 @@ describe("runner E2E report aggregation", () => {
|
||||
workflowRunUrl: "https://example.test/actions/runs/forged",
|
||||
},
|
||||
runIds: ["run-2"],
|
||||
turnTimings: [
|
||||
{
|
||||
turn: 1,
|
||||
submittedAt: "2026-08-26T00:00:00.000Z",
|
||||
runStartedAt: "2026-08-26T00:00:00.100Z",
|
||||
runFinishedAt: "2026-08-26T00:00:01.000Z",
|
||||
schedulerLatencyMs: 100,
|
||||
runDurationMs: 900,
|
||||
responseLatencyMs: 1_000,
|
||||
runId: "run-2",
|
||||
leaseAcquisitionOutcome: "created",
|
||||
},
|
||||
],
|
||||
usage: {
|
||||
inputTokens: 1_250,
|
||||
outputTokens: 75,
|
||||
@@ -201,6 +214,9 @@ describe("runner E2E report aggregation", () => {
|
||||
);
|
||||
expect(dashboard).toContain("message_contains");
|
||||
expect(dashboard).toContain("Matchers and test context");
|
||||
expect(dashboard).toContain("Scheduler");
|
||||
expect(dashboard).toContain("Run duration");
|
||||
expect(dashboard).toContain("100ms");
|
||||
expect(dashboard).toContain("Campaign billing summary");
|
||||
expect(dashboard).toContain("LLM reported subtotal");
|
||||
expect(dashboard).toContain("Agent execution time");
|
||||
|
||||
+461
-11
@@ -44,6 +44,9 @@ interface IssueRecord {
|
||||
status: string;
|
||||
workMode?: string;
|
||||
assigneeAgentId?: string | null;
|
||||
projectId?: string | null;
|
||||
projectWorkspaceId?: string | null;
|
||||
executionWorkspaceId?: string | null;
|
||||
executionRunId?: string | null;
|
||||
checkoutRunId?: string | null;
|
||||
}
|
||||
@@ -66,6 +69,9 @@ interface RunRecord {
|
||||
continuationAttempt?: number;
|
||||
retryOfRunId?: string | null;
|
||||
runnerInstanceId?: string | null;
|
||||
nativeSessionId?: string | null;
|
||||
processPid?: number | null;
|
||||
processStartedAt?: string | null;
|
||||
contextSnapshot?: Record<string, unknown> | null;
|
||||
runnerProfileJson?: Record<string, unknown> | null;
|
||||
usageJson?: Record<string, unknown> | null;
|
||||
@@ -80,13 +86,17 @@ interface RunRecord {
|
||||
|
||||
interface EnvironmentLeaseRecord {
|
||||
id: string;
|
||||
status?: string;
|
||||
leasePolicy?: string;
|
||||
providerLeaseId?: string | null;
|
||||
executionWorkspaceId?: string | null;
|
||||
metadata?: Record<string, unknown> | null;
|
||||
issueId?: string | null;
|
||||
heartbeatRunId?: string | null;
|
||||
provider?: string | null;
|
||||
acquiredAt?: string | null;
|
||||
releasedAt?: string | null;
|
||||
updatedAt?: string | null;
|
||||
metadata?: Record<string, unknown> | null;
|
||||
}
|
||||
|
||||
interface InteractionRecord {
|
||||
@@ -290,6 +300,7 @@ async function createTaskThroughUi(input: {
|
||||
title: string;
|
||||
prompt: string;
|
||||
workMode: "standard" | "planning" | "ask";
|
||||
projectName?: string;
|
||||
}) {
|
||||
const issuesUrl = `/${encodeURIComponent(input.issuePrefix)}/issues`;
|
||||
const newTask = input.page.getByRole("button", { name: "New Task" }).first();
|
||||
@@ -336,9 +347,29 @@ async function createTaskThroughUi(input: {
|
||||
.getByPlaceholder("Search assignees...")
|
||||
.fill(input.agentName);
|
||||
await input.page.getByText(input.agentName, { exact: true }).last().click();
|
||||
if (input.projectName) {
|
||||
const dialog = input.page.getByRole("dialog");
|
||||
await dialog.getByRole("button", { name: "Project", exact: true }).click();
|
||||
await dialog.getByPlaceholder("Search projects...").fill(input.projectName);
|
||||
await dialog.getByText(input.projectName, { exact: true }).last().click();
|
||||
}
|
||||
const submittedAtMs = Date.now();
|
||||
await input.page
|
||||
.getByRole("button", { name: "Create Task", exact: true })
|
||||
.click();
|
||||
return submittedAtMs;
|
||||
}
|
||||
|
||||
async function submitTaskReply(page: Page, body: string): Promise<number> {
|
||||
const composer = page.getByTestId("task-chat-composer-input").last();
|
||||
await expect(composer).toBeVisible({ timeout: 30_000 });
|
||||
await composer
|
||||
.locator('[contenteditable="true"], textarea')
|
||||
.first()
|
||||
.fill(body);
|
||||
const submittedAtMs = Date.now();
|
||||
await page.getByTestId("task-chat-composer-send").last().click();
|
||||
return submittedAtMs;
|
||||
}
|
||||
|
||||
function matchingRuns(runs: RunRecord[], issue: IssueRecord) {
|
||||
@@ -524,7 +555,11 @@ for (const execution of executions) {
|
||||
page,
|
||||
request,
|
||||
}, testInfo) => {
|
||||
test.setTimeout(deadlineMs + 90_000);
|
||||
test.setTimeout(
|
||||
execution.task.flow === "warm_three_turn"
|
||||
? deadlineMs
|
||||
: deadlineMs + 90_000,
|
||||
);
|
||||
const startedAtMs = Date.now();
|
||||
const startedAt = new Date(startedAtMs).toISOString();
|
||||
const nonce = `${randomBytes(6).toString("hex")}-${attempt}`;
|
||||
@@ -541,6 +576,8 @@ for (const execution of executions) {
|
||||
let selectedRuns: RunRecord[] = [];
|
||||
let runtimeLeases: EnvironmentLeaseRecord[] = [];
|
||||
let matcherResults: MatcherResult[] = [];
|
||||
let turnTimings: NonNullable<RunnerE2EResult["turnTimings"]> | undefined;
|
||||
const turnSubmissionTimesMs: number[] = [];
|
||||
const screenshots: NonNullable<RunnerE2EResult["screenshots"]> = [];
|
||||
let primaryError: unknown;
|
||||
let failureClassOverride: FailureClass | undefined;
|
||||
@@ -746,14 +783,17 @@ for (const execution of executions) {
|
||||
throw new Error(
|
||||
"Created fixture company did not return an issue prefix",
|
||||
);
|
||||
await createTaskThroughUi({
|
||||
page,
|
||||
issuePrefix,
|
||||
agentName: fixtures.agent.name,
|
||||
title,
|
||||
prompt,
|
||||
workMode: execution.task.workMode,
|
||||
});
|
||||
turnSubmissionTimesMs.push(
|
||||
await createTaskThroughUi({
|
||||
page,
|
||||
issuePrefix,
|
||||
agentName: fixtures.agent.name,
|
||||
title,
|
||||
prompt,
|
||||
workMode: execution.task.workMode,
|
||||
projectName: fixtures.project?.name,
|
||||
}),
|
||||
);
|
||||
|
||||
const deadlineAt = startedAtMs + deadlineMs;
|
||||
issue = await pollUntil({
|
||||
@@ -781,6 +821,16 @@ for (const execution of executions) {
|
||||
`UI-created issue work mode was ${String(issue.workMode)}; expected ${execution.task.workMode}`,
|
||||
);
|
||||
}
|
||||
if (fixtures.project) {
|
||||
if (
|
||||
issue.projectId !== fixtures.project.id ||
|
||||
issue.projectWorkspaceId !== fixtures.project.primaryWorkspace?.id
|
||||
) {
|
||||
throw new Error(
|
||||
`Warm task did not retain its project/workspace scope: ${JSON.stringify({ projectId: issue.projectId, projectWorkspaceId: issue.projectWorkspaceId })}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
await page.goto(
|
||||
`/${encodeURIComponent(issuePrefix)}/issues/${encodeURIComponent(issue.identifier ?? issue.id)}`,
|
||||
@@ -827,6 +877,7 @@ for (const execution of executions) {
|
||||
|
||||
let planLifecycleEvidence: Record<string, unknown> | null = null;
|
||||
let questionLifecycleEvidence: Record<string, unknown> | null = null;
|
||||
let warmLifecycleEvidence: Record<string, unknown> | null = null;
|
||||
let expectedQuestionResolution: {
|
||||
interactionId: string;
|
||||
optionId: string;
|
||||
@@ -1214,12 +1265,165 @@ for (const execution of executions) {
|
||||
.last()
|
||||
.click();
|
||||
planLifecycleEvidence = { interaction, plan };
|
||||
} else if (execution.task.flow === "warm_three_turn") {
|
||||
const followups = execution.task.buildFollowupMessages?.(nonce);
|
||||
if (!followups || !fixtures.project?.primaryWorkspace?.id) {
|
||||
throw new Error(
|
||||
`Warm fixture ${execution.task.id} is missing its project or follow-up messages`,
|
||||
);
|
||||
}
|
||||
const workspaceFile = path.join(
|
||||
workspacePath,
|
||||
`daytona-warm-${nonce}.txt`,
|
||||
);
|
||||
const turnEvidence: Array<Record<string, unknown>> = [];
|
||||
for (const completedTurn of [1, 2] as const) {
|
||||
const turnDeadlineAt = Math.min(
|
||||
deadlineAt,
|
||||
turnSubmissionTimesMs[completedTurn - 1]! +
|
||||
(execution.task.turnTimeoutMs ?? 10 * 60_000),
|
||||
);
|
||||
const waitingState = await pollUntil({
|
||||
label: `warm Daytona turn ${completedTurn} review state for issue ${issue.id}`,
|
||||
deadlineAt: turnDeadlineAt,
|
||||
load: loadTaskState,
|
||||
accept: ({ currentIssue, taskRuns }) =>
|
||||
currentIssue.status === "in_review" &&
|
||||
taskRuns.length === completedTurn &&
|
||||
taskRuns.every((run) => run.status === "succeeded"),
|
||||
reject: ({ taskRuns }) =>
|
||||
definitiveRunFailure(taskRuns) ??
|
||||
(taskRuns.length > completedTurn
|
||||
? `warm turn ${completedTurn} dispatched duplicate runs`
|
||||
: undefined),
|
||||
});
|
||||
const expectedPrefix = `${Array.from(
|
||||
{ length: completedTurn },
|
||||
(_, index) => `T${index + 1}_${nonce}`,
|
||||
).join("\n")}\n`;
|
||||
const hostContent = await readFile(workspaceFile, "utf8");
|
||||
if (hostContent !== expectedPrefix) {
|
||||
throw new Error(
|
||||
`Host workspace was not finalized after warm turn ${completedTurn}: expected ${JSON.stringify(expectedPrefix)}, observed ${JSON.stringify(hostContent)}`,
|
||||
);
|
||||
}
|
||||
if (
|
||||
waitingState.currentIssue.projectId !== fixtures.project.id ||
|
||||
waitingState.currentIssue.projectWorkspaceId !==
|
||||
fixtures.project.primaryWorkspace.id ||
|
||||
!waitingState.currentIssue.executionWorkspaceId
|
||||
) {
|
||||
throw new Error(
|
||||
`Warm turn ${completedTurn} lost its project execution-workspace scope`,
|
||||
);
|
||||
}
|
||||
const completedRunIds = new Set(
|
||||
waitingState.taskRuns.map((candidate) => candidate.id),
|
||||
);
|
||||
const retainedTurnLeases = await pollUntil({
|
||||
label: `retained Daytona leases after warm turn ${completedTurn}`,
|
||||
deadlineAt: Math.min(turnDeadlineAt, Date.now() + 30_000),
|
||||
intervalMs: 500,
|
||||
load: () =>
|
||||
api.get<EnvironmentLeaseRecord[]>(
|
||||
`/api/environments/${fixtures!.environment.id}/leases`,
|
||||
),
|
||||
accept: (leases) => {
|
||||
const runOrder = new Map(
|
||||
waitingState.taskRuns.map((candidate, index) => [
|
||||
candidate.id,
|
||||
index,
|
||||
]),
|
||||
);
|
||||
const completed = leases
|
||||
.filter(
|
||||
(lease) =>
|
||||
lease.heartbeatRunId &&
|
||||
completedRunIds.has(lease.heartbeatRunId),
|
||||
)
|
||||
.sort(
|
||||
(left, right) =>
|
||||
(runOrder.get(left.heartbeatRunId ?? "") ?? 0) -
|
||||
(runOrder.get(right.heartbeatRunId ?? "") ?? 0),
|
||||
);
|
||||
return (
|
||||
completed.length === completedTurn &&
|
||||
completed.every(
|
||||
(lease) =>
|
||||
lease.status === "retained" &&
|
||||
lease.leasePolicy === "reuse_by_environment" &&
|
||||
typeof lease.providerLeaseId === "string" &&
|
||||
record(lease.metadata).sandboxState === "started",
|
||||
) &&
|
||||
completed
|
||||
.slice(1)
|
||||
.every(
|
||||
(lease) =>
|
||||
record(lease.metadata).resumedFromState === "started",
|
||||
)
|
||||
);
|
||||
},
|
||||
});
|
||||
const turnRunOrder = new Map(
|
||||
waitingState.taskRuns.map((candidate, index) => [
|
||||
candidate.id,
|
||||
index,
|
||||
]),
|
||||
);
|
||||
const completedLeases = retainedTurnLeases
|
||||
.filter(
|
||||
(lease) =>
|
||||
lease.heartbeatRunId &&
|
||||
completedRunIds.has(lease.heartbeatRunId),
|
||||
)
|
||||
.sort(
|
||||
(left, right) =>
|
||||
(turnRunOrder.get(left.heartbeatRunId ?? "") ?? 0) -
|
||||
(turnRunOrder.get(right.heartbeatRunId ?? "") ?? 0),
|
||||
);
|
||||
if (
|
||||
new Set(completedLeases.map((lease) => lease.providerLeaseId))
|
||||
.size !== 1
|
||||
) {
|
||||
throw new Error(
|
||||
`Warm turn ${completedTurn} replaced its Daytona sandbox`,
|
||||
);
|
||||
}
|
||||
turnEvidence.push({
|
||||
turn: completedTurn,
|
||||
issue: waitingState.currentIssue,
|
||||
run: waitingState.taskRuns.at(-1),
|
||||
hostContent,
|
||||
leases: completedLeases,
|
||||
});
|
||||
await page.goto(
|
||||
`/${encodeURIComponent(issuePrefix)}/issues/${encodeURIComponent(issue.identifier ?? issue.id)}`,
|
||||
{ waitUntil: "domcontentloaded" },
|
||||
);
|
||||
await captureScreenshot(
|
||||
`warm-turn-${completedTurn}`,
|
||||
`Warm Daytona turn ${completedTurn} awaiting review`,
|
||||
`warm-turn-${completedTurn}.png`,
|
||||
);
|
||||
turnSubmissionTimesMs.push(
|
||||
await submitTaskReply(page, followups[completedTurn - 1]),
|
||||
);
|
||||
}
|
||||
warmLifecycleEvidence = { turns: turnEvidence };
|
||||
}
|
||||
|
||||
const taskMatchers = execution.task.buildMatchers(nonce, execution);
|
||||
const terminalDeadlineAt =
|
||||
execution.task.flow === "warm_three_turn"
|
||||
? Math.min(
|
||||
deadlineAt,
|
||||
turnSubmissionTimesMs.at(-1)! +
|
||||
(execution.task.turnTimeoutMs ?? 10 * 60_000),
|
||||
)
|
||||
: deadlineAt;
|
||||
let terminal = await pollUntil({
|
||||
label: `issue ${issue.id} and heartbeat run terminal state`,
|
||||
deadlineAt,
|
||||
deadlineAt: terminalDeadlineAt,
|
||||
load: loadTaskState,
|
||||
accept: ({ currentIssue, taskRuns }) =>
|
||||
currentIssue.status === execution.task.expectedTerminalState.issue &&
|
||||
@@ -1289,6 +1493,45 @@ for (const execution of executions) {
|
||||
),
|
||||
);
|
||||
selectedRuns = sortRunsChronologically(selectedRuns);
|
||||
if (execution.task.flow === "warm_three_turn") {
|
||||
turnTimings = selectedRuns.map((candidate, index) => {
|
||||
const submittedAtMs = turnSubmissionTimesMs[index]!;
|
||||
const runStartedAtMs = candidate.startedAt
|
||||
? Date.parse(candidate.startedAt)
|
||||
: Number.NaN;
|
||||
const runFinishedAtMs = candidate.finishedAt
|
||||
? Date.parse(candidate.finishedAt)
|
||||
: Number.NaN;
|
||||
const acquisition = record(
|
||||
record(candidate.contextSnapshot).paperclipEnvironment,
|
||||
).sandboxLeaseAcquisition;
|
||||
const acquisitionOutcome = record(acquisition).outcome;
|
||||
return {
|
||||
turn: index + 1,
|
||||
submittedAt: new Date(submittedAtMs).toISOString(),
|
||||
runStartedAt: candidate.startedAt ?? null,
|
||||
runFinishedAt: candidate.finishedAt ?? null,
|
||||
schedulerLatencyMs: Number.isFinite(runStartedAtMs)
|
||||
? Math.max(0, runStartedAtMs - submittedAtMs)
|
||||
: null,
|
||||
runDurationMs:
|
||||
Number.isFinite(runStartedAtMs) &&
|
||||
Number.isFinite(runFinishedAtMs)
|
||||
? Math.max(0, runFinishedAtMs - runStartedAtMs)
|
||||
: null,
|
||||
responseLatencyMs: Number.isFinite(runFinishedAtMs)
|
||||
? Math.max(0, runFinishedAtMs - submittedAtMs)
|
||||
: null,
|
||||
runId: candidate.id,
|
||||
leaseAcquisitionOutcome:
|
||||
acquisitionOutcome === "created" ||
|
||||
acquisitionOutcome === "resumed" ||
|
||||
acquisitionOutcome === "replacement"
|
||||
? acquisitionOutcome
|
||||
: "unknown",
|
||||
};
|
||||
});
|
||||
}
|
||||
const finalRun = selectedRuns.at(-1)!;
|
||||
const run =
|
||||
selectedRuns.find(
|
||||
@@ -1500,10 +1743,31 @@ for (const execution of executions) {
|
||||
interactions: terminal.interactions,
|
||||
},
|
||||
};
|
||||
const fileObservations = Object.fromEntries(
|
||||
await Promise.all(
|
||||
taskMatchers
|
||||
.filter(
|
||||
(matcher) =>
|
||||
matcher.kind === "file_exists" ||
|
||||
matcher.kind === "file_exact" ||
|
||||
matcher.kind === "file_contains",
|
||||
)
|
||||
.map(async (matcher) => [
|
||||
matcher.path,
|
||||
await readFile(
|
||||
path.isAbsolute(matcher.path)
|
||||
? matcher.path
|
||||
: path.join(workspacePath, matcher.path),
|
||||
"utf8",
|
||||
).catch(() => undefined),
|
||||
]),
|
||||
),
|
||||
);
|
||||
matcherResults = await Promise.all(
|
||||
taskMatchers.map((matcher) =>
|
||||
evaluateMatcher(matcher, {
|
||||
...matcherObservation,
|
||||
files: fileObservations,
|
||||
// Multi-run tasks intentionally retain earlier waiting/revision
|
||||
// replies. Exact completion text belongs to the chronological
|
||||
// final run, while occurrence checks still span every agent
|
||||
@@ -1571,6 +1835,190 @@ for (const execution of executions) {
|
||||
"expected a Daytona sandbox lease on the run context",
|
||||
);
|
||||
}
|
||||
if (execution.task.flow === "warm_three_turn") {
|
||||
const runEnvironmentContexts = selectedRuns.map((candidate) =>
|
||||
record(record(candidate.contextSnapshot).paperclipEnvironment),
|
||||
);
|
||||
const leaseIds = runEnvironmentContexts.map((entry) => entry.leaseId);
|
||||
const acquisitionOutcomes = runEnvironmentContexts.map(
|
||||
(entry) => record(entry.sandboxLeaseAcquisition).outcome,
|
||||
);
|
||||
const projectWorkspaceIds = selectedRuns.map(
|
||||
(candidate) =>
|
||||
record(record(candidate.contextSnapshot).paperclipWorkspace)
|
||||
.workspaceId,
|
||||
);
|
||||
const executionWorkspaceIds = selectedRuns.map(
|
||||
(candidate) => record(candidate.contextSnapshot).executionWorkspaceId,
|
||||
);
|
||||
if (
|
||||
leaseIds.some(
|
||||
(leaseId) => typeof leaseId !== "string" || leaseId.length === 0,
|
||||
)
|
||||
) {
|
||||
invariantFailures.push(
|
||||
`expected a persisted Daytona lease row for every warm turn; observed ${JSON.stringify(leaseIds)}`,
|
||||
);
|
||||
}
|
||||
if (
|
||||
JSON.stringify(acquisitionOutcomes) !==
|
||||
JSON.stringify(["created", "resumed", "resumed"])
|
||||
) {
|
||||
invariantFailures.push(
|
||||
`expected warm lease outcomes created,resumed,resumed; observed ${JSON.stringify(acquisitionOutcomes)}`,
|
||||
);
|
||||
}
|
||||
if (
|
||||
!fixtures.project?.primaryWorkspace?.id ||
|
||||
projectWorkspaceIds.some(
|
||||
(workspaceId) =>
|
||||
workspaceId !== fixtures!.project!.primaryWorkspace!.id,
|
||||
) ||
|
||||
new Set(executionWorkspaceIds).size !== 1 ||
|
||||
executionWorkspaceIds[0] !== issue.executionWorkspaceId ||
|
||||
issue.projectId !== fixtures.project.id ||
|
||||
issue.projectWorkspaceId !== fixtures.project.primaryWorkspace.id ||
|
||||
!issue.executionWorkspaceId
|
||||
) {
|
||||
invariantFailures.push(
|
||||
`warm task did not preserve project, project-workspace, and execution-workspace identity: ${JSON.stringify({ projectWorkspaceIds, executionWorkspaceIds, projectId: issue.projectId, projectWorkspaceId: issue.projectWorkspaceId, executionWorkspaceId: issue.executionWorkspaceId })}`,
|
||||
);
|
||||
}
|
||||
if (execution.profile.generation === "native") {
|
||||
const stableIdentityFields: Array<{
|
||||
label: string;
|
||||
values: unknown[];
|
||||
}> = [
|
||||
{
|
||||
label: "native session",
|
||||
values: selectedRuns.map(
|
||||
(candidate) => candidate.nativeSessionId,
|
||||
),
|
||||
},
|
||||
{
|
||||
label: "runner instance",
|
||||
values: selectedRuns.map(
|
||||
(candidate) => candidate.runnerInstanceId,
|
||||
),
|
||||
},
|
||||
{
|
||||
label: "provider session",
|
||||
values: selectedRuns.map((candidate) => candidate.sessionIdAfter),
|
||||
},
|
||||
{
|
||||
label: "runner pid",
|
||||
values: selectedRuns.map((candidate) => candidate.processPid),
|
||||
},
|
||||
{
|
||||
label: "runner process fingerprint",
|
||||
values: selectedRuns.map(
|
||||
(candidate) => candidate.processStartedAt,
|
||||
),
|
||||
},
|
||||
];
|
||||
for (const { label, values } of stableIdentityFields) {
|
||||
if (
|
||||
values.some(
|
||||
(value) =>
|
||||
value === null ||
|
||||
value === undefined ||
|
||||
String(value).length === 0,
|
||||
) ||
|
||||
new Set(values).size !== 1
|
||||
) {
|
||||
invariantFailures.push(
|
||||
`expected one stable ${label} across native warm turns; observed ${JSON.stringify(values)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (
|
||||
turnTimings?.length !== 3 ||
|
||||
turnTimings.some(
|
||||
(timing) =>
|
||||
timing.runStartedAt === null ||
|
||||
timing.runFinishedAt === null ||
|
||||
timing.schedulerLatencyMs === null ||
|
||||
timing.runDurationMs === null ||
|
||||
timing.responseLatencyMs === null ||
|
||||
timing.responseLatencyMs >
|
||||
(execution.task.turnTimeoutMs ?? 10 * 60_000),
|
||||
)
|
||||
) {
|
||||
invariantFailures.push(
|
||||
`warm turn timing data was incomplete or exceeded its structural deadline: ${JSON.stringify(turnTimings)}`,
|
||||
);
|
||||
}
|
||||
const retainedLeases = await pollUntil({
|
||||
label: `retained warm Daytona lease for issue ${issue.id}`,
|
||||
deadlineAt: Math.min(deadlineAt, Date.now() + 30_000),
|
||||
intervalMs: 500,
|
||||
load: () =>
|
||||
api.get<EnvironmentLeaseRecord[]>(
|
||||
`/api/environments/${fixtures!.environment.id}/leases`,
|
||||
),
|
||||
accept: (leases) => {
|
||||
const warmLeases = leases.filter(
|
||||
(lease) =>
|
||||
lease.issueId === issue!.id &&
|
||||
selectedRuns.some(
|
||||
(candidate) => candidate.id === lease.heartbeatRunId,
|
||||
),
|
||||
);
|
||||
return (
|
||||
warmLeases.length === 3 &&
|
||||
warmLeases.every(
|
||||
(lease) =>
|
||||
lease.status === "retained" &&
|
||||
lease.leasePolicy === "reuse_by_environment" &&
|
||||
typeof lease.providerLeaseId === "string" &&
|
||||
record(lease.metadata).sandboxState === "started",
|
||||
)
|
||||
);
|
||||
},
|
||||
});
|
||||
const selectedRunOrder = new Map(
|
||||
selectedRuns.map((candidate, index) => [candidate.id, index]),
|
||||
);
|
||||
const warmLeases = retainedLeases
|
||||
.filter(
|
||||
(lease) =>
|
||||
lease.issueId === issue!.id &&
|
||||
selectedRunOrder.has(lease.heartbeatRunId ?? ""),
|
||||
)
|
||||
.sort(
|
||||
(left, right) =>
|
||||
(selectedRunOrder.get(left.heartbeatRunId ?? "") ?? 0) -
|
||||
(selectedRunOrder.get(right.heartbeatRunId ?? "") ?? 0),
|
||||
);
|
||||
const providerLeaseIds = warmLeases.map(
|
||||
(lease) => lease.providerLeaseId,
|
||||
);
|
||||
const resumedFromStates = warmLeases
|
||||
.slice(1)
|
||||
.map((lease) => record(lease.metadata).resumedFromState);
|
||||
if (
|
||||
new Set(providerLeaseIds).size !== 1 ||
|
||||
typeof providerLeaseIds[0] !== "string" ||
|
||||
JSON.stringify(resumedFromStates) !==
|
||||
JSON.stringify(["started", "started"])
|
||||
) {
|
||||
invariantFailures.push(
|
||||
`expected one continuously-started Daytona sandbox; observed ${JSON.stringify({ providerLeaseIds, resumedFromStates })}`,
|
||||
);
|
||||
}
|
||||
warmLifecycleEvidence = {
|
||||
...(warmLifecycleEvidence ?? {}),
|
||||
leaseIds,
|
||||
acquisitionOutcomes,
|
||||
projectWorkspaceIds,
|
||||
executionWorkspaceIds,
|
||||
providerLeaseIds,
|
||||
resumedFromStates,
|
||||
retainedLeases: warmLeases,
|
||||
turnTimings,
|
||||
};
|
||||
}
|
||||
if (execution.environment.id === "local") {
|
||||
const runLogContent = String(record(runLog).content ?? "");
|
||||
// The log endpoint returns NDJSON, so quotes inside each `chunk` are
|
||||
@@ -1723,6 +2171,7 @@ for (const execution of executions) {
|
||||
interactions: terminal.interactions,
|
||||
planLifecycleEvidence,
|
||||
questionLifecycleEvidence,
|
||||
warmLifecycleEvidence,
|
||||
matcherResults,
|
||||
invariantFailures,
|
||||
runEvents,
|
||||
@@ -1940,6 +2389,7 @@ for (const execution of executions) {
|
||||
issueId: issue?.id,
|
||||
issueIdentifier: issue?.identifier ?? null,
|
||||
runIds: selectedRuns.map((run) => run.id),
|
||||
...(turnTimings ? { turnTimings } : {}),
|
||||
startedAt,
|
||||
finishedAt: new Date(finishedAtMs).toISOString(),
|
||||
durationMs: finishedAtMs - startedAtMs,
|
||||
|
||||
@@ -341,6 +341,23 @@ describe("runner E2E matchers", () => {
|
||||
expect(result?.detail).toContain("observed 2");
|
||||
});
|
||||
|
||||
it("matches finalized workspace files byte-for-byte", async () => {
|
||||
const [matched, extraLine] = await evaluateMatchers(
|
||||
[
|
||||
{ kind: "file_exact", path: "continuity.txt", expected: "T1\nT2\n" },
|
||||
{ kind: "file_exact", path: "duplicate.txt", expected: "T1\nT2\n" },
|
||||
],
|
||||
{
|
||||
files: {
|
||||
"continuity.txt": "T1\nT2\n",
|
||||
"duplicate.txt": "T1\nT2\nT2\n",
|
||||
},
|
||||
},
|
||||
);
|
||||
expect(matched?.passed).toBe(true);
|
||||
expect(extraLine?.passed).toBe(false);
|
||||
});
|
||||
|
||||
it("normalizes ordered fragments and evaluates nested JSON Schema", async () => {
|
||||
const results = await evaluateMatchers(
|
||||
[
|
||||
|
||||
@@ -13,7 +13,8 @@ export type RunnerTaskFlow =
|
||||
| "single_turn"
|
||||
| "plan_revision_acceptance"
|
||||
| "question_resume_completion"
|
||||
| "plan_approval_completion";
|
||||
| "plan_approval_completion"
|
||||
| "warm_three_turn";
|
||||
|
||||
export interface SecretReference {
|
||||
type: "secret_ref";
|
||||
@@ -73,6 +74,8 @@ export interface RunnerProfileFixture {
|
||||
|
||||
export interface EnvironmentFixture {
|
||||
id: RunnerEnvironmentId;
|
||||
/** Distinguishes materially different configurations that share a provider ID. */
|
||||
configurationKey?: string;
|
||||
label: string;
|
||||
groups: readonly string[];
|
||||
driver: "local" | "sandbox";
|
||||
@@ -103,6 +106,7 @@ export type Matcher =
|
||||
| { kind: "runtime_mode"; expected: RunnerGeneration }
|
||||
| { kind: "environment"; expected: RunnerEnvironmentId }
|
||||
| { kind: "file_exists"; path: string }
|
||||
| { kind: "file_exact"; path: string; expected: string }
|
||||
| { kind: "file_contains"; path: string; expected: string }
|
||||
| { kind: "artifact_exists"; name: string; mimeType?: string }
|
||||
| { kind: "json_path"; path: string; expected: unknown }
|
||||
@@ -124,6 +128,8 @@ export interface RunnerTaskFixture {
|
||||
buildPrompt(nonce: string): string;
|
||||
buildVisibleMarker(nonce: string): string;
|
||||
buildRevisionRequest?(nonce: string): string;
|
||||
buildFollowupMessages?(nonce: string): readonly [string, string];
|
||||
turnTimeoutMs?: number;
|
||||
buildQuestionAnswer?(nonce: string): {
|
||||
optionLabel: string;
|
||||
expectedMarker: string;
|
||||
@@ -260,6 +266,17 @@ export interface RunnerE2EResult {
|
||||
issueId?: string;
|
||||
issueIdentifier?: string | null;
|
||||
runIds?: string[];
|
||||
turnTimings?: Array<{
|
||||
turn: number;
|
||||
submittedAt: string;
|
||||
runStartedAt: string | null;
|
||||
runFinishedAt: string | null;
|
||||
schedulerLatencyMs: number | null;
|
||||
runDurationMs: number | null;
|
||||
responseLatencyMs: number | null;
|
||||
runId: string;
|
||||
leaseAcquisitionOutcome: "created" | "resumed" | "replacement" | "unknown";
|
||||
}>;
|
||||
startedAt: string;
|
||||
finishedAt: string;
|
||||
durationMs: number;
|
||||
|
||||
@@ -4,7 +4,7 @@ import { describe, expect, it } from "vitest";
|
||||
|
||||
const repositoryRoot = path.resolve(import.meta.dirname, "../..");
|
||||
const ordinaryPrTrustedWorkflowRevision =
|
||||
"a0a78ee60946a5f79f85b2bd0584fc766fae43bb";
|
||||
"03609aa6ecc9a047ed53d6b6469d8be554fbc46d";
|
||||
const fullStackTestNeeds =
|
||||
/needs:\s*\[\s*authorize,\s*target_lock,\s*catalog,\s*daytona_image,\s*build_runner_artifacts,\s*build_remote_provider_pack,?\s*\]/u;
|
||||
const buildRunnerNeeds =
|
||||
|
||||
Reference in new issue
Block a user