fix(runner): persist warm Daytona workspaces

This commit is contained in:
Dotta committed 2026-09-05 10:52:22 -05:00
1 parent 3da58b185e
commit ee11cd65d9
44 files changed
+4019 -359

No files matched your search

+11 -4
View File
@@ -58,9 +58,13 @@ registry therefore discovers that row through the public environments API.
This still provides full isolation because every cell starts a new Paperclip
instance and database.
Daytona creates a sandbox environment through the public API. Keep
`reuseLease:false`, `runnerLifecycleMode:"per_turn"`, short provider cleanup
backstops, a Daytona secret reference, and an immutable image digest. Teardown
Daytona creates sandbox environments through the public API. The core fixture
keeps `reuseLease:false` and `runnerLifecycleMode:"per_turn"`. The dedicated
warm-continuity fixture uses `reuseLease:true` and
`runnerLifecycleMode:"warm"`; its distinct `configurationKey` is part of the
suite fingerprint even though both fixtures report `environmentId:"daytona"`.
Keep short provider cleanup backstops, a Daytona secret reference, and an
immutable image digest. Teardown
must delete the environment with reusable-lease destruction and must fail the
cell if cleanup cannot be confirmed. Keep CPU, memory, and disk explicit: lease
metadata and the per-test public-list-price runtime estimate depend on that
@@ -93,7 +97,10 @@ marker factories. `question_resume_completion` must define the deterministic
browser answer and prove exactly two successful runs with no pending
interaction. `plan_approval_completion` must target the exact two-step
canonical Plan revision, capture its pending UI, approve in the browser, and
prove exactly two successful runs.
prove exactly two successful runs. `warm_three_turn` provides exactly two
browser follow-up messages, preserves one project/execution-workspace scope,
verifies host file contents after every turn, and finishes within three
ten-minute turn deadlines.
Every selected case runs in its own isolated Paperclip process, and independent
cases may run concurrently. Follow-up turns inside one case retain their shared
+16 -3
View File
@@ -68,10 +68,11 @@ pnpm test:e2e:runner -- --group native --environment local
pnpm test:e2e:runner -- --profile runner-codex --case message-marker
pnpm test:e2e:runner -- --case plan-revise-accept --group local
pnpm test:e2e:runner -- --case ask-question --group native
pnpm test:e2e:runner -- --suite daytona-warm-continuity
pnpm test:e2e:runner -- --all
```
The catalog contains three suites. `core-compatibility` (**Core Runner
The catalog contains four suites. `core-compatibility` (**Core Runner
Compatibility**) is seven major runner profiles × local/Daytona × three
workflows: 42 cells. Its cases are:
@@ -107,7 +108,19 @@ duplicating the final response. The second workflow restarts the isolated
Paperclip server while the interaction is waiting, reloads that state, and
then resumes it. The suite has no Daytona cells.
The complete catalog is 66 cells (45 local and 21 Daytona) and 114 expected
`daytona-warm-continuity` (**Daytona Warm Continuity**) is exactly two paid
cells: legacy Codex and Runner Codex against one reusable warm Daytona
configuration. Each cell creates a real project with a primary local-path
workspace through the API, selects it in the browser task dialog, and performs
three browser-driven turns on one issue. Every turn reads and extends the same
nonce file, verifies host copy-back, records scheduler/run/end-to-end timing,
and asserts `created`, `resumed`, `resumed` lease acquisition on one sandbox.
Runner Codex additionally proves stable native session, provider session,
runner instance, PID, and process-start identity. Each turn is bounded to ten
minutes, the cell to thirty minutes, and cleanup explicitly deletes the
sandbox rather than waiting for Daytona's idle timeout.
The complete catalog is 68 cells (45 local and 23 Daytona) and 120 expected
paid agent turns. Follow-up steps remain ordered within their cell; all other
cells are independent. Narrow selectors are strongly recommended while
developing fixtures.
@@ -351,7 +364,7 @@ Set `RUNNER_E2E_AWS_ENABLED=true` to route paid cells to the repository-scoped
ephemeral AWS RunsOn fleet selected by
`runs-on/fleet=paperclip-public-pr-x64/env=public-ci`. Any other value uses the
proven GitHub-hosted `ubuntu-latest` target. Set `RUNNER_E2E_MAX_PARALLEL` to an
integer from 1–100 on AWS (default 100); use at least 66 to run the current
integer from 1–100 on AWS (default 100); use at least 68 to run the current
complete catalog in one wave. The fallback runner retains its 1–57 limit and
default of 32. Multi-turn steps are sequential inside their cell while
independent cells overlap. Artifacts and merged HTML/JUnit/normalized reports
+80 -13
View File
@@ -10,7 +10,10 @@ import {
runnerProfiles,
runnerSuites,
runnerTasks,
daytonaWarmContinuityTask,
daytonaWarmEnvironment,
isImmutableDaytonaImage,
suiteDefinitionHash,
validateRunnerCatalog,
} from "./catalog.js";
import {
@@ -21,7 +24,7 @@ import {
} from "./selectors.js";
describe("runner E2E catalog", () => {
it("validates the core, local-integrity, and breadth suites", () => {
it("validates the core, local-integrity, breadth, and warm suites", () => {
expect(runnerProfiles).toHaveLength(7);
expect(openRouterBreadthProfiles).toHaveLength(4);
expect(runnerEnvironments).toHaveLength(2);
@@ -29,10 +32,10 @@ describe("runner E2E catalog", () => {
expect(localIntegrityTasks).toHaveLength(2);
expect(openRouterBreadthTasks).toHaveLength(3);
expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([
42, 14, 10,
42, 14, 10, 2,
]);
expect(validateRunnerCatalog()).toHaveLength(66);
expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(66);
expect(validateRunnerCatalog()).toHaveLength(68);
expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(68);
expect(
runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"),
).toHaveLength(42);
@@ -46,18 +49,81 @@ describe("runner E2E catalog", () => {
(entry) => entry.suite.id === "openrouter-model-breadth",
),
).toHaveLength(10);
expect(
runnerMatrix.filter(
(entry) => entry.suite.id === "daytona-warm-continuity",
),
).toHaveLength(2);
expect(
runnerMatrix.reduce(
(total, execution) => total + execution.task.expectedRunCount,
0,
),
).toBe(114);
).toBe(120);
expect(
runnerTasks.find((task) => task.id === "plan-revise-accept")
?.attemptTimeoutMs,
).toEqual({ local: 8 * 60_000, daytona: 12 * 60_000 });
});
it("defines the warm Daytona continuity fixture as exactly two Codex cells", () => {
expect(daytonaWarmEnvironment).toMatchObject({
id: "daytona",
configurationKey: "warm-reuse-v1",
groups: ["daytona", "warm"],
});
expect(
daytonaWarmEnvironment.buildEnvironment({
secretRefs: {
DAYTONA_API_KEY: {
type: "secret_ref",
secretId: "22222222-2222-4222-8222-222222222222",
version: "latest",
},
},
daytonaImage: `runner@sha256:${"a".repeat(64)}`,
executionId: "warm",
}),
).toMatchObject({
config: {
reuseLease: true,
runnerLifecycleMode: "warm",
autoStopInterval: 5,
autoArchiveInterval: 15,
autoDeleteInterval: 60,
},
});
expect(daytonaWarmContinuityTask).toMatchObject({
flow: "warm_three_turn",
expectedRunCount: 3,
turnTimeoutMs: 600_000,
});
expect(
daytonaWarmContinuityTask.buildFollowupMessages?.("nonce"),
).toHaveLength(2);
const cells = runnerMatrix.filter(
(entry) => entry.suite.id === "daytona-warm-continuity",
);
expect(cells.map((entry) => entry.profile.id)).toEqual([
"legacy-codex",
"runner-codex",
]);
expect(cells.every((entry) => entry.environment.id === "daytona")).toBe(
true,
);
const suite = runnerSuites.find(
(candidate) => candidate.id === "daytona-warm-continuity",
)!;
expect(
suiteDefinitionHash({
...suite,
environments: [
{ ...daytonaWarmEnvironment, configurationKey: "changed" },
],
}),
).not.toBe(suiteDefinitionHash(suite));
});
it("derives the qualified local native OpenCode profiles from the ranked snapshot", () => {
expect(openRouterBreadthExcludedModelIds).toEqual(["xiaomi/mimo-v2.5"]);
expect(openRouterBreadthExcludedExecutionIds).toEqual([
@@ -181,9 +247,10 @@ describe("runner E2E catalog", () => {
question?.buildPrompt("nonce"),
...breadthTasks,
]) {
expect(prompt).toContain("then emit exactly");
const terminalTextInstruction = prompt?.match(/then emit (?:exactly|only)/)?.[0];
expect(terminalTextInstruction).toBeDefined();
expect(prompt!.indexOf("paperclip_finish exactly once")).toBeLessThan(
prompt!.indexOf("then emit exactly"),
prompt!.indexOf(terminalTextInstruction!),
);
expect(prompt).toContain("Wait for that tool call to succeed");
}
@@ -336,7 +403,7 @@ describe("runner E2E catalog", () => {
"those two tool calls form one indivisible response sequence",
);
expect(task!.buildPrompt("nonce")).toContain(
"Do not emit assistant text, end the heartbeat, or stop after write_document alone",
"Do not emit assistant text, end the response or heartbeat, or stop after write_document alone",
);
expect(task!.buildPrompt("nonce")).toContain(
"one atomic issue PATCH with status `done` and that exact comment",
@@ -433,7 +500,7 @@ describe("runner E2E selectors", () => {
"daytona",
]);
const selected = selectRunnerExecutions(options);
expect(selected).toHaveLength(12);
expect(selected).toHaveLength(13);
expect(
selected.every(
(entry) =>
@@ -443,7 +510,7 @@ describe("runner E2E selectors", () => {
).toBe(true);
});
it("rejects groups outside the advertised four", () => {
it("rejects unknown groups", () => {
const options = parseRunnerSelectors(["--group", "codex"]);
expect(() => selectRunnerExecutions(options)).toThrow("Unknown group");
});
@@ -452,10 +519,10 @@ describe("runner E2E selectors", () => {
const jobs = buildMatrixJobs(
selectRunnerExecutions(parseRunnerSelectors(["--all"])),
);
expect(jobs).toHaveLength(66);
expect(jobs.filter((job) => job.needsDaytona)).toHaveLength(21);
expect(jobs).toHaveLength(68);
expect(jobs.filter((job) => job.needsDaytona)).toHaveLength(23);
expect(jobs.filter((job) => !job.needsDaytona)).toHaveLength(45);
expect(new Set(jobs.map((job) => job.executionId)).size).toBe(66);
expect(new Set(jobs.map((job) => job.executionId)).size).toBe(68);
expect(
jobs.find(
(job) =>
+139 -4
View File
@@ -27,6 +27,7 @@ const SELECTABLE_GROUPS = [
"native",
"local",
"daytona",
"warm",
"core",
"breadth",
] as const;
@@ -374,6 +375,50 @@ export const runnerEnvironments: readonly EnvironmentFixture[] = [
},
] as const;
export const daytonaWarmEnvironment: EnvironmentFixture = {
id: "daytona",
configurationKey: "warm-reuse-v1",
label: "Daytona warm reusable sandbox",
groups: ["daytona", "warm"],
driver: "sandbox",
provider: "daytona",
credential: "DAYTONA_API_KEY",
lifecycle: {
setup: "create_via_api",
probe: "run_context_via_api",
cleanup: "delete_via_api_and_destroy_leases",
},
expectedExecutionTarget: { kind: "remote", transport: "sandbox" },
buildEnvironment(input) {
if (!isImmutableDaytonaImage(input.daytonaImage)) {
throw new Error(
"PAPERCLIP_E2E_DAYTONA_IMAGE must be an immutable image digest",
);
}
return {
name: `Runner E2E Daytona warm ${input.executionId}`,
description: "Ephemeral reusable Daytona runner E2E environment",
driver: "sandbox",
config: {
provider: "daytona",
apiKey: requiredDaytonaSecret(input),
image: input.daytonaImage,
cpu: 4,
memory: 4,
disk: 10,
reuseLease: true,
runnerLifecycleMode: "warm",
autoStopInterval: 5,
autoArchiveInterval: 15,
autoDeleteInterval: 60,
timeoutMs: 300_000,
livenessTimeoutMs: 30_000,
},
envVars: {},
};
},
};
export const runnerTasks: readonly RunnerTaskFixture[] = [
{
id: "message-marker",
@@ -719,6 +764,80 @@ const localEnvironment = runnerEnvironments.find(
(environment) => environment.id === "local",
)!;
function warmTurnMarker(turn: 1 | 2 | 3, nonce: string) {
return `PAPERCLIP_E2E_WARM_T${turn}_${nonce}`;
}
function warmWorkspaceLine(turn: 1 | 2 | 3, nonce: string) {
return `T${turn}_${nonce}`;
}
function warmTurnInstructions(turn: 1 | 2 | 3, nonce: string) {
const file = `daytona-warm-${nonce}.txt`;
const lines = Array.from({ length: turn }, (_, index) =>
warmWorkspaceLine((index + 1) as 1 | 2 | 3, nonce),
);
const marker = warmTurnMarker(turn, nonce);
const finalTurn = turn === 3;
return [
`This is warm Daytona continuity turn ${turn} of 3. Work only in the current execution workspace.`,
turn === 1
? `Create ${file} with exactly this one line followed by a newline: ${lines[0]}`
: `Before changing anything, read ${file} and verify its content is exactly ${lines.slice(0, -1).join("\\n")} followed by a newline. Then append exactly ${lines.at(-1)} followed by a newline.`,
`After the write, verify ${file} contains exactly these lines, once each and in order: ${lines.join(" | ")}.`,
`In a native runner, call paperclip_finish exactly once with {reportedWorkDisposition:"${finalTurn ? "done" : "needs_review"}",summary:"${marker}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[{commandOrCheck:"read ${file}",status:"passed"}]}. Wait for that tool call to succeed, then emit exactly ${marker} once as the complete user-facing final response.`,
`In a legacy runner, make exactly one public-API completion write after verification: PATCH /api/issues/$PAPERCLIP_TASK_ID with {"status":"${finalTurn ? "done" : "in_review"}","comment":"${marker}"}. Include Authorization and X-Paperclip-Run-Id. Do not POST a separate comment.`,
`Do not include ${marker} in any other visible response or write. Do not recreate, truncate, reorder, or duplicate prior lines.`,
].join("\n");
}
export const daytonaWarmContinuityTask: RunnerTaskFixture = {
id: "warm-three-turn",
label: "Warm three-turn workspace continuity",
groups: ["warm"],
workMode: "standard",
flow: "warm_three_turn",
expectedRunCount: 3,
attemptTimeoutMs: { local: 30 * 60_000, daytona: 30 * 60_000 },
turnTimeoutMs: 10 * 60_000,
expectedTerminalState: { issue: "done", run: "succeeded" },
buildTitle: (nonce) => `Runner E2E warm Daytona continuity ${nonce}`,
buildVisibleMarker: (nonce) => warmTurnMarker(3, nonce),
buildPrompt: (nonce) => warmTurnInstructions(1, nonce),
buildFollowupMessages: (nonce) => [
warmTurnInstructions(2, nonce),
warmTurnInstructions(3, nonce),
],
buildMatchers(nonce, execution) {
const markers = ([1, 2, 3] as const).map((turn) =>
warmTurnMarker(turn, nonce),
);
return [
{ kind: "message_exact", expected: markers[2] },
...markers.map(
(expected) =>
({ kind: "message_occurrences", expected, count: 1 }) as const,
),
{ kind: "message_ordered", expected: markers },
{
kind: "file_exact",
path: `daytona-warm-${nonce}.txt`,
expected: `${([1, 2, 3] as const)
.map((turn) => warmWorkspaceLine(turn, nonce))
.join("\n")}\n`,
},
{ kind: "issue_status", expected: "done" },
{ kind: "run_status", expected: "succeeded" },
{ kind: "runtime_mode", expected: execution.profile.expectedRuntimeMode },
{ kind: "environment", expected: "daytona" },
];
},
};
const codexContinuityProfiles = runnerProfiles.filter((profile) =>
["legacy-codex", "runner-codex"].includes(profile.id),
);
export const runnerSuites: readonly RunnerSuiteFixture[] = [
{
id: "core-compatibility",
@@ -762,6 +881,17 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
excludedExecutionIds: openRouterBreadthExcludedExecutionIds,
},
},
{
id: "daytona-warm-continuity",
label: "Daytona Warm Continuity",
description:
"Three browser-driven turns on one reusable Daytona sandbox for legacy and native Codex.",
groups: ["daytona", "warm"],
profiles: codexContinuityProfiles,
environments: [daytonaWarmEnvironment],
tasks: [daytonaWarmContinuityTask],
expectedMatrixSize: 2,
},
] as const;
export function suiteDefinitionHash(suite: RunnerSuiteFixture) {
@@ -774,7 +904,10 @@ export function suiteDefinitionHash(suite: RunnerSuiteFixture) {
model: profile.model,
qualification: profile.modelQualification,
})),
environments: suite.environments.map((environment) => environment.id),
environments: suite.environments.map((environment) => ({
id: environment.id,
configurationKey: environment.configurationKey ?? "default",
})),
tasks: suite.tasks.map((task) => ({
id: task.id,
flow: task.flow,
@@ -869,6 +1002,7 @@ export function validateRunnerCatalog(): MatrixExecution[] {
...runnerTasks,
...localIntegrityTasks,
...openRouterBreadthTasks,
daytonaWarmContinuityTask,
];
for (const [label, values] of [
["suite", runnerSuites],
@@ -888,6 +1022,7 @@ export function validateRunnerCatalog(): MatrixExecution[] {
...runnerSuites,
...allProfiles,
...runnerEnvironments,
daytonaWarmEnvironment,
...allTasks,
]) {
const unknownGroups = fixture.groups.filter(
@@ -916,7 +1051,7 @@ export function validateRunnerCatalog(): MatrixExecution[] {
]),
);
for (const environment of runnerEnvironments) {
for (const environment of [...runnerEnvironments, daytonaWarmEnvironment]) {
const payload = environment.buildEnvironment({
secretRefs: sampleRefs,
daytonaImage:
@@ -968,8 +1103,8 @@ export function validateRunnerCatalog(): MatrixExecution[] {
);
}
}
if (matrix.length !== 66)
throw new Error(`Expected 66 runner executions; received ${matrix.length}`);
if (matrix.length !== 68)
throw new Error(`Expected 68 runner executions; received ${matrix.length}`);
return matrix;
}
+13
View File
@@ -196,6 +196,18 @@ function renderCase(
</tr>`,
)
.join("");
const turnTimingRows = (entry?.result.turnTimings ?? [])
.map(
(timing) => `<tr>
<td>${timing.turn}</td>
<td><code>${html(timing.runId)}</code></td>
<td>${html(timing.leaseAcquisitionOutcome)}</td>
<td>${html(timing.schedulerLatencyMs === null ? "unavailable" : durationLabel(timing.schedulerLatencyMs))}</td>
<td>${html(timing.runDurationMs === null ? "unavailable" : durationLabel(timing.runDurationMs))}</td>
<td>${html(timing.responseLatencyMs === null ? "unavailable" : durationLabel(timing.responseLatencyMs))}</td>
</tr>`,
)
.join("");
const gallery = screenshots.length
? `<div class="gallery" aria-label="Screenshots for ${html(execution.id)}">${screenshots
.map(
@@ -268,6 +280,7 @@ function renderCase(
: ""
}
<p class="detail">${html(detail)}</p>
${turnTimingRows ? `<div class="matcher-wrap"><table class="matchers"><thead><tr><th>Turn</th><th>Run</th><th>Lease</th><th>Scheduler</th><th>Run duration</th><th>Response</th></tr></thead><tbody>${turnTimingRows}</tbody></table></div>` : ""}
${matcherRows ? `<div class="matcher-wrap"><table class="matchers"><thead><tr><th>Result</th><th>Matcher</th><th>Expectation</th><th>Detail</th></tr></thead><tbody>${matcherRows}</tbody></table></div>` : `<p class="detail">No matcher result was recorded.</p>`}
${entry ? `<details class="usage"><summary>Usage and billing metadata</summary><pre>${html(JSON.stringify({ billing, rawUsage: entry.result.usage ?? null }, null, 2))}</pre></details>` : ""}
${links}
+2 -2
View File
@@ -195,8 +195,8 @@ describe("runner E2E campaign history", () => {
expect(index).toContain("Runner E2E campaigns");
expect(index).toContain("complete-green");
expect(index).toContain("complete-red");
expect(index).toContain("66/66 passed");
expect(index).toContain("65/66 passed");
expect(index).toContain("68/68 passed");
expect(index).toContain("67/68 passed");
expect(index).toContain("Open report&nbsp;→");
expect(index).toContain(
"campaigns/complete-red/public-images/campaign-summary.png",
+76
View File
@@ -70,4 +70,80 @@ describe("live runner fixtures", () => {
"DELETE /api/environments/environment-1?destroyReusableSandboxLeases=true",
);
});
it("creates a primary project workspace for reusable Daytona scope", async () => {
const calls: string[] = [];
const api = {
async post(path: string, data?: Record<string, unknown>) {
calls.push(`POST ${path}`);
if (path === "/api/plugins/install") {
return {
id: "plugin-daytona",
pluginKey: "paperclip.daytona-sandbox-provider",
status: "ready",
};
}
if (path === "/api/companies") {
return { id: "company-1", name: "Runner E2E" };
}
if (path.endsWith("/environments")) {
return { id: "environment-1", driver: "sandbox" };
}
if (path.endsWith("/agents")) {
return { id: "agent-1", name: "Agent", companyId: "company-1" };
}
if (path.endsWith("/projects")) {
expect(data).toMatchObject({
executionWorkspacePolicy: {
enabled: true,
defaultMode: "shared_workspace",
environmentId: "environment-1",
},
workspace: {
sourceType: "local_path",
cwd: "/tmp/workspace",
isPrimary: true,
},
});
return {
id: "project-1",
name: data?.name,
primaryWorkspace: {
id: "project-workspace-1",
cwd: "/tmp/workspace",
},
};
}
throw new Error(`Unexpected POST ${path}`);
},
async postSensitive(_path: string, data?: Record<string, unknown>) {
return { id: `secret-${String(data?.key).toLowerCase()}` };
},
async delete() {},
} as unknown as RunnerApi;
const execution = runnerMatrix.find(
(candidate) =>
candidate.id ===
"daytona-warm-continuity.runner-codex.daytona.warm-three-turn",
)!;
const fixtures = await setupLiveFixtures({
api,
execution,
executionNonce: "nonce",
workspacePath: "/tmp/workspace",
credentials: {
OPENAI_API_KEY: "openai-test-value",
DAYTONA_API_KEY: "daytona-test-value",
},
daytonaImage:
"ghcr.io/paperclip/image@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
});
expect(fixtures.project?.primaryWorkspace?.id).toBe("project-workspace-1");
expect(
calls.indexOf("POST /api/companies/company-1/environments"),
).toBeLessThan(calls.indexOf("POST /api/companies/company-1/projects"));
await fixtures.teardown();
});
});
+48
View File
@@ -32,12 +32,21 @@ interface AgentRecord {
name: string;
companyId: string;
}
interface ProjectRecord {
id: string;
name: string;
primaryWorkspace?: {
id: string;
cwd?: string | null;
} | null;
}
export interface LiveFixtureValues {
company: CompanyRecord;
secretRefs: SecretReferenceMap;
environment: EnvironmentRecord;
agent: AgentRecord;
project?: ProjectRecord;
teardown(): Promise<void>;
}
@@ -210,12 +219,51 @@ export async function setupLiveFixtures(input: {
},
});
if (execution.environment.configurationKey === "warm-reuse-v1") {
registry.register<ProjectRecord>({
id: "project",
dependencies: ["company", "environment"],
async setup(resolved) {
const company = value<CompanyRecord>(resolved, "company");
const environment = value<EnvironmentRecord>(resolved, "environment");
return api.post<ProjectRecord>(
`/api/companies/${company.id}/projects`,
{
name: `Runner E2E warm project ${input.executionNonce}`,
description:
"Ephemeral project anchoring a reusable Daytona execution workspace",
executionWorkspacePolicy: {
enabled: true,
defaultMode: "shared_workspace",
sharedWorkspaceConcurrency: "serialize",
allowIssueOverride: false,
environmentId: environment.id,
workspaceStrategy: { type: "project_primary" },
},
workspace: {
name: "Primary",
sourceType: "local_path",
cwd: input.workspacePath,
isPrimary: true,
},
},
);
},
async teardown() {
// The isolated instance is deleted after provider resources are gone.
},
});
}
const setup = await registry.setupAll();
return {
company: value<CompanyRecord>(setup.values, "company"),
secretRefs: value<SecretReferenceMap>(setup.values, "secrets"),
environment: value<EnvironmentRecord>(setup.values, "environment"),
agent: value<AgentRecord>(setup.values, "agent"),
...(setup.values.has("project")
? { project: value<ProjectRecord>(setup.values, "project") }
: {}),
teardown: setup.teardown,
};
}
+4 -1
View File
@@ -116,6 +116,7 @@ export async function evaluateMatcher(
passed = actual === matcher.expected;
} else if (
matcher.kind === "file_exists" ||
matcher.kind === "file_exact" ||
matcher.kind === "file_contains"
) {
try {
@@ -124,7 +125,9 @@ export async function evaluateMatcher(
(await readFile(matcher.path, "utf8"));
passed =
matcher.kind === "file_exists" ||
String(actual).includes(matcher.expected);
(matcher.kind === "file_exact"
? String(actual) === matcher.expected
: String(actual).includes(matcher.expected));
} catch {
actual = undefined;
passed = false;
+16
View File
@@ -44,6 +44,19 @@ describe("runner E2E report aggregation", () => {
workflowRunUrl: "https://example.test/actions/runs/forged",
},
runIds: ["run-2"],
turnTimings: [
{
turn: 1,
submittedAt: "2026-08-26T00:00:00.000Z",
runStartedAt: "2026-08-26T00:00:00.100Z",
runFinishedAt: "2026-08-26T00:00:01.000Z",
schedulerLatencyMs: 100,
runDurationMs: 900,
responseLatencyMs: 1_000,
runId: "run-2",
leaseAcquisitionOutcome: "created",
},
],
usage: {
inputTokens: 1_250,
outputTokens: 75,
@@ -201,6 +214,9 @@ describe("runner E2E report aggregation", () => {
);
expect(dashboard).toContain("message_contains");
expect(dashboard).toContain("Matchers and test context");
expect(dashboard).toContain("Scheduler");
expect(dashboard).toContain("Run duration");
expect(dashboard).toContain("100ms");
expect(dashboard).toContain("Campaign billing summary");
expect(dashboard).toContain("LLM reported subtotal");
expect(dashboard).toContain("Agent execution time");
+461 -11
View File
@@ -44,6 +44,9 @@ interface IssueRecord {
status: string;
workMode?: string;
assigneeAgentId?: string | null;
projectId?: string | null;
projectWorkspaceId?: string | null;
executionWorkspaceId?: string | null;
executionRunId?: string | null;
checkoutRunId?: string | null;
}
@@ -66,6 +69,9 @@ interface RunRecord {
continuationAttempt?: number;
retryOfRunId?: string | null;
runnerInstanceId?: string | null;
nativeSessionId?: string | null;
processPid?: number | null;
processStartedAt?: string | null;
contextSnapshot?: Record<string, unknown> | null;
runnerProfileJson?: Record<string, unknown> | null;
usageJson?: Record<string, unknown> | null;
@@ -80,13 +86,17 @@ interface RunRecord {
interface EnvironmentLeaseRecord {
id: string;
status?: string;
leasePolicy?: string;
providerLeaseId?: string | null;
executionWorkspaceId?: string | null;
metadata?: Record<string, unknown> | null;
issueId?: string | null;
heartbeatRunId?: string | null;
provider?: string | null;
acquiredAt?: string | null;
releasedAt?: string | null;
updatedAt?: string | null;
metadata?: Record<string, unknown> | null;
}
interface InteractionRecord {
@@ -290,6 +300,7 @@ async function createTaskThroughUi(input: {
title: string;
prompt: string;
workMode: "standard" | "planning" | "ask";
projectName?: string;
}) {
const issuesUrl = `/${encodeURIComponent(input.issuePrefix)}/issues`;
const newTask = input.page.getByRole("button", { name: "New Task" }).first();
@@ -336,9 +347,29 @@ async function createTaskThroughUi(input: {
.getByPlaceholder("Search assignees...")
.fill(input.agentName);
await input.page.getByText(input.agentName, { exact: true }).last().click();
if (input.projectName) {
const dialog = input.page.getByRole("dialog");
await dialog.getByRole("button", { name: "Project", exact: true }).click();
await dialog.getByPlaceholder("Search projects...").fill(input.projectName);
await dialog.getByText(input.projectName, { exact: true }).last().click();
}
const submittedAtMs = Date.now();
await input.page
.getByRole("button", { name: "Create Task", exact: true })
.click();
return submittedAtMs;
}
async function submitTaskReply(page: Page, body: string): Promise<number> {
const composer = page.getByTestId("task-chat-composer-input").last();
await expect(composer).toBeVisible({ timeout: 30_000 });
await composer
.locator('[contenteditable="true"], textarea')
.first()
.fill(body);
const submittedAtMs = Date.now();
await page.getByTestId("task-chat-composer-send").last().click();
return submittedAtMs;
}
function matchingRuns(runs: RunRecord[], issue: IssueRecord) {
@@ -524,7 +555,11 @@ for (const execution of executions) {
page,
request,
}, testInfo) => {
test.setTimeout(deadlineMs + 90_000);
test.setTimeout(
execution.task.flow === "warm_three_turn"
? deadlineMs
: deadlineMs + 90_000,
);
const startedAtMs = Date.now();
const startedAt = new Date(startedAtMs).toISOString();
const nonce = `${randomBytes(6).toString("hex")}-${attempt}`;
@@ -541,6 +576,8 @@ for (const execution of executions) {
let selectedRuns: RunRecord[] = [];
let runtimeLeases: EnvironmentLeaseRecord[] = [];
let matcherResults: MatcherResult[] = [];
let turnTimings: NonNullable<RunnerE2EResult["turnTimings"]> | undefined;
const turnSubmissionTimesMs: number[] = [];
const screenshots: NonNullable<RunnerE2EResult["screenshots"]> = [];
let primaryError: unknown;
let failureClassOverride: FailureClass | undefined;
@@ -746,14 +783,17 @@ for (const execution of executions) {
throw new Error(
"Created fixture company did not return an issue prefix",
);
await createTaskThroughUi({
page,
issuePrefix,
agentName: fixtures.agent.name,
title,
prompt,
workMode: execution.task.workMode,
});
turnSubmissionTimesMs.push(
await createTaskThroughUi({
page,
issuePrefix,
agentName: fixtures.agent.name,
title,
prompt,
workMode: execution.task.workMode,
projectName: fixtures.project?.name,
}),
);
const deadlineAt = startedAtMs + deadlineMs;
issue = await pollUntil({
@@ -781,6 +821,16 @@ for (const execution of executions) {
`UI-created issue work mode was ${String(issue.workMode)}; expected ${execution.task.workMode}`,
);
}
if (fixtures.project) {
if (
issue.projectId !== fixtures.project.id ||
issue.projectWorkspaceId !== fixtures.project.primaryWorkspace?.id
) {
throw new Error(
`Warm task did not retain its project/workspace scope: ${JSON.stringify({ projectId: issue.projectId, projectWorkspaceId: issue.projectWorkspaceId })}`,
);
}
}
await page.goto(
`/${encodeURIComponent(issuePrefix)}/issues/${encodeURIComponent(issue.identifier ?? issue.id)}`,
@@ -827,6 +877,7 @@ for (const execution of executions) {
let planLifecycleEvidence: Record<string, unknown> | null = null;
let questionLifecycleEvidence: Record<string, unknown> | null = null;
let warmLifecycleEvidence: Record<string, unknown> | null = null;
let expectedQuestionResolution: {
interactionId: string;
optionId: string;
@@ -1214,12 +1265,165 @@ for (const execution of executions) {
.last()
.click();
planLifecycleEvidence = { interaction, plan };
} else if (execution.task.flow === "warm_three_turn") {
const followups = execution.task.buildFollowupMessages?.(nonce);
if (!followups || !fixtures.project?.primaryWorkspace?.id) {
throw new Error(
`Warm fixture ${execution.task.id} is missing its project or follow-up messages`,
);
}
const workspaceFile = path.join(
workspacePath,
`daytona-warm-${nonce}.txt`,
);
const turnEvidence: Array<Record<string, unknown>> = [];
for (const completedTurn of [1, 2] as const) {
const turnDeadlineAt = Math.min(
deadlineAt,
turnSubmissionTimesMs[completedTurn - 1]! +
(execution.task.turnTimeoutMs ?? 10 * 60_000),
);
const waitingState = await pollUntil({
label: `warm Daytona turn ${completedTurn} review state for issue ${issue.id}`,
deadlineAt: turnDeadlineAt,
load: loadTaskState,
accept: ({ currentIssue, taskRuns }) =>
currentIssue.status === "in_review" &&
taskRuns.length === completedTurn &&
taskRuns.every((run) => run.status === "succeeded"),
reject: ({ taskRuns }) =>
definitiveRunFailure(taskRuns) ??
(taskRuns.length > completedTurn
? `warm turn ${completedTurn} dispatched duplicate runs`
: undefined),
});
const expectedPrefix = `${Array.from(
{ length: completedTurn },
(_, index) => `T${index + 1}_${nonce}`,
).join("\n")}\n`;
const hostContent = await readFile(workspaceFile, "utf8");
if (hostContent !== expectedPrefix) {
throw new Error(
`Host workspace was not finalized after warm turn ${completedTurn}: expected ${JSON.stringify(expectedPrefix)}, observed ${JSON.stringify(hostContent)}`,
);
}
if (
waitingState.currentIssue.projectId !== fixtures.project.id ||
waitingState.currentIssue.projectWorkspaceId !==
fixtures.project.primaryWorkspace.id ||
!waitingState.currentIssue.executionWorkspaceId
) {
throw new Error(
`Warm turn ${completedTurn} lost its project execution-workspace scope`,
);
}
const completedRunIds = new Set(
waitingState.taskRuns.map((candidate) => candidate.id),
);
const retainedTurnLeases = await pollUntil({
label: `retained Daytona leases after warm turn ${completedTurn}`,
deadlineAt: Math.min(turnDeadlineAt, Date.now() + 30_000),
intervalMs: 500,
load: () =>
api.get<EnvironmentLeaseRecord[]>(
`/api/environments/${fixtures!.environment.id}/leases`,
),
accept: (leases) => {
const runOrder = new Map(
waitingState.taskRuns.map((candidate, index) => [
candidate.id,
index,
]),
);
const completed = leases
.filter(
(lease) =>
lease.heartbeatRunId &&
completedRunIds.has(lease.heartbeatRunId),
)
.sort(
(left, right) =>
(runOrder.get(left.heartbeatRunId ?? "") ?? 0) -
(runOrder.get(right.heartbeatRunId ?? "") ?? 0),
);
return (
completed.length === completedTurn &&
completed.every(
(lease) =>
lease.status === "retained" &&
lease.leasePolicy === "reuse_by_environment" &&
typeof lease.providerLeaseId === "string" &&
record(lease.metadata).sandboxState === "started",
) &&
completed
.slice(1)
.every(
(lease) =>
record(lease.metadata).resumedFromState === "started",
)
);
},
});
const turnRunOrder = new Map(
waitingState.taskRuns.map((candidate, index) => [
candidate.id,
index,
]),
);
const completedLeases = retainedTurnLeases
.filter(
(lease) =>
lease.heartbeatRunId &&
completedRunIds.has(lease.heartbeatRunId),
)
.sort(
(left, right) =>
(turnRunOrder.get(left.heartbeatRunId ?? "") ?? 0) -
(turnRunOrder.get(right.heartbeatRunId ?? "") ?? 0),
);
if (
new Set(completedLeases.map((lease) => lease.providerLeaseId))
.size !== 1
) {
throw new Error(
`Warm turn ${completedTurn} replaced its Daytona sandbox`,
);
}
turnEvidence.push({
turn: completedTurn,
issue: waitingState.currentIssue,
run: waitingState.taskRuns.at(-1),
hostContent,
leases: completedLeases,
});
await page.goto(
`/${encodeURIComponent(issuePrefix)}/issues/${encodeURIComponent(issue.identifier ?? issue.id)}`,
{ waitUntil: "domcontentloaded" },
);
await captureScreenshot(
`warm-turn-${completedTurn}`,
`Warm Daytona turn ${completedTurn} awaiting review`,
`warm-turn-${completedTurn}.png`,
);
turnSubmissionTimesMs.push(
await submitTaskReply(page, followups[completedTurn - 1]),
);
}
warmLifecycleEvidence = { turns: turnEvidence };
}
const taskMatchers = execution.task.buildMatchers(nonce, execution);
const terminalDeadlineAt =
execution.task.flow === "warm_three_turn"
? Math.min(
deadlineAt,
turnSubmissionTimesMs.at(-1)! +
(execution.task.turnTimeoutMs ?? 10 * 60_000),
)
: deadlineAt;
let terminal = await pollUntil({
label: `issue ${issue.id} and heartbeat run terminal state`,
deadlineAt,
deadlineAt: terminalDeadlineAt,
load: loadTaskState,
accept: ({ currentIssue, taskRuns }) =>
currentIssue.status === execution.task.expectedTerminalState.issue &&
@@ -1289,6 +1493,45 @@ for (const execution of executions) {
),
);
selectedRuns = sortRunsChronologically(selectedRuns);
if (execution.task.flow === "warm_three_turn") {
turnTimings = selectedRuns.map((candidate, index) => {
const submittedAtMs = turnSubmissionTimesMs[index]!;
const runStartedAtMs = candidate.startedAt
? Date.parse(candidate.startedAt)
: Number.NaN;
const runFinishedAtMs = candidate.finishedAt
? Date.parse(candidate.finishedAt)
: Number.NaN;
const acquisition = record(
record(candidate.contextSnapshot).paperclipEnvironment,
).sandboxLeaseAcquisition;
const acquisitionOutcome = record(acquisition).outcome;
return {
turn: index + 1,
submittedAt: new Date(submittedAtMs).toISOString(),
runStartedAt: candidate.startedAt ?? null,
runFinishedAt: candidate.finishedAt ?? null,
schedulerLatencyMs: Number.isFinite(runStartedAtMs)
? Math.max(0, runStartedAtMs - submittedAtMs)
: null,
runDurationMs:
Number.isFinite(runStartedAtMs) &&
Number.isFinite(runFinishedAtMs)
? Math.max(0, runFinishedAtMs - runStartedAtMs)
: null,
responseLatencyMs: Number.isFinite(runFinishedAtMs)
? Math.max(0, runFinishedAtMs - submittedAtMs)
: null,
runId: candidate.id,
leaseAcquisitionOutcome:
acquisitionOutcome === "created" ||
acquisitionOutcome === "resumed" ||
acquisitionOutcome === "replacement"
? acquisitionOutcome
: "unknown",
};
});
}
const finalRun = selectedRuns.at(-1)!;
const run =
selectedRuns.find(
@@ -1500,10 +1743,31 @@ for (const execution of executions) {
interactions: terminal.interactions,
},
};
const fileObservations = Object.fromEntries(
await Promise.all(
taskMatchers
.filter(
(matcher) =>
matcher.kind === "file_exists" ||
matcher.kind === "file_exact" ||
matcher.kind === "file_contains",
)
.map(async (matcher) => [
matcher.path,
await readFile(
path.isAbsolute(matcher.path)
? matcher.path
: path.join(workspacePath, matcher.path),
"utf8",
).catch(() => undefined),
]),
),
);
matcherResults = await Promise.all(
taskMatchers.map((matcher) =>
evaluateMatcher(matcher, {
...matcherObservation,
files: fileObservations,
// Multi-run tasks intentionally retain earlier waiting/revision
// replies. Exact completion text belongs to the chronological
// final run, while occurrence checks still span every agent
@@ -1571,6 +1835,190 @@ for (const execution of executions) {
"expected a Daytona sandbox lease on the run context",
);
}
if (execution.task.flow === "warm_three_turn") {
const runEnvironmentContexts = selectedRuns.map((candidate) =>
record(record(candidate.contextSnapshot).paperclipEnvironment),
);
const leaseIds = runEnvironmentContexts.map((entry) => entry.leaseId);
const acquisitionOutcomes = runEnvironmentContexts.map(
(entry) => record(entry.sandboxLeaseAcquisition).outcome,
);
const projectWorkspaceIds = selectedRuns.map(
(candidate) =>
record(record(candidate.contextSnapshot).paperclipWorkspace)
.workspaceId,
);
const executionWorkspaceIds = selectedRuns.map(
(candidate) => record(candidate.contextSnapshot).executionWorkspaceId,
);
if (
leaseIds.some(
(leaseId) => typeof leaseId !== "string" || leaseId.length === 0,
)
) {
invariantFailures.push(
`expected a persisted Daytona lease row for every warm turn; observed ${JSON.stringify(leaseIds)}`,
);
}
if (
JSON.stringify(acquisitionOutcomes) !==
JSON.stringify(["created", "resumed", "resumed"])
) {
invariantFailures.push(
`expected warm lease outcomes created,resumed,resumed; observed ${JSON.stringify(acquisitionOutcomes)}`,
);
}
if (
!fixtures.project?.primaryWorkspace?.id ||
projectWorkspaceIds.some(
(workspaceId) =>
workspaceId !== fixtures!.project!.primaryWorkspace!.id,
) ||
new Set(executionWorkspaceIds).size !== 1 ||
executionWorkspaceIds[0] !== issue.executionWorkspaceId ||
issue.projectId !== fixtures.project.id ||
issue.projectWorkspaceId !== fixtures.project.primaryWorkspace.id ||
!issue.executionWorkspaceId
) {
invariantFailures.push(
`warm task did not preserve project, project-workspace, and execution-workspace identity: ${JSON.stringify({ projectWorkspaceIds, executionWorkspaceIds, projectId: issue.projectId, projectWorkspaceId: issue.projectWorkspaceId, executionWorkspaceId: issue.executionWorkspaceId })}`,
);
}
if (execution.profile.generation === "native") {
const stableIdentityFields: Array<{
label: string;
values: unknown[];
}> = [
{
label: "native session",
values: selectedRuns.map(
(candidate) => candidate.nativeSessionId,
),
},
{
label: "runner instance",
values: selectedRuns.map(
(candidate) => candidate.runnerInstanceId,
),
},
{
label: "provider session",
values: selectedRuns.map((candidate) => candidate.sessionIdAfter),
},
{
label: "runner pid",
values: selectedRuns.map((candidate) => candidate.processPid),
},
{
label: "runner process fingerprint",
values: selectedRuns.map(
(candidate) => candidate.processStartedAt,
),
},
];
for (const { label, values } of stableIdentityFields) {
if (
values.some(
(value) =>
value === null ||
value === undefined ||
String(value).length === 0,
) ||
new Set(values).size !== 1
) {
invariantFailures.push(
`expected one stable ${label} across native warm turns; observed ${JSON.stringify(values)}`,
);
}
}
}
if (
turnTimings?.length !== 3 ||
turnTimings.some(
(timing) =>
timing.runStartedAt === null ||
timing.runFinishedAt === null ||
timing.schedulerLatencyMs === null ||
timing.runDurationMs === null ||
timing.responseLatencyMs === null ||
timing.responseLatencyMs >
(execution.task.turnTimeoutMs ?? 10 * 60_000),
)
) {
invariantFailures.push(
`warm turn timing data was incomplete or exceeded its structural deadline: ${JSON.stringify(turnTimings)}`,
);
}
const retainedLeases = await pollUntil({
label: `retained warm Daytona lease for issue ${issue.id}`,
deadlineAt: Math.min(deadlineAt, Date.now() + 30_000),
intervalMs: 500,
load: () =>
api.get<EnvironmentLeaseRecord[]>(
`/api/environments/${fixtures!.environment.id}/leases`,
),
accept: (leases) => {
const warmLeases = leases.filter(
(lease) =>
lease.issueId === issue!.id &&
selectedRuns.some(
(candidate) => candidate.id === lease.heartbeatRunId,
),
);
return (
warmLeases.length === 3 &&
warmLeases.every(
(lease) =>
lease.status === "retained" &&
lease.leasePolicy === "reuse_by_environment" &&
typeof lease.providerLeaseId === "string" &&
record(lease.metadata).sandboxState === "started",
)
);
},
});
const selectedRunOrder = new Map(
selectedRuns.map((candidate, index) => [candidate.id, index]),
);
const warmLeases = retainedLeases
.filter(
(lease) =>
lease.issueId === issue!.id &&
selectedRunOrder.has(lease.heartbeatRunId ?? ""),
)
.sort(
(left, right) =>
(selectedRunOrder.get(left.heartbeatRunId ?? "") ?? 0) -
(selectedRunOrder.get(right.heartbeatRunId ?? "") ?? 0),
);
const providerLeaseIds = warmLeases.map(
(lease) => lease.providerLeaseId,
);
const resumedFromStates = warmLeases
.slice(1)
.map((lease) => record(lease.metadata).resumedFromState);
if (
new Set(providerLeaseIds).size !== 1 ||
typeof providerLeaseIds[0] !== "string" ||
JSON.stringify(resumedFromStates) !==
JSON.stringify(["started", "started"])
) {
invariantFailures.push(
`expected one continuously-started Daytona sandbox; observed ${JSON.stringify({ providerLeaseIds, resumedFromStates })}`,
);
}
warmLifecycleEvidence = {
...(warmLifecycleEvidence ?? {}),
leaseIds,
acquisitionOutcomes,
projectWorkspaceIds,
executionWorkspaceIds,
providerLeaseIds,
resumedFromStates,
retainedLeases: warmLeases,
turnTimings,
};
}
if (execution.environment.id === "local") {
const runLogContent = String(record(runLog).content ?? "");
// The log endpoint returns NDJSON, so quotes inside each `chunk` are
@@ -1723,6 +2171,7 @@ for (const execution of executions) {
interactions: terminal.interactions,
planLifecycleEvidence,
questionLifecycleEvidence,
warmLifecycleEvidence,
matcherResults,
invariantFailures,
runEvents,
@@ -1940,6 +2389,7 @@ for (const execution of executions) {
issueId: issue?.id,
issueIdentifier: issue?.identifier ?? null,
runIds: selectedRuns.map((run) => run.id),
...(turnTimings ? { turnTimings } : {}),
startedAt,
finishedAt: new Date(finishedAtMs).toISOString(),
durationMs: finishedAtMs - startedAtMs,
+17
View File
@@ -341,6 +341,23 @@ describe("runner E2E matchers", () => {
expect(result?.detail).toContain("observed 2");
});
it("matches finalized workspace files byte-for-byte", async () => {
const [matched, extraLine] = await evaluateMatchers(
[
{ kind: "file_exact", path: "continuity.txt", expected: "T1\nT2\n" },
{ kind: "file_exact", path: "duplicate.txt", expected: "T1\nT2\n" },
],
{
files: {
"continuity.txt": "T1\nT2\n",
"duplicate.txt": "T1\nT2\nT2\n",
},
},
);
expect(matched?.passed).toBe(true);
expect(extraLine?.passed).toBe(false);
});
it("normalizes ordered fragments and evaluates nested JSON Schema", async () => {
const results = await evaluateMatchers(
[
+18 -1
View File
@@ -13,7 +13,8 @@ export type RunnerTaskFlow =
| "single_turn"
| "plan_revision_acceptance"
| "question_resume_completion"
| "plan_approval_completion";
| "plan_approval_completion"
| "warm_three_turn";
export interface SecretReference {
type: "secret_ref";
@@ -73,6 +74,8 @@ export interface RunnerProfileFixture {
export interface EnvironmentFixture {
id: RunnerEnvironmentId;
/** Distinguishes materially different configurations that share a provider ID. */
configurationKey?: string;
label: string;
groups: readonly string[];
driver: "local" | "sandbox";
@@ -103,6 +106,7 @@ export type Matcher =
| { kind: "runtime_mode"; expected: RunnerGeneration }
| { kind: "environment"; expected: RunnerEnvironmentId }
| { kind: "file_exists"; path: string }
| { kind: "file_exact"; path: string; expected: string }
| { kind: "file_contains"; path: string; expected: string }
| { kind: "artifact_exists"; name: string; mimeType?: string }
| { kind: "json_path"; path: string; expected: unknown }
@@ -124,6 +128,8 @@ export interface RunnerTaskFixture {
buildPrompt(nonce: string): string;
buildVisibleMarker(nonce: string): string;
buildRevisionRequest?(nonce: string): string;
buildFollowupMessages?(nonce: string): readonly [string, string];
turnTimeoutMs?: number;
buildQuestionAnswer?(nonce: string): {
optionLabel: string;
expectedMarker: string;
@@ -260,6 +266,17 @@ export interface RunnerE2EResult {
issueId?: string;
issueIdentifier?: string | null;
runIds?: string[];
turnTimings?: Array<{
turn: number;
submittedAt: string;
runStartedAt: string | null;
runFinishedAt: string | null;
schedulerLatencyMs: number | null;
runDurationMs: number | null;
responseLatencyMs: number | null;
runId: string;
leaseAcquisitionOutcome: "created" | "resumed" | "replacement" | "unknown";
}>;
startedAt: string;
finishedAt: string;
durationMs: number;
+1 -1
View File
@@ -4,7 +4,7 @@ import { describe, expect, it } from "vitest";
const repositoryRoot = path.resolve(import.meta.dirname, "../..");
const ordinaryPrTrustedWorkflowRevision =
"a0a78ee60946a5f79f85b2bd0584fc766fae43bb";
"03609aa6ecc9a047ed53d6b6469d8be554fbc46d";
const fullStackTestNeeds =
/needs:\s*\[\s*authorize,\s*target_lock,\s*catalog,\s*daytona_image,\s*build_runner_artifacts,\s*build_remote_provider_pack,?\s*\]/u;
const buildRunnerNeeds =