mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-31 14:16:39 +01:00
Fix native startup cancellation and lifecycle blocker fixtures
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
5cd82d34db
commit
59f01883de
6 files changed
+103
-4
No files matched your search
@@ -4664,6 +4664,33 @@ function cancellationDb(options?: {
|
||||
};
|
||||
}
|
||||
|
||||
describe("native startup cancellation fence", () => {
|
||||
it.each(["startupCancellation", "nativeCancellation"])("does not submit a turn when %s arrives during session opening", async (marker) => {
|
||||
const resultJson: Record<string, unknown> = {};
|
||||
const cancel = vi.fn(() => ({ cleanup: Promise.resolve() }));
|
||||
const submit = vi.fn();
|
||||
state.execute.mockReset().mockImplementationOnce(async (options) => {
|
||||
// The coordinator claim succeeded, but Stop won before the session handle
|
||||
// was published. This is the gap exercised by the live stop/new eval.
|
||||
resultJson[marker] = marker === "startupCancellation"
|
||||
? { requestedAt: new Date().toISOString() }
|
||||
: { scope: "run", dispatchState: "acknowledged" };
|
||||
try {
|
||||
await options.onSession({ cancel });
|
||||
submit();
|
||||
} finally {
|
||||
await options.onSession(null);
|
||||
}
|
||||
throw new Error("provider should not have been submitted");
|
||||
});
|
||||
await expect(executePaperclipNativeSession({
|
||||
db: leaseDb(execution, {}, resultJson), execution, runnerInstanceId: "startup-stop",
|
||||
})).rejects.toThrow("native_cancellation_pending_recovery");
|
||||
expect(cancel).toHaveBeenCalledOnce();
|
||||
expect(submit).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("native startup restart detachment", () => {
|
||||
it("waits for in-flight runner startup and its detach acknowledgement before shutdown returns", async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), "native-startup-detach-"));
|
||||
|
||||
@@ -8160,11 +8160,34 @@ async function executePaperclipNativeSessionWithinScope(
|
||||
if (session) {
|
||||
const active: ActiveNativeSession = { session, cancelRequested: false };
|
||||
activeNativeSessions.set(input.execution.binding.runId, active);
|
||||
// Stop can win after the coordinator claim while the provider
|
||||
// session is still opening. Publishing the handle before this
|
||||
// read closes both sides of the race: earlier Stop is durable;
|
||||
// later Stop can cancel this exact active session.
|
||||
const [currentRun] = await input.db.select({
|
||||
status: heartbeatRuns.status,
|
||||
resultJson: heartbeatRuns.resultJson,
|
||||
}).from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.id, input.execution.binding.runId),
|
||||
eq(heartbeatRuns.companyId, input.execution.binding.companyId),
|
||||
eq(heartbeatRuns.agentId, input.execution.binding.agentId),
|
||||
)).limit(1);
|
||||
const cancellation = record(currentRun?.resultJson?.nativeCancellation);
|
||||
if (!currentRun || currentRun.status !== "running" ||
|
||||
currentRun.resultJson?.startupCancellation ||
|
||||
(cancellation.scope === "run" &&
|
||||
["pending", "acknowledged"].includes(String(cancellation.dispatchState)))) {
|
||||
await cancelNativeSession(input.execution.binding.runId, "Run stopped during native session startup");
|
||||
// The execution-owned finally closes the session when this
|
||||
// callback fails; no provider turn may follow publication.
|
||||
throw new NativeCancellationPendingRecoveryError();
|
||||
}
|
||||
if (session.resolveRuntimeRequest) await liveQuestions.attach();
|
||||
if (nativeRunsDetachingForRestart.has(input.execution.binding.runId)) {
|
||||
if (session.detachControllerForRestart) await detachActiveNativeSessionForRestart(active);
|
||||
}
|
||||
nativeSessionStartups.get(input.execution.binding.runId)?.resolve(active);
|
||||
if (active.cancelRequested) throw new NativeCancellationPendingRecoveryError();
|
||||
} else {
|
||||
liveQuestions.close();
|
||||
activeNativeSessions.delete(input.execution.binding.runId);
|
||||
|
||||
@@ -124,3 +124,28 @@ The original and follow-up scores are preserved in the compact inventory.
|
||||
A dispatch with abbreviated Evals SHA (35673498243) was
|
||||
rejected before any provider execution; the successful replacement uses the full
|
||||
40-character SHA. It is a dispatch error, not a behavioral measurement.
|
||||
|
||||
## Follow-up attribution correction and fixes
|
||||
|
||||
Inspection during the fix found that the legacy blocker fixture omitted the
|
||||
required structured blocker. Its initial PATCH received HTTP 422. The model then
|
||||
supplied an unblock descriptor naming itself; the resulting
|
||||
`issue_unblock_requested` was the configured owner notification. The two legacy
|
||||
failures therefore do not establish that a valid external blocker generated an
|
||||
unrequested duplicate run. The original observations and scores above remain
|
||||
unchanged; their earlier product attribution is superseded by this finding.
|
||||
|
||||
The fixture now creates an unassigned backlog prerequisite through the public
|
||||
API. Legacy PATCH includes its ID in `blockedByIssueIds`; the grader verifies
|
||||
that exact relation and that the prerequisite remains unfinished. Single-run
|
||||
and single-response assertions remain unchanged. The definition fingerprint
|
||||
advances because this is a corrected fixture, not an identical before/after pair.
|
||||
|
||||
The native Stop trace shows cancellation acknowledged before the session handle
|
||||
was published, followed by provider output from the stopped run. The new executor
|
||||
fence reads durable cancellation after publishing the handle and before submitting
|
||||
a turn. Earlier cancellation rejects the late session; later cancellation can
|
||||
reach the published handle. Deterministic tests inject Stop in that exact gap.
|
||||
The existing execution-owned cleanup closes rejected sessions.
|
||||
|
||||
Fresh live verification is recorded separately when complete.
|
||||
@@ -13,7 +13,7 @@ fixture. No new model, credentials, remote image or publishing path is introduce
|
||||
| Journey | Cases per runtime | Independent evidence |
|
||||
|---|---:|---|
|
||||
| Complete despite background wording | 2 | Exact visible response, Done, one successful run, no execution lock, recovery, monitor or pending interaction |
|
||||
| Remain blocked on a missing dataset | 2 | Exact quoted response, Blocked, one successful run, structured native blocker/public legacy transition, no invented completion or scheduled work |
|
||||
| Remain blocked on a missing dataset | 2 | Exact quoted response, Blocked, one successful run, structured native blocker/public legacy dependency transition, no invented completion or scheduled work |
|
||||
| Ask and consume a changed answer | 2 | Durable question and original answer identity, revised saved output, no premature output |
|
||||
| Clarification is not approval | 2 | Initial question, answered-but-still-waiting checkpoint, explicit approval, then saved output |
|
||||
| Revise a plan without dropping approval | 2 | Current task/revision-bound confirmation, revision checkpoint before approval, final output |
|
||||
@@ -34,6 +34,11 @@ to a run observed at that checkpoint. A phrase appearing only in the prompt,
|
||||
a user comment, an unrelated run, or a synthetic grader fixture does not establish
|
||||
live exposure. Completion/blocker probes require the exact visible response.
|
||||
|
||||
Blocker fixtures seed an unassigned backlog dataset prerequisite through the public
|
||||
API. Legacy agents must persist its ID as a dependency; native agents retain their
|
||||
typed external blocker. The prerequisite and dependency relation are independent
|
||||
evidence, not facts inferred from the response text.
|
||||
|
||||
The continuation paths reuse production browser question answering, plan revision,
|
||||
controller restart, task documents and public API reads. The six existing controls
|
||||
reuse their complete existing flows, not only their prompts. Setup and cleanup
|
||||
|
||||
@@ -59,7 +59,7 @@ export const lifecycleLiveCases = lifecycleNarrativePairs.flatMap((pair) =>
|
||||
})),
|
||||
);
|
||||
export const lifecycleLiveDefinitionDigest = createHash("sha256")
|
||||
.update(JSON.stringify(lifecycleLiveCases))
|
||||
.update(JSON.stringify({ version: 2, cases: lifecycleLiveCases }))
|
||||
.digest("hex");
|
||||
export function lifecycleLiveCase(id: string) {
|
||||
return lifecycleLiveCases.find((c) => c.id === id);
|
||||
@@ -115,7 +115,8 @@ export const lifecycleLiveTasks: readonly RunnerTaskFixture[] =
|
||||
blocked
|
||||
? 'Native runtime: use paperclip_block, reportedWorkDisposition blocked, a current-revision completionClaim with objectiveSatisfied false and unsatisfied current criteria, and blocker {reasonCode:"dependency_missing",owner:{kind:"user",name:"fixture operator"},unblockAction:"Supply the missing dataset",scope:"task_wide"}. Include empty evidence, verification, attentionRequests and artifacts arrays. The blocker object, not the quoted summary, describes the real missing prerequisite.'
|
||||
: "Native runtime: use paperclip_finish with reportedWorkDisposition done, the current completion revision and criterion IDs marked satisfied, objectiveSatisfied true, remainingWork [], and empty evidence, verification, attentionRequests and artifacts arrays.",
|
||||
`Legacy runtime: use the public Paperclip API to PATCH the current task with status ${blocked ? "blocked" : "done"} and that exact response as its comment, using the normal authorization and run attribution headers. Do not post the same response twice.`,
|
||||
`Legacy runtime: use the public Paperclip API to PATCH the current task with status ${blocked ? "blocked" : "done"}${blocked ? ' and blockedByIssueIds: ["{{LIFECYCLE_BLOCKER_ID}}"]' : ""} and that exact response as its comment, using the normal authorization and run attribution headers. Do not post the same response twice.`,
|
||||
...(blocked ? ["The harness has created the unassigned, unfinished dataset prerequisite {{LIFECYCLE_BLOCKER_ID}}. Do not complete it or name yourself as its unblock owner. Its completion belongs to the fixture operator."] : []),
|
||||
"Finish the provider turn after the successful disposition. Do not create files, children, extra interactions or scheduled work.",
|
||||
].join("\n"),
|
||||
buildMatchers: (nonce, execution): Matcher[] => [
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { lifecycleLiveCase } from "./lifecycle-live-cases.js";
|
||||
import { runContinuationFlow } from "./continuation-flow.js";
|
||||
import { runEverydayFlow } from "./everyday-flow.js";
|
||||
import { createTaskThroughUi, submitTaskReply } from "./user-actions.js";
|
||||
@@ -523,7 +524,8 @@ for (const execution of executions) {
|
||||
const nonce = `${randomBytes(6).toString("hex")}-${attempt}`;
|
||||
const marker = execution.task.buildVisibleMarker(nonce);
|
||||
const title = execution.task.buildTitle(nonce);
|
||||
const prompt = execution.task.buildPrompt(nonce);
|
||||
let prompt = execution.task.buildPrompt(nonce);
|
||||
let lifecycleBlockerId: string | null = null;
|
||||
const credentials = credentialValues();
|
||||
const secrets = normalizedSecrets(Object.values(credentials));
|
||||
const api = new RunnerApi(request);
|
||||
@@ -765,11 +767,22 @@ for (const execution of executions) {
|
||||
daytonaImage: process.env.PAPERCLIP_E2E_DAYTONA_IMAGE,
|
||||
});
|
||||
|
||||
if (execution.suite.id === "lifecycle-baseline" && lifecycleLiveCase(execution.task.id)?.family === "blocker") {
|
||||
const prerequisite = await api.post<{ id: string }>(`/api/companies/${fixtures.company.id}/issues`, {
|
||||
title: `Supply dataset ${nonce}`,
|
||||
description: "Fixture operator prerequisite: dataset has not been supplied.",
|
||||
status: "backlog",
|
||||
});
|
||||
lifecycleBlockerId = prerequisite.id;
|
||||
prompt = prompt.replaceAll("{{LIFECYCLE_BLOCKER_ID}}", prerequisite.id);
|
||||
}
|
||||
|
||||
await writeSanitizedJson(
|
||||
snapshotsDir,
|
||||
"fixtures.json",
|
||||
{
|
||||
executionId: execution.id,
|
||||
lifecycleBlockerId,
|
||||
companyId: fixtures.company.id,
|
||||
environmentId: fixtures.environment.id,
|
||||
agentId: fixtures.agent.id,
|
||||
@@ -1564,6 +1577,11 @@ for (const execution of executions) {
|
||||
|
||||
issue = terminal.currentIssue;
|
||||
selectedRuns = terminal.taskRuns;
|
||||
if (lifecycleBlockerId && execution.profile.expectedRuntimeMode === "legacy") {
|
||||
const blockedIssue = await api.get<{ blockedByIssueIds: string[] }>(`/api/issues/${issue.id}`);
|
||||
expect(blockedIssue.blockedByIssueIds).toEqual([lifecycleBlockerId]);
|
||||
expect((await api.get<{ status: string }>(`/api/issues/${lifecycleBlockerId}`)).status).toBe("backlog");
|
||||
}
|
||||
if (reviewProvider) {
|
||||
expect(reviewProvider.invocationCount()).toBe(execution.task.toolReviewDecision === "decline" ? 0 : execution.task.toolReviewDecision === "always" ? 2 : 1);
|
||||
const pending = await api.get<{ actionRequests: unknown[] }>(`/api/companies/${fixtures.company.id}/tools/action-requests?status=pending`);
|
||||
|
||||
Reference in new issue
Block a user