mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-27 11:20:42 +01:00
fix(recovery): avoid duplicate terminal run status notifications (#17614)
Retain pending status deliveries when Native or legacy terminal recovery projects the same state. Preserve completion time on unchanged Native terminal status. Real state, metadata, error, explicit patch, and retained-file changes still notify. The publisher acknowledgement, accepted results, retry deadlines, recovery holds, ownership checks, and workspace cleanup remain intact. Correct the queued Retry browser fixture race by releasing its old-process guard at the exact Retry request, requiring a successful response, and retaining fixture cleanup after failure. Validated by 159 focused tests, 653 wider Native recovery tests, six real Chromium/PostgreSQL browser cases, five queued Retry repeats, build/typecheck, exact-head Apex 5/5, and green required CI. The original detached-button CI timeout is documented in the PR. The additional full local repository Vitest command is still running and is not claimed complete. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
e335265568
commit
7ebf98eafd
6 files changed
+262
-27
No files matched your search
@@ -246,6 +246,17 @@ The benchmark loads the baseline authorization module with current dependencies,
|
||||
compares complete workspace rows, and prints five warm samples and query plans.
|
||||
It measures SQL execution, not the full repository admission or agent startup.
|
||||
|
||||
Native finalization retries preserve a run's terminal completion time when its
|
||||
status is unchanged. A retry mints another status delivery only when the current
|
||||
run's status, finalization metadata, phase, error fields, or missing completion
|
||||
time changes. An unchanged projection retains its existing pending delivery ID,
|
||||
including after a publish failure. Status delivery stays at least once; retry
|
||||
deadlines, recovery, accepted results, and workspace cleanup still run normally.
|
||||
Legacy recovery also retains the pending delivery when status is unchanged and
|
||||
no patch was supplied. Explicit patches, new terminal statuses, and newly
|
||||
retained workspace evidence still create a delivery. File retention, ownership
|
||||
checks, failed write conditions, lock cleanup, and recovery holds still apply.
|
||||
|
||||
Agent lists read the complete company roster once and use that same snapshot for
|
||||
termination filtering and organization-chain normalization. Detail and plugin runtime reads
|
||||
load only identity, company, name, status, and manager from other company agents
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { agents, companies, createDb, environmentLeases, environments, executionWorkspaces, heartbeatRuns, issueRecoveryActions, issues, projects } from "@paperclipai/db";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { recordLegacyWorkspaceRestoreFailure, legacyExecutionNeedsReconciliationWithEvidence, terminalizeLegacyExecution } from "../services/legacy-execution-recovery.js";
|
||||
@@ -8,6 +8,7 @@ import { hasConversationContinuationPolicy, conversationRecoveryActionPredicate
|
||||
import { preserveWorkspaceRestoreRecoveryMetadataSql } from "../services/legacy-workspace-restore-recovery.js";
|
||||
import { hasConfirmedSandboxStopAndRetain, prepareSandboxStopAndRetain, settleStopOnlyCleanup } from "../services/sandbox-stop-and-retain.js";
|
||||
import { environmentService } from "../services/environments.js";
|
||||
import { deliverExecutionStatuses } from "../services/execution-status-delivery.js";
|
||||
import { hasRequiredWorkspaceRecovery } from "../services/workspace-restore-recovery-state.js";
|
||||
|
||||
const externalTestDatabaseUrl = process.env.PAPERCLIP_TEST_DATABASE_URL?.trim();
|
||||
@@ -45,6 +46,66 @@ const support = externalTestDatabaseUrl ? { supported: true } : await getEmbedde
|
||||
} };
|
||||
const readLease = async (id: string) => (await db.select().from(environmentLeases).where(eq(environmentLeases.id, id)))[0];
|
||||
|
||||
it("preserves pending and acknowledged status delivery on unpatched legacy recovery retries", async () => {
|
||||
await deliverExecutionStatuses(db, { publish: vi.fn() });
|
||||
const f = await seed();
|
||||
const failed = (await terminalizeLegacyExecution({ db, run: f.run, status: "failed", patch }))!;
|
||||
const holds = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId));
|
||||
expect(holds).toHaveLength(1);
|
||||
expect(hasRequiredWorkspaceRecovery(failed.resultJson)).toBe(true);
|
||||
const repeated = (await terminalizeLegacyExecution({ db, run: failed, status: "failed" }))!;
|
||||
expect(repeated.executionStatusDeliveryId).toBe(failed.executionStatusDeliveryId);
|
||||
const publish = vi.fn();
|
||||
await deliverExecutionStatuses(db, { publish });
|
||||
await terminalizeLegacyExecution({ db, run: failed, status: "failed" });
|
||||
await deliverExecutionStatuses(db, { publish });
|
||||
expect(publish.mock.calls.filter(([event]) => event.payload.runId === f.runId)).toHaveLength(1);
|
||||
const after = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId));
|
||||
expect(after).toHaveLength(1);
|
||||
expect(after[0]).toMatchObject({ id: holds[0].id, status: holds[0].status, outcome: holds[0].outcome,
|
||||
ownerType: holds[0].ownerType, wakePolicy: null, evidence: { ...holds[0].evidence,
|
||||
automaticRecovery: expect.objectContaining({ policy: "preserve_without_replay_v1", replay: "blocked",
|
||||
runId: f.runId, actionOutcome: "unknown" }),
|
||||
} });
|
||||
expect(await readLease(f.lease.id)).toMatchObject({ status: "pending_cleanup", leasePolicy: "retain_on_failure" });
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0].resultJson).toEqual(failed.resultJson);
|
||||
});
|
||||
|
||||
it("delivers new retained workspace evidence even when legacy run status is unchanged", async () => {
|
||||
await deliverExecutionStatuses(db, { publish: vi.fn() });
|
||||
const f = await seed();
|
||||
const failed = (await terminalizeLegacyExecution({ db, run: f.run, status: "failed", patch }))!;
|
||||
const publish = vi.fn();
|
||||
await deliverExecutionStatuses(db, { publish });
|
||||
const [added] = await db.insert(environmentLeases).values({ ...f.lease, id: randomUUID(), providerLeaseId: randomUUID() }).returning();
|
||||
const updated = (await terminalizeLegacyExecution({ db, run: failed, status: "failed" }))!;
|
||||
expect(updated.executionStatusDeliveryId).toBeTruthy();
|
||||
expect(updated.executionStatusDeliveryId).not.toBe(failed.executionStatusDeliveryId);
|
||||
expect(updated.resultJson?.workspaceRestoreRecovery).toMatchObject({ leaseIds: expect.arrayContaining([f.lease.id, added.id]) });
|
||||
await deliverExecutionStatuses(db, { publish });
|
||||
expect(publish.mock.calls.filter(([event]) => event.payload.runId === f.runId)).toHaveLength(2);
|
||||
await terminalizeLegacyExecution({ db, run: updated, status: "failed" });
|
||||
await deliverExecutionStatuses(db, { publish });
|
||||
expect(publish.mock.calls.filter(([event]) => event.payload.runId === f.runId)).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("retains write conditions and delivers an explicit patch on unchanged legacy status", async () => {
|
||||
await deliverExecutionStatuses(db, { publish: vi.fn() });
|
||||
const f = await seed();
|
||||
const failed = (await terminalizeLegacyExecution({ db, run: f.run, status: "failed", patch }))!;
|
||||
const publish = vi.fn();
|
||||
await deliverExecutionStatuses(db, { publish });
|
||||
const holds = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId));
|
||||
expect(await terminalizeLegacyExecution({ db, run: failed, status: "failed", writeConditions: [sql`false`] })).toBeNull();
|
||||
await deliverExecutionStatuses(db, { publish });
|
||||
expect(publish.mock.calls.filter(([event]) => event.payload.runId === f.runId)).toHaveLength(1);
|
||||
expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId))).toEqual(holds);
|
||||
const patched = (await terminalizeLegacyExecution({ db, run: failed, status: "failed", patch: { error: "Retained workspace still needs repair" } }))!;
|
||||
expect(patched.error).toBe("Retained workspace still needs repair");
|
||||
await deliverExecutionStatuses(db, { publish });
|
||||
expect(publish.mock.calls.filter(([event]) => event.payload.runId === f.runId)).toHaveLength(2);
|
||||
});
|
||||
|
||||
async function stopAndRetain(f: Awaited<ReturnType<typeof seed>>, retry = false) {
|
||||
let stopping = (await prepareSandboxStopAndRetain(db, f.lease))!;
|
||||
expect(stopping.status).toBe("pending_cleanup");
|
||||
|
||||
@@ -115,7 +115,8 @@ export async function terminalizeLegacyExecution(input: {
|
||||
.set({
|
||||
status: input.recordRestoreFailureOnly ? current.status : status,
|
||||
...patch,
|
||||
executionStatusDeliveryId: randomUUID(),
|
||||
executionStatusDeliveryId: input.recordRestoreFailureOnly || current.status !== status || Object.keys(patch ?? {}).length > 0
|
||||
? randomUUID() : current.executionStatusDeliveryId,
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(
|
||||
@@ -131,10 +132,15 @@ export async function terminalizeLegacyExecution(input: {
|
||||
const retainedLeaseIds = await preserveLegacyWorkspaceRestoreSources(tx as unknown as Db,
|
||||
input.recordRestoreFailureOnly ? { ...updated, status: "failed" } : updated, input.workspaceRestoreSource);
|
||||
if (retainedLeaseIds.length) {
|
||||
[updated] = await tx.update(heartbeatRuns).set({ resultJson: {
|
||||
const retainedResult = {
|
||||
...updated.resultJson,
|
||||
workspaceRestoreRecovery: { schema: LEGACY_WORKSPACE_RECOVERY_SCHEMA, leaseIds: retainedLeaseIds },
|
||||
} }).where(and(eq(heartbeatRuns.id, updated.id), eq(heartbeatRuns.companyId, updated.companyId))).returning();
|
||||
workspaceRestoreRecovery: { schema: LEGACY_WORKSPACE_RECOVERY_SCHEMA, leaseIds: [...retainedLeaseIds].sort() },
|
||||
};
|
||||
[updated] = await tx.update(heartbeatRuns).set({
|
||||
resultJson: retainedResult,
|
||||
executionStatusDeliveryId: sql`case when ${heartbeatRuns.resultJson} is distinct from ${JSON.stringify(retainedResult)}::jsonb
|
||||
then ${randomUUID()}::uuid else ${heartbeatRuns.executionStatusDeliveryId} end`,
|
||||
}).where(and(eq(heartbeatRuns.id, updated.id), eq(heartbeatRuns.companyId, updated.companyId))).returning();
|
||||
}
|
||||
const needsRecovery = input.recordRestoreFailureOnly
|
||||
? retainedLeaseIds.length > 0 || hasRequiredWorkspaceRecovery(updated.resultJson)
|
||||
|
||||
@@ -1608,33 +1608,42 @@ export async function finalizeNativeRun(input: {
|
||||
statusVersionAfter: Number(committed.issue.statusVersion),
|
||||
workspaceFinalizeStatus: input.workspaceFinalizeStatus,
|
||||
};
|
||||
const projectedStatus = terminalState === "succeeded" ? "succeeded" as const
|
||||
: terminalState === "cancelled" ? "cancelled" as const : "failed" as const;
|
||||
const projectedResult = sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb)
|
||||
|| ${JSON.stringify(finalizationMetadata)}::jsonb
|
||||
|| ${clearExecutionFailure ? recoveredExecutionFailureMetadata() : sql`'{}'::jsonb`}`;
|
||||
const projectedError = providerFailure ? sql`coalesce(${heartbeatRuns.error}, ${providerFailure.errorMessage})`
|
||||
: clearExecutionFailure ? sql`null` : heartbeatRuns.error;
|
||||
const projectedErrorCode = providerFailure ? sql`coalesce(${heartbeatRuns.errorCode}, ${providerFailure.errorCode})`
|
||||
: clearExecutionFailure ? sql`null` : heartbeatRuns.errorCode;
|
||||
const projectionChanged = or(
|
||||
sql`${heartbeatRuns.nativePhase} is distinct from ${finalizationPhase}`,
|
||||
sql`${heartbeatRuns.resultJson} is distinct from ${projectedResult}`,
|
||||
sql`${heartbeatRuns.error} is distinct from ${projectedError}`,
|
||||
sql`${heartbeatRuns.errorCode} is distinct from ${projectedErrorCode}`,
|
||||
...(input.projectRunStatus ? [
|
||||
sql`${heartbeatRuns.status} is distinct from ${projectedStatus}`,
|
||||
isNull(heartbeatRuns.finishedAt),
|
||||
] : []),
|
||||
);
|
||||
const projectRun = (tx: Db) => tx
|
||||
.update(heartbeatRuns)
|
||||
.set({
|
||||
executionStatusDeliveryId: randomUUID(),
|
||||
executionStatusDeliveryId: sql`case when ${projectionChanged}
|
||||
then ${randomUUID()}::uuid else ${heartbeatRuns.executionStatusDeliveryId} end`,
|
||||
...(input.projectRunStatus
|
||||
? {
|
||||
status:
|
||||
terminalState === "succeeded"
|
||||
? "succeeded"
|
||||
: terminalState === "cancelled"
|
||||
? "cancelled"
|
||||
: "failed",
|
||||
finishedAt: now,
|
||||
status: projectedStatus,
|
||||
finishedAt: sql`case when ${heartbeatRuns.status} is distinct from ${projectedStatus}
|
||||
then ${now.toISOString()}::timestamptz else coalesce(${heartbeatRuns.finishedAt}, ${now.toISOString()}::timestamptz) end`,
|
||||
}
|
||||
: {}),
|
||||
nativePhase: finalizationPhase,
|
||||
nativePhaseUpdatedAt: now,
|
||||
...(clearExecutionFailure ? { error: null, errorCode: null } : {}),
|
||||
...(providerFailure ? {
|
||||
// Recovery can finalize before heartbeat saves the adapter result.
|
||||
// Retain any independently recorded execution/cleanup failure.
|
||||
error: sql`coalesce(${heartbeatRuns.error}, ${providerFailure.errorMessage})`,
|
||||
errorCode: sql`coalesce(${heartbeatRuns.errorCode}, ${providerFailure.errorCode})`,
|
||||
} : {}),
|
||||
resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb)
|
||||
|| ${JSON.stringify(finalizationMetadata)}::jsonb
|
||||
|| ${clearExecutionFailure ? recoveredExecutionFailureMetadata() : sql`'{}'::jsonb`}`,
|
||||
...(providerFailure ? { error: projectedError, errorCode: projectedErrorCode } : {}),
|
||||
resultJson: projectedResult,
|
||||
updatedAt: now,
|
||||
})
|
||||
.where(eq(heartbeatRuns.id, run.id))
|
||||
|
||||
@@ -0,0 +1,132 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { agents, companies, completionContracts, createDb, heartbeatRuns, issues,
|
||||
nativeRunFinalizations, nativeRunResults } from "@paperclipai/db";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
|
||||
import { deliverExecutionStatuses } from "../execution-status-delivery.js";
|
||||
import { buildIssueMonitorTriggeredPatch, normalizeIssueExecutionPolicy } from "../issue-execution-policy.js";
|
||||
import { PaperclipRunnerToolAuthority } from "./paperclip-runner-tool-authority.js";
|
||||
import { finalizeNativeRun } from "./native-run-finalizer.js";
|
||||
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
describe.skipIf(!support.supported)("native terminal status projection", () => {
|
||||
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
beforeAll(async () => {
|
||||
temporary = await startEmbeddedPostgresTestDatabase("native-terminal-status-");
|
||||
db = createDb(temporary.connectionString);
|
||||
});
|
||||
afterAll(async () => { await temporary?.cleanup(); });
|
||||
|
||||
async function fixture() {
|
||||
await deliverExecutionStatuses(db, { publish: vi.fn() });
|
||||
const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID();
|
||||
const contractId = randomUUID(), resultId = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: "Monitor status", issuePrefix: `S${companyId.slice(0, 7)}` });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Worker", adapterType: "paperclip_runner", status: "active" });
|
||||
await db.insert(issues).values({ id: issueId, companyId, title: "Saved check", status: "in_progress", assigneeAgentId: agentId });
|
||||
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, nativeIssueId: issueId,
|
||||
runtimeMode: "native", status: "running", contextSnapshot: { issueId } });
|
||||
await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId));
|
||||
const authority = new PaperclipRunnerToolAuthority(db, { companyId, agentId, issueId, runId });
|
||||
await authority.execute({ tool: "set_task_monitor", callId: randomUUID(), arguments: {
|
||||
idempotencyKey: "saved-monitor", monitor: { nextCheckAt: new Date(Date.now() + 60_000).toISOString(), notes: "Check the saved result." },
|
||||
} });
|
||||
await db.insert(completionContracts).values({ id: contractId, companyId, issueId, revision: 1,
|
||||
schemaVersion: "paperclip.completion-contract.v1", policyVersion: "test", risk: "standard",
|
||||
completionAuthority: "server_arbiter", incompleteCriteriaPolicy: "preserve_non_terminal",
|
||||
contractJson: { revision: "test", criteria: [] }, canonicalSha256: randomUUID(),
|
||||
createdByActorType: "system", createdByActorId: "test" });
|
||||
const accepted = { result: { schema: "paperclip.run_result.v1", reportedWorkDisposition: "yielded",
|
||||
summary: "Wait for the saved check.", completionClaim: { contractRevision: "test", objectiveSatisfied: false,
|
||||
criteria: [], remainingWork: [{ description: "Check again", blocksCompletion: true }] },
|
||||
continuation: { kind: "monitor", summary: "Wait for the check", idempotencyKey: "provider-selected" },
|
||||
evidence: [], verification: [], attentionRequests: [], artifacts: [] },
|
||||
terminal: { schema: "paperclip.prp.terminal.v1", runTerminalState: "succeeded", turnTerminalState: "completed", reportedWorkDisposition: "yielded" } };
|
||||
const finishedAt = new Date("2020-01-01T00:00:00Z");
|
||||
await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt, completionContractId: contractId }).where(eq(heartbeatRuns.id, runId));
|
||||
await db.insert(nativeRunResults).values({ id: resultId, companyId, issueId, runId, completionContractId: contractId,
|
||||
serverFingerprint: randomUUID(), canonicalSha256: randomUUID(), schemaStatus: "accepted", resultJson: accepted });
|
||||
await db.insert(nativeRunFinalizations).values({ companyId, issueId, runId, phase: "retryable_failure", resultId });
|
||||
const [task] = await db.select().from(issues).where(eq(issues.id, issueId));
|
||||
await db.update(issues).set({ ...buildIssueMonitorTriggeredPatch({ issue: task!,
|
||||
policy: normalizeIssueExecutionPolicy(task!.executionPolicy), triggeredAt: new Date() }), executionRunId: null, checkoutRunId: null,
|
||||
}).where(eq(issues.id, issueId));
|
||||
const finalize = () => finalizeNativeRun({ db, runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
|
||||
const read = async () => (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)))[0]!;
|
||||
return { runId, companyId, issueId, finishedAt, resultId, accepted, finalize, read };
|
||||
}
|
||||
|
||||
it("preserves the original completion time while retrying an accepted monitor result", async () => {
|
||||
const f = await fixture();
|
||||
await f.finalize();
|
||||
expect((await f.read()).finishedAt).toEqual(f.finishedAt);
|
||||
await f.finalize();
|
||||
expect((await f.read()).finishedAt).toEqual(f.finishedAt);
|
||||
const [coordinator] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId));
|
||||
expect(coordinator).toMatchObject({ phase: "retryable_failure", failureCode: "monitor_wait_authority_lost", resultId: f.resultId });
|
||||
expect(coordinator.nextAttemptAt!.getTime()).toBeGreaterThan(Date.now());
|
||||
expect((await db.select().from(nativeRunResults).where(eq(nativeRunResults.id, f.resultId)))[0].resultJson).toEqual(f.accepted);
|
||||
});
|
||||
|
||||
it("retains an unacknowledged status delivery across unchanged finalization retries", async () => {
|
||||
const f = await fixture();
|
||||
await f.finalize();
|
||||
const before = await f.read();
|
||||
expect(before.executionStatusDeliveryId).toBeTruthy();
|
||||
await f.finalize();
|
||||
expect((await f.read()).executionStatusDeliveryId).toBe(before.executionStatusDeliveryId);
|
||||
const publish = vi.fn();
|
||||
await expect(deliverExecutionStatuses(db, { publish, failpoint: () => { throw new Error("after publish"); } })).rejects.toThrow("after publish");
|
||||
await f.finalize();
|
||||
expect((await f.read()).executionStatusDeliveryId).toBe(before.executionStatusDeliveryId);
|
||||
await deliverExecutionStatuses(db, { publish });
|
||||
const deliveries = publish.mock.calls.filter(([event]) => event.payload.runId === f.runId);
|
||||
expect(deliveries).toHaveLength(2);
|
||||
expect(deliveries[0][0].payload.deliveryId).toBe(deliveries[1][0].payload.deliveryId);
|
||||
});
|
||||
|
||||
it("preserves a pending status notification created by another producer", async () => {
|
||||
const f = await fixture();
|
||||
await f.finalize();
|
||||
const newer = randomUUID();
|
||||
await db.update(heartbeatRuns).set({ executionStatusDeliveryId: newer }).where(eq(heartbeatRuns.id, f.runId));
|
||||
await f.finalize();
|
||||
expect((await f.read()).executionStatusDeliveryId).toBe(newer);
|
||||
});
|
||||
|
||||
it.each(["metadata", "status", "missing_completion_time"])("broadcasts a changed terminal projection after acknowledgement (%s)", async change => {
|
||||
const f = await fixture();
|
||||
await f.finalize();
|
||||
const publish = vi.fn();
|
||||
await deliverExecutionStatuses(db, { publish });
|
||||
if (change === "metadata") {
|
||||
await finalizeNativeRun({ db, runId: f.runId, workspaceFinalizeStatus: "failed", projectRunStatus: true });
|
||||
expect((await f.read()).resultJson?.workspaceFinalizeStatus).toBe("failed");
|
||||
} else {
|
||||
await db.update(heartbeatRuns).set(change === "status" ? { status: "failed" } : { finishedAt: null })
|
||||
.where(eq(heartbeatRuns.id, f.runId));
|
||||
await f.finalize();
|
||||
expect(await f.read()).toMatchObject({ status: "succeeded", finishedAt: expect.any(Date) });
|
||||
}
|
||||
await deliverExecutionStatuses(db, { publish });
|
||||
expect(publish.mock.calls.filter(([event]) => event.payload.runId === f.runId)).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("does not broadcast an unchanged terminal projection after acknowledgement or a new DB client", async () => {
|
||||
const f = await fixture();
|
||||
await f.finalize();
|
||||
const publish = vi.fn();
|
||||
await deliverExecutionStatuses(db, { publish });
|
||||
await f.finalize();
|
||||
await deliverExecutionStatuses(db, { publish });
|
||||
const reopened = createDb(temporary.connectionString);
|
||||
try {
|
||||
await finalizeNativeRun({ db: reopened, runId: f.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
|
||||
await deliverExecutionStatuses(reopened, { publish });
|
||||
} finally { await reopened.$client.end(); }
|
||||
expect(publish.mock.calls.filter(([event]) => event.payload.runId === f.runId)).toHaveLength(1);
|
||||
expect((await f.read()).executionStatusDeliveryId).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -92,8 +92,18 @@ for (const action of ["task_retry", "retry_after_reload", "inbox_retry", "messag
|
||||
await expect(page.getByRole("button", { name: "Steer", exact: true })).toHaveCount(0);
|
||||
const retry = page.getByRole("button", { name: "Retry", exact: true });
|
||||
await expect(retry).toBeEnabled();
|
||||
await db.update(heartbeatRuns).set({ processPid: 999999999 }).where(eq(heartbeatRuns.id, sourceRunId));
|
||||
await retry.click();
|
||||
await page.route((url) => url.pathname === `/api/issues/${issue.id}/retry`, async route => {
|
||||
if (route.request().method() !== "POST") return route.continue();
|
||||
await db.update(heartbeatRuns).set({ processPid: 999999999 }).where(eq(heartbeatRuns.id, sourceRunId));
|
||||
await route.continue();
|
||||
}, { times: 1 });
|
||||
const [response] = await Promise.all([
|
||||
page.waitForResponse(response => response.request().method() === "POST" &&
|
||||
new URL(response.url()).pathname === `/api/issues/${issue.id}/retry` &&
|
||||
response.request().postDataJSON()?.observedRunId === sourceRunId),
|
||||
retry.click(),
|
||||
]);
|
||||
expect(response.ok(), `${response.status()} ${await response.text()}`).toBe(true);
|
||||
} else if (action === "automatic_message") {
|
||||
// No Retry, duplicate message, or status change: the saved input runs.
|
||||
} else if (action === "message") {
|
||||
@@ -131,9 +141,15 @@ for (const action of ["task_retry", "retry_after_reload", "inbox_retry", "messag
|
||||
await page.reload();
|
||||
await expect(page.getByText("Answered the pending follow-up once.", { exact: false })).toBeVisible();
|
||||
} finally {
|
||||
await request.patch(`/api/companies/${company.id}`, { data: { status: "archived" } });
|
||||
await closeRegisteredClients(url);
|
||||
await rm(root, { recursive: true, force: true });
|
||||
try {
|
||||
await request.patch(`/api/companies/${company.id}`, { data: { status: "archived" } });
|
||||
} finally {
|
||||
try {
|
||||
await closeRegisteredClients(url);
|
||||
} finally {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
Reference in new issue
Block a user