|
|
|
@@ -1,3 +1,22 @@
|
|
|
|
|
import { createAdmissionTransactionScope, createWakeAdmissionReader, createWakeAdmissionWriter } from "../modules/wake-queue/adapters/postgres.js";
|
|
|
|
|
import { createAdmitWakeBehindIssueExecution } from "../modules/wake-queue/application/use-cases.js";
|
|
|
|
|
import { filterZombieCoalesceTarget } from "./heartbeat.js";
|
|
|
|
|
import { mergeCoalescedContextSnapshot } from "./heartbeat/run-preparation.js";
|
|
|
|
|
import { recoveryService } from "./recovery/index.js";
|
|
|
|
|
import { inspectWorkspaceSource } from "./workspace-source-inspection.js";
|
|
|
|
|
import { appendHeartbeatRunEvent } from "./heartbeat-run-events.js";
|
|
|
|
|
import { isPreDispatchExecutionWaitVerified } from "./pre-dispatch-execution-wait.js";
|
|
|
|
|
import { heartbeatRunEvents } from "@paperclipai/db";
|
|
|
|
|
import { createHeartbeatQueue, shouldDeferFollowupWakeForSameIssue, shouldQueueFollowupForRunningIssueWake } from "./heartbeat/queue.js";
|
|
|
|
|
import { createHeartbeatQueueTestDependencies } from "../__tests__/helpers/heartbeat-queue.js";
|
|
|
|
|
import { recordRequiredWorkspaceRestoreFailure } from "./legacy-execution-recovery.js";
|
|
|
|
|
import { admitExplicitNativeContinuation } from "./explicit-native-continuation.js";
|
|
|
|
|
import { withQueuedCommentIdsInWakePayload } from "./issue-queued-comment-queue.js";
|
|
|
|
|
import * as recoverySources from "./workspace-recovery-source.js";
|
|
|
|
|
import { requestedRunCancellation } from "./run-cancellation.js";
|
|
|
|
|
import { createHeartbeatRunControl } from "./heartbeat/run-control.js";
|
|
|
|
|
import { callbacks } from "./heartbeat/__tests__/run-control-fixtures.js";
|
|
|
|
|
import { adapterExecutionControls, createAdapterExecutionControl } from "./adapter-execution-control.js";
|
|
|
|
|
import { resolveExternalTestDatabaseUrl } from "../__tests__/helpers/external-test-database.js";
|
|
|
|
|
import { DELIVERY_QUEUES, subscribeDeliveryWork } from "./delivery-work-notifications.js";
|
|
|
|
|
import { prepareHeartbeatWorkspace } from "./heartbeat/workspace-preparation.js";
|
|
|
|
@@ -12,7 +31,7 @@ import { projectTaskInterruption } from "./task-interruption-presentation.js";
|
|
|
|
|
import * as nativeWorkspaceSync from "./native-runtime/native-workspace-sync.js";
|
|
|
|
|
import { GitShallowMetadataError } from "@paperclipai/adapter-utils/git-workspace-sync";
|
|
|
|
|
import { readSystemRecovery } from "./system-recovery-state.js";
|
|
|
|
|
import { adoptSystemRecovery } from "./system-recovery.js";
|
|
|
|
|
import { adoptSystemRecovery, updateSystemRecovery } from "./system-recovery.js";
|
|
|
|
|
import { getStorageService } from "../storage/index.js";
|
|
|
|
|
import { attachPreservedWorkspaceSnapshots } from "./workspace-recovery-snapshot.js";
|
|
|
|
|
import { adoptHistoricalWorkspaceRecoveries, historicalRecoveryCounts } from "./workspace-recovery-backfill.js";
|
|
|
|
@@ -33,10 +52,10 @@ import { resumePreservedResultFinalization } from "./preserved-result-finalizati
|
|
|
|
|
import type { AdapterSandboxExecutionTarget } from "@paperclipai/adapter-utils/execution-target";
|
|
|
|
|
import type { EnvironmentLease } from "@paperclipai/shared";
|
|
|
|
|
import { startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js";
|
|
|
|
|
import { prepareNativeWorkspaceSync } from "./native-runtime/native-workspace-sync.js";
|
|
|
|
|
import { prepareNativeWorkspaceSync, resumeNativeWorkspaceSync } from "./native-runtime/native-workspace-sync.js";
|
|
|
|
|
import { nativeWorkspaceFinalizationLockKey } from "./native-runtime/native-workspace-finalization-ownership.js";
|
|
|
|
|
import { remoteTerminationReceipt } from "./remote-execution-termination.js";
|
|
|
|
|
import { prepareStoppedSandboxForRecovery } from "./sandbox-stop-and-retain.js";
|
|
|
|
|
import { prepareStoppedSandboxForRecovery, prepareSandboxStopAndRetain, settleStopOnlyCleanup, readStopOnlyCleanup } from "./sandbox-stop-and-retain.js";
|
|
|
|
|
import { recoverRetainedWorkspace, requestWorkspaceRecovery, scheduleRecoveredWorkspaceCleanup, reconcileRetainedWorkspaces, retainedWorkspaceCandidates } from "./workspace-recovery.js";
|
|
|
|
|
import * as activity from "./activity-log.js";
|
|
|
|
|
import { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js";
|
|
|
|
@@ -79,8 +98,8 @@ describe("automatic retained workspace recovery", () => {
|
|
|
|
|
if (priorHome === undefined) delete process.env.PAPERCLIP_HOME; else process.env.PAPERCLIP_HOME = priorHome;
|
|
|
|
|
if (priorInstance === undefined) delete process.env.PAPERCLIP_INSTANCE_ID; else process.env.PAPERCLIP_INSTANCE_ID = priorInstance;
|
|
|
|
|
});
|
|
|
|
|
async function seed(mode = "native", nativeDescriptor = true, resourceDisposition: "destroy" | "keep_running" = "destroy") {
|
|
|
|
|
const companyId = randomUUID(), issueId = randomUUID(), agentId = randomUUID(), runId = randomUUID(), actionId = randomUUID();
|
|
|
|
|
async function seed(mode = "native", nativeDescriptor = true, resourceDisposition: "destroy" | "keep_running" = "destroy", reusable = false, ephemeral = false) {
|
|
|
|
|
const companyId = randomUUID(), issueId = randomUUID(), agentId = randomUUID(), runId = randomUUID(), actionId: string = randomUUID();
|
|
|
|
|
await db.insert(companies).values({ id: companyId, name: "Workspace recovery", defaultResponsibleUserId: "board", issuePrefix: `R${companyId.slice(0, 6)}` });
|
|
|
|
|
await db.insert(agents).values({ id: agentId, companyId, name: "Worker", adapterType: "paperclip_runner", status: "idle", runtimeConfig: { heartbeat: { maxConcurrentRuns: 1 } } });
|
|
|
|
|
await db.insert(issues).values({ id: issueId, companyId, assigneeAgentId: agentId, status: "blocked", title: "Recover existing work" });
|
|
|
|
@@ -91,10 +110,13 @@ describe("automatic retained workspace recovery", () => {
|
|
|
|
|
const [environment] = await db.insert(environments).values({ name: runId, driver: "sandbox" }).returning();
|
|
|
|
|
const providerLeaseId = randomUUID(), remote = path.join(root, runId, "remote"), host = path.join(root, runId, "host");
|
|
|
|
|
await mkdir(host, { recursive: true }); await writeFile(path.join(host, "existing.txt"), "base\n");
|
|
|
|
|
const [lease] = await db.insert(environmentLeases).values({ companyId, issueId, heartbeatRunId: runId,
|
|
|
|
|
environmentId: environment.id, provider: "daytona", providerLeaseId, leasePolicy: "retain_on_failure", status: "released",
|
|
|
|
|
cleanupStatus: "success", releasedAt: new Date(), metadata: { remoteCwd: remote, driver: "sandbox", pluginId: randomUUID(), sandboxProviderPlugin: true },
|
|
|
|
|
}).returning();
|
|
|
|
|
const leaseValues = { companyId, issueId, heartbeatRunId: runId,
|
|
|
|
|
environmentId: environment.id, provider: "daytona", providerLeaseId,
|
|
|
|
|
metadata: { remoteCwd: remote, driver: "sandbox", pluginId: randomUUID(), sandboxProviderPlugin: true, ...(reusable ? { agentId } : {}) } };
|
|
|
|
|
const acquired = reusable ? await environmentService(db).acquireLease({ ...leaseValues, leasePolicy: "reuse_by_environment" }) : null;
|
|
|
|
|
const [lease] = acquired ? await db.select().from(environmentLeases).where(eq(environmentLeases.id, acquired.id))
|
|
|
|
|
: await db.insert(environmentLeases).values({ ...leaseValues, leasePolicy: ephemeral ? "ephemeral" : "retain_on_failure", status: "released",
|
|
|
|
|
cleanupStatus: "success", releasedAt: new Date() }).returning();
|
|
|
|
|
const target: AdapterSandboxExecutionTarget = { kind: "remote", transport: "sandbox", providerKey: "test", leaseId: lease.id,
|
|
|
|
|
remoteCwd: remote, timeoutMs: 30_000, sandboxLeaseAcquisition: { outcome: "created", providerLeaseId },
|
|
|
|
|
runner: { execute: input => new Promise((resolve, reject) => {
|
|
|
|
@@ -144,6 +166,207 @@ describe("automatic retained workspace recovery", () => {
|
|
|
|
|
const action = async (id: string) => (await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, id)))[0]!;
|
|
|
|
|
const run = async (id: string) => (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)))[0]!;
|
|
|
|
|
|
|
|
|
|
async function warmHandoffSource(hops = 1, beforeStop?: (lease: typeof environmentLeases.$inferSelect) => Promise<void>) {
|
|
|
|
|
const prior = await seed("native", true, "keep_running", true), source = await run(prior.runId);
|
|
|
|
|
let previousTarget = targets.get(prior.lease.id)!, previousLeaseId = prior.lease.id;
|
|
|
|
|
let previousRunId: string = prior.runId;
|
|
|
|
|
let lease = prior.lease, current = source;
|
|
|
|
|
const leaseIds = [previousLeaseId];
|
|
|
|
|
for (let hop = 0; hop < hops; hop++) {
|
|
|
|
|
expect(await resumeNativeWorkspaceSync({ db, runId: previousRunId, target: previousTarget })).toBe(true);
|
|
|
|
|
await db.update(heartbeatRuns).set({ status: "succeeded", errorCode: null, resultJson: null }).where(eq(heartbeatRuns.id, previousRunId));
|
|
|
|
|
await db.update(nativeRunFinalizations).set({ phase: "committed" }).where(eq(nativeRunFinalizations.runId, previousRunId));
|
|
|
|
|
await db.delete(issueRecoveryActions).where(eq(issueRecoveryActions.id, prior.actionId));
|
|
|
|
|
await environmentService(db).releaseLease(previousLeaseId, "released", { cleanupStatus: "success" });
|
|
|
|
|
const [released] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, previousLeaseId));
|
|
|
|
|
[current] = await db.insert(heartbeatRuns).values({ companyId: prior.companyId, agentId: source.agentId,
|
|
|
|
|
issueId: prior.issueId, nativeIssueId: prior.issueId, runtimeMode: "native", status: "cancelled", finishedAt: new Date(),
|
|
|
|
|
contextSnapshot: { issueId: prior.issueId } }).returning();
|
|
|
|
|
await db.insert(nativeRunFinalizations).values({ companyId: prior.companyId, issueId: prior.issueId, runId: current.id, phase: "terminal_failure" });
|
|
|
|
|
const acquired = await environmentService(db).acquireLease({ companyId: prior.companyId, environmentId: prior.lease.environmentId!,
|
|
|
|
|
issueId: prior.issueId, heartbeatRunId: current.id, provider: "daytona", providerLeaseId: prior.providerLeaseId,
|
|
|
|
|
leasePolicy: "reuse_by_environment", replacesReusableLeaseId: previousLeaseId,
|
|
|
|
|
metadata: { ...released.metadata, sandboxLeaseAcquisition: { outcome: "resumed" } } });
|
|
|
|
|
[lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, acquired.id));
|
|
|
|
|
const target = { ...previousTarget, leaseId: lease.id, sandboxLeaseAcquisition: { outcome: "resumed" as const, providerLeaseId: prior.providerLeaseId } };
|
|
|
|
|
targets.set(lease.id, target);
|
|
|
|
|
expect(await prepareNativeWorkspaceSync({ db, companyId: prior.companyId, workspaceId: prior.issueId,
|
|
|
|
|
workspaceLocalDir: prior.host, runId: current.id, lease: lease as EnvironmentLease, target, resourceDisposition: "keep_running" })).not.toBeNull();
|
|
|
|
|
leaseIds.push(lease.id); previousRunId = current.id; previousLeaseId = lease.id; previousTarget = target;
|
|
|
|
|
}
|
|
|
|
|
[lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, lease.id));
|
|
|
|
|
await beforeStop?.(lease);
|
|
|
|
|
await writeFile(path.join(prior.remote, "after-warm-stop.txt"), "stopped turn work\n");
|
|
|
|
|
await recordRequiredWorkspaceRestoreFailure(db, await run(current.id), { workspaceRestoreFailure: "restore_failed" }, lease);
|
|
|
|
|
const [action] = await db.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.companyId, prior.companyId),
|
|
|
|
|
eq(issueRecoveryActions.fingerprint, `native-workspace:${current.id}`)));
|
|
|
|
|
const [retained] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, lease.id));
|
|
|
|
|
const stopping = readStopOnlyCleanup(retained) ? retained : await prepareSandboxStopAndRetain(db, retained);
|
|
|
|
|
expect(stopping).not.toBeNull();
|
|
|
|
|
expect(await settleStopOnlyCleanup(db, stopping!, { attemptId: String(stopping!.metadata!.pendingCleanupAttemptId),
|
|
|
|
|
receipt: { providerLeaseId: prior.providerLeaseId, state: "stopped" } })).not.toBeNull();
|
|
|
|
|
return { ...prior, priorRunId: prior.runId, priorLeaseId: prior.lease.id, leaseIds, runId: current.id, lease, actionId: action.id,
|
|
|
|
|
input: { ...prior.input, runId: current.id, actionId: action.id } };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
describe("verified warm allocation handoffs", () => {
|
|
|
|
|
let original: { db: typeof db; root: string; connectionString: string; home: string | undefined };
|
|
|
|
|
let isolated: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
|
|
|
|
|
let isolatedDb: typeof db | undefined;
|
|
|
|
|
let isolatedRoot: string | undefined;
|
|
|
|
|
beforeAll(async () => {
|
|
|
|
|
original = { db, root, connectionString, home: process.env.PAPERCLIP_HOME };
|
|
|
|
|
isolated = await startEmbeddedPostgresTestDatabase("workspace-warm-handoff-");
|
|
|
|
|
isolatedDb = createDb(isolated.connectionString);
|
|
|
|
|
isolatedRoot = await mkdtemp(path.join(os.tmpdir(), "workspace-warm-handoff-files-"));
|
|
|
|
|
db = isolatedDb; root = isolatedRoot; connectionString = isolated.connectionString;
|
|
|
|
|
process.env.PAPERCLIP_HOME = path.join(root, "home");
|
|
|
|
|
}, 30_000);
|
|
|
|
|
afterAll(async () => {
|
|
|
|
|
await isolatedDb?.$client.end(); await isolated?.cleanup();
|
|
|
|
|
if (isolatedRoot) await rm(isolatedRoot, { recursive: true, force: true });
|
|
|
|
|
if (original) {
|
|
|
|
|
db = original.db; root = original.root; connectionString = original.connectionString;
|
|
|
|
|
if (original.home === undefined) delete process.env.PAPERCLIP_HOME; else process.env.PAPERCLIP_HOME = original.home;
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
it.each([1, 2])("recovers a stopped native source after %s verified warm predecessor handoffs", async hops => {
|
|
|
|
|
const f = await warmHandoffSource(hops);
|
|
|
|
|
expect((await run(f.priorRunId)).runnerProfileJson?.nativeWorkspaceSync).toMatchObject({ state: "finalized", leaseId: f.priorLeaseId });
|
|
|
|
|
const [previous] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.priorLeaseId));
|
|
|
|
|
expect(previous).toMatchObject({ status: "expired", cleanupStatus: "success", providerLeaseId: f.providerLeaseId });
|
|
|
|
|
await recoverRetainedWorkspace(f.input);
|
|
|
|
|
expect(readSystemRecovery((await action(f.actionId)).evidence)?.state).toBe("complete");
|
|
|
|
|
expect(await hasVerifiedWorkspaceRepair(db, await run(f.runId))).toBe(true);
|
|
|
|
|
expect(await readFile(path.join(f.host, "after-warm-stop.txt"), "utf8")).toBe("stopped turn work\n");
|
|
|
|
|
const source = await run(f.runId), freshProviderLeaseId = randomUUID();
|
|
|
|
|
const [fresh] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: source.agentId,
|
|
|
|
|
issueId: f.issueId, nativeIssueId: f.issueId, runtimeMode: "native", status: "succeeded", finishedAt: new Date() }).returning();
|
|
|
|
|
const remoteCwd = path.join(root, fresh.id, "remote");
|
|
|
|
|
const freshLease = await environmentService(db).acquireLease({ companyId: f.companyId, environmentId: f.lease.environmentId!,
|
|
|
|
|
issueId: f.issueId, heartbeatRunId: fresh.id, provider: "daytona", providerLeaseId: freshProviderLeaseId,
|
|
|
|
|
metadata: { ...f.lease.metadata, remoteCwd } });
|
|
|
|
|
const freshTarget = { ...targets.get(f.lease.id)!, leaseId: freshLease.id, remoteCwd,
|
|
|
|
|
sandboxLeaseAcquisition: { outcome: "created" as const, providerLeaseId: freshProviderLeaseId } };
|
|
|
|
|
expect(await prepareNativeWorkspaceSync({ db, companyId: f.companyId, workspaceId: f.issueId,
|
|
|
|
|
workspaceLocalDir: f.host, runId: fresh.id, lease: freshLease, target: freshTarget, resourceDisposition: "keep_running" })).not.toBeNull();
|
|
|
|
|
expect(await resumeNativeWorkspaceSync({ db, runId: fresh.id, target: freshTarget })).toBe(true);
|
|
|
|
|
expect(await inspectWorkspaceSource(db, { companyId: f.companyId, issueId: f.issueId, runId: f.runId, actionId: f.actionId }))
|
|
|
|
|
.toMatchObject({ status: "verified" });
|
|
|
|
|
}, 30_000);
|
|
|
|
|
|
|
|
|
|
it.each(["active", "active_run", "active_control", "foreign_company", "environment", "workspace", "task", "agent", "unfinalized", "owner", "missing_proof", "conflicting_proof", "dangling_proof", "extra_history"])(
|
|
|
|
|
"keeps ambiguous warm source history fenced: %s", async fault => {
|
|
|
|
|
const f = await warmHandoffSource();
|
|
|
|
|
const [previous] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.priorLeaseId));
|
|
|
|
|
if (fault === "active") await db.update(environmentLeases).set({ status: "active" }).where(eq(environmentLeases.id, previous.id));
|
|
|
|
|
if (fault === "active_run") await db.update(heartbeatRuns).set({ status: "running" }).where(eq(heartbeatRuns.id, f.priorRunId));
|
|
|
|
|
if (fault === "active_control") adapterExecutionControls.set(f.priorRunId, createAdapterExecutionControl());
|
|
|
|
|
if (fault === "foreign_company") {
|
|
|
|
|
const [foreign] = await db.insert(companies).values({ name: "Foreign owner", issuePrefix: `F${randomUUID().slice(0, 6)}` }).returning();
|
|
|
|
|
await db.update(environmentLeases).set({ companyId: foreign.id }).where(eq(environmentLeases.id, previous.id));
|
|
|
|
|
}
|
|
|
|
|
if (fault === "environment") {
|
|
|
|
|
const [other] = await db.insert(environments).values({ name: randomUUID(), driver: "sandbox" }).returning();
|
|
|
|
|
await db.update(environmentLeases).set({ environmentId: other.id }).where(eq(environmentLeases.id, previous.id));
|
|
|
|
|
}
|
|
|
|
|
if (fault === "workspace") {
|
|
|
|
|
const [other] = await db.insert(executionWorkspaces).values({ companyId: f.companyId, mode: "isolated_workspace", strategyType: "test", name: "Other workspace" }).returning();
|
|
|
|
|
await db.update(environmentLeases).set({ executionWorkspaceId: other.id }).where(eq(environmentLeases.id, previous.id));
|
|
|
|
|
}
|
|
|
|
|
if (fault === "task") {
|
|
|
|
|
const [other] = await db.insert(issues).values({ companyId: f.companyId, title: "Other task" }).returning();
|
|
|
|
|
await db.update(environmentLeases).set({ issueId: other.id }).where(eq(environmentLeases.id, previous.id));
|
|
|
|
|
}
|
|
|
|
|
if (fault === "agent") {
|
|
|
|
|
const [other] = await db.insert(agents).values({ companyId: f.companyId, name: "Other agent", adapterType: "paperclip_runner" }).returning();
|
|
|
|
|
await db.update(heartbeatRuns).set({ agentId: other.id }).where(eq(heartbeatRuns.id, f.priorRunId));
|
|
|
|
|
}
|
|
|
|
|
if (fault === "unfinalized") {
|
|
|
|
|
const prior = await run(f.priorRunId), reference = prior.runnerProfileJson!.nativeWorkspaceSync as Record<string, unknown>;
|
|
|
|
|
await db.update(heartbeatRuns).set({ runnerProfileJson: { ...prior.runnerProfileJson, nativeWorkspaceSync: { ...reference, state: "prepared", finalHostSha256: null } } }).where(eq(heartbeatRuns.id, prior.id));
|
|
|
|
|
}
|
|
|
|
|
if (fault === "owner") await db.update(nativeRunFinalizations).set({ leaseOwner: randomUUID() }).where(eq(nativeRunFinalizations.runId, f.priorRunId));
|
|
|
|
|
if (fault === "missing_proof") {
|
|
|
|
|
const metadata = { ...previous.metadata }; delete metadata.reusableLeaseSuccessor;
|
|
|
|
|
await db.update(environmentLeases).set({ metadata }).where(eq(environmentLeases.id, previous.id));
|
|
|
|
|
}
|
|
|
|
|
if (fault === "conflicting_proof") await db.update(environmentLeases).set({ metadata: { ...previous.metadata,
|
|
|
|
|
reusableLeaseSuccessor: { ...(previous.metadata!.reusableLeaseSuccessor as Record<string, unknown>), successorRunId: randomUUID() } } }).where(eq(environmentLeases.id, previous.id));
|
|
|
|
|
if (fault === "dangling_proof") await db.update(environmentLeases).set({ metadata: { ...previous.metadata, reusableLeasePredecessor: {} } }).where(eq(environmentLeases.id, previous.id));
|
|
|
|
|
if (fault === "extra_history") await db.insert(environmentLeases).values({ companyId: f.companyId, environmentId: previous.environmentId,
|
|
|
|
|
heartbeatRunId: f.priorRunId, issueId: f.issueId, provider: "daytona", providerLeaseId: f.providerLeaseId,
|
|
|
|
|
status: "expired", leasePolicy: "reuse_by_environment", cleanupStatus: "success", metadata: previous.metadata });
|
|
|
|
|
try {
|
|
|
|
|
expect(await inspectWorkspaceSource(db, { companyId: f.companyId, issueId: f.issueId, runId: f.runId, actionId: f.actionId }))
|
|
|
|
|
.toMatchObject({ status: "blocked", failureCode: "allocation_ownership_ambiguous" });
|
|
|
|
|
await expect(readFile(path.join(f.host, "after-warm-stop.txt"))).rejects.toMatchObject({ code: "ENOENT" });
|
|
|
|
|
} finally { adapterExecutionControls.get(f.priorRunId)?.finish(); adapterExecutionControls.delete(f.priorRunId); }
|
|
|
|
|
}, 30_000);
|
|
|
|
|
|
|
|
|
|
it("keeps server handoff receipts across metadata updates and same-run reacquisition", async () => {
|
|
|
|
|
const f = await warmHandoffSource(1, async lease => {
|
|
|
|
|
const proof = lease.metadata!.reusableLeasePredecessor;
|
|
|
|
|
expect(proof).toMatchObject({ schema: "paperclip.reusable-lease-handoff.v1", successorLeaseId: lease.id });
|
|
|
|
|
const forged = { schema: "forged-provider-metadata" };
|
|
|
|
|
const updated = await environmentService(db).updateLeaseMetadata(lease.id, { ...lease.metadata,
|
|
|
|
|
reusableLeasePredecessor: forged, reusableLeaseSuccessor: forged, marker: "updated" });
|
|
|
|
|
expect(updated!.metadata?.reusableLeasePredecessor).toEqual(proof);
|
|
|
|
|
expect(updated!.metadata?.reusableLeaseSuccessor).toBeUndefined();
|
|
|
|
|
await environmentService(db).releaseLease(lease.id, "released", { cleanupStatus: "success" });
|
|
|
|
|
const reacquired = await environmentService(db).acquireLease({ companyId: lease.companyId, environmentId: lease.environmentId!,
|
|
|
|
|
issueId: lease.issueId, heartbeatRunId: lease.heartbeatRunId, provider: lease.provider, providerLeaseId: lease.providerLeaseId,
|
|
|
|
|
leasePolicy: "reuse_by_environment", reusesReusableLeaseId: lease.id,
|
|
|
|
|
metadata: { ...updated!.metadata, reusableLeasePredecessor: forged, reusableLeaseSuccessor: forged } });
|
|
|
|
|
expect(reacquired.id).toBe(lease.id);
|
|
|
|
|
expect(reacquired.metadata?.reusableLeasePredecessor).toEqual(proof);
|
|
|
|
|
expect(reacquired.metadata?.reusableLeaseSuccessor).toBeUndefined();
|
|
|
|
|
});
|
|
|
|
|
await recoverRetainedWorkspace(f.input);
|
|
|
|
|
expect(await hasVerifiedWorkspaceRepair(db, await run(f.runId))).toBe(true);
|
|
|
|
|
}, 30_000);
|
|
|
|
|
|
|
|
|
|
it("does not manufacture handoff authority from injected fresh or orphan lease metadata", async () => {
|
|
|
|
|
const f = await warmHandoffSource(), source = await run(f.runId);
|
|
|
|
|
const forged = { schema: "paperclip.reusable-lease-handoff.v1", successorLeaseId: f.lease.id };
|
|
|
|
|
const values = { companyId: f.companyId, environmentId: f.lease.environmentId!, issueId: f.issueId,
|
|
|
|
|
heartbeatRunId: source.id, provider: "daytona", providerLeaseId: randomUUID(),
|
|
|
|
|
metadata: { agentId: source.agentId, reusableLeasePredecessor: forged, reusableLeaseSuccessor: forged } };
|
|
|
|
|
const fresh = await environmentService(db).acquireLease(values);
|
|
|
|
|
const orphan = await environmentService(db).insertPendingCleanupLease({ ...values, providerLeaseId: randomUUID(), failureReason: "test" });
|
|
|
|
|
for (const lease of [fresh, orphan]) {
|
|
|
|
|
expect(lease.metadata?.reusableLeasePredecessor).toBeUndefined();
|
|
|
|
|
expect(lease.metadata?.reusableLeaseSuccessor).toBeUndefined();
|
|
|
|
|
}
|
|
|
|
|
}, 30_000);
|
|
|
|
|
|
|
|
|
|
it("rolls back both handoff links and keeps provider mismatches or retained sources fenced", async () => {
|
|
|
|
|
const f = await warmHandoffSource(), source = await run(f.runId), service = environmentService(db);
|
|
|
|
|
const values = { companyId: f.companyId, environmentId: f.lease.environmentId!, issueId: f.issueId,
|
|
|
|
|
heartbeatRunId: f.priorRunId, provider: "daytona", providerLeaseId: randomUUID(),
|
|
|
|
|
leasePolicy: "reuse_by_environment" as const, metadata: { agentId: source.agentId } };
|
|
|
|
|
const prior = await service.acquireLease(values);
|
|
|
|
|
await service.releaseLease(prior.id, "released", { cleanupStatus: "success" });
|
|
|
|
|
const [before] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, prior.id));
|
|
|
|
|
const rollback = new Error("rollback_handoff");
|
|
|
|
|
await expect(db.transaction(async tx => {
|
|
|
|
|
const acquired = await environmentService(tx as unknown as typeof db).acquireLease({ ...values, heartbeatRunId: f.runId, replacesReusableLeaseId: prior.id });
|
|
|
|
|
expect(acquired.metadata?.reusableLeasePredecessor).toMatchObject({ previousLeaseId: prior.id, successorLeaseId: acquired.id });
|
|
|
|
|
throw rollback;
|
|
|
|
|
})).rejects.toBe(rollback);
|
|
|
|
|
expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, prior.id)))[0]).toEqual(before);
|
|
|
|
|
expect(await db.select().from(environmentLeases).where(eq(environmentLeases.providerLeaseId, values.providerLeaseId))).toHaveLength(1);
|
|
|
|
|
await expect(service.acquireLease({ ...values, heartbeatRunId: f.runId, provider: "foreign-provider", replacesReusableLeaseId: prior.id })).rejects.toThrow("ownership changed");
|
|
|
|
|
await expect(service.acquireLease({ ...values, providerLeaseId: f.providerLeaseId, replacesReusableLeaseId: f.lease.id })).rejects.toThrow("ownership changed");
|
|
|
|
|
const [later] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: source.agentId, issueId: f.issueId,
|
|
|
|
|
nativeIssueId: f.issueId, runtimeMode: "native", status: "succeeded", finishedAt: new Date() }).returning();
|
|
|
|
|
await service.acquireLease({ ...values, heartbeatRunId: later.id, providerLeaseId: f.providerLeaseId });
|
|
|
|
|
expect(await inspectWorkspaceSource(db, { companyId: f.companyId, issueId: f.issueId, runId: f.runId, actionId: f.actionId }))
|
|
|
|
|
.toMatchObject({ status: "blocked", failureCode: "allocation_ownership_ambiguous" });
|
|
|
|
|
}, 30_000);
|
|
|
|
|
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
it("unblocks a pending workspace selection only after a superseded retry's real files are preserved", async () => {
|
|
|
|
|
const f = await seed();
|
|
|
|
|
await db.delete(issueRecoveryActions).where(eq(issueRecoveryActions.id, f.actionId));
|
|
|
|
@@ -1211,4 +1434,546 @@ describe("automatic retained workspace recovery", () => {
|
|
|
|
|
await restarted.$client.end();
|
|
|
|
|
}
|
|
|
|
|
}, 30_000);
|
|
|
|
|
async function queuedResetFixture(nativeDescriptor = true, ephemeral = false) {
|
|
|
|
|
const f = await seed("native", nativeDescriptor, "keep_running", false, ephemeral), original = await run(f.runId);
|
|
|
|
|
await instanceSettingsService(db).updateExperimental({ enableAgentChat: true });
|
|
|
|
|
await db.delete(issueRecoveryActions).where(eq(issueRecoveryActions.id, f.actionId));
|
|
|
|
|
await db.update(issues).set({ conversationAgentId: original.agentId, conversationUserId: "operator", conversationState: "active", status: "in_progress" }).where(eq(issues.id, f.issueId));
|
|
|
|
|
await db.update(heartbeatRuns).set({ status: "cancelled", nativePhase: "terminal_failure", errorCode: "cancelled", resultJson: {
|
|
|
|
|
cancelledByActorType: "user", cancelledByUserId: "operator",
|
|
|
|
|
cancellation: requestedRunCancellation({ cancelledByActorType: "user", cancelledByUserId: "operator" }, "Stop"),
|
|
|
|
|
executionCancellation: { state: "requested", requestedAt: new Date().toISOString() },
|
|
|
|
|
nativeCancellation: { schema: "paperclip.native-cancellation.v1", runId: f.runId, companyId: f.companyId, issueId: f.issueId,
|
|
|
|
|
scope: "run", reasonCode: "cancellation_run_only", dispatchState: "acknowledged", dispatched: true,
|
|
|
|
|
intentAuditId: randomUUID(), acknowledgementAuditId: randomUUID() },
|
|
|
|
|
} }).where(eq(heartbeatRuns.id, f.runId));
|
|
|
|
|
await db.update(environmentLeases).set({ status: "active", releasedAt: null, cleanupStatus: null, metadata: f.lease.metadata }).where(eq(environmentLeases.id, f.lease.id));
|
|
|
|
|
const [reset] = await db.insert(issueComments).values({ companyId: f.companyId, issueId: f.issueId,
|
|
|
|
|
authorType: "user", authorUserId: "operator", body: "/new" }).returning();
|
|
|
|
|
expect(await getExecutionBlocker(db, f.companyId, f.issueId, { conversationResetCommentId: reset.id })).toBeNull();
|
|
|
|
|
const context = { source: "issue.comment", issueId: f.issueId, wakeReason: "issue_commented", wakeCommentId: reset.id, wakeCommentIds: [reset.id] };
|
|
|
|
|
const payload = withQueuedCommentIdsInWakePayload({ issueId: f.issueId, commentId: reset.id, mutation: "comment", _paperclipWakeContext: context }, [reset.id]);
|
|
|
|
|
const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: original.agentId,
|
|
|
|
|
source: "on_demand", triggerDetail: "manual", reason: "issue_commented", status: "claimed", claimedAt: new Date(),
|
|
|
|
|
requestedByActorType: "user", requestedByActorId: "operator", payload }).returning();
|
|
|
|
|
const [queued] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: original.agentId,
|
|
|
|
|
invocationSource: "on_demand", triggerDetail: "manual", status: "queued", runtimeMode: "legacy", responsibleUserId: "operator",
|
|
|
|
|
wakeupRequestId: wake.id, sessionIdBefore: "saved-native-session-before-reset", contextSnapshot: context }).returning();
|
|
|
|
|
await db.update(agentWakeupRequests).set({ runId: queued.id }).where(eq(agentWakeupRequests.id, wake.id));
|
|
|
|
|
await db.update(issues).set({ executionRunId: queued.id, executionAgentNameKey: "Worker", executionLockedAt: new Date() }).where(eq(issues.id, f.issueId));
|
|
|
|
|
const deps = callbacks(db), control = createHeartbeatRunControl(db, deps), execution = createAdapterExecutionControl();
|
|
|
|
|
const queueDeps = createHeartbeatQueueTestDependencies(db);
|
|
|
|
|
queueDeps.activeRunExecutions = deps.activeRunExecutions;
|
|
|
|
|
queueDeps.getAgent.mockResolvedValue((await db.select().from(agents).where(eq(agents.id, original.agentId)))[0]);
|
|
|
|
|
queueDeps.instanceSettings.getExperimental.mockResolvedValue({ enableAgentChat: true, enableNativeRunner: true } as Awaited<ReturnType<typeof queueDeps.instanceSettings.getExperimental>>);
|
|
|
|
|
const queue = createHeartbeatQueue(db, queueDeps);
|
|
|
|
|
vi.mocked(deps.enqueueWakeup).mockImplementation(queue.enqueueWakeup);
|
|
|
|
|
await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: original.agentId, status: "running", invocationSource: "assignment" });
|
|
|
|
|
deps.activeRunExecutions.add(f.runId); adapterExecutionControls.set(f.runId, execution);
|
|
|
|
|
return { f, original, reset, wake, queued, deps, control, execution, queueDeps, queue };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function coalesceQueuedReset(t: Awaited<ReturnType<typeof queuedResetFixture>>) {
|
|
|
|
|
const { f, original, reset, wake, queued } = t;
|
|
|
|
|
const [followup] = await db.insert(issueComments).values({ companyId: f.companyId, issueId: f.issueId,
|
|
|
|
|
authorType: "user", authorUserId: "operator", body: "/new" }).returning();
|
|
|
|
|
const incoming = { source: "issue.comment", issueId: f.issueId, wakeReason: "issue_commented",
|
|
|
|
|
wakeCommentId: followup.id, wakeCommentIds: [followup.id] };
|
|
|
|
|
await db.transaction(async tx => {
|
|
|
|
|
const scope = createAdmissionTransactionScope(f.companyId, tx as unknown as typeof db);
|
|
|
|
|
const actor = { companyId: f.companyId, wakeupRequestId: wake.id, requestedByActorType: "user", requestedByActorId: "operator" };
|
|
|
|
|
expect(await createWakeAdmissionReader().matchesActiveWakeActor(scope, actor)).toBe(true);
|
|
|
|
|
expect(await createWakeAdmissionReader().matchesActiveWakeActor(scope, { ...actor, requestedByActorId: "other-user" })).toBe(false);
|
|
|
|
|
await createWakeAdmissionWriter().coalesceIntoActiveExecutionRun(scope, {
|
|
|
|
|
companyId: f.companyId, agentId: original.agentId, activeExecutionRunId: queued.id,
|
|
|
|
|
mergedContextSnapshot: mergeCoalescedContextSnapshot((await run(queued.id)).contextSnapshot, incoming,
|
|
|
|
|
{ preserveExistingInteractionContinuation: true }),
|
|
|
|
|
source: "on_demand", triggerDetail: "manual",
|
|
|
|
|
payload: withQueuedCommentIdsInWakePayload({ issueId: f.issueId, mutation: "comment", _paperclipWakeContext: incoming }, [followup.id]),
|
|
|
|
|
requestedByActorType: "user", requestedByActorId: "operator", idempotencyKey: null,
|
|
|
|
|
});
|
|
|
|
|
});
|
|
|
|
|
const [originalWake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id));
|
|
|
|
|
const [receipt] = await db.select().from(agentWakeupRequests).where(and(eq(agentWakeupRequests.runId, queued.id), eq(agentWakeupRequests.status, "coalesced")));
|
|
|
|
|
expect(originalWake.payload?.commentId).toBe(reset.id);
|
|
|
|
|
expect((await run(queued.id)).contextSnapshot).toMatchObject({ wakeCommentId: followup.id,
|
|
|
|
|
wakeCommentIds: [reset.id, followup.id] });
|
|
|
|
|
return { followup, receipt };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function ordinaryMixedQueuedFixture(kind: "active" | "deferred" | "deferred-different" | "system-deferred") {
|
|
|
|
|
const t = await queuedResetFixture(true, kind !== "active"), { f, original, reset, wake, queued, queueDeps, queue, control } = t;
|
|
|
|
|
await db.update(issues).set({ conversationAgentId: null, conversationUserId: null, conversationState: null,
|
|
|
|
|
executionRunId: null, executionAgentNameKey: null, executionLockedAt: null }).where(eq(issues.id, f.issueId));
|
|
|
|
|
await db.update(heartbeatRuns).set({ wakeupRequestId: null }).where(eq(heartbeatRuns.id, queued.id));
|
|
|
|
|
await db.update(agentWakeupRequests).set({ runId: null }).where(eq(agentWakeupRequests.id, wake.id));
|
|
|
|
|
await db.delete(heartbeatRuns).where(eq(heartbeatRuns.id, queued.id));
|
|
|
|
|
await db.delete(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id));
|
|
|
|
|
const reader = createWakeAdmissionReader(), sameActor = vi.spyOn(reader, "matchesActiveWakeActor");
|
|
|
|
|
queueDeps.wakeQueue.createAdmissionTransactionScope.mockImplementation(createAdmissionTransactionScope);
|
|
|
|
|
queueDeps.wakeQueue.admitWakeBehindIssueExecution.mockImplementation(createAdmitWakeBehindIssueExecution({
|
|
|
|
|
reader, writer: createWakeAdmissionWriter(), helpers: { filterZombieCoalesceTarget, mergeCoalescedContextSnapshot,
|
|
|
|
|
shouldDeferFollowupWakeForSameIssue, shouldQueueFollowupForRunningIssueWake },
|
|
|
|
|
}));
|
|
|
|
|
queueDeps.liveRunExecutions.has.mockImplementation(id => t.deps.activeRunExecutions.has(id));
|
|
|
|
|
const stopped = await run(f.runId);
|
|
|
|
|
if (kind !== "active") {
|
|
|
|
|
await db.update(heartbeatRuns).set({ status: "running", finishedAt: null, errorCode: null,
|
|
|
|
|
nativePhase: "running", resultJson: null }).where(eq(heartbeatRuns.id, f.runId));
|
|
|
|
|
await db.update(nativeRunFinalizations).set({ phase: "running" }).where(eq(nativeRunFinalizations.runId, f.runId));
|
|
|
|
|
await db.update(issues).set({ executionRunId: f.runId, executionAgentNameKey: "worker", executionLockedAt: new Date() }).where(eq(issues.id, f.issueId));
|
|
|
|
|
}
|
|
|
|
|
const enqueue = async (comment: typeof reset) => queue.enqueueWakeup(original.agentId, {
|
|
|
|
|
source: "automation", triggerDetail: "system", reason: "issue_commented", requestedByActorType: "user", requestedByActorId: comment.authorUserId!,
|
|
|
|
|
payload: { issueId: f.issueId, commentId: comment.id, mutation: "comment", resumeIntent: true, followUpRequested: true },
|
|
|
|
|
contextSnapshot: { issueId: f.issueId, taskId: f.issueId, commentId: comment.id, wakeCommentId: comment.id,
|
|
|
|
|
wakeCommentIds: [comment.id], source: "issue.comment", wakeReason: "issue_commented", resumeIntent: true, followUpRequested: true },
|
|
|
|
|
});
|
|
|
|
|
if (kind === "system-deferred") expect(await queue.enqueueWakeup(original.agentId, {
|
|
|
|
|
source: "automation", triggerDetail: "system", reason: "issue_updated", requestedByActorType: "system", requestedByActorId: "second-user",
|
|
|
|
|
payload: { issueId: f.issueId, mutation: "update" },
|
|
|
|
|
contextSnapshot: { issueId: f.issueId, taskId: f.issueId, source: "issue.updated", wakeReason: "issue_updated", forceFreshSession: true },
|
|
|
|
|
})).toBeNull();
|
|
|
|
|
const first = await enqueue(reset);
|
|
|
|
|
const [followup] = await db.insert(issueComments).values({ companyId: f.companyId, issueId: f.issueId,
|
|
|
|
|
authorType: "user", authorUserId: "second-user", body: "Please also preserve the second user's input." }).returning();
|
|
|
|
|
const second = await enqueue(followup);
|
|
|
|
|
const ids = [reset.id, followup.id];
|
|
|
|
|
let admitted = first;
|
|
|
|
|
if (kind === "active") { expect(first?.status).toBe("queued"); expect(second?.id).toBe(first!.id); }
|
|
|
|
|
else {
|
|
|
|
|
expect(first).toBeNull(); expect(second).toBeNull();
|
|
|
|
|
const [pending] = await db.select().from(agentWakeupRequests).where(and(eq(agentWakeupRequests.companyId, f.companyId),
|
|
|
|
|
eq(agentWakeupRequests.agentId, original.agentId), eq(agentWakeupRequests.status, "deferred_issue_execution")));
|
|
|
|
|
expect(pending.payload?._paperclipWakeContext).toMatchObject({ wakeCommentIds: ids });
|
|
|
|
|
expect(pending.requestedByActorType).toBe(kind === "system-deferred" ? "system" : "user");
|
|
|
|
|
expect(pending.requestedByActorId).toBe(kind === "system-deferred" ? "second-user" : "operator");
|
|
|
|
|
await db.update(heartbeatRuns).set({ status: "cancelled", nativePhase: "terminal_failure", finishedAt: new Date(),
|
|
|
|
|
errorCode: stopped.errorCode, resultJson: stopped.resultJson }).where(eq(heartbeatRuns.id, f.runId));
|
|
|
|
|
await db.update(nativeRunFinalizations).set({ phase: "terminal_failure" }).where(eq(nativeRunFinalizations.runId, f.runId));
|
|
|
|
|
const [currentLease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id));
|
|
|
|
|
const stopping = await prepareSandboxStopAndRetain(db, currentLease);
|
|
|
|
|
expect(stopping).not.toBeNull();
|
|
|
|
|
expect(await settleStopOnlyCleanup(db, stopping!, { attemptId: String(stopping!.metadata!.pendingCleanupAttemptId),
|
|
|
|
|
receipt: { providerLeaseId: f.providerLeaseId, state: "stopped" } })).not.toBeNull();
|
|
|
|
|
await control.acknowledgeRemoteStop(f.runId, f.companyId);
|
|
|
|
|
const [last] = await db.insert(issueComments).values({ companyId: f.companyId, issueId: f.issueId,
|
|
|
|
|
authorType: "user", authorUserId: kind === "deferred-different" ? "third-user" : "operator", body: "Resume after Stop and preserve all three inputs in their original order." }).returning();
|
|
|
|
|
expect(last.createdAt.getTime()).toBeGreaterThan((await run(f.runId)).finishedAt!.getTime());
|
|
|
|
|
ids.push(last.id); admitted = await enqueue(last);
|
|
|
|
|
}
|
|
|
|
|
expect(admitted?.status).toBe("queued");
|
|
|
|
|
const current = await run(admitted!.id);
|
|
|
|
|
const [claimedWake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, current.wakeupRequestId!));
|
|
|
|
|
expect(current.contextSnapshot?.wakeCommentIds).toEqual(ids);
|
|
|
|
|
expect(sameActor).not.toHaveBeenCalled();
|
|
|
|
|
expect(queueDeps.wakeQueue.admitWakeBehindIssueExecution.mock.calls.every(([, input]) => input.durableReceipt === undefined)).toBe(true);
|
|
|
|
|
expect(queueDeps.executeRun).not.toHaveBeenCalled();
|
|
|
|
|
return { ...t, queued: current, wake: claimedWake, followup, expectedIds: ids, latestAuthor: kind === "deferred-different" ? "third-user" : kind === "active" ? "second-user" : "operator" };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function exerciseQueuedReset(nativeDescriptor: boolean, coalesced = false, ordinary?: "active" | "deferred" | "deferred-different" | "system-deferred") {
|
|
|
|
|
const mixed = ordinary ? await ordinaryMixedQueuedFixture(ordinary) : null;
|
|
|
|
|
const t = mixed ?? await queuedResetFixture(nativeDescriptor);
|
|
|
|
|
const { f, original, reset, wake, queued, deps, control, execution, queueDeps, queue } = t;
|
|
|
|
|
const followup = coalesced ? (await coalesceQueuedReset(t)).followup : null;
|
|
|
|
|
const expectedIds = mixed?.expectedIds ?? (followup ? [reset.id, followup.id] : [reset.id]);
|
|
|
|
|
const responsible = queued.responsibleUserId;
|
|
|
|
|
try {
|
|
|
|
|
expect(await queue.claimQueuedRun(queued)).toBeNull();
|
|
|
|
|
expect((await run(queued.id)).status).toBe("queued");
|
|
|
|
|
await recordRequiredWorkspaceRestoreFailure(db, await run(f.runId), { workspaceRestoreFailure: "restore_failed" }, f.lease);
|
|
|
|
|
const [preservation] = await db.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.companyId, f.companyId),
|
|
|
|
|
eq(issueRecoveryActions.sourceIssueId, f.issueId), eq(issueRecoveryActions.fingerprint, `native-workspace:${f.runId}`)));
|
|
|
|
|
f.actionId = preservation.id; f.input.actionId = preservation.id;
|
|
|
|
|
await recoverRetainedWorkspace(f.input);
|
|
|
|
|
expect(readSystemRecovery((await action(f.actionId)).evidence)?.failure?.code).toBe("agent_stop_unconfirmed");
|
|
|
|
|
await db.update(environmentLeases).set({ status: "released", releasedAt: new Date(), cleanupStatus: "success", metadata: { ...f.lease.metadata,
|
|
|
|
|
remoteExecutionTermination: remoteTerminationReceipt(f.lease, { providerLeaseId: f.providerLeaseId, state: "stopped" }) } }).where(eq(environmentLeases.id, f.lease.id));
|
|
|
|
|
await control.acknowledgeRemoteStop(f.runId, f.companyId);
|
|
|
|
|
expect((await run(f.runId)).resultJson?.executionCancellation).toMatchObject({ state: "acknowledged", proof: "provider_termination_receipt" });
|
|
|
|
|
deps.activeRunExecutions.delete(f.runId); execution.finish(); adapterExecutionControls.delete(f.runId);
|
|
|
|
|
await control.resumeRemoteStopComments(await run(f.runId));
|
|
|
|
|
expect(readSystemRecovery((await action(f.actionId)).evidence)!.nextAttemptAt!).toBeLessThanOrEqual(Date.now());
|
|
|
|
|
const onRecovered = vi.fn(async (current: Awaited<ReturnType<typeof run>>) => {
|
|
|
|
|
if (nativeDescriptor) expect(await inspectWorkspaceSource(db, { companyId: f.companyId, issueId: f.issueId,
|
|
|
|
|
runId: f.runId, actionId: f.actionId })).toMatchObject({ status: "verified", scope: "recorded_source_identity" });
|
|
|
|
|
await Promise.all([control.resumeRemoteStopComments(current), control.resumeRemoteStopComments(current)]);
|
|
|
|
|
});
|
|
|
|
|
const repair = async () => { await Promise.all(await reconcileRetainedWorkspaces(db, f.environmentRuntime, onRecovered) ?? []); };
|
|
|
|
|
if (!nativeDescriptor) await repair();
|
|
|
|
|
expect(await queue.startNextQueuedRunForAgent(original.agentId)).toEqual([]);
|
|
|
|
|
expect(await queue.claimQueuedRun(await run(queued.id))).toBeNull();
|
|
|
|
|
const [heldWake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id));
|
|
|
|
|
expect(await isPreDispatchExecutionWaitVerified(db, await run(queued.id))).toBe(true);
|
|
|
|
|
const admissionEvents = await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, queued.id));
|
|
|
|
|
expect(admissionEvents.map(event => event.eventType)).toEqual(["lifecycle"]);
|
|
|
|
|
expect((await run(queued.id)).nextEventSeq).toBe(2);
|
|
|
|
|
expect((await run(queued.id)).sessionIdBefore).toBe(queued.sessionIdBefore);
|
|
|
|
|
expect((await run(queued.id)).responsibleUserId).toBe(responsible);
|
|
|
|
|
const scheduleRecoveryRetry = vi.fn(async (_runId: string) => null);
|
|
|
|
|
const recoveryDependencies = { enqueueWakeup: vi.fn(async () => null), scheduleRecoveryRetry, liveRunExecutions: { has: () => false } };
|
|
|
|
|
const [task] = await db.select().from(issues).where(eq(issues.id, f.issueId));
|
|
|
|
|
const recover = (database: typeof db) => recoveryService(database, recoveryDependencies).reconcileStrandedAssignedIssues({ issueCreatedAtGte: task.createdAt });
|
|
|
|
|
if (nativeDescriptor) {
|
|
|
|
|
const rollback = new Error("restore_counterfactual_skipped_wake");
|
|
|
|
|
await expect(db.transaction(async tx => {
|
|
|
|
|
await tx.update(agentWakeupRequests).set({ status: "skipped" }).where(eq(agentWakeupRequests.id, wake.id));
|
|
|
|
|
await recover(tx as unknown as typeof db);
|
|
|
|
|
expect(scheduleRecoveryRetry).toHaveBeenCalledWith(queued.id);
|
|
|
|
|
throw rollback;
|
|
|
|
|
})).rejects.toBe(rollback);
|
|
|
|
|
scheduleRecoveryRetry.mockClear();
|
|
|
|
|
await recover(db);
|
|
|
|
|
expect(scheduleRecoveryRetry).not.toHaveBeenCalledWith(queued.id);
|
|
|
|
|
}
|
|
|
|
|
expect(queueDeps.executeRun).not.toHaveBeenCalled();
|
|
|
|
|
if (nativeDescriptor) await repair();
|
|
|
|
|
expect(onRecovered).toHaveBeenCalled();
|
|
|
|
|
expect(readSystemRecovery((await action(f.actionId)).evidence)?.state).toBe("complete");
|
|
|
|
|
if (nativeDescriptor) expect(await readFile(path.join(f.host, "saved.txt"), "utf8")).toBe("finished work\n");
|
|
|
|
|
await Promise.all([control.resumeRemoteStopComments(await run(f.runId)),
|
|
|
|
|
control.resumeRemoteStopComments(await run(f.runId))]);
|
|
|
|
|
const [afterWake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id));
|
|
|
|
|
expect(heldWake.status).toBe(nativeDescriptor ? "deferred_issue_execution" : "skipped");
|
|
|
|
|
if (!nativeDescriptor) {
|
|
|
|
|
expect(afterWake.status).toBe("skipped");
|
|
|
|
|
expect(deps.enqueueWakeup).not.toHaveBeenCalled();
|
|
|
|
|
expect(queueDeps.executeRun).not.toHaveBeenCalled();
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
expect(afterWake).toMatchObject({ status: "coalesced", runId: expect.any(String) });
|
|
|
|
|
expect(afterWake.runId).not.toBe(queued.id);
|
|
|
|
|
expect((await action(f.actionId)).evidence).toMatchObject({ automaticRecovery: { replay: "explicit_user_continuation" } });
|
|
|
|
|
expect(await run(queued.id)).toMatchObject({ status: "cancelled", errorCode: "execution_reconciliation_required", startedAt: null });
|
|
|
|
|
const successors = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId),
|
|
|
|
|
sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${f.issueId}`, eq(heartbeatRuns.status, "queued")));
|
|
|
|
|
expect(successors).toHaveLength(1);
|
|
|
|
|
expect(successors[0].contextSnapshot?.wakeCommentIds).toEqual(expectedIds);
|
|
|
|
|
if (mixed) expect(deps.enqueueWakeup).toHaveBeenCalledWith(original.agentId, expect.objectContaining({ requestedByActorType: "user", requestedByActorId: mixed.latestAuthor }), wake.id);
|
|
|
|
|
await recover(db);
|
|
|
|
|
expect(scheduleRecoveryRetry).not.toHaveBeenCalledWith(queued.id);
|
|
|
|
|
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, queued.id))).toHaveLength(0);
|
|
|
|
|
expect((await run(queued.id)).nextEventSeq).toBe(2);
|
|
|
|
|
expect(queueDeps.executeRun).not.toHaveBeenCalled();
|
|
|
|
|
} finally { deps.activeRunExecutions.delete(f.runId); execution.finish(); adapterExecutionControls.delete(f.runId); }
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
it.each([
|
|
|
|
|
{ nativeDescriptor: true, title: "preserves an already queued reset until real native copyback replays its saved wake" },
|
|
|
|
|
{ nativeDescriptor: false, title: "does not replay a snapshot-only source without a verified native baseline" },
|
|
|
|
|
])("$title", async ({ nativeDescriptor }) => exerciseQueuedReset(nativeDescriptor), 30_000);
|
|
|
|
|
|
|
|
|
|
it("preserves coalesced queued user comments until real native copyback replays the original wake", async () => {
|
|
|
|
|
await exerciseQueuedReset(true, true);
|
|
|
|
|
}, 30_000);
|
|
|
|
|
|
|
|
|
|
it("preserves each ordinary mixed-author comment across queued Stop and native copyback", async () => {
|
|
|
|
|
await exerciseQueuedReset(true, false, "active");
|
|
|
|
|
}, 30_000);
|
|
|
|
|
|
|
|
|
|
it("preserves merged-deferred mixed-author comments across queued Stop and native copyback", async () => {
|
|
|
|
|
await exerciseQueuedReset(true, false, "deferred");
|
|
|
|
|
}, 30_000);
|
|
|
|
|
|
|
|
|
|
it("preserves mixed-author comments merged into a system-owned deferred carrier across Stop and native copyback", async () => {
|
|
|
|
|
await exerciseQueuedReset(true, false, "system-deferred");
|
|
|
|
|
}, 30_000);
|
|
|
|
|
|
|
|
|
|
it("preserves mixed-author deferred inputs when the resuming third user differs from the carrier", async () => {
|
|
|
|
|
await exerciseQueuedReset(true, false, "deferred-different");
|
|
|
|
|
}, 30_000);
|
|
|
|
|
|
|
|
|
|
it.each(["missing_primary", "agent_namespace", "system_namespace", "foreign_link", "foreign_carrier_run", "foreign_receipt_user", "array_only"] as const)("rejects mixed-author queue authority forgery: %s", async fault => {
|
|
|
|
|
const t = await ordinaryMixedQueuedFixture("system-deferred"), { f, wake, queued, deps, execution, queueDeps, queue, followup } = t;
|
|
|
|
|
try {
|
|
|
|
|
await recordRequiredWorkspaceRestoreFailure(db, await run(f.runId), { workspaceRestoreFailure: "restore_failed" }, f.lease);
|
|
|
|
|
deps.activeRunExecutions.delete(f.runId); execution.finish(); adapterExecutionControls.delete(f.runId);
|
|
|
|
|
const sourceIds = queued.contextSnapshot!.queuedCommentSourceWakeIds as string[];
|
|
|
|
|
const [receipt] = await db.select().from(agentWakeupRequests).where(and(eq(agentWakeupRequests.companyId, f.companyId),
|
|
|
|
|
eq(agentWakeupRequests.status, "coalesced"), eq(agentWakeupRequests.requestedByActorType, "user"),
|
|
|
|
|
sql`${agentWakeupRequests.payload}->>'commentId' = ${followup.id}`));
|
|
|
|
|
expect(receipt).toBeDefined();
|
|
|
|
|
if (fault === "missing_primary") await db.delete(agentWakeupRequests).where(eq(agentWakeupRequests.id, receipt.id));
|
|
|
|
|
if (fault === "agent_namespace" || fault === "system_namespace") await db.update(agentWakeupRequests).set({ requestedByActorType: fault === "agent_namespace" ? "agent" : "system", requestedByActorId: "second-user" }).where(eq(agentWakeupRequests.id, receipt.id));
|
|
|
|
|
if (fault === "foreign_link") await db.update(agentWakeupRequests).set({ payload: { ...receipt.payload, coalescedIntoWakeupRequestId: randomUUID() } }).where(eq(agentWakeupRequests.id, receipt.id));
|
|
|
|
|
if (fault === "foreign_carrier_run") await db.update(agentWakeupRequests).set({ runId: f.runId }).where(eq(agentWakeupRequests.id, sourceIds[0]));
|
|
|
|
|
if (fault === "foreign_receipt_user") await db.update(agentWakeupRequests).set({ requestedByActorId: "operator" }).where(eq(agentWakeupRequests.id, receipt.id));
|
|
|
|
|
if (fault === "array_only") {
|
|
|
|
|
const [injected] = await db.insert(issueComments).values({ companyId: f.companyId, issueId: f.issueId, authorType: "user", authorUserId: "second-user", body: "An array is not a saved admission receipt." }).returning();
|
|
|
|
|
await db.update(heartbeatRuns).set({ contextSnapshot: mergeCoalescedContextSnapshot(queued.contextSnapshot,
|
|
|
|
|
{ issueId: f.issueId, wakeReason: "issue_commented", wakeCommentId: injected.id, wakeCommentIds: [injected.id] }, { preserveExistingInteractionContinuation: true }) }).where(eq(heartbeatRuns.id, queued.id));
|
|
|
|
|
await db.update(agentWakeupRequests).set({ payload: withQueuedCommentIdsInWakePayload(receipt.payload!, [followup.id, injected.id]) }).where(eq(agentWakeupRequests.id, receipt.id));
|
|
|
|
|
}
|
|
|
|
|
expect(await queue.claimQueuedRun(await run(queued.id))).toBeNull();
|
|
|
|
|
const [after] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id));
|
|
|
|
|
expect(after.status).toBe("skipped");
|
|
|
|
|
expect(queueDeps.executeRun).not.toHaveBeenCalled(); expect(deps.enqueueWakeup).not.toHaveBeenCalled();
|
|
|
|
|
} finally { deps.activeRunExecutions.delete(f.runId); execution.finish(); adapterExecutionControls.delete(f.runId); }
|
|
|
|
|
}, 30_000);
|
|
|
|
|
|
|
|
|
|
it("preserves ordinary no-issue deferred merge semantics", async () => {
|
|
|
|
|
const f = await seed(), original = await run(f.runId), writer = createWakeAdmissionWriter();
|
|
|
|
|
const [carrier] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: original.agentId,
|
|
|
|
|
source: "automation", status: "deferred_issue_execution", payload: { text: "original" }, requestedByActorType: "system" }).returning();
|
|
|
|
|
await db.transaction(tx => writer.mergeIntoExistingDeferredWake(createAdmissionTransactionScope(f.companyId, tx as unknown as typeof db), {
|
|
|
|
|
companyId: f.companyId, existingDeferredWakeId: carrier.id, mergedPayload: { text: "merged" }, nextCoalescedCount: 1,
|
|
|
|
|
coalescedReceipt: { agentId: original.agentId, source: "automation", triggerDetail: null, reason: null, payload: { text: "saved" }, requestedByActorType: "system", requestedByActorId: null, idempotencyKey: null, runId: null },
|
|
|
|
|
}));
|
|
|
|
|
const [after] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, carrier.id));
|
|
|
|
|
expect(after.payload).toEqual({ text: "merged" });
|
|
|
|
|
const [receipt] = await db.select().from(agentWakeupRequests).where(and(eq(agentWakeupRequests.companyId, f.companyId), eq(agentWakeupRequests.status, "coalesced")));
|
|
|
|
|
expect(receipt.payload).toEqual({ text: "saved", coalescedIntoWakeupRequestId: carrier.id });
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
it("rolls back saved primary and carrier overwrite if incoming mixed-author receipt persistence fails", async () => {
|
|
|
|
|
const f = await seed(), original = await run(f.runId), writer = createWakeAdmissionWriter();
|
|
|
|
|
const [first] = await db.insert(issueComments).values({ companyId: f.companyId, issueId: f.issueId, authorType: "user", authorUserId: "operator", body: "First saved input" }).returning();
|
|
|
|
|
const [second] = await db.insert(issueComments).values({ companyId: f.companyId, issueId: f.issueId, authorType: "user", authorUserId: "second-user", body: "Second saved input" }).returning();
|
|
|
|
|
const [carrier] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: original.agentId,
|
|
|
|
|
source: "automation", status: "deferred_issue_execution", payload: { issueId: f.issueId, commentId: first.id }, requestedByActorType: "user", requestedByActorId: "operator" }).returning();
|
|
|
|
|
const merge = (tx: Parameters<Parameters<typeof db.transaction>[0]>[0], id?: string) => writer.mergeIntoExistingDeferredWake(createAdmissionTransactionScope(f.companyId, tx as unknown as typeof db), {
|
|
|
|
|
companyId: f.companyId, existingDeferredWakeId: carrier.id, mergedPayload: { issueId: f.issueId, commentId: second.id }, nextCoalescedCount: 1,
|
|
|
|
|
coalescedReceipt: { ...(id ? { id } : {}), agentId: original.agentId, source: "automation", triggerDetail: null, reason: "issue_commented", payload: { issueId: f.issueId, commentId: second.id, coalescedIntoWakeupRequestId: "untrusted" }, requestedByActorType: "user", requestedByActorId: "second-user", idempotencyKey: null, runId: null },
|
|
|
|
|
});
|
|
|
|
|
await expect(db.transaction(tx => merge(tx, carrier.id))).rejects.toThrow();
|
|
|
|
|
expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.companyId)))).toEqual([carrier]);
|
|
|
|
|
await db.transaction(tx => merge(tx)); await db.transaction(tx => merge(tx));
|
|
|
|
|
const proofs = await db.select().from(agentWakeupRequests).where(and(eq(agentWakeupRequests.companyId, f.companyId), eq(agentWakeupRequests.status, "coalesced")));
|
|
|
|
|
expect(proofs.filter(receipt => receipt.payload?.commentId === first.id)).toHaveLength(1);
|
|
|
|
|
expect(proofs.every(receipt => receipt.payload?.coalescedIntoWakeupRequestId === carrier.id)).toBe(true);
|
|
|
|
|
expect(proofs.find(receipt => receipt.payload?.commentId === first.id)?.requestedByActorId).toBe("operator");
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
it.each(["missing_receipt", "wrong_receipt_run", "wrong_receipt_agent", "wrong_receipt_company", "wrong_receipt_user", "wrong_receipt_issue", "terminal_receipt", "foreign_comment_user", "deleted_followup", "unrecorded_comment", "unrecorded_original"] as const)("keeps coalesced Stop recovery input closed without saved receipt authority: %s", async fault => {
|
|
|
|
|
const t = await queuedResetFixture(), { f, wake, queued, deps, execution, queueDeps, queue } = t;
|
|
|
|
|
try {
|
|
|
|
|
const { followup, receipt } = await coalesceQueuedReset(t);
|
|
|
|
|
await recordRequiredWorkspaceRestoreFailure(db, await run(f.runId), { workspaceRestoreFailure: "restore_failed" }, f.lease);
|
|
|
|
|
deps.activeRunExecutions.delete(f.runId); execution.finish(); adapterExecutionControls.delete(f.runId);
|
|
|
|
|
if (fault === "missing_receipt") await db.delete(agentWakeupRequests).where(eq(agentWakeupRequests.id, receipt.id));
|
|
|
|
|
if (fault === "wrong_receipt_run") await db.update(agentWakeupRequests).set({ runId: f.runId }).where(eq(agentWakeupRequests.id, receipt.id));
|
|
|
|
|
if (fault === "wrong_receipt_agent") await db.update(agentWakeupRequests).set({ agentId: (await db.insert(agents).values({ companyId: f.companyId, name: "Other", role: "engineer" }).returning())[0].id }).where(eq(agentWakeupRequests.id, receipt.id));
|
|
|
|
|
if (fault === "wrong_receipt_company") await db.update(agentWakeupRequests).set({ companyId: (await db.insert(companies).values({ name: "Other", issuePrefix: `OT${randomUUID().slice(0, 6)}` }).returning())[0].id }).where(eq(agentWakeupRequests.id, receipt.id));
|
|
|
|
|
if (fault === "wrong_receipt_user") await db.update(agentWakeupRequests).set({ requestedByActorId: "other-user" }).where(eq(agentWakeupRequests.id, receipt.id));
|
|
|
|
|
if (fault === "wrong_receipt_issue") await db.update(agentWakeupRequests).set({ payload: { ...receipt.payload, issueId: randomUUID() } }).where(eq(agentWakeupRequests.id, receipt.id));
|
|
|
|
|
if (fault === "terminal_receipt") await db.update(agentWakeupRequests).set({ status: "cancelled" }).where(eq(agentWakeupRequests.id, receipt.id));
|
|
|
|
|
if (fault === "foreign_comment_user") await db.update(issueComments).set({ authorUserId: "other-user" }).where(eq(issueComments.id, followup.id));
|
|
|
|
|
if (fault === "deleted_followup") await db.update(issueComments).set({ deletedAt: new Date() }).where(eq(issueComments.id, followup.id));
|
|
|
|
|
if (fault === "unrecorded_comment") {
|
|
|
|
|
const [injected] = await db.insert(issueComments).values({ companyId: f.companyId, issueId: f.issueId, authorType: "user", authorUserId: "operator", body: "/new" }).returning();
|
|
|
|
|
await db.update(heartbeatRuns).set({ contextSnapshot: mergeCoalescedContextSnapshot((await run(queued.id)).contextSnapshot,
|
|
|
|
|
{ issueId: f.issueId, wakeReason: "issue_commented", wakeCommentId: injected.id, wakeCommentIds: [injected.id] },
|
|
|
|
|
{ preserveExistingInteractionContinuation: true }) }).where(eq(heartbeatRuns.id, queued.id));
|
|
|
|
|
}
|
|
|
|
|
if (fault === "unrecorded_original") await db.update(agentWakeupRequests).set({ payload: { ...wake.payload, commentId: randomUUID() } }).where(eq(agentWakeupRequests.id, wake.id));
|
|
|
|
|
expect(await queue.claimQueuedRun(await run(queued.id))).toBeNull();
|
|
|
|
|
const [afterWake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id));
|
|
|
|
|
expect(afterWake.status).toBe("skipped");
|
|
|
|
|
expect(queueDeps.executeRun).not.toHaveBeenCalled();
|
|
|
|
|
expect(deps.enqueueWakeup).not.toHaveBeenCalled();
|
|
|
|
|
expect((await run(queued.id)).status).toBe("cancelled");
|
|
|
|
|
} finally { deps.activeRunExecutions.delete(f.runId); execution.finish(); adapterExecutionControls.delete(f.runId); }
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
it.each(["system_actor", "wrong_actor", "deleted_comment", "foreign_comment", "foreign_agent", "foreign_company", "foreign_run", "terminal_wake", "started", "provider_event", "provider_lease", "resolved_runtime"] as const)("keeps queued Stop recovery input closed without paired predispatch authority: %s", async fault => {
|
|
|
|
|
const t = await queuedResetFixture(), { f, original, reset, wake, queued, deps, execution, queueDeps, queue } = t;
|
|
|
|
|
try {
|
|
|
|
|
await recordRequiredWorkspaceRestoreFailure(db, await run(f.runId), { workspaceRestoreFailure: "restore_failed" }, f.lease);
|
|
|
|
|
deps.activeRunExecutions.delete(f.runId); execution.finish(); adapterExecutionControls.delete(f.runId);
|
|
|
|
|
if (fault === "system_actor") await db.update(agentWakeupRequests).set({ requestedByActorType: "system" }).where(eq(agentWakeupRequests.id, wake.id));
|
|
|
|
|
if (fault === "wrong_actor") await db.update(agentWakeupRequests).set({ requestedByActorId: "other-user" }).where(eq(agentWakeupRequests.id, wake.id));
|
|
|
|
|
if (fault === "deleted_comment") await db.update(issueComments).set({ deletedAt: new Date() }).where(eq(issueComments.id, reset.id));
|
|
|
|
|
if (fault === "foreign_comment") await db.update(issueComments).set({ issueId: (await db.insert(issues).values({ companyId: f.companyId, title: "Other" }).returning())[0].id }).where(eq(issueComments.id, reset.id));
|
|
|
|
|
if (fault === "foreign_agent") await db.update(agentWakeupRequests).set({ agentId: (await db.insert(agents).values({ companyId: f.companyId, name: "Other", role: "engineer" }).returning())[0].id }).where(eq(agentWakeupRequests.id, wake.id));
|
|
|
|
|
if (fault === "foreign_company") await db.update(agentWakeupRequests).set({ companyId: (await db.insert(companies).values({ name: "Other", issuePrefix: "OT" }).returning())[0].id }).where(eq(agentWakeupRequests.id, wake.id));
|
|
|
|
|
if (fault === "foreign_run") await db.update(agentWakeupRequests).set({ runId: f.runId }).where(eq(agentWakeupRequests.id, wake.id));
|
|
|
|
|
if (fault === "terminal_wake") await db.update(agentWakeupRequests).set({ status: "cancelled", finishedAt: new Date(), error: "Operator cancelled" }).where(eq(agentWakeupRequests.id, wake.id));
|
|
|
|
|
if (fault === "started") await db.update(heartbeatRuns).set({ startedAt: new Date() }).where(eq(heartbeatRuns.id, queued.id));
|
|
|
|
|
if (fault === "provider_event") await appendHeartbeatRunEvent(db, { companyId: f.companyId, agentId: original.agentId, runId: queued.id,
|
|
|
|
|
eventType: "adapter.invoke", stream: "system", level: "info", message: "Controlled provider execution evidence" });
|
|
|
|
|
if (fault === "provider_lease") await db.insert(environmentLeases).values({ companyId: f.companyId, heartbeatRunId: queued.id, environmentId: f.lease.environmentId, provider: "daytona", providerLeaseId: randomUUID() });
|
|
|
|
|
if (fault === "resolved_runtime") await db.update(heartbeatRuns).set({ runtimeModeResolvedAt: new Date() }).where(eq(heartbeatRuns.id, queued.id));
|
|
|
|
|
const [beforeWake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id));
|
|
|
|
|
expect(await queue.claimQueuedRun(await run(queued.id))).toBeNull();
|
|
|
|
|
const [afterWake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id));
|
|
|
|
|
expect(afterWake.status).not.toBe("deferred_issue_execution");
|
|
|
|
|
if (["foreign_agent", "foreign_company", "foreign_run", "terminal_wake"].includes(fault)) expect(afterWake).toEqual(beforeWake);
|
|
|
|
|
else expect(afterWake.status).toBe("skipped");
|
|
|
|
|
expect(queueDeps.executeRun).not.toHaveBeenCalled();
|
|
|
|
|
expect(deps.enqueueWakeup).not.toHaveBeenCalled();
|
|
|
|
|
} finally { deps.activeRunExecutions.delete(f.runId); execution.finish(); adapterExecutionControls.delete(f.runId); }
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
async function stoppedEvidenceFixture() {
|
|
|
|
|
const f = await seed();
|
|
|
|
|
const original = await run(f.runId);
|
|
|
|
|
await db.update(heartbeatRuns).set({ status: "cancelled", nativePhase: "terminal_failure", errorCode: "cancelled", resultJson: {
|
|
|
|
|
...original.resultJson, cancelledByActorType: "user", cancelledByUserId: "operator",
|
|
|
|
|
cancellation: requestedRunCancellation({ cancelledByActorType: "user", cancelledByUserId: "operator" }, "Stop"),
|
|
|
|
|
executionCancellation: { state: "requested", requestedAt: new Date().toISOString() },
|
|
|
|
|
nativeCancellation: { schema: "paperclip.native-cancellation.v1", runId: f.runId, companyId: f.companyId, issueId: f.issueId,
|
|
|
|
|
scope: "run", reasonCode: "cancellation_run_only", dispatchState: "acknowledged", dispatched: true,
|
|
|
|
|
intentAuditId: randomUUID(), acknowledgementAuditId: randomUUID() },
|
|
|
|
|
} }).where(eq(heartbeatRuns.id, f.runId));
|
|
|
|
|
await db.update(issues).set({ conversationAgentId: original.agentId, conversationUserId: "operator", conversationState: "active", status: "in_progress" }).where(eq(issues.id, f.issueId));
|
|
|
|
|
const [reset] = await db.insert(issueComments).values({ companyId: f.companyId, issueId: f.issueId,
|
|
|
|
|
authorType: "user", authorUserId: "operator", body: "/new" }).returning();
|
|
|
|
|
await db.delete(issueRecoveryActions).where(eq(issueRecoveryActions.id, f.actionId));
|
|
|
|
|
await recordRequiredWorkspaceRestoreFailure(db, await run(f.runId), { workspaceRestoreFailure: "restore_failed" }, f.lease);
|
|
|
|
|
const [preservation] = await db.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.companyId, f.companyId),
|
|
|
|
|
eq(issueRecoveryActions.sourceIssueId, f.issueId), eq(issueRecoveryActions.fingerprint, `native-workspace:${f.runId}`)));
|
|
|
|
|
expect(preservation).toMatchObject({ status: "resolved", evidence: { automaticRecovery: { policy: "preserve_without_replay_v1", replay: "blocked", runId: f.runId } } });
|
|
|
|
|
f.actionId = preservation.id; f.input.actionId = preservation.id;
|
|
|
|
|
const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: original.agentId,
|
|
|
|
|
source: "on_demand", reason: "issue_execution_deferred", status: "deferred_issue_execution", requestedByActorType: "user", requestedByActorId: "operator",
|
|
|
|
|
payload: withQueuedCommentIdsInWakePayload({ issueId: f.issueId, commentId: reset.id,
|
|
|
|
|
_paperclipWakeContext: { issueId: f.issueId, wakeReason: "issue_commented", wakeCommentId: reset.id, wakeCommentIds: [reset.id] } }, [reset.id]) }).returning();
|
|
|
|
|
const deps = callbacks(db), control = createHeartbeatRunControl(db, deps);
|
|
|
|
|
const execution = createAdapterExecutionControl();
|
|
|
|
|
deps.activeRunExecutions.add(f.runId); adapterExecutionControls.set(f.runId, execution);
|
|
|
|
|
const cleanup = () => { deps.activeRunExecutions.delete(f.runId); execution.finish(); adapterExecutionControls.delete(f.runId); };
|
|
|
|
|
return { f, original, reset, wake, deps, control, execution, cleanup };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function heldStoppedEvidence() {
|
|
|
|
|
const prepared = await stoppedEvidenceFixture();
|
|
|
|
|
await recoverRetainedWorkspace(prepared.f.input);
|
|
|
|
|
const held = readSystemRecovery((await action(prepared.f.actionId)).evidence)!;
|
|
|
|
|
expect(held.state).toBe("awaiting_evidence");
|
|
|
|
|
await prepared.control.acknowledgeRemoteStop(prepared.f.runId, prepared.f.companyId);
|
|
|
|
|
prepared.cleanup();
|
|
|
|
|
return { ...prepared, held };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
it("advances saved workspace recovery on new Stop evidence after executor cleanup", async () => {
|
|
|
|
|
const { f, original, reset, deps, control, execution } = await stoppedEvidenceFixture();
|
|
|
|
|
try {
|
|
|
|
|
await recoverRetainedWorkspace(f.input);
|
|
|
|
|
const held = readSystemRecovery((await action(f.actionId)).evidence)!;
|
|
|
|
|
expect(held).toMatchObject({ state: "awaiting_evidence", failure: { code: "agent_stop_unconfirmed", retryClass: "recheck_evidence" } });
|
|
|
|
|
expect(held.nextAttemptAt!).toBeGreaterThan(Date.now() + 20_000);
|
|
|
|
|
expect(await getExecutionBlocker(db, f.companyId, f.issueId, { conversationResetCommentId: reset.id })).not.toBeNull();
|
|
|
|
|
await control.acknowledgeRemoteStop(f.runId, f.companyId);
|
|
|
|
|
expect((await run(f.runId)).resultJson?.executionCancellation).toMatchObject({ state: "acknowledged", proof: "provider_termination_receipt" });
|
|
|
|
|
await control.resumeRemoteStopComments(await run(f.runId));
|
|
|
|
|
expect(readSystemRecovery((await action(f.actionId)).evidence)).toEqual(held);
|
|
|
|
|
deps.activeRunExecutions.delete(f.runId); execution.finish(); adapterExecutionControls.delete(f.runId);
|
|
|
|
|
const at = Date.now(); let events = 0;
|
|
|
|
|
const stop = subscribeDeliveryWork(db, DELIVERY_QUEUES.workspaceRecovery, () => { events += 1; });
|
|
|
|
|
try { await control.resumeRemoteStopComments(await run(f.runId)); } finally { stop(); }
|
|
|
|
|
const advanced = readSystemRecovery((await action(f.actionId)).evidence)!;
|
|
|
|
|
expect(advanced.nextAttemptAt!).toBeLessThanOrEqual(Date.now());
|
|
|
|
|
expect(advanced).toMatchObject({ generationId: held.generationId, attempts: held.attempts, retriesUsed: held.retriesUsed,
|
|
|
|
|
firstFailureAt: held.firstFailureAt, deadlineAt: held.deadlineAt, providerCooldownAt: held.providerCooldownAt });
|
|
|
|
|
expect(events).toBeGreaterThan(0);
|
|
|
|
|
await recoverRetainedWorkspace(f.input);
|
|
|
|
|
expect(readSystemRecovery((await action(f.actionId)).evidence)?.state).toBe("complete");
|
|
|
|
|
expect(await readFile(path.join(f.host, "saved.txt"), "utf8")).toBe("finished work\n");
|
|
|
|
|
expect(await getExecutionBlocker(db, f.companyId, f.issueId)).not.toBeNull();
|
|
|
|
|
const successorRunId = randomUUID();
|
|
|
|
|
const admitted = await admitExplicitNativeContinuation({ db, companyId: f.companyId, issueId: f.issueId, agentId: original.agentId,
|
|
|
|
|
actorType: "user", actorId: "operator", reason: "issue_commented", commentId: reset.id, successorRunId });
|
|
|
|
|
expect(admitted).toMatchObject({ previousRunId: f.runId, commentId: reset.id });
|
|
|
|
|
expect((await action(f.actionId)).evidence).toMatchObject({ automaticRecovery: { replay: "explicit_user_continuation" } });
|
|
|
|
|
expect(await getExecutionBlocker(db, f.companyId, f.issueId, { conversationResetCommentId: reset.id })).toBeNull();
|
|
|
|
|
expect(Date.now() - at).toBeLessThan(30_000);
|
|
|
|
|
expect(deps.enqueueWakeup).not.toHaveBeenCalled();
|
|
|
|
|
} finally { deps.activeRunExecutions.delete(f.runId); execution.finish(); adapterExecutionControls.delete(f.runId); }
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
it.each(["active_adapter", "active_execution", "missing_native_ack", "wrong_native_run", "missing_physical_receipt", "wrong_allocation", "coordinator_owner", "coordinator_observed", "foreign_action_run", "expired_deadline", "exhausted", "retry_budget", "unrelated_failure"] as const)("does not advance Stop-evidence work without current authority: %s", async fault => {
|
|
|
|
|
const t = await heldStoppedEvidence(), { f, deps, control, execution } = t;
|
|
|
|
|
try {
|
|
|
|
|
if (fault === "active_adapter") adapterExecutionControls.set(f.runId, execution);
|
|
|
|
|
if (fault === "active_execution") deps.activeRunExecutions.add(f.runId);
|
|
|
|
|
if (fault === "missing_native_ack") await db.update(heartbeatRuns).set({ resultJson: sql`jsonb_set(${heartbeatRuns.resultJson}, '{nativeCancellation,dispatchState}', '"pending"'::jsonb)` }).where(eq(heartbeatRuns.id, f.runId));
|
|
|
|
|
if (fault === "wrong_native_run") await db.update(heartbeatRuns).set({ resultJson: sql`jsonb_set(${heartbeatRuns.resultJson}, '{nativeCancellation,runId}', ${JSON.stringify(randomUUID())}::jsonb)` }).where(eq(heartbeatRuns.id, f.runId));
|
|
|
|
|
if (fault === "missing_physical_receipt") await db.update(environmentLeases).set({ metadata: sql`${environmentLeases.metadata} - 'remoteExecutionTermination'` }).where(eq(environmentLeases.id, f.lease.id));
|
|
|
|
|
if (fault === "wrong_allocation") await db.update(environmentLeases).set({ metadata: sql`jsonb_set(${environmentLeases.metadata}, '{remoteExecutionTermination,providerLeaseId}', ${JSON.stringify(randomUUID())}::jsonb)` }).where(eq(environmentLeases.id, f.lease.id));
|
|
|
|
|
if (fault === "coordinator_owner") await db.update(nativeRunFinalizations).set({ leaseOwner: randomUUID(), leaseExpiresAt: new Date(Date.now() + 60_000) }).where(eq(nativeRunFinalizations.runId, f.runId));
|
|
|
|
|
if (fault === "coordinator_observed") await db.update(nativeRunFinalizations).set({ phase: "observed" }).where(eq(nativeRunFinalizations.runId, f.runId));
|
|
|
|
|
if (fault === "foreign_action_run") await db.update(issueRecoveryActions).set({ evidence: sql`jsonb_set(${issueRecoveryActions.evidence}, '{runId}', ${JSON.stringify(randomUUID())}::jsonb)` }).where(eq(issueRecoveryActions.id, f.actionId));
|
|
|
|
|
if (fault === "expired_deadline") await updateSystemRecovery(db, f.input, t.held, { ...t.held, revision: t.held.revision + 1, deadlineAt: Date.now() - 1 });
|
|
|
|
|
if (fault === "exhausted") await updateSystemRecovery(db, f.input, t.held, { ...t.held, revision: t.held.revision + 1, state: "exhausted", nextAttemptAt: null });
|
|
|
|
|
if (fault === "retry_budget") await updateSystemRecovery(db, f.input, t.held, { ...t.held, revision: t.held.revision + 1, retriesUsed: 5 });
|
|
|
|
|
if (fault === "unrelated_failure") await updateSystemRecovery(db, f.input, t.held, failSystemRecovery(t.held, new SystemRecoveryFailure("provider_unavailable", "retry_transient")));
|
|
|
|
|
const before = (await action(f.actionId)).evidence;
|
|
|
|
|
await control.resumeRemoteStopComments(await run(f.runId));
|
|
|
|
|
expect((await action(f.actionId)).evidence).toEqual(before);
|
|
|
|
|
if (fault !== "exhausted" && fault !== "retry_budget") expect(deps.enqueueWakeup).not.toHaveBeenCalled();
|
|
|
|
|
} finally { t.cleanup(); }
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
it("consumes identical Stop-evidence only once across another actual failed attempt", async () => {
|
|
|
|
|
const t = await heldStoppedEvidence(), { f, control, execution } = t;
|
|
|
|
|
try {
|
|
|
|
|
await control.resumeRemoteStopComments(await run(f.runId));
|
|
|
|
|
expect((await action(f.actionId)).evidence.workspaceStopEvidence).toMatchObject({ consumed: true });
|
|
|
|
|
adapterExecutionControls.set(f.runId, execution);
|
|
|
|
|
await recoverRetainedWorkspace(f.input);
|
|
|
|
|
t.cleanup();
|
|
|
|
|
const before = (await action(f.actionId)).evidence;
|
|
|
|
|
expect(readSystemRecovery(before)?.nextAttemptAt).toBeGreaterThan(Date.now() + 20_000);
|
|
|
|
|
await control.resumeRemoteStopComments(await run(f.runId));
|
|
|
|
|
await control.resumeRemoteStopComments(await run(f.runId));
|
|
|
|
|
expect((await action(f.actionId)).evidence).toEqual(before);
|
|
|
|
|
} finally { t.cleanup(); }
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
it("keeps the provider cooldown and retry window when advancing new Stop-evidence", async () => {
|
|
|
|
|
const t = await heldStoppedEvidence(), { f, control } = t;
|
|
|
|
|
try {
|
|
|
|
|
const cooldown = Date.now() + 5_000;
|
|
|
|
|
const state = failSystemRecovery(t.held, new SystemRecoveryFailure("agent_stop_unconfirmed", "recheck_evidence"), { retryAfterAt: cooldown });
|
|
|
|
|
await updateSystemRecovery(db, f.input, t.held, state);
|
|
|
|
|
await control.resumeRemoteStopComments(await run(f.runId));
|
|
|
|
|
const after = readSystemRecovery((await action(f.actionId)).evidence)!;
|
|
|
|
|
expect(after.nextAttemptAt).toBe(cooldown);
|
|
|
|
|
expect(after).toMatchObject({ generationId: state.generationId, attempts: state.attempts, retriesUsed: state.retriesUsed, firstFailureAt: state.firstFailureAt, deadlineAt: state.deadlineAt, providerCooldownAt: cooldown });
|
|
|
|
|
await control.resumeRemoteStopComments(await run(f.runId));
|
|
|
|
|
expect(readSystemRecovery((await action(f.actionId)).evidence)).toEqual(after);
|
|
|
|
|
await recoverRetainedWorkspace(f.input);
|
|
|
|
|
expect(readSystemRecovery((await action(f.actionId)).evidence)).toEqual(after);
|
|
|
|
|
} finally { t.cleanup(); }
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
it.each(["current", "stale_generation", "lost_receipt"] as const)("retains a post-cleanup Stop-evidence event before worker failure commits: %s", async variant => {
|
|
|
|
|
const t = await stoppedEvidenceFixture(), { f, control } = t;
|
|
|
|
|
const originalReader = recoverySources.inspectWorkspaceRecoverySource;
|
|
|
|
|
let observed = false, release!: () => void;
|
|
|
|
|
const barrier = new Promise<void>(resolve => { release = resolve; });
|
|
|
|
|
const reader = vi.spyOn(recoverySources, "inspectWorkspaceRecoverySource").mockImplementationOnce(async (...args) => {
|
|
|
|
|
try { return await originalReader(...args); }
|
|
|
|
|
catch (error) { observed = true; await barrier; throw error; }
|
|
|
|
|
});
|
|
|
|
|
const worker = recoverRetainedWorkspace(f.input);
|
|
|
|
|
try {
|
|
|
|
|
await expect.poll(() => observed, { timeout: 5_000 }).toBe(true);
|
|
|
|
|
const running = readSystemRecovery((await action(f.actionId)).evidence)!;
|
|
|
|
|
expect(running.state).toBe("running");
|
|
|
|
|
await control.acknowledgeRemoteStop(f.runId, f.companyId); t.cleanup();
|
|
|
|
|
await control.resumeRemoteStopComments(await run(f.runId));
|
|
|
|
|
const pending = await action(f.actionId);
|
|
|
|
|
expect(readSystemRecovery(pending.evidence)).toEqual(running);
|
|
|
|
|
expect(pending.evidence.workspaceStopEvidence).toMatchObject({ sourceRunId: f.runId, generationId: running.generationId, consumed: false });
|
|
|
|
|
if (variant === "stale_generation") await db.update(issueRecoveryActions).set({ evidence: sql`jsonb_set(${issueRecoveryActions.evidence}, '{workspaceStopEvidence,generationId}', ${JSON.stringify(randomUUID())}::jsonb)` }).where(eq(issueRecoveryActions.id, f.actionId));
|
|
|
|
|
if (variant === "lost_receipt") await db.update(environmentLeases).set({ metadata: sql`${environmentLeases.metadata} - 'remoteExecutionTermination'` }).where(eq(environmentLeases.id, f.lease.id));
|
|
|
|
|
release(); await worker;
|
|
|
|
|
const after = await action(f.actionId), state = readSystemRecovery(after.evidence)!;
|
|
|
|
|
expect(state.state).toBe("awaiting_evidence");
|
|
|
|
|
expect(state).toMatchObject({ attempts: running.attempts, retriesUsed: running.retriesUsed, generationId: running.generationId });
|
|
|
|
|
if (variant === "current") {
|
|
|
|
|
expect(state.nextAttemptAt!).toBeLessThanOrEqual(Date.now());
|
|
|
|
|
expect(after.evidence.workspaceStopEvidence).toMatchObject({ consumed: true });
|
|
|
|
|
await recoverRetainedWorkspace(f.input);
|
|
|
|
|
expect(readSystemRecovery((await action(f.actionId)).evidence)?.state).toBe("complete");
|
|
|
|
|
expect(await readFile(path.join(f.host, "saved.txt"), "utf8")).toBe("finished work\n");
|
|
|
|
|
} else {
|
|
|
|
|
expect(state.nextAttemptAt!).toBeGreaterThan(Date.now() + 20_000);
|
|
|
|
|
expect(after.evidence.workspaceStopEvidence).toMatchObject({ consumed: false });
|
|
|
|
|
}
|
|
|
|
|
} finally { release(); await worker; reader.mockRestore(); t.cleanup(); }
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
});
|