mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-16 11:59:26 +01:00
fix(recovery): expire each abandoned-files receipt on its own and never reference a deleted run or task
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
926a9119b0
commit
bb43f52702
2 files changed
+128
-19
No files matched your search
@@ -1,7 +1,7 @@
|
|||||||
import { JsonRpcCallError } from "@paperclipai/plugin-sdk";
|
import { JsonRpcCallError } from "@paperclipai/plugin-sdk";
|
||||||
import { randomUUID } from "node:crypto";
|
import { randomUUID } from "node:crypto";
|
||||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
import { eq } from "drizzle-orm";
|
import { eq, sql } from "drizzle-orm";
|
||||||
import {
|
import {
|
||||||
activityLog,
|
activityLog,
|
||||||
agents,
|
agents,
|
||||||
@@ -29,7 +29,7 @@ import { nextEnvironmentCleanupAt } from "../services/environment-cleanup-state.
|
|||||||
import { releasableRetainedOrphanCondition, settleRetainedOrphanSandbox } from "../services/environment-retained-orphan-settlement.ts";
|
import { releasableRetainedOrphanCondition, settleRetainedOrphanSandbox } from "../services/environment-retained-orphan-settlement.ts";
|
||||||
import {
|
import {
|
||||||
ABANDONED_WORKSPACE_FILES_PARKED_KEY, ABANDONED_WORKSPACE_FILES_PARKED_REASON, ABANDONED_WORKSPACE_FILES_RETENTION_DAYS,
|
ABANDONED_WORKSPACE_FILES_PARKED_KEY, ABANDONED_WORKSPACE_FILES_PARKED_REASON, ABANDONED_WORKSPACE_FILES_RETENTION_DAYS,
|
||||||
expireAbandonedWorkspaceFileHolds, releaseAbandonedWorkspaceFiles,
|
expireAbandonedWorkspaceFileHolds, nextAbandonedWorkspaceFileHoldExpiryAt, releaseAbandonedWorkspaceFiles,
|
||||||
} from "../services/remote-workspace-abandoned-files-hold.ts";
|
} from "../services/remote-workspace-abandoned-files-hold.ts";
|
||||||
|
|
||||||
vi.mock("../services/native-runtime/native-harness-backup-storage.js", () => ({
|
vi.mock("../services/native-runtime/native-harness-backup-storage.js", () => ({
|
||||||
@@ -369,6 +369,52 @@ describe.skipIf(!support.supported)("sandbox that holds files from an abandoned
|
|||||||
expect(await entries("workspace.abandoned_files_removed")).toEqual([]);
|
expect(await entries("workspace.abandoned_files_removed")).toEqual([]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("releases a receipt whose run was deleted and records it without the run reference", async () => {
|
||||||
|
const seeded = await seed();
|
||||||
|
await abandon(seeded);
|
||||||
|
await setReceipts(seeded, receipt => ({ ...receipt, runId: randomUUID(), retainUntil: new Date(Date.now() - 1_000).toISOString() }));
|
||||||
|
expect(await expireAbandonedWorkspaceFileHolds(db)).toEqual({ expired: 1 });
|
||||||
|
const [ended] = await entries("workspace.abandoned_files_released");
|
||||||
|
expect(ended).toMatchObject({ entityType: "issue", entityId: seeded.issueId, runId: null });
|
||||||
|
expect(await nativeWorkspaceExportPendingForLease(db, seeded.lease)).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("releases a receipt whose task was deleted without writing a task entry", async () => {
|
||||||
|
const seeded = await seed();
|
||||||
|
await abandon(seeded);
|
||||||
|
await setReceipts(seeded, receipt => ({ ...receipt, runId: randomUUID(), retainUntil: new Date(Date.now() - 1_000).toISOString() }));
|
||||||
|
await db.update(executionWorkspaces).set({ sourceIssueId: null }).where(eq(executionWorkspaces.id, seeded.executionWorkspaceId));
|
||||||
|
expect(await expireAbandonedWorkspaceFileHolds(db)).toEqual({ expired: 1 });
|
||||||
|
expect(await entries("workspace.abandoned_files_released")).toEqual([]);
|
||||||
|
expect(await nativeWorkspaceExportPendingForLease(db, seeded.lease)).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps expiring later receipts when one fails, and retries the failed one after a backoff", async () => {
|
||||||
|
const seeded = await seed();
|
||||||
|
await abandon(seeded);
|
||||||
|
await abandonOnOtherSandbox(seeded, "/later-root");
|
||||||
|
await setReceipts(seeded, receipt => ({ ...receipt, retainUntil: new Date(Date.now() - 1_000).toISOString() }));
|
||||||
|
await db.execute(sql`create function reject_abandoned_epoch_one() returns trigger language plpgsql as $$
|
||||||
|
begin if new.details->>'abandonedEpoch' = '1' then raise exception 'test activity failure'; end if; return new; end $$`);
|
||||||
|
await db.execute(sql`create trigger reject_abandoned_epoch_one before insert on activity_log
|
||||||
|
for each row execute function reject_abandoned_epoch_one()`);
|
||||||
|
try {
|
||||||
|
const before = Date.now();
|
||||||
|
expect(await expireAbandonedWorkspaceFileHolds(db)).toEqual({ expired: 1 });
|
||||||
|
const receipts = (await readWorkspace(seeded)).metadata!.remoteWorkspaceAbandonedEpochs as Array<Record<string, unknown>>;
|
||||||
|
const failed = receipts.find(receipt => receipt.epoch === 1)!;
|
||||||
|
expect(failed.released).toBeUndefined();
|
||||||
|
expect(failed.expiryAttempts).toBe(1);
|
||||||
|
expect(receipts.find(receipt => receipt.epoch !== 1)!.released).toMatchObject({ reason: "retention_expired" });
|
||||||
|
expect(await nativeWorkspaceExportPendingForLease(db, seeded.lease)).toBe(true);
|
||||||
|
expect(await nextAbandonedWorkspaceFileHoldExpiryAt(db)).toBeGreaterThanOrEqual(before + 5 * 60_000);
|
||||||
|
} finally {
|
||||||
|
await db.execute(sql`drop trigger reject_abandoned_epoch_one on activity_log`);
|
||||||
|
await db.execute(sql`drop function reject_abandoned_epoch_one()`);
|
||||||
|
}
|
||||||
|
expect(await expireAbandonedWorkspaceFileHolds(db, new Date(Date.now() + 10 * 60_000))).toEqual({ expired: 1 });
|
||||||
|
});
|
||||||
|
|
||||||
it("wakes the cleanup worker at the receipt's retention end, also after a restart, and then goes quiet", async () => {
|
it("wakes the cleanup worker at the receipt's retention end, also after a restart, and then goes quiet", async () => {
|
||||||
const seeded = await seed();
|
const seeded = await seed();
|
||||||
await abandon(seeded);
|
await abandon(seeded);
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { and, eq, inArray, sql, type SQL } from "drizzle-orm";
|
import { and, eq, inArray, sql, type SQL } from "drizzle-orm";
|
||||||
import { environmentLeases, executionWorkspaces, heartbeatRuns, type Db } from "@paperclipai/db";
|
import { environmentLeases, executionWorkspaces, heartbeatRuns, issues, type Db } from "@paperclipai/db";
|
||||||
import { logger } from "../middleware/logger.js";
|
import { logger } from "../middleware/logger.js";
|
||||||
import { logActivity } from "./activity-log.js";
|
import { logActivity } from "./activity-log.js";
|
||||||
import { writeEnvironmentCleanup } from "./environment-cleanup-notifications.js";
|
import { writeEnvironmentCleanup } from "./environment-cleanup-notifications.js";
|
||||||
@@ -12,6 +12,21 @@ export const ABANDONED_WORKSPACE_FILES_PARKED_KEY = "abandonedWorkspaceFilesPark
|
|||||||
export const ABANDONED_WORKSPACE_FILES_PARKED_REASON = "abandoned_workspace_files_held";
|
export const ABANDONED_WORKSPACE_FILES_PARKED_REASON = "abandoned_workspace_files_held";
|
||||||
export const ABANDONED_WORKSPACE_FILES_RELEASED_REASON = "abandoned_workspace_files_released";
|
export const ABANDONED_WORKSPACE_FILES_RELEASED_REASON = "abandoned_workspace_files_released";
|
||||||
export const ABANDONED_WORKSPACE_FILES_RELEASE_KEY = "abandonedWorkspaceFilesRelease";
|
export const ABANDONED_WORKSPACE_FILES_RELEASE_KEY = "abandonedWorkspaceFilesRelease";
|
||||||
|
const ABANDONED_WORKSPACE_FILES_EXPIRY_BACKOFF_MS = 5 * 60_000;
|
||||||
|
const ABANDONED_WORKSPACE_FILES_EXPIRY_MAX_BACKOFF_MS = 6 * 60 * 60_000;
|
||||||
|
|
||||||
|
async function existingIssueId(db: Db, companyId: string, issueId: string | null): Promise<string | null> {
|
||||||
|
if (!issueId) return null;
|
||||||
|
const [issue] = await db.select({ id: issues.id }).from(issues).where(and(eq(issues.companyId, companyId), eq(issues.id, issueId)));
|
||||||
|
return issue?.id ?? null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function existingRunId(db: Db, companyId: string, runId: string | null | undefined): Promise<string | undefined> {
|
||||||
|
if (!runId) return undefined;
|
||||||
|
const [run] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns)
|
||||||
|
.where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, runId)));
|
||||||
|
return run?.id;
|
||||||
|
}
|
||||||
|
|
||||||
function days(count: number) {
|
function days(count: number) {
|
||||||
return `${count} ${count === 1 ? "day" : "days"}`;
|
return `${count} ${count === 1 ? "day" : "days"}`;
|
||||||
@@ -64,6 +79,11 @@ export async function parkAbandonedWorkspaceFilesLease(db: Db, lease: Pick<typeo
|
|||||||
return parked.length > 0;
|
return parked.length > 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function receiptDueAt(receipt: Receipt): number {
|
||||||
|
const retry = typeof receipt.expiryRetryAt === "string" ? Date.parse(receipt.expiryRetryAt) : Number.NaN;
|
||||||
|
return Math.max(receiptRetainUntil(receipt), Number.isFinite(retry) ? retry : 0);
|
||||||
|
}
|
||||||
|
|
||||||
function receiptRetainUntil(receipt: Receipt): number {
|
function receiptRetainUntil(receipt: Receipt): number {
|
||||||
const explicit = typeof receipt.retainUntil === "string" ? Date.parse(receipt.retainUntil) : Number.NaN;
|
const explicit = typeof receipt.retainUntil === "string" ? Date.parse(receipt.retainUntil) : Number.NaN;
|
||||||
if (Number.isFinite(explicit)) return explicit;
|
if (Number.isFinite(explicit)) return explicit;
|
||||||
@@ -95,10 +115,11 @@ export async function releaseAbandonedWorkspaceFiles(db: Db, input: {
|
|||||||
remoteWorkspaceAbandonedEpochs: receipts.map(receipt => receipt === target ? { ...target, released } : receipt) } })
|
remoteWorkspaceAbandonedEpochs: receipts.map(receipt => receipt === target ? { ...target, released } : receipt) } })
|
||||||
.where(and(eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, workspace.id)));
|
.where(and(eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, workspace.id)));
|
||||||
const providerLeaseId = target.location!.providerLeaseId as string;
|
const providerLeaseId = target.location!.providerLeaseId as string;
|
||||||
const runId = typeof target.runId === "string" ? target.runId : undefined;
|
const recordedRunId = typeof target.runId === "string" ? target.runId : undefined;
|
||||||
const [run] = runId ? await tx.select({ issueId: heartbeatRuns.issueId }).from(heartbeatRuns)
|
const [run] = recordedRunId ? await tx.select({ id: heartbeatRuns.id, issueId: heartbeatRuns.issueId }).from(heartbeatRuns)
|
||||||
.where(and(eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, runId))) : [];
|
.where(and(eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, recordedRunId))) : [];
|
||||||
const issueId = workspace.sourceIssueId ?? run?.issueId ?? null;
|
const runId = run?.id;
|
||||||
|
const issueId = await existingIssueId(tx, input.companyId, workspace.sourceIssueId ?? run?.issueId ?? null);
|
||||||
const keptDays = Math.max(1, Math.round((now.getTime() - Date.parse(String(target.abandonedAt ?? now.toISOString()))) / 86_400_000));
|
const keptDays = Math.max(1, Math.round((now.getTime() - Date.parse(String(target.abandonedAt ?? now.toISOString()))) / 86_400_000));
|
||||||
const marker = { issueId, workspaceId: workspace.id, abandonedEpoch: input.epoch, reason: input.reason, keptDays, runId: runId ?? null };
|
const marker = { issueId, workspaceId: workspace.id, abandonedEpoch: input.epoch, reason: input.reason, keptDays, runId: runId ?? null };
|
||||||
const unparked = await writeEnvironmentCleanup(tx, write => write.update(environmentLeases).set({
|
const unparked = await writeEnvironmentCleanup(tx, write => write.update(environmentLeases).set({
|
||||||
@@ -117,10 +138,10 @@ export async function releaseAbandonedWorkspaceFiles(db: Db, input: {
|
|||||||
? `Unsaved files from an interrupted run were discarded.${scheduled ? " They are scheduled for removal." : ""}`
|
? `Unsaved files from an interrupted run were discarded.${scheduled ? " They are scheduled for removal." : ""}`
|
||||||
: "Unsaved files from an interrupted run were saved to the task's recovered files.";
|
: "Unsaved files from an interrupted run were saved to the task's recovered files.";
|
||||||
logger.info({ companyId: input.companyId, workspaceId: workspace.id, abandonedEpoch: input.epoch, reason: input.reason,
|
logger.info({ companyId: input.companyId, workspaceId: workspace.id, abandonedEpoch: input.epoch, reason: input.reason,
|
||||||
actorType: input.actorType, actorId: input.actorId, leasesHandedToCleanup: unparked.length }, "abandoned workspace files released");
|
actorType: input.actorType, actorId: input.actorId, leasesHandedToCleanup: unparked.length, issueId, runId: runId ?? null,
|
||||||
await logActivity(tx, { companyId: input.companyId, actorType: input.actorType, actorId: input.actorId,
|
summary }, "abandoned workspace files released");
|
||||||
action: "workspace.abandoned_files_released", entityType: issueId ? "issue" : "execution_workspace",
|
if (issueId) await logActivity(tx, { companyId: input.companyId, actorType: input.actorType, actorId: input.actorId,
|
||||||
entityId: issueId ?? workspace.id, runId,
|
action: "workspace.abandoned_files_released", entityType: "issue", entityId: issueId, runId,
|
||||||
details: { summary, reason: input.reason, workspaceId: workspace.id, abandonedEpoch: input.epoch,
|
details: { summary, reason: input.reason, workspaceId: workspace.id, abandonedEpoch: input.epoch,
|
||||||
retainUntil: new Date(receiptRetainUntil(target)).toISOString(), leasesHandedToCleanup: unparked.length } });
|
retainUntil: new Date(receiptRetainUntil(target)).toISOString(), leasesHandedToCleanup: unparked.length } });
|
||||||
return true;
|
return true;
|
||||||
@@ -135,29 +156,61 @@ export async function expireAbandonedWorkspaceFileHolds(db: Db, now = new Date()
|
|||||||
where receipt->>'schema' = ${REMOTE_WORKSPACE_ABANDONED_EPOCH_SCHEMA}
|
where receipt->>'schema' = ${REMOTE_WORKSPACE_ABANDONED_EPOCH_SCHEMA}
|
||||||
and coalesce(jsonb_typeof(receipt->'released'), 'null') = 'null')`);
|
and coalesce(jsonb_typeof(receipt->'released'), 'null') = 'null')`);
|
||||||
let expired = 0;
|
let expired = 0;
|
||||||
|
let failed = 0;
|
||||||
const held = new Map<string, number>();
|
const held = new Map<string, number>();
|
||||||
for (const row of rows) {
|
for (const row of rows) {
|
||||||
for (const receipt of Array.isArray(row.receipts) ? row.receipts : []) {
|
for (const receipt of Array.isArray(row.receipts) ? row.receipts : []) {
|
||||||
if (!unreleased(receipt)) continue;
|
if (!unreleased(receipt)) continue;
|
||||||
if (receiptRetainUntil(receipt) > now.getTime() || typeof receipt.epoch !== "number") {
|
if (receiptDueAt(receipt) > now.getTime() || typeof receipt.epoch !== "number") {
|
||||||
held.set(row.companyId, (held.get(row.companyId) ?? 0) + 1);
|
held.set(row.companyId, (held.get(row.companyId) ?? 0) + 1);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if (await releaseAbandonedWorkspaceFiles(db, { companyId: row.companyId, workspaceId: row.id, epoch: receipt.epoch,
|
try {
|
||||||
reason: "retention_expired", actorType: "system", actorId: "workspace-retention", now })) expired += 1;
|
if (await releaseAbandonedWorkspaceFiles(db, { companyId: row.companyId, workspaceId: row.id, epoch: receipt.epoch,
|
||||||
|
reason: "retention_expired", actorType: "system", actorId: "workspace-retention", now })) expired += 1;
|
||||||
|
} catch (error) {
|
||||||
|
failed += 1;
|
||||||
|
const retryAt = await deferAbandonedWorkspaceFilesExpiry(db, row.companyId, row.id, receipt, now);
|
||||||
|
logger.warn({ err: error, companyId: row.companyId, workspaceId: row.id, abandonedEpoch: receipt.epoch, retryAt },
|
||||||
|
"abandoned workspace file expiry failed; it retries later");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (expired > 0 || held.size > 0) {
|
if (expired > 0 || failed > 0 || held.size > 0) {
|
||||||
const parked = await db.select({ companyId: environmentLeases.companyId, count: sql<number>`count(*)::int` })
|
const parked = await db.select({ companyId: environmentLeases.companyId, count: sql<number>`count(*)::int` })
|
||||||
.from(environmentLeases).where(sql`${environmentLeases.metadata} ? ${ABANDONED_WORKSPACE_FILES_PARKED_KEY}`)
|
.from(environmentLeases).where(sql`${environmentLeases.metadata} ? ${ABANDONED_WORKSPACE_FILES_PARKED_KEY}`)
|
||||||
.groupBy(environmentLeases.companyId);
|
.groupBy(environmentLeases.companyId);
|
||||||
logger.info({ expired, retentionDays: ABANDONED_WORKSPACE_FILES_RETENTION_DAYS,
|
logger.info({ expired, failed, retentionDays: ABANDONED_WORKSPACE_FILES_RETENTION_DAYS,
|
||||||
heldReceiptsByCompany: Object.fromEntries(held), parkedLeasesByCompany: Object.fromEntries(parked.map(p => [p.companyId, p.count])) },
|
heldReceiptsByCompany: Object.fromEntries(held), parkedLeasesByCompany: Object.fromEntries(parked.map(p => [p.companyId, p.count])) },
|
||||||
"abandoned workspace file holds checked");
|
"abandoned workspace file holds checked");
|
||||||
}
|
}
|
||||||
return { expired };
|
return { expired };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function deferAbandonedWorkspaceFilesExpiry(db: Db, companyId: string, workspaceId: string, receipt: Receipt, now: Date) {
|
||||||
|
const attempts = (typeof receipt.expiryAttempts === "number" ? receipt.expiryAttempts : 0) + 1;
|
||||||
|
const retryAt = new Date(now.getTime() + Math.min(ABANDONED_WORKSPACE_FILES_EXPIRY_MAX_BACKOFF_MS,
|
||||||
|
ABANDONED_WORKSPACE_FILES_EXPIRY_BACKOFF_MS * 2 ** (attempts - 1))).toISOString();
|
||||||
|
try {
|
||||||
|
await db.transaction(async transaction => {
|
||||||
|
const tx = transaction as unknown as Db;
|
||||||
|
const [workspace] = await tx.select({ metadata: executionWorkspaces.metadata }).from(executionWorkspaces)
|
||||||
|
.where(and(eq(executionWorkspaces.companyId, companyId), eq(executionWorkspaces.id, workspaceId))).for("update");
|
||||||
|
const receipts = Array.isArray(workspace?.metadata?.remoteWorkspaceAbandonedEpochs)
|
||||||
|
? workspace!.metadata!.remoteWorkspaceAbandonedEpochs as unknown[] : [];
|
||||||
|
if (!workspace) return;
|
||||||
|
await tx.update(executionWorkspaces).set({ metadata: { ...workspace.metadata,
|
||||||
|
remoteWorkspaceAbandonedEpochs: receipts.map(value => unreleased(value) && value.epoch === receipt.epoch
|
||||||
|
&& value.location?.providerLeaseId === receipt.location?.providerLeaseId
|
||||||
|
? { ...value, expiryAttempts: attempts, expiryRetryAt: retryAt } : value) } })
|
||||||
|
.where(and(eq(executionWorkspaces.companyId, companyId), eq(executionWorkspaces.id, workspaceId)));
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
logger.warn({ err: error, companyId, workspaceId }, "abandoned workspace file expiry retry could not be recorded");
|
||||||
|
}
|
||||||
|
return retryAt;
|
||||||
|
}
|
||||||
|
|
||||||
function receiptExpirySql() {
|
function receiptExpirySql() {
|
||||||
return sql`coalesce(case when receipt->>'retainUntil' ~ '^[0-9]{4}-[0-9]{2}-[0-9]{2}T' then (receipt->>'retainUntil')::timestamptz end,
|
return sql`coalesce(case when receipt->>'retainUntil' ~ '^[0-9]{4}-[0-9]{2}-[0-9]{2}T' then (receipt->>'retainUntil')::timestamptz end,
|
||||||
case when receipt->>'abandonedAt' ~ '^[0-9]{4}-[0-9]{2}-[0-9]{2}T'
|
case when receipt->>'abandonedAt' ~ '^[0-9]{4}-[0-9]{2}-[0-9]{2}T'
|
||||||
@@ -165,8 +218,14 @@ function receiptExpirySql() {
|
|||||||
'epoch'::timestamptz)`;
|
'epoch'::timestamptz)`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function receiptDueSql() {
|
||||||
|
return sql`greatest(${receiptExpirySql()},
|
||||||
|
case when receipt->>'expiryRetryAt' ~ '^[0-9]{4}-[0-9]{2}-[0-9]{2}T' then (receipt->>'expiryRetryAt')::timestamptz
|
||||||
|
else 'epoch'::timestamptz end)`;
|
||||||
|
}
|
||||||
|
|
||||||
export async function nextAbandonedWorkspaceFileHoldExpiryAt(db: Pick<Db, "execute">): Promise<number | null> {
|
export async function nextAbandonedWorkspaceFileHoldExpiryAt(db: Pick<Db, "execute">): Promise<number | null> {
|
||||||
const [row] = await db.execute<{ next: string | Date | null }>(sql`select min(${receiptExpirySql()}) as next
|
const [row] = await db.execute<{ next: string | Date | null }>(sql`select min(${receiptDueSql()}) as next
|
||||||
from execution_workspaces workspace
|
from execution_workspaces workspace
|
||||||
cross join lateral jsonb_array_elements(case when jsonb_typeof(workspace.metadata->'remoteWorkspaceAbandonedEpochs') = 'array'
|
cross join lateral jsonb_array_elements(case when jsonb_typeof(workspace.metadata->'remoteWorkspaceAbandonedEpochs') = 'array'
|
||||||
then workspace.metadata->'remoteWorkspaceAbandonedEpochs' else '[]'::jsonb end) receipt
|
then workspace.metadata->'remoteWorkspaceAbandonedEpochs' else '[]'::jsonb end) receipt
|
||||||
@@ -189,11 +248,15 @@ export async function recordAbandonedWorkspaceFilesRemoved(db: Db, lease: Pick<t
|
|||||||
issueId?: string | null; workspaceId?: string; abandonedEpoch?: number; reason?: string; keptDays?: number; runId?: string | null;
|
issueId?: string | null; workspaceId?: string; abandonedEpoch?: number; reason?: string; keptDays?: number; runId?: string | null;
|
||||||
} | undefined;
|
} | undefined;
|
||||||
if (!marker || (marker.reason !== "retention_expired" && marker.reason !== "discarded")) return;
|
if (!marker || (marker.reason !== "retention_expired" && marker.reason !== "discarded")) return;
|
||||||
|
const issueId = await existingIssueId(db, lease.companyId, marker.issueId ?? null);
|
||||||
|
const runId = await existingRunId(db, lease.companyId, marker.runId);
|
||||||
const summary = marker.reason === "retention_expired"
|
const summary = marker.reason === "retention_expired"
|
||||||
? `Unsaved files from an interrupted run were kept for ${days(marker.keptDays ?? ABANDONED_WORKSPACE_FILES_RETENTION_DAYS)} and have now been removed.`
|
? `Unsaved files from an interrupted run were kept for ${days(marker.keptDays ?? ABANDONED_WORKSPACE_FILES_RETENTION_DAYS)} and have now been removed.`
|
||||||
: "Discarded unsaved files from an interrupted run have now been removed.";
|
: "Discarded unsaved files from an interrupted run have now been removed.";
|
||||||
|
logger.info({ companyId: lease.companyId, leaseId: lease.id, workspaceId: marker.workspaceId, issueId, summary },
|
||||||
|
"abandoned workspace files removed");
|
||||||
|
if (!issueId) return;
|
||||||
await logActivity(db, { companyId: lease.companyId, actorType: "system", actorId: "workspace-retention",
|
await logActivity(db, { companyId: lease.companyId, actorType: "system", actorId: "workspace-retention",
|
||||||
action: "workspace.abandoned_files_removed", entityType: marker.issueId ? "issue" : "execution_workspace",
|
action: "workspace.abandoned_files_removed", entityType: "issue", entityId: issueId, runId,
|
||||||
entityId: marker.issueId ?? marker.workspaceId ?? lease.id, runId: marker.runId ?? undefined,
|
|
||||||
details: { summary, reason: marker.reason, workspaceId: marker.workspaceId, abandonedEpoch: marker.abandonedEpoch, leaseId: lease.id } });
|
details: { summary, reason: marker.reason, workspaceId: marker.workspaceId, abandonedEpoch: marker.abandonedEpoch, leaseId: lease.id } });
|
||||||
}
|
}
|
||||||
Reference in new issue
Block a user