diff --git a/packages/db/src/client.ts b/packages/db/src/client.ts index 1f7c25f6c59..6e6df8060c9 100644 --- a/packages/db/src/client.ts +++ b/packages/db/src/client.ts @@ -648,6 +648,7 @@ const MIGRATIONS_WITH_TOP_LEVEL_STATEMENTS = new Set([ "0460_workspace_activity_context_index.sql", "0461_created_from_legacy_origin_run_index.sql", "0470_heartbeat_runs_queued_created_index.sql", + "0472_runner_api_capture_pending_deletes.sql", "0469_zippy_photon.sql", diff --git a/packages/db/src/migrations/0472_runner_api_capture_pending_deletes.sql b/packages/db/src/migrations/0472_runner_api_capture_pending_deletes.sql new file mode 100644 index 00000000000..eb3908d85b5 --- /dev/null +++ b/packages/db/src/migrations/0472_runner_api_capture_pending_deletes.sql @@ -0,0 +1,2 @@ +ALTER TABLE "runner_api_response_reservations" ADD COLUMN IF NOT EXISTS "pending_delete_object_key" text;--> statement-breakpoint +CREATE INDEX CONCURRENTLY IF NOT EXISTS "runner_api_response_reservations_asset_idx" ON "runner_api_response_reservations" USING btree ("asset_id"); diff --git a/packages/db/src/migrations/meta/0467_snapshot.json b/packages/db/src/migrations/meta/0472_snapshot.json similarity index 99% rename from packages/db/src/migrations/meta/0467_snapshot.json rename to packages/db/src/migrations/meta/0472_snapshot.json index 3167bbd1e50..ccbd3f28a41 100644 --- a/packages/db/src/migrations/meta/0467_snapshot.json +++ b/packages/db/src/migrations/meta/0472_snapshot.json @@ -1,6 +1,6 @@ { - "id": "426d1f42-0983-4c6b-bb8d-5c798eee915a", - "prevId": "593f81c4-2b4d-4fb4-a3e3-848b19bc9b0f", + "id": "77704cfc-2cbe-41de-9d2c-5225b8466646", + "prevId": "671a422a-0b93-4155-b097-9d35a4fd2b2b", "version": "7", "dialect": "postgresql", "tables": { @@ -1417,6 +1417,34 @@ "concurrently": false, "method": "btree", "with": {} + }, + "activity_log_company_handled_for_created_idx": { + "name": "activity_log_company_handled_for_created_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "(\"details\" ->> 'handledForUserId')", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "(\"activity_log\".\"details\" ->> 'handledForUserId') IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} } }, "foreignKeys": { @@ -16340,6 +16368,18 @@ "primaryKey": false, "notNull": false }, + "default_decision_owner_user_id": { + "name": "default_decision_owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "budget_owner_user_id": { + "name": "budget_owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, "require_board_approval_for_new_agents": { "name": "require_board_approval_for_new_agents", "type": "boolean", @@ -22302,6 +22342,18 @@ "type": "timestamp with time zone", "primaryKey": false, "notNull": false + }, + "dispatching_at": { + "name": "dispatching_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "dispatch_lease_expires_at": { + "name": "dispatch_lease_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false } }, "indexes": { @@ -35606,6 +35658,34 @@ "method": "btree", "with": {} }, + "heartbeat_runs_queued_created_idx": { + "name": "heartbeat_runs_queued_created_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"heartbeat_runs\".\"status\" = 'queued'", + "concurrently": true, + "method": "btree", + "with": {} + }, "heartbeat_runs_cost_accounting_pending_idx": { "name": "heartbeat_runs_cost_accounting_pending_idx", "columns": [ @@ -36235,6 +36315,12 @@ "primaryKey": false, "notNull": false }, + "fingerprint": { + "name": "fingerprint", + "type": "text", + "primaryKey": false, + "notNull": false + }, "created_at": { "name": "created_at", "type": "timestamp with time zone", @@ -51040,6 +51126,12 @@ "primaryKey": false, "notNull": false }, + "decision_owner_user_id": { + "name": "decision_owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, "target_date": { "name": "target_date", "type": "date", @@ -53594,6 +53686,12 @@ "primaryKey": false, "notNull": true }, + "pending_delete_object_key": { + "name": "pending_delete_object_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, "created_at": { "name": "created_at", "type": "timestamp with time zone", @@ -53617,6 +53715,21 @@ "concurrently": false, "method": "btree", "with": {} + }, + "runner_api_response_reservations_asset_idx": { + "name": "runner_api_response_reservations_asset_idx", + "columns": [ + { + "expression": "asset_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} } }, "foreignKeys": { @@ -62996,6 +63109,12 @@ "notNull": true, "default": false }, + "time_zone": { + "name": "time_zone", + "type": "text", + "primaryKey": false, + "notNull": false + }, "created_at": { "name": "created_at", "type": "timestamp with time zone", diff --git a/packages/db/src/migrations/meta/_journal.json b/packages/db/src/migrations/meta/_journal.json index e433eeb4219..7238172ae7a 100644 --- a/packages/db/src/migrations/meta/_journal.json +++ b/packages/db/src/migrations/meta/_journal.json @@ -3291,6 +3291,13 @@ "when": 1794495402720, "tag": "0471_controller_instances_dispatch_marker", "breakpoints": true + }, + { + "idx": 472, + "version": "7", + "when": 1794503924304, + "tag": "0472_runner_api_capture_pending_deletes", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/db/src/schema/runner_api_response_reservations.ts b/packages/db/src/schema/runner_api_response_reservations.ts index 32519a7a866..70783e91e7c 100644 --- a/packages/db/src/schema/runner_api_response_reservations.ts +++ b/packages/db/src/schema/runner_api_response_reservations.ts @@ -1,4 +1,4 @@ -import { pgTable, uuid, bigint, timestamp, index } from "drizzle-orm/pg-core"; +import { pgTable, uuid, bigint, text, timestamp, index } from "drizzle-orm/pg-core"; import { companies } from "./companies.js"; import { heartbeatRuns } from "./heartbeat_runs.js"; import { assets } from "./assets.js"; @@ -9,5 +9,9 @@ export const runnerApiResponseReservations = pgTable("runner_api_response_reserv runId: uuid("run_id").references(() => heartbeatRuns.id, { onDelete: "set null" }), assetId: uuid("asset_id").references(() => assets.id, { onDelete: "cascade" }), reservedBytes: bigint("reserved_bytes", { mode: "number" }).notNull(), + pendingDeleteObjectKey: text("pending_delete_object_key"), createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), -}, table => ({ companyIdx: index("runner_api_response_reservations_company_idx").on(table.companyId) })); +}, table => ({ + companyIdx: index("runner_api_response_reservations_company_idx").on(table.companyId), + assetIdx: index("runner_api_response_reservations_asset_idx").on(table.assetId), +})); diff --git a/server/src/services/native-runtime/paperclip-runner-tool-authority.ts b/server/src/services/native-runtime/paperclip-runner-tool-authority.ts index 82f3e3f3550..af33f5154d9 100644 --- a/server/src/services/native-runtime/paperclip-runner-tool-authority.ts +++ b/server/src/services/native-runtime/paperclip-runner-tool-authority.ts @@ -76,7 +76,8 @@ import { badRequest, forbidden, notFound, HttpError } from "../../errors.js"; import { casesDisabledError } from "../cases-disabled.js"; import { runnerApiCatalog, searchRunnerApi, withCompanyPagePaths } from "./runner-api-catalog.js"; import { assertRunnerApiDispatchable, executeRunnerApi, validateRunnerApiCall, RUNNER_API_MAX_BYTES, type RunnerApiCall, type RunnerApiFile, type RunnerApiIo } from "./runner-api-client.js"; -import { acquireRunnerApiResponseSlot, runnerApiCompanyCaptureMaxBytes, RUNNER_API_RESPONSE_MAX_BYTES, RUNNER_API_RESPONSE_RUN_MAX_BYTES, RunnerApiResponseLimitError } from "./runner-api-response-limits.js"; +import { finishPendingRunnerApiCaptureDeletes, markRunnerApiCaptureEvictions, RUNNER_API_CAPTURE_EVICTION_PASSES, runnerApiCaptureCleanupStalled, runnerApiCaptureUsageBytes } from "./runner-api-capture-retention.js"; +import { acquireRunnerApiResponseSlot, RUNNER_API_RESPONSE_DEADLINE_MS, RunnerApiCaptureReservationReclaimedError, runnerApiCompanyCaptureMaxBytes, RUNNER_API_RESPONSE_MAX_BYTES, RUNNER_API_RESPONSE_RUN_MAX_BYTES, RunnerApiResponseLimitError } from "./runner-api-response-limits.js"; import { and, desc, eq, inArray, isNull, notInArray, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { @@ -1505,9 +1506,17 @@ export class PaperclipRunnerToolAuthority { if (!reservationId) throw new Error("API response storage reservation is missing"); const storage = this.binding.storage ?? getStorageService(); storageAttempted = true; - const saved = await storage.putFile({ companyId: this.binding.companyId, namespace: caseId ? `cases/${caseId}/runner-api` : "runner-api", originalFilename: contentType.includes("json") ? "response.json" : "response.bin", contentType, ...(Buffer.isBuffer(bytes) ? { body: bytes } : { body: createReadStream(bytes.path), byteSize: bytes.byteSize, sha256: bytes.sha256 }) }); + const saved = await storage.putFile({ companyId: this.binding.companyId, namespace: caseId ? `cases/${caseId}/runner-api` : "runner-api", originalFilename: contentType.includes("json") ? "response.json" : "response.bin", contentType, signal: AbortSignal.timeout(RUNNER_API_RESPONSE_DEADLINE_MS), ...(Buffer.isBuffer(bytes) ? { body: bytes } : { body: createReadStream(bytes.path), byteSize: bytes.byteSize, sha256: bytes.sha256 }) }); const asset = await this.db.transaction(async tx => { await this.#lockApiCaptureCompany(tx as unknown as Db); + const [held] = await tx.select({ id: runnerApiResponseReservations.id }).from(runnerApiResponseReservations).where(and( + eq(runnerApiResponseReservations.id, reservationId!), eq(runnerApiResponseReservations.companyId, this.binding.companyId), + isNull(runnerApiResponseReservations.assetId))).for("update"); + if (!held) { + await tx.insert(runnerApiResponseReservations).values({ companyId: this.binding.companyId, runId: null, + reservedBytes: saved.byteSize, pendingDeleteObjectKey: saved.objectKey }); + return null; + } const asset = await assetService(tx as unknown as Db).create(this.binding.companyId, { ...saved, createdByAgentId: this.binding.agentId }); await tx.update(runnerApiResponseReservations).set({ assetId: asset.id, reservedBytes: asset.byteSize }).where(and(eq(runnerApiResponseReservations.id, reservationId!), eq(runnerApiResponseReservations.companyId, this.binding.companyId))); return asset; @@ -1524,6 +1533,11 @@ export class PaperclipRunnerToolAuthority { } throw error; }); + if (!asset) { + storageAttempted = false; + await finishPendingRunnerApiCaptureDeletes(this.db, storage, this.binding.companyId); + throw new RunnerApiCaptureReservationReclaimedError(); + } const activity = await persistActivity(this.db, { companyId: this.binding.companyId, actorType: "agent", actorId: this.binding.agentId, agentId: this.binding.agentId, runId: this.binding.runId, issueId: this.binding.issueId, action: "asset.created", entityType: "asset", entityId: asset.id, details: { source, byteSize: saved.byteSize } }); publishActivity(activity.publication); return { artifactId: asset.id, url: `/api/assets/${asset.id}/content`, contentType, byteSize: saved.byteSize, sha256: saved.sha256 }; @@ -1537,43 +1551,43 @@ export class PaperclipRunnerToolAuthority { async #reserveApiResponseCapture() { const release = acquireRunnerApiResponseSlot(this.binding.companyId); - let id: string; + let id: string | undefined; + let cleanupFailing = false; try { - id = await this.db.transaction(async tx => { - await this.#lockApiCaptureCompany(tx as unknown as Db); - const locked = await this.#lockAuthorizedMutationContext(tx as unknown as Db); - const resultJson = record(locked.run.resultJson); - const used = resultJson.apiResponseCaptureBytes ?? 0; - if (typeof used !== "number" || !Number.isSafeInteger(used) || used < 0 - || used + RUNNER_API_RESPONSE_MAX_BYTES > RUNNER_API_RESPONSE_RUN_MAX_BYTES) { - throw new RunnerApiResponseLimitError("api_response_capture_limit", "This run has insufficient capture budget (4 GiB per run). Read an existing asset or narrow the query."); + for (let attempt = 0; id === undefined; attempt++) { + id = await this.db.transaction(async tx => { + await this.#lockApiCaptureCompany(tx as unknown as Db); + const locked = await this.#lockAuthorizedMutationContext(tx as unknown as Db); + const resultJson = record(locked.run.resultJson); + const used = resultJson.apiResponseCaptureBytes ?? 0; + if (typeof used !== "number" || !Number.isSafeInteger(used) || used < 0 + || used + RUNNER_API_RESPONSE_MAX_BYTES > RUNNER_API_RESPONSE_RUN_MAX_BYTES) { + throw new RunnerApiResponseLimitError("api_response_capture_limit", "This run has insufficient capture budget (4 GiB per run). Read an existing asset or narrow the query."); + } + const companyBytes = await runnerApiCaptureUsageBytes(tx as unknown as Db, this.binding.companyId); + const overBy = companyBytes + RUNNER_API_RESPONSE_MAX_BYTES - runnerApiCompanyCaptureMaxBytes(); + if (Number.isSafeInteger(companyBytes) && companyBytes >= 0 && overBy > 0 && attempt < RUNNER_API_CAPTURE_EVICTION_PASSES) { + const marked = await markRunnerApiCaptureEvictions(tx as unknown as Db, this.binding.companyId, overBy); + if (marked.releasedBytes >= overBy) return this.#insertApiCaptureReservation(tx as unknown as Db, resultJson, used); + return undefined; + } + if (cleanupFailing && overBy > 0) { + throw new RunnerApiResponseLimitError("api_response_company_storage_limit", "Saved API responses for this company cannot be stored right now because file storage keeps failing to delete old ones. Paperclip keeps retrying the cleanup automatically. Read a saved response you already have, ask for less data, or try again later."); + } + if (!Number.isSafeInteger(companyBytes) || companyBytes < 0 || overBy > 0) { + throw new RunnerApiResponseLimitError("api_response_company_storage_limit", "Saved API responses for this company are using all of their storage, and none can be removed yet: they belong to runs that are still working, are attached to tasks, or are still being deleted. Read a saved response you already have, ask for less data, or try again in a few minutes."); + } + return this.#insertApiCaptureReservation(tx as unknown as Db, resultJson, used); + }); + if (id === undefined) { + const cleanup = await finishPendingRunnerApiCaptureDeletes(this.db, this.binding.storage ?? getStorageService(), this.binding.companyId); + cleanupFailing = cleanup.failed > 0 && await runnerApiCaptureCleanupStalled(this.db, this.binding.companyId); } - // One company lock covers all runs/processes. Stored assets include - // snapshots from before reservations existed; attached reservations - // are excluded so completed snapshots are counted exactly once. - const [usage] = await tx.select({ bytes: sql` - coalesce((select sum(${assets.byteSize}) from ${assets} - where ${assets.companyId} = ${this.binding.companyId} - and (${assets.objectKey} like ${`${this.binding.companyId}/runner-api/%`} - or exists (select 1 from ${runnerApiResponseReservations} capture - where capture.asset_id = ${assets.id} and capture.company_id = ${this.binding.companyId}))), 0) - + coalesce((select sum(${runnerApiResponseReservations.reservedBytes}) from ${runnerApiResponseReservations} - where ${runnerApiResponseReservations.companyId} = ${this.binding.companyId} - and ${runnerApiResponseReservations.assetId} is null), 0)` }).from(heartbeatRuns).where(eq(heartbeatRuns.id, this.binding.runId)); - const companyBytes = Number(usage.bytes); - if (!Number.isSafeInteger(companyBytes) || companyBytes < 0 || companyBytes + RUNNER_API_RESPONSE_MAX_BYTES > runnerApiCompanyCaptureMaxBytes()) { - throw new RunnerApiResponseLimitError("api_response_company_storage_limit", "Company API snapshot storage quota is exhausted. Read an existing asset or ask the operator to remove old snapshots or increase the quota."); - } - // Reserve worst-case bytes durably before creating a temporary file. - // Interrupted/failed captures keep the reservation, preventing retry - // loops and process restarts from resetting this run's disk-I/O budget. - await tx.update(heartbeatRuns).set({ resultJson: { ...resultJson, apiResponseCaptureBytes: used + RUNNER_API_RESPONSE_MAX_BYTES } }).where(eq(heartbeatRuns.id, this.binding.runId)); - const [reservation] = await tx.insert(runnerApiResponseReservations).values({ companyId: this.binding.companyId, runId: this.binding.runId, reservedBytes: RUNNER_API_RESPONSE_MAX_BYTES }).returning({ id: runnerApiResponseReservations.id }); - return reservation.id; - }); + } } catch (error) { release(); throw error; } + const reservationId = id; let settled = false; - return { id, settle: async (completedBytes?: number, cleaned?: boolean, storageAttempted?: boolean) => { + return { id: reservationId, settle: async (completedBytes?: number, cleaned?: boolean, storageAttempted?: boolean) => { if (settled) return; settled = true; try { @@ -1581,8 +1595,8 @@ export class PaperclipRunnerToolAuthority { await this.#lockApiCaptureCompany(tx as unknown as Db); // Reclaim a handled pre-storage failure only after disk cleanup. // Crashes, failed cleanup, and ambiguous storage failures keep their - // reservation until an operator reconciles the possible orphan data. - if (cleaned && !storageAttempted) await tx.delete(runnerApiResponseReservations).where(and(eq(runnerApiResponseReservations.id, id), isNull(runnerApiResponseReservations.assetId))); + // reservation until company admission reclaims it after the run ends. + if (cleaned && !storageAttempted) await tx.delete(runnerApiResponseReservations).where(and(eq(runnerApiResponseReservations.id, reservationId), isNull(runnerApiResponseReservations.assetId))); if (completedBytes === undefined) return; const [run] = await tx.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, this.binding.runId), eq(heartbeatRuns.companyId, this.binding.companyId))).for("update"); if (!run) return; @@ -1595,6 +1609,12 @@ export class PaperclipRunnerToolAuthority { } }; } + async #insertApiCaptureReservation(tx: Db, resultJson: Record, used: number): Promise { + await tx.update(heartbeatRuns).set({ resultJson: { ...resultJson, apiResponseCaptureBytes: used + RUNNER_API_RESPONSE_MAX_BYTES } }).where(eq(heartbeatRuns.id, this.binding.runId)); + const [reservation] = await tx.insert(runnerApiResponseReservations).values({ companyId: this.binding.companyId, runId: this.binding.runId, reservedBytes: RUNNER_API_RESPONSE_MAX_BYTES }).returning({ id: runnerApiResponseReservations.id }); + return reservation.id; + } + async #dotOwnsApiAsset(assetId: string): Promise { const [capture] = await this.db.select({ id: runnerApiResponseReservations.id }).from(runnerApiResponseReservations).where(and( eq(runnerApiResponseReservations.companyId, this.binding.companyId), eq(runnerApiResponseReservations.runId, this.binding.runId), diff --git a/server/src/services/native-runtime/runner-api-capture-retention.ts b/server/src/services/native-runtime/runner-api-capture-retention.ts new file mode 100644 index 00000000000..08f5cfd8bbc --- /dev/null +++ b/server/src/services/native-runtime/runner-api-capture-retention.ts @@ -0,0 +1,136 @@ +import { and, asc, eq, gt, inArray, isNotNull, isNull, lt, sql } from "drizzle-orm"; +import { activityLog, assets, runnerApiResponseReservations, type Db } from "@paperclipai/db"; +import type { StorageService } from "../../storage/types.js"; +import { logger } from "../../middleware/logger.js"; +import { incrementToolRuntimeMetricCounter } from "../tool-runtime-metrics.js"; +import { RUNNER_API_RESPONSE_DEADLINE_MS } from "./runner-api-response-limits.js"; + +const ACTIVE_RUN_STATUSES = ["queued", "running", "scheduled_retry"] as const; +const TERMINAL_RUN_STATUSES = ["succeeded", "interrupted", "failed", "cancelled", "timed_out"] as const; +export const CAPTURE_EVICTION_MIN_AGE_MS = 60 * 60_000; +export const RUNNER_API_CAPTURE_EVICTION_PASSES = 2; +export const RUNNER_API_CAPTURE_RETENTION_ACTOR = "api-response-retention"; +const EVICTION_PAGE_SIZE = 200; +const PENDING_DELETE_BATCH_SIZE = 50; +export const STALE_CAPTURE_RESERVATION_MS = 2 * RUNNER_API_RESPONSE_DEADLINE_MS; +export const CAPTURE_EVICTION_DELETE_TIMEOUT_MS = 30_000; +export const CAPTURE_CLEANUP_STALLED_MS = 5 * 60_000; +export const RUNNER_API_CAPTURE_DELETE_FAILURE_METRIC = "runner_api_capture_delete_failed"; + +function activeRunStatusList() { + return sql.join(ACTIVE_RUN_STATUSES.map(status => sql`${status}`), sql`, `); +} + +function terminalRunStatusList() { + return sql.join(TERMINAL_RUN_STATUSES.map(status => sql`${status}`), sql`, `); +} + +export async function runnerApiCaptureUsageBytes(tx: Db, companyId: string): Promise { + const [row] = await tx.execute<{ bytes: string | number }>(sql`select + coalesce((select sum(${assets.byteSize}) from ${assets} + where ${assets.companyId} = ${companyId} + and (${assets.objectKey} like ${`${companyId}/runner-api/%`} + or exists (select 1 from ${runnerApiResponseReservations} capture + where capture.asset_id = ${assets.id} and capture.company_id = ${companyId}))), 0) + + coalesce((select sum(${runnerApiResponseReservations.reservedBytes}) from ${runnerApiResponseReservations} + where ${runnerApiResponseReservations.companyId} = ${companyId} + and ${runnerApiResponseReservations.assetId} is null), 0) as bytes`); + return Number(row?.bytes ?? 0); +} + +export async function markRunnerApiCaptureEvictions(tx: Db, companyId: string, neededBytes: number, now = new Date()) { + const released = await tx.delete(runnerApiResponseReservations).where(and( + eq(runnerApiResponseReservations.companyId, companyId), + isNull(runnerApiResponseReservations.assetId), + isNull(runnerApiResponseReservations.pendingDeleteObjectKey), + lt(runnerApiResponseReservations.createdAt, new Date(now.getTime() - STALE_CAPTURE_RESERVATION_MS)), + sql`not exists (select 1 from heartbeat_runs run where run.id = ${runnerApiResponseReservations.runId} + and run.status in (${activeRunStatusList()}))`, + )).returning({ bytes: runnerApiResponseReservations.reservedBytes }); + let reclaimedBytes = released.reduce((sum, row) => sum + row.bytes, 0); + const [pending] = await tx.select({ bytes: sql`coalesce(sum(${runnerApiResponseReservations.reservedBytes}), 0)` }) + .from(runnerApiResponseReservations).where(and( + eq(runnerApiResponseReservations.companyId, companyId), isNotNull(runnerApiResponseReservations.pendingDeleteObjectKey), + )); + let pendingBytes = Number(pending?.bytes ?? 0); + while (reclaimedBytes + pendingBytes < neededBytes) { + const candidates = await tx.select({ id: assets.id, objectKey: assets.objectKey, byteSize: assets.byteSize, + runId: sql`(select capture.run_id from runner_api_response_reservations capture + where capture.asset_id = "assets"."id" and capture.run_id is not null limit 1)` }).from(assets).where(and( + eq(assets.companyId, companyId), + lt(assets.createdAt, new Date(now.getTime() - CAPTURE_EVICTION_MIN_AGE_MS)), + sql`(${assets.objectKey} like ${`${companyId}/runner-api/%`} + or exists (select 1 from runner_api_response_reservations capture + where capture.asset_id = ${assets.id} and capture.company_id = ${companyId}))`, + sql`not exists (select 1 from runner_api_response_reservations capture + join heartbeat_runs run on run.id = capture.run_id + where capture.asset_id = ${assets.id} and run.status not in (${terminalRunStatusList()}))`, + sql`not exists (select 1 from issue_attachments link where link.asset_id = ${assets.id})`, + sql`not exists (select 1 from case_attachments link where link.asset_id = ${assets.id})`, + sql`not exists (select 1 from company_logos link where link.asset_id = ${assets.id})`, + )).orderBy(asc(assets.createdAt), asc(assets.id)).limit(EVICTION_PAGE_SIZE); + const chosen: typeof candidates = []; + for (const candidate of candidates) { + if (reclaimedBytes + pendingBytes >= neededBytes) break; + chosen.push(candidate); + pendingBytes += candidate.byteSize; + } + if (chosen.length) { + await tx.delete(assets).where(and(eq(assets.companyId, companyId), inArray(assets.id, chosen.map(candidate => candidate.id)))); + await tx.insert(runnerApiResponseReservations).values(chosen.map(candidate => ({ + companyId, runId: null, assetId: null, reservedBytes: candidate.byteSize, pendingDeleteObjectKey: candidate.objectKey, + }))); + await tx.insert(activityLog).values(chosen.map(candidate => ({ + companyId, actorType: "system", actorId: RUNNER_API_CAPTURE_RETENTION_ACTOR, action: "asset.evicted", + entityType: "asset", entityId: candidate.id, runId: candidate.runId, + details: { source: "runner.call_api", reason: "company_capture_storage_full", byteSize: candidate.byteSize, runId: candidate.runId }, + }))); + } + if (candidates.length < EVICTION_PAGE_SIZE) break; + } + return { releasedBytes: reclaimedBytes, pendingBytes }; +} + +export async function finishPendingRunnerApiCaptureDeletes(db: Db, storage: StorageService, companyId: string, + timeoutMs = CAPTURE_EVICTION_DELETE_TIMEOUT_MS) { + const signal = AbortSignal.timeout(timeoutMs); + let deletedCount = 0; + let failedCount = 0; + let after: string | undefined; + while (!signal.aborted) { + const pending = await db.select({ id: runnerApiResponseReservations.id, objectKey: runnerApiResponseReservations.pendingDeleteObjectKey }) + .from(runnerApiResponseReservations).where(and( + eq(runnerApiResponseReservations.companyId, companyId), isNotNull(runnerApiResponseReservations.pendingDeleteObjectKey), + after ? gt(runnerApiResponseReservations.id, after) : undefined, + )).orderBy(asc(runnerApiResponseReservations.id)).limit(PENDING_DELETE_BATCH_SIZE); + if (!pending.length) break; + after = pending[pending.length - 1].id; + const results = await Promise.all(pending.map(row => storage.deleteObject(companyId, row.objectKey!, { signal, allVersions: true }) + .then(() => ({ id: row.id, error: null as unknown }), (error: unknown) => ({ id: row.id, error })))); + const deleted = results.filter(result => result.error === null).map(result => result.id); + const failures = results.filter(result => result.error !== null); + if (failures.length) { + failedCount += failures.length; + logger.warn({ companyId, failedDeletes: failures.length, err: failures[0].error }, + "Failed to delete evicted API response snapshots; they stay counted and will be retried"); + await Promise.all(failures.map(() => incrementToolRuntimeMetricCounter(db, { companyId, metric: RUNNER_API_CAPTURE_DELETE_FAILURE_METRIC }) + .catch(() => undefined))); + } + if (deleted.length) { + await db.delete(runnerApiResponseReservations).where(and( + eq(runnerApiResponseReservations.companyId, companyId), inArray(runnerApiResponseReservations.id, deleted), + )); + deletedCount += deleted.length; + } + if (pending.length < PENDING_DELETE_BATCH_SIZE) break; + } + return { deleted: deletedCount, failed: failedCount }; +} + +export async function runnerApiCaptureCleanupStalled(db: Db, companyId: string, now = new Date()) { + const [row] = await db.select({ id: runnerApiResponseReservations.id }).from(runnerApiResponseReservations).where(and( + eq(runnerApiResponseReservations.companyId, companyId), isNotNull(runnerApiResponseReservations.pendingDeleteObjectKey), + lt(runnerApiResponseReservations.createdAt, new Date(now.getTime() - CAPTURE_CLEANUP_STALLED_MS)), + )).limit(1); + return Boolean(row); +} diff --git a/server/src/services/native-runtime/runner-api-response-limits.ts b/server/src/services/native-runtime/runner-api-response-limits.ts index 7ae151a1994..936cd2d7b31 100644 --- a/server/src/services/native-runtime/runner-api-response-limits.ts +++ b/server/src/services/native-runtime/runner-api-response-limits.ts @@ -37,3 +37,7 @@ export function acquireRunnerApiResponseSlot(companyId: string): () => void { else activeByCompany.delete(companyId); }; } + +export class RunnerApiCaptureReservationReclaimedError extends Error { + constructor() { super("The saved API response expired before it could be stored. Run the read again."); } +} diff --git a/server/src/services/native-runtime/runner-api.integration.test.ts b/server/src/services/native-runtime/runner-api.integration.test.ts index c5fc2fb2a67..fc851306508 100644 --- a/server/src/services/native-runtime/runner-api.integration.test.ts +++ b/server/src/services/native-runtime/runner-api.integration.test.ts @@ -2,8 +2,8 @@ import { existsSync } from "node:fs"; import { createHash, randomUUID } from "node:crypto"; import { chmod, writeFile, symlink, mkdir, open } from "node:fs/promises"; import { join, dirname } from "node:path"; -import { and, eq } from "drizzle-orm"; -import { agents, agentApiKeys, agentWakeupRequests, assets, authUsers, documents, heartbeatRuns, issues, projects, routineDocuments, routineRevisions, routines, runnerApiResponseReservations, companySkills, companySkillVersions, companySkillTestRuns, companyMemberships, principalPermissionGrants, budgetPolicies, companies } from "@paperclipai/db"; +import { and, eq, isNotNull } from "drizzle-orm"; +import { agents, agentApiKeys, agentWakeupRequests, activityLog, assets, authUsers, issueAttachments, cases, caseAttachments, companyLogos, toolRuntimeMetricCounters, documents, heartbeatRuns, issues, projects, routineDocuments, routineRevisions, routines, runnerApiResponseReservations, companySkills, companySkillVersions, companySkillTestRuns, companyMemberships, principalPermissionGrants, budgetPolicies, companies } from "@paperclipai/db"; import { initializeRunIdentity } from "../run-identity.js"; import { beforeAll, afterAll, beforeEach, afterEach, describe, expect, it, vi } from "vitest"; import * as nativeSetupReadiness from "./setup-readiness.js"; @@ -16,6 +16,7 @@ import { registerRunnerPrpAuthority } from "../../realtime/runner-prp-ws.js"; import { createLocalAgentJwt } from "../../agent-auth-jwt.js"; import { RUNNER_API_RESPONSE_MAX_BYTES, RUNNER_API_RESPONSE_RUN_MAX_BYTES } from "./runner-api-response-limits.js"; import { PaperclipRunnerToolAuthority } from "./paperclip-runner-tool-authority.js"; +import { finishPendingRunnerApiCaptureDeletes, RUNNER_API_CAPTURE_DELETE_FAILURE_METRIC } from "./runner-api-capture-retention.js"; import { prepareHeartbeatTaskContext } from "../heartbeat/task-context.js"; import { createHeartbeatRunPreparation } from "../heartbeat/run-preparation.js"; import { createAssignedMcpTools } from "./assigned-mcp-tools.js"; @@ -1055,27 +1056,278 @@ else if(m.id!==undefined) send({id:m.id,result:{}}); expect(await call()).toMatchObject({ ok: false, error: "api_response_capture_limit" }); }); - it("counts old snapshots and other-run reservations against the company quota, and reclaims deletions", async () => { + it("evicts the oldest finished snapshots instead of blocking a read when the company quota is full", async () => { const fixture = await server.fixture(); vi.stubEnv("PAPERCLIP_RUNNER_API_COMPANY_CAPTURE_MAX_BYTES", String(2 * RUNNER_API_RESPONSE_MAX_BYTES)); - const [oldSnapshot] = await server.db.insert(assets).values({ companyId: fixture.companyId, provider: "local_disk", - objectKey: `${fixture.companyId}/runner-api/old-snapshot`, contentType: "text/plain", byteSize: RUNNER_API_RESPONSE_MAX_BYTES, - sha256: "old-snapshot", createdByAgentId: fixture.agentId }).returning(); - const [reservation] = await server.db.insert(runnerApiResponseReservations).values({ companyId: fixture.companyId, runId: null, reservedBytes: RUNNER_API_RESPONSE_MAX_BYTES }).returning(); + const stored = await server.storage.putFile({ companyId: fixture.companyId, namespace: "runner-api", originalFilename: "response.json", + contentType: "application/json", body: Buffer.from("{}") }); + const [oldest] = await server.db.insert(assets).values({ ...stored, companyId: fixture.companyId, byteSize: RUNNER_API_RESPONSE_MAX_BYTES, + createdByAgentId: fixture.agentId, createdAt: new Date(Date.now() - 2 * 86_400_000) }).returning(); + const finishedRunId = randomUUID(); + await server.db.insert(heartbeatRuns).values({ id: finishedRunId, companyId: fixture.companyId, agentId: fixture.agentId, + status: "succeeded", runtimeMode: "native", nativeIssueId: fixture.blockerId, contextSnapshot: { issueId: fixture.blockerId } }); + const [newer] = await server.db.insert(assets).values({ companyId: fixture.companyId, provider: "local_disk", + objectKey: `${fixture.companyId}/runner-api/newer-snapshot`, contentType: "text/plain", byteSize: RUNNER_API_RESPONSE_MAX_BYTES / 2, + sha256: "newer-snapshot", createdByAgentId: fixture.agentId, createdAt: new Date(Date.now() - 86_400_000) }).returning(); + await server.db.insert(runnerApiResponseReservations).values({ companyId: fixture.companyId, runId: finishedRunId, assetId: newer.id, reservedBytes: newer.byteSize }); await server.db.update(projects).set({ description: "x".repeat(32 * 1024) }).where(eq(projects.id, fixture.projectId)); - // A fresh authority instance cannot reset reservations from another run/process. const authority = new PaperclipRunnerToolAuthority(server.db, fixture); - const call = () => authority.execute({ tool: "call_api", callId: randomUUID(), arguments: { + const result: any = await authority.execute({ tool: "call_api", callId: randomUUID(), arguments: { + operationId: "GET /api/projects/{id}", pathParams: { id: fixture.projectId }, + } }); + expect(result).toMatchObject({ ok: true }); + const remaining = await server.db.select({ id: assets.id }).from(assets).where(eq(assets.companyId, fixture.companyId)); + expect(remaining.map(row => row.id)).not.toContain(oldest.id); + expect(remaining.map(row => row.id)).toContain(newer.id); + expect(remaining.map(row => row.id)).toContain(result.artifact.artifactId); + expect((await server.storage.headObject(fixture.companyId, oldest.objectKey)).exists).toBe(false); + }); + + it("keeps an evicted snapshot counted until its stored object is deleted, then retries the delete", async () => { + const fixture = await server.fixture(); + vi.stubEnv("PAPERCLIP_RUNNER_API_COMPANY_CAPTURE_MAX_BYTES", String(2 * RUNNER_API_RESPONSE_MAX_BYTES)); + const stored = await server.storage.putFile({ companyId: fixture.companyId, namespace: "runner-api", originalFilename: "response.json", + contentType: "application/json", body: Buffer.from("{}") }); + const [oldest] = await server.db.insert(assets).values({ ...stored, companyId: fixture.companyId, byteSize: RUNNER_API_RESPONSE_MAX_BYTES * 1.5, + createdByAgentId: fixture.agentId, createdAt: new Date(Date.now() - 86_400_000) }).returning(); + await server.db.update(projects).set({ description: "x".repeat(32 * 1024) }).where(eq(projects.id, fixture.projectId)); + const call = () => fixture.authority.execute({ tool: "call_api", callId: randomUUID(), arguments: { + operationId: "GET /api/projects/{id}", pathParams: { id: fixture.projectId }, + } }); + const spy = vi.spyOn(server.storage, "deleteObject").mockRejectedValue(new Error("storage unavailable")); + try { + expect(await call()).toMatchObject({ ok: false, error: "api_response_company_storage_limit" }); + } finally { spy.mockRestore(); } + expect(await server.db.select().from(runnerApiResponseReservations).where(eq(runnerApiResponseReservations.companyId, fixture.companyId))).toEqual([ + expect.objectContaining({ assetId: null, pendingDeleteObjectKey: oldest.objectKey, reservedBytes: oldest.byteSize }), + ]); + expect((await server.storage.headObject(fixture.companyId, oldest.objectKey)).exists).toBe(true); + expect(await call()).toMatchObject({ ok: true }); + expect((await server.storage.headObject(fixture.companyId, oldest.objectKey)).exists).toBe(false); + const reservations = await server.db.select().from(runnerApiResponseReservations).where(eq(runnerApiResponseReservations.companyId, fixture.companyId)); + expect(reservations.every(row => row.pendingDeleteObjectKey === null)).toBe(true); + }); + + it("reclaims enough space when eviction spans more than one page of small snapshots", async () => { + const fixture = await server.fixture(); + vi.stubEnv("PAPERCLIP_RUNNER_API_COMPANY_CAPTURE_MAX_BYTES", String(2 * RUNNER_API_RESPONSE_MAX_BYTES)); + const size = 4 * 1024 * 1024; + const start = Date.now() - 86_400_000; + const small = await server.db.insert(assets).values(Array.from({ length: 500 }, (_, index) => ({ companyId: fixture.companyId, provider: "local_disk", + objectKey: `${fixture.companyId}/runner-api/small-${index}`, contentType: "text/plain", byteSize: size, + sha256: `small-${index}`, createdByAgentId: fixture.agentId, createdAt: new Date(start + index * 1000) }))).returning({ id: assets.id }); + await server.db.update(projects).set({ description: "x".repeat(32 * 1024) }).where(eq(projects.id, fixture.projectId)); + expect(await fixture.authority.execute({ tool: "call_api", callId: randomUUID(), arguments: { + operationId: "GET /api/projects/{id}", pathParams: { id: fixture.projectId }, + } })).toMatchObject({ ok: true }); + const remaining = new Set((await server.db.select({ id: assets.id }).from(assets).where(eq(assets.companyId, fixture.companyId))).map(row => row.id)); + const evicted = small.filter(row => !remaining.has(row.id)).length; + expect(evicted).toBeGreaterThan(200); + expect(evicted).toBeLessThan(500); + expect(small.slice(0, evicted).every(row => !remaining.has(row.id))).toBe(true); + expect(await server.db.select().from(runnerApiResponseReservations).where(and(eq(runnerApiResponseReservations.companyId, fixture.companyId), + isNotNull(runnerApiResponseReservations.pendingDeleteObjectKey)))).toEqual([]); + }); + + it("bounds a stalled snapshot delete and keeps it for a later retry", async () => { + const fixture = await server.fixture(); + const objectKey = `${fixture.companyId}/runner-api/stalled-delete`; + await server.db.insert(runnerApiResponseReservations).values({ companyId: fixture.companyId, runId: null, reservedBytes: 1, pendingDeleteObjectKey: objectKey }); + const stalled = { ...server.storage, deleteObject: (_companyId: string, _key: string, options?: { signal?: AbortSignal }) => + new Promise((_, reject) => options?.signal?.addEventListener("abort", () => reject(options.signal!.reason))) }; + const startedAt = Date.now(); + expect(await finishPendingRunnerApiCaptureDeletes(server.db, stalled as typeof server.storage, fixture.companyId, 50)).toEqual({ deleted: 0, failed: 1 }); + expect(await server.db.select().from(toolRuntimeMetricCounters).where(and(eq(toolRuntimeMetricCounters.companyId, fixture.companyId), + eq(toolRuntimeMetricCounters.metric, RUNNER_API_CAPTURE_DELETE_FAILURE_METRIC)))).toEqual([expect.objectContaining({ count: 1 })]); + expect(Date.now() - startedAt).toBeLessThan(5_000); + expect(await server.db.select().from(runnerApiResponseReservations).where(eq(runnerApiResponseReservations.companyId, fixture.companyId))).toEqual([ + expect.objectContaining({ pendingDeleteObjectKey: objectKey }), + ]); + const deleteSpy = vi.spyOn(server.storage, "deleteObject"); + try { + expect(await finishPendingRunnerApiCaptureDeletes(server.db, server.storage, fixture.companyId)).toEqual({ deleted: 1, failed: 0 }); + expect(deleteSpy).toHaveBeenCalledWith(fixture.companyId, objectKey, expect.objectContaining({ allVersions: true })); + } finally { deleteSpy.mockRestore(); } + expect(await server.db.select().from(runnerApiResponseReservations).where(eq(runnerApiResponseReservations.companyId, fixture.companyId))).toEqual([]); + }); + + it("retries earlier pending deletes before evicting more snapshots", async () => { + const fixture = await server.fixture(); + vi.stubEnv("PAPERCLIP_RUNNER_API_COMPANY_CAPTURE_MAX_BYTES", String(2 * RUNNER_API_RESPONSE_MAX_BYTES)); + const stored = await server.storage.putFile({ companyId: fixture.companyId, namespace: "runner-api", originalFilename: "response.json", + contentType: "application/json", body: Buffer.from("{}") }); + await server.db.insert(runnerApiResponseReservations).values({ companyId: fixture.companyId, runId: null, + reservedBytes: RUNNER_API_RESPONSE_MAX_BYTES, pendingDeleteObjectKey: stored.objectKey }); + const [survivor] = await server.db.insert(assets).values({ companyId: fixture.companyId, provider: "local_disk", + objectKey: `${fixture.companyId}/runner-api/survivor`, contentType: "text/plain", byteSize: RUNNER_API_RESPONSE_MAX_BYTES / 2, + sha256: "survivor", createdByAgentId: fixture.agentId, createdAt: new Date(Date.now() - 86_400_000) }).returning(); + await server.db.update(projects).set({ description: "x".repeat(32 * 1024) }).where(eq(projects.id, fixture.projectId)); + expect(await fixture.authority.execute({ tool: "call_api", callId: randomUUID(), arguments: { + operationId: "GET /api/projects/{id}", pathParams: { id: fixture.projectId }, + } })).toMatchObject({ ok: true }); + expect((await server.db.select({ id: assets.id }).from(assets).where(eq(assets.companyId, fixture.companyId))).map(row => row.id)).toContain(survivor.id); + expect((await server.storage.headObject(fixture.companyId, stored.objectKey)).exists).toBe(false); + }); + + it("says storage cleanup is failing when old snapshot deletes keep failing", async () => { + const fixture = await server.fixture(); + vi.stubEnv("PAPERCLIP_RUNNER_API_COMPANY_CAPTURE_MAX_BYTES", String(2 * RUNNER_API_RESPONSE_MAX_BYTES)); + await server.db.insert(runnerApiResponseReservations).values({ companyId: fixture.companyId, runId: null, + reservedBytes: RUNNER_API_RESPONSE_MAX_BYTES * 1.5, pendingDeleteObjectKey: `${fixture.companyId}/runner-api/undeletable`, + createdAt: new Date(Date.now() - 3_600_000) }); + await server.db.update(projects).set({ description: "x".repeat(32 * 1024) }).where(eq(projects.id, fixture.projectId)); + const spy = vi.spyOn(server.storage, "deleteObject").mockRejectedValue(new Error("storage unavailable")); + let result: any; + try { + result = await fixture.authority.execute({ tool: "call_api", callId: randomUUID(), arguments: { + operationId: "GET /api/projects/{id}", pathParams: { id: fixture.projectId }, + } }); + } finally { spy.mockRestore(); } + expect(result).toMatchObject({ ok: false, error: "api_response_company_storage_limit" }); + expect(result.message).toMatch(/keeps failing to delete/); + expect(result.message).not.toMatch(/operator|reservation|snapshot|quota/i); + }); + + it("rejects and removes an upload that finishes after its reservation was reclaimed", async () => { + const fixture = await server.fixture(); + await server.db.update(projects).set({ description: "x".repeat(32 * 1024) }).where(eq(projects.id, fixture.projectId)); + const before = (await server.db.select({ id: assets.id }).from(assets).where(eq(assets.companyId, fixture.companyId))).length; + const putFile = server.storage.putFile.bind(server.storage); + let objectKey = ""; + let uploadSignal: AbortSignal | undefined; + const spy = vi.spyOn(server.storage, "putFile").mockImplementationOnce(async input => { + uploadSignal = input.signal; + const saved = await putFile(input); + objectKey = saved.objectKey; + await server.db.delete(runnerApiResponseReservations).where(eq(runnerApiResponseReservations.companyId, fixture.companyId)); + return saved; + }); + try { + await expect(fixture.authority.execute({ tool: "call_api", callId: randomUUID(), arguments: { + operationId: "GET /api/projects/{id}", pathParams: { id: fixture.projectId }, + } })).rejects.toThrow("expired before it could be stored"); + } finally { spy.mockRestore(); } + expect(uploadSignal).toBeInstanceOf(AbortSignal); + expect((await server.db.select({ id: assets.id }).from(assets).where(eq(assets.companyId, fixture.companyId))).length).toBe(before); + expect((await server.storage.headObject(fixture.companyId, objectKey)).exists).toBe(false); + expect(await server.db.select().from(runnerApiResponseReservations).where(eq(runnerApiResponseReservations.companyId, fixture.companyId))).toEqual([]); + }); + + it("evicts only expired snapshots of finished runs and records each eviction", async () => { + const fixture = await server.fixture(); + vi.stubEnv("PAPERCLIP_RUNNER_API_COMPANY_CAPTURE_MAX_BYTES", String(2 * RUNNER_API_RESPONSE_MAX_BYTES)); + const finishedRunId = randomUUID(); + await server.db.insert(heartbeatRuns).values({ id: finishedRunId, companyId: fixture.companyId, agentId: fixture.agentId, + status: "cancelled", runtimeMode: "native", nativeIssueId: fixture.blockerId, contextSnapshot: { issueId: fixture.blockerId } }); + const [recent] = await server.db.insert(assets).values({ companyId: fixture.companyId, provider: "local_disk", + objectKey: `${fixture.companyId}/runner-api/recent-finished`, contentType: "text/plain", byteSize: RUNNER_API_RESPONSE_MAX_BYTES * 1.5, + sha256: "recent-finished", createdByAgentId: fixture.agentId, createdAt: new Date(Date.now() - 5 * 60_000) }).returning(); + await server.db.insert(runnerApiResponseReservations).values({ companyId: fixture.companyId, runId: finishedRunId, assetId: recent.id, reservedBytes: recent.byteSize }); + await server.db.update(projects).set({ description: "x".repeat(32 * 1024) }).where(eq(projects.id, fixture.projectId)); + const call = () => fixture.authority.execute({ tool: "call_api", callId: randomUUID(), arguments: { operationId: "GET /api/projects/{id}", pathParams: { id: fixture.projectId }, } }); expect(await call()).toMatchObject({ ok: false, error: "api_response_company_storage_limit" }); - await server.db.delete(assets).where(eq(assets.id, oldSnapshot.id)); - const first: any = await call(); - expect(first).toMatchObject({ ok: true }); - expect(await call()).toMatchObject({ ok: false, error: "api_response_company_storage_limit" }); - await server.db.delete(assets).where(eq(assets.id, first.artifact.artifactId)); - expect(await server.db.select().from(runnerApiResponseReservations).where(eq(runnerApiResponseReservations.companyId, fixture.companyId))).toEqual([expect.objectContaining({ id: reservation.id })]); + expect((await server.db.select({ id: assets.id }).from(assets).where(eq(assets.id, recent.id)))).toHaveLength(1); + await server.db.update(assets).set({ createdAt: new Date(Date.now() - 2 * 3_600_000) }).where(eq(assets.id, recent.id)); expect(await call()).toMatchObject({ ok: true }); + expect((await server.db.select({ id: assets.id }).from(assets).where(eq(assets.id, recent.id)))).toHaveLength(0); + expect(await server.db.select().from(activityLog).where(and(eq(activityLog.companyId, fixture.companyId), eq(activityLog.action, "asset.evicted")))).toEqual([ + expect.objectContaining({ actorType: "system", entityType: "asset", entityId: recent.id, runId: finishedRunId }), + ]); + }); + + it("runs another eviction pass when a concurrent capture takes the freed space", async () => { + const fixture = await server.fixture(); + vi.stubEnv("PAPERCLIP_RUNNER_API_COMPANY_CAPTURE_MAX_BYTES", String(2 * RUNNER_API_RESPONSE_MAX_BYTES)); + const expired = async (name: string, ageMs: number) => (await server.db.insert(assets).values({ companyId: fixture.companyId, provider: "local_disk", + objectKey: `${fixture.companyId}/runner-api/${name}`, contentType: "text/plain", byteSize: RUNNER_API_RESPONSE_MAX_BYTES, + sha256: name, createdByAgentId: fixture.agentId, createdAt: new Date(Date.now() - ageMs) }).returning())[0]; + const older = await expired("older", 3 * 3_600_000); + const newer = await expired("newer", 2 * 3_600_000); + const otherRunId = randomUUID(); + await server.db.insert(heartbeatRuns).values({ id: otherRunId, companyId: fixture.companyId, agentId: fixture.agentId, + status: "running", runtimeMode: "native", nativeIssueId: fixture.blockerId, contextSnapshot: { issueId: fixture.blockerId } }); + await server.db.update(projects).set({ description: "x".repeat(32 * 1024) }).where(eq(projects.id, fixture.projectId)); + const deleteObject = server.storage.deleteObject.bind(server.storage); + let stolen = false; + const spy = vi.spyOn(server.storage, "deleteObject").mockImplementation(async (companyId, key, options) => { + if (!stolen) { + stolen = true; + await server.db.insert(runnerApiResponseReservations).values({ companyId: fixture.companyId, runId: otherRunId, reservedBytes: RUNNER_API_RESPONSE_MAX_BYTES }); + } + return deleteObject(companyId, key, options); + }); + let result: any; + try { + result = await fixture.authority.execute({ tool: "call_api", callId: randomUUID(), arguments: { + operationId: "GET /api/projects/{id}", pathParams: { id: fixture.projectId }, + } }); + } finally { spy.mockRestore(); } + expect(stolen).toBe(true); + expect(result).toMatchObject({ ok: true }); + const remaining = (await server.db.select({ id: assets.id }).from(assets).where(eq(assets.companyId, fixture.companyId))).map(row => row.id); + expect(remaining).not.toContain(older.id); + expect(remaining).not.toContain(newer.id); + }); + + it("releases stale reservations left by interrupted captures of finished runs", async () => { + const fixture = await server.fixture(); + vi.stubEnv("PAPERCLIP_RUNNER_API_COMPANY_CAPTURE_MAX_BYTES", String(2 * RUNNER_API_RESPONSE_MAX_BYTES)); + const finishedRunId = randomUUID(); + await server.db.insert(heartbeatRuns).values({ id: finishedRunId, companyId: fixture.companyId, agentId: fixture.agentId, + status: "failed", runtimeMode: "native", nativeIssueId: fixture.blockerId, contextSnapshot: { issueId: fixture.blockerId } }); + const longAgo = new Date(Date.now() - 3_600_000); + await server.db.insert(runnerApiResponseReservations).values([ + { companyId: fixture.companyId, runId: null, reservedBytes: RUNNER_API_RESPONSE_MAX_BYTES, createdAt: longAgo }, + { companyId: fixture.companyId, runId: finishedRunId, reservedBytes: RUNNER_API_RESPONSE_MAX_BYTES, createdAt: longAgo }, + ]); + await server.db.update(projects).set({ description: "x".repeat(32 * 1024) }).where(eq(projects.id, fixture.projectId)); + const call = () => fixture.authority.execute({ tool: "call_api", callId: randomUUID(), arguments: { + operationId: "GET /api/projects/{id}", pathParams: { id: fixture.projectId }, + } }); + expect(await call()).toMatchObject({ ok: true }); + expect(await call()).toMatchObject({ ok: true }); + const reservations = await server.db.select().from(runnerApiResponseReservations).where(eq(runnerApiResponseReservations.companyId, fixture.companyId)); + expect(reservations.every(row => row.runId === fixture.runId && row.assetId)).toBe(true); + }); + + it("keeps snapshots of active runs, attached snapshots, and in-flight reservations when the company quota is full", async () => { + const fixture = await server.fixture(); + vi.stubEnv("PAPERCLIP_RUNNER_API_COMPANY_CAPTURE_MAX_BYTES", String(2 * RUNNER_API_RESPONSE_MAX_BYTES)); + const activeRunId = randomUUID(); + await server.db.insert(heartbeatRuns).values({ id: activeRunId, companyId: fixture.companyId, agentId: fixture.agentId, + status: "running", runtimeMode: "native", nativeIssueId: fixture.blockerId, contextSnapshot: { issueId: fixture.blockerId } }); + const longAgo = new Date(Date.now() - 86_400_000); + const [held] = await server.db.insert(assets).values({ companyId: fixture.companyId, provider: "local_disk", + objectKey: `${fixture.companyId}/runner-api/active-run-snapshot`, contentType: "text/plain", byteSize: RUNNER_API_RESPONSE_MAX_BYTES / 2, + sha256: "active-run-snapshot", createdByAgentId: fixture.agentId, createdAt: longAgo }).returning(); + await server.db.insert(runnerApiResponseReservations).values([ + { companyId: fixture.companyId, runId: activeRunId, assetId: held.id, reservedBytes: held.byteSize }, + { companyId: fixture.companyId, runId: null, reservedBytes: RUNNER_API_RESPONSE_MAX_BYTES / 2 }, + ]); + const [attached] = await server.db.insert(assets).values({ companyId: fixture.companyId, provider: "local_disk", + objectKey: `${fixture.companyId}/runner-api/attached-snapshot`, contentType: "text/plain", byteSize: RUNNER_API_RESPONSE_MAX_BYTES / 2, + sha256: "attached-snapshot", createdByAgentId: fixture.agentId, createdAt: longAgo }).returning(); + await server.db.insert(issueAttachments).values({ companyId: fixture.companyId, issueId: fixture.issueId, assetId: attached.id }); + const linked = async (name: string) => (await server.db.insert(assets).values({ companyId: fixture.companyId, provider: "local_disk", + objectKey: `${fixture.companyId}/runner-api/${name}`, contentType: "text/plain", byteSize: 1024 * 1024, + sha256: name, createdByAgentId: fixture.agentId, createdAt: new Date(longAgo.getTime() - 1000) }).returning())[0]; + const caseAsset = await linked("case-snapshot"); + const logoAsset = await linked("logo-snapshot"); + const caseId = randomUUID(); + await server.db.insert(cases).values({ id: caseId, companyId: fixture.companyId, caseNumber: 1, identifier: `CASE-${caseId}`, caseType: "work", title: "Retention" }); + await server.db.insert(caseAttachments).values({ companyId: fixture.companyId, caseId, assetId: caseAsset.id }); + await server.db.insert(companyLogos).values({ companyId: fixture.companyId, assetId: logoAsset.id }); + await server.db.update(projects).set({ description: "x".repeat(32 * 1024) }).where(eq(projects.id, fixture.projectId)); + const result: any = await fixture.authority.execute({ tool: "call_api", callId: randomUUID(), arguments: { + operationId: "GET /api/projects/{id}", pathParams: { id: fixture.projectId }, + } }); + expect(result).toMatchObject({ ok: false, error: "api_response_company_storage_limit" }); + expect(result.message).not.toMatch(/operator/i); + const remaining = await server.db.select({ id: assets.id }).from(assets).where(eq(assets.companyId, fixture.companyId)); + expect(remaining.map(row => row.id)).toEqual(expect.arrayContaining([held.id, attached.id, caseAsset.id, logoAsset.id])); + expect(await server.db.select().from(runnerApiResponseReservations).where(eq(runnerApiResponseReservations.companyId, fixture.companyId))).toHaveLength(2); }); it("keeps company storage reserved after an ambiguous upload failure", async () => {