diff --git a/doc/database-backups.md b/doc/database-backups.md index 0faaab6e99c..20e827da842 100644 --- a/doc/database-backups.md +++ b/doc/database-backups.md @@ -81,6 +81,17 @@ Environment overrides: and publishes no partial backup. The whole restore, including local decrypt, digest and extraction, has a 30-minute outer limit (`native_harness_storage_restore_deadline_exceeded`). Stop ends the restore. + The per-turn backup that saves provider state after an accepted turn follows the + same rule. Its upload and its verifying read-back each have a five-minute idle + limit instead of a fixed five-minute total, so a large backup or a slow store + keeps saving while the store makes progress. For an upload, progress is a chunk + the store takes or, when the store sends the file itself (S3), a request it + answers, a part it takes or a resend that starts its wait. An upload with no + progress for five minutes fails with `native_harness_storage_upload_stalled`, a + stalled read-back with `native_harness_storage_download_stalled`, and neither + publishes a backup. The whole backup, including archive capture, encryption and + any storage-outage wait, has a 30-minute outer limit + (`native_harness_storage_persist_deadline_exceeded`). - `PAPERCLIP_NATIVE_RUNNER_EVIDENCE_RETENTION_DAYS=` sets how long native session continuity-break archives and quarantined session state are kept (see `doc/recoverable-disk-cleanup.md`). The default is `7` and the minimum is `1`. A diff --git a/server/src/services/native-runtime/native-harness-backup-storage.ts b/server/src/services/native-runtime/native-harness-backup-storage.ts index 09a050ab3f0..ab84982c818 100644 --- a/server/src/services/native-runtime/native-harness-backup-storage.ts +++ b/server/src/services/native-runtime/native-harness-backup-storage.ts @@ -5,7 +5,7 @@ import { mkdtemp, readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; import os from "node:os"; import { pipeline } from "node:stream/promises"; -import { Transform } from "node:stream"; +import { Readable, Transform } from "node:stream"; import { and, eq, inArray, isNotNull, or, sql } from "drizzle-orm"; import { environmentLeases, heartbeatRuns, issues, nativeHarnessBackups, nativeRunFinalizations, type Db } from "@paperclipai/db"; import { createLocalEncryptedCipher } from "../../secrets/local-encrypted-provider.js"; @@ -14,6 +14,7 @@ import { lockNativeRunnerCheckpointOwner, validateNativeRunnerCheckpointOwner, t import { loadConfig } from "../../config.js"; import { getStorageService } from "../../storage/index.js"; import { storageOutageWaitFor, waitOutStorageOutages } from "../../storage/storage-unavailable.js"; +import { observeStorageProgress } from "../../storage/request-stats.js"; import { readNativeHarnessBackupStamp, verifyNativeHarnessBackupStamp } from "./native-harness-backup-stamp.js"; import { captureNativeHarnessArchive, restoreNativeHarnessArchive, MAX_NATIVE_HARNESS_OBJECT_BYTES } from "./native-harness-backup-archive.js"; import { removeNativeHarnessBackup } from "./native-harness-backup-cleanup.js"; @@ -201,8 +202,13 @@ type PersistNativeHarnessBackupInput = { providerArtifacts?: NativeProviderArtifactSource; }; -export function persistNativeHarnessBackup(input: PersistNativeHarnessBackupInput): Promise { - return waitOutStorageOutages(() => storageOutageWaitFor("saving the provider session", () => persistNativeHarnessBackupOnce(input))); +export function persistNativeHarnessBackup(input: PersistNativeHarnessBackupInput & { + transferIdleTimeoutMs?: number; persistMaxMs?: number; +}): Promise { + return withTransferProgressDeadline({ storage: input.storage, signal: input.signal, idleTimeoutMs: input.transferIdleTimeoutMs, + maxMs: input.persistMaxMs ?? NATIVE_HARNESS_PERSIST_MAX_MS, deadlineCode: NATIVE_HARNESS_PERSIST_DEADLINE_EXCEEDED }, + (storage, signal) => waitOutStorageOutages(() => storageOutageWaitFor("saving the provider session", + () => persistNativeHarnessBackupOnce({ ...input, storage, signal })))); } async function persistNativeHarnessBackupOnce(input: PersistNativeHarnessBackupInput): Promise { @@ -298,39 +304,57 @@ async function persistNativeHarnessBackupOnce(input: PersistNativeHarnessBackupI /** How long a durable read may go without progress. */ export const NATIVE_HARNESS_DOWNLOAD_IDLE_TIMEOUT_MS = 300_000; +/** How long a durable write may go without progress. */ +export const NATIVE_HARNESS_UPLOAD_IDLE_TIMEOUT_MS = 300_000; /** Outer bound for one whole restore or verification, including local steps. */ export const NATIVE_HARNESS_RESTORE_MAX_MS = 30 * 60_000; +/** Outer bound for one whole backup, including capture, encryption and read-back. */ +export const NATIVE_HARNESS_PERSIST_MAX_MS = 30 * 60_000; export const NATIVE_HARNESS_DOWNLOAD_STALLED = "native_harness_storage_download_stalled"; +export const NATIVE_HARNESS_UPLOAD_STALLED = "native_harness_storage_upload_stalled"; export const NATIVE_HARNESS_RESTORE_DEADLINE_EXCEEDED = "native_harness_storage_restore_deadline_exceeded"; +export const NATIVE_HARNESS_PERSIST_DEADLINE_EXCEEDED = "native_harness_storage_persist_deadline_exceeded"; -/** - * Bound each durable read by inactivity, not by a short total time. A large or - * slow backup keeps restoring while bytes flow through the read; a read that - * makes no progress for the idle window aborts the whole restore with a named - * error. A generous outer bound keeps a hung local step diagnosable. Caller - * cancellation (Stop) keeps its own reason. - */ async function withDownloadProgressDeadline( input: { storage: StorageService; signal?: AbortSignal; downloadIdleTimeoutMs?: number; restoreMaxMs?: number }, run: (storage: StorageService, signal: AbortSignal | undefined) => Promise, ): Promise { - const idleMs = input.downloadIdleTimeoutMs; + return withTransferProgressDeadline({ storage: input.storage, signal: input.signal, idleTimeoutMs: input.downloadIdleTimeoutMs, + maxMs: input.restoreMaxMs ?? Math.max(input.downloadIdleTimeoutMs ?? 0, NATIVE_HARNESS_RESTORE_MAX_MS), + deadlineCode: NATIVE_HARNESS_RESTORE_DEADLINE_EXCEEDED }, run); +} + +/** + * Bound each durable read and write by inactivity, not by a short total time. + * A large or slow backup keeps moving while the store makes progress; a + * transfer that makes none for the idle window aborts the whole operation with + * a named error. A generous outer bound keeps a hung local step diagnosable. + * Caller cancellation (Stop) keeps its own reason. + */ +async function withTransferProgressDeadline( + input: { storage: StorageService; signal?: AbortSignal; idleTimeoutMs?: number; maxMs: number; deadlineCode: string }, + run: (storage: StorageService, signal: AbortSignal | undefined) => Promise, +): Promise { + const idleMs = input.idleTimeoutMs; if (idleMs === undefined) return run(input.storage, input.signal); if (!Number.isSafeInteger(idleMs) || idleMs <= 0) throw new Error("native_harness_storage_idle_timeout_invalid"); const stall = new AbortController(); - const outer = setTimeout(() => stall.abort(new Error(NATIVE_HARNESS_RESTORE_DEADLINE_EXCEEDED)), - input.restoreMaxMs ?? Math.max(idleMs, NATIVE_HARNESS_RESTORE_MAX_MS)); + const outer = setTimeout(() => stall.abort(new Error(input.deadlineCode)), input.maxMs); outer.unref?.(); const signal = input.signal ? AbortSignal.any([input.signal, stall.signal]) : stall.signal; const base = input.storage; - const getObject: StorageService["getObject"] = async (companyId, objectKey, options) => { + const idleTimer = (code: string) => { let timer: NodeJS.Timeout | undefined; const disarm = () => { if (timer) clearTimeout(timer); timer = undefined; }; const arm = () => { disarm(); - timer = setTimeout(() => stall.abort(new Error(NATIVE_HARNESS_DOWNLOAD_STALLED)), idleMs); + timer = setTimeout(() => stall.abort(new Error(code)), idleMs); timer.unref?.(); }; + return { arm, disarm }; + }; + const getObject: StorageService["getObject"] = async (companyId, objectKey, options) => { + const { arm, disarm } = idleTimer(NATIVE_HARNESS_DOWNLOAD_STALLED); arm(); let object: Awaited>; try { @@ -348,10 +372,30 @@ async function withDownloadProgressDeadline( pipeline(object.stream, observed, { signal }).catch(() => disarm()); return { ...object, stream: observed }; }; + const putFile: StorageService["putFile"] = async request => { + const { arm, disarm } = idleTimer(NATIVE_HARNESS_UPLOAD_STALLED); + const withSignal = { signal: request.signal ? AbortSignal.any([request.signal, signal]) : signal }; + arm(); + try { + // A provider that sends the file source itself (S3) never reads the + // stream; each request the store answers, part it takes or resend it + // waits for renews the window instead. After the last of them the window + // stays armed until the store answers, so a write that never completes + // is named too. + return await observeStorageProgress(arm, async () => { + if (!(request.body instanceof Readable)) return base.putFile({ ...request, ...withSignal }); + // A provider that reads the stream renews the window with each chunk it takes. + const observed = new Transform({ transform(bytes, _encoding, callback) { arm(); callback(null, bytes); } }); + pipeline(request.body, observed, { signal }).catch(() => undefined); + return base.putFile({ ...request, body: observed, ...withSignal } as typeof request); + }); + } finally { disarm(); } + }; // Forward every other member to the real service so its own state stays intact. const storage = new Proxy(base, { get(target, property) { if (property === "getObject") return getObject; + if (property === "putFile") return putFile; const value = Reflect.get(target, property, target) as unknown; return typeof value === "function" ? (value as (...args: unknown[]) => unknown).bind(target) : value; }, diff --git a/server/src/services/native-runtime/native-harness-backup-upload-deadline.integration.test.ts b/server/src/services/native-runtime/native-harness-backup-upload-deadline.integration.test.ts new file mode 100644 index 00000000000..e1b128464de --- /dev/null +++ b/server/src/services/native-runtime/native-harness-backup-upload-deadline.integration.test.ts @@ -0,0 +1,188 @@ +import { createHash, randomBytes, randomUUID } from "node:crypto"; +import { mkdir, readFile, stat, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { Readable } from "node:stream"; +import { and, eq } from "drizzle-orm"; +import { nativeHarnessBackups, nativeRunFinalizations } from "@paperclipai/db"; +import { afterEach, expect, it, vi } from "vitest"; +import { describeEmbeddedPostgres, nativeRestartFixture } from "../../__tests__/helpers/native-restart.js"; +import { createLocalDiskStorageProvider } from "../../storage/local-disk-provider.js"; +import { createStorageService } from "../../storage/service.js"; +import { recordStorageRequest } from "../../storage/request-stats.js"; +import type { StorageService } from "../../storage/types.js"; +import { nativeHarnessBackupOwner, persistNativeHarnessBackup, restoreDurableNativeHarnessBackup } from "./native-harness-backup-storage.js"; +import { removeNativeHarnessBackup } from "./native-harness-backup-cleanup.js"; + +/** A store that takes one upload chunk every `delayMs`, optionally no more after `stallAfter` chunks. */ +function throttledUploads(storage: StorageService, delayMs: number, stallAfter?: number) { + let taken = 0; + const writes: string[] = []; + const throttled = new Proxy(storage, { get(target, property) { + if (property !== "putFile") { + const value = Reflect.get(target, property, target) as unknown; + return typeof value === "function" ? (value as (...args: unknown[]) => unknown).bind(target) : value; + } + return async (input: Parameters[0]) => { + writes.push(input.objectKey ?? ""); + if (!(input.body instanceof Readable)) return target.putFile(input); + const body = input.body, signal = input.signal; + async function* slow() { + for await (const part of body as AsyncIterable) { + if (stallAfter !== undefined && taken >= stallAfter) { + await new Promise(resolve => signal?.addEventListener("abort", () => resolve(), { once: true })); + signal?.throwIfAborted(); + } + await new Promise(resolve => setTimeout(resolve, delayMs)); + signal?.throwIfAborted(); + taken++; + yield part; + } + } + return target.putFile({ ...input, body: Readable.from(slow()) } as typeof input); + }; + } }); + return { storage: throttled, writes }; +} + +/** + * A store that, like S3 with a file source, never reads the stream body: it + * sends `requests` requests of `delayMs` each, recording each answer as the S3 + * provider does, then stores the file. With `stallAfter`, a request never ends. + */ +function fileSourceUploads(storage: StorageService, requests: number, delayMs: number, stallAfter?: number) { + return new Proxy(storage, { get(target, property) { + if (property !== "putFile") { + const value = Reflect.get(target, property, target) as unknown; + return typeof value === "function" ? (value as (...args: unknown[]) => unknown).bind(target) : value; + } + return async (input: Parameters[0]) => { + if (!("sourceFile" in input) || !input.sourceFile) throw new Error("expected a file source"); + if (input.body instanceof Readable) input.body.destroy(); + for (let sent = 0; sent < requests; sent++) { + if (stallAfter !== undefined && sent >= stallAfter) { + await new Promise(resolve => input.signal?.addEventListener("abort", () => resolve(), { once: true })); + } + await new Promise(resolve => setTimeout(resolve, delayMs)); + input.signal?.throwIfAborted(); + recordStorageRequest({ $metadata: { attempts: 1 } }, false); + } + const { body: _stream, byteSize: _size, sha256: _sha, sourceFile, ...rest } = input; + return target.putFile({ ...rest, body: await readFile(sourceFile) }); + }; + } }); +} + +describeEmbeddedPostgres("durable native backup upload deadlines", () => { + const environment = nativeRestartFixture(); + const ownedHomes: string[] = []; + afterEach(async () => { + for (const home of ownedHomes.splice(0)) await removeNativeHarnessBackup(home); + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + }); + /** A backup large enough that its upload is many store-sized chunks. */ + async function fixture() { + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "52".repeat(32)); + const f = await environment.seedRun(randomUUID()); + const { companyId, runtimeRoot, successor } = environment.current; + await f.db.update(nativeRunFinalizations).set({ controllerBootId: successor.bootId, controllerGeneration: 1, + leaseOwner: "backup-writer", leaseExpiresAt: new Date(Date.now() + 120_000) }).where(eq(nativeRunFinalizations.runId, f.runId)); + const owner = await nativeHarnessBackupOwner(f.db, { companyId, issueId: f.issueId, runId: f.runId, bootId: successor.bootId }); + const home = join(runtimeRoot, randomUUID()), root = join(home, "pending"); + ownedHomes.push(home); + await mkdir(join(root, "acpx", "sessions"), { recursive: true }); + const identity = { normalizedSessionId: randomUUID(), providerSessionId: randomUUID(), provider: "acpx" }; + // Incompressible bytes, so the encrypted object spans many file-stream chunks. + await writeFile(join(root, "acpx", "sessions", "history.json"), + JSON.stringify({ ...identity, turns: randomBytes(768 * 1024).toString("base64") })); + await writeFile(join(root, "manifest.json"), + JSON.stringify({ schema: "paperclip.native-harness-backup.v1", ...identity, directories: ["acpx"] }), { mode: 0o600 }); + const scopeDigest = createHash("sha256").update(randomUUID()).digest("hex"); + const storage = createStorageService(createLocalDiskStorageProvider(join(runtimeRoot, "durable-objects"))); + const current = async () => f.db.select().from(nativeHarnessBackups).where(and( + eq(nativeHarnessBackups.scopeDigest, scopeDigest), eq(nativeHarnessBackups.status, "current"))); + return { db: f.db, companyId, owner, root, scopeDigest, storage, identity, current }; + } + function replacementRoot() { + const root = join(environment.current.runtimeRoot, randomUUID(), "failover-backups"); + ownedHomes.push(root); + return root; + } + + it("keeps saving a backup whose upload outlasts the idle window while the store keeps taking bytes", async () => { + const f = await fixture(), idle = 200, slow = throttledUploads(f.storage, 40); + // The former fixed total deadline aborted this same progressing upload with + // the generic "The operation was aborted" seen after accepted staging turns. + await expect(persistNativeHarnessBackup({ ...f, storage: slow.storage, signal: AbortSignal.timeout(idle) })) + .rejects.toThrow("The operation was aborted"); + expect(await f.current()).toHaveLength(0); + const startedAt = Date.now(); + slow.writes.splice(0); + await persistNativeHarnessBackup({ ...f, storage: slow.storage, transferIdleTimeoutMs: idle }); + expect(Date.now() - startedAt).toBeGreaterThan(idle * 2); + expect(slow.writes.some(key => key.includes("/native-harness-backups/"))).toBe(true); + expect(await f.current()).toHaveLength(1); + const restored = replacementRoot(); + expect(await restoreDurableNativeHarnessBackup({ ...f, backupRoot: restored })).toBe(true); + expect(JSON.parse(await readFile(join(restored, "current/acpx/sessions/history.json"), "utf8"))).toMatchObject(f.identity); + }); + + it("names an upload that stops taking bytes and publishes no backup", { timeout: 10_000 }, async () => { + const f = await fixture(); + await expect(persistNativeHarnessBackup({ ...f, storage: throttledUploads(f.storage, 5, 2).storage, + transferIdleTimeoutMs: 250 })).rejects.toThrow("native_harness_storage_upload_stalled"); + expect(await f.current()).toHaveLength(0); + // The local source is untouched and saves once the store responds again. + await persistNativeHarnessBackup({ ...f, transferIdleTimeoutMs: 250 }); + expect(await f.current()).toHaveLength(1); + }); + + it("keeps a file-source upload going while the store answers its requests, and names one that stops", { timeout: 10_000 }, async () => { + const f = await fixture(), idle = 200; + // Ten requests of 60 ms each: longer than the idle window in total, but each answer renews it. + const startedAt = Date.now(); + await persistNativeHarnessBackup({ ...f, storage: fileSourceUploads(f.storage, 10, 60), transferIdleTimeoutMs: idle }); + expect(Date.now() - startedAt).toBeGreaterThan(idle * 2); + expect(await f.current()).toHaveLength(1); + await expect(persistNativeHarnessBackup({ ...f, storage: fileSourceUploads(f.storage, 10, 20, 3), + transferIdleTimeoutMs: idle })).rejects.toThrow("native_harness_storage_upload_stalled"); + }); + + it("names a read-back that stops sending bytes and publishes no backup", { timeout: 10_000 }, async () => { + const f = await fixture(); + const stalledReads = new Proxy(f.storage, { get(target, property) { + if (property === "getObject") return async (companyId: string, objectKey: string, options?: { signal?: AbortSignal }) => { + const object = await target.getObject(companyId, objectKey, options); + object.stream.destroy(); + return { ...object, stream: Readable.from((async function* () { + await new Promise(resolve => options?.signal?.addEventListener("abort", () => resolve(), { once: true })); + options?.signal?.throwIfAborted(); + })()) }; + }; + const value = Reflect.get(target, property, target) as unknown; + return typeof value === "function" ? (value as (...args: unknown[]) => unknown).bind(target) : value; + } }); + await expect(persistNativeHarnessBackup({ ...f, storage: stalledReads, transferIdleTimeoutMs: 250 })) + .rejects.toThrow("native_harness_storage_download_stalled"); + expect(await f.current()).toHaveLength(0); + }); + + it("stops at the outer bound while an upload still progresses", async () => { + const f = await fixture(); + await expect(persistNativeHarnessBackup({ ...f, storage: throttledUploads(f.storage, 40).storage, + transferIdleTimeoutMs: 5_000, persistMaxMs: 150 })).rejects.toThrow("native_harness_storage_persist_deadline_exceeded"); + expect(await f.current()).toHaveLength(0); + }); + + it("lets Stop end a progressing upload with its own reason", async () => { + const f = await fixture(), stop = new AbortController(); + setTimeout(() => stop.abort(new Error("caller_stopped")), 80); + const failure = await persistNativeHarnessBackup({ ...f, storage: throttledUploads(f.storage, 40).storage, + signal: stop.signal, transferIdleTimeoutMs: 5_000 }).then(() => null, (error: unknown) => error); + const reason = failure as Error & { cause?: unknown }; + expect(failure).toBeInstanceOf(Error); + expect([reason.message, (reason.cause as Error | undefined)?.message]).toContain("caller_stopped"); + expect(await f.current()).toHaveLength(0); + await expect(stat(f.root)).resolves.toBeTruthy(); + }); +}); diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index dc8015649b8..29858a3d5f8 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -3188,6 +3188,10 @@ describe("remote provider pack manifest", () => { await driver.mock.calls.at(-1)![0].checkpoint!("settled"); expect(providerStateUnsaved.mark).not.toHaveBeenCalled(); expect(harnessStorage.persistNativeHarnessBackup).toHaveBeenCalledTimes(1); + // The per-turn backup is bounded by transfer inactivity, not a fixed total time. + const [persisted] = vi.mocked(harnessStorage.persistNativeHarnessBackup).mock.calls[0]!; + expect(persisted.transferIdleTimeoutMs).toBe(harnessStorage.NATIVE_HARNESS_UPLOAD_IDLE_TIMEOUT_MS); + expect(persisted.signal).toBeUndefined(); expect(checkpointReadResets).toBe(0); expect(inspectionResets).toBe(0); return; diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index f8bc84eb968..68857191dc4 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -23,8 +23,8 @@ import { readClaimedNativeStopRecovery } from "./native-cancellation-request.js" import { publishNativeHarnessBackup } from "./native-harness-backup-publication.js"; import { restartRecoveryMeasurementFor, speedBudgetAdapter } from "./native-speed-budgets.js"; import { getStartupRecoveryState } from "../../startup-recovery-state.js"; -import { assertNativeHarnessBackupOwner, durableNativeHarnessStorage, NATIVE_HARNESS_DOWNLOAD_IDLE_TIMEOUT_MS, nativeHarnessBackupOwner, persistNativeHarnessBackup, - restoreDurableNativeHarnessBackup } from "./native-harness-backup-storage.js"; +import { assertNativeHarnessBackupOwner, durableNativeHarnessStorage, NATIVE_HARNESS_DOWNLOAD_IDLE_TIMEOUT_MS, NATIVE_HARNESS_UPLOAD_IDLE_TIMEOUT_MS, + nativeHarnessBackupOwner, persistNativeHarnessBackup, restoreDurableNativeHarnessBackup } from "./native-harness-backup-storage.js"; import { restoreNativeProviderArtifacts, retainNativeProviderArtifacts, retainExistingNativeProviderArtifacts, retainReadyNativeProviderArtifacts, NativeProviderArtifactTransferTimeoutError, nativeProviderArtifactTransferBounds, type NativeProviderArtifactReceipt } from "./native-provider-artifacts.js"; @@ -18395,7 +18395,7 @@ async function createRunnerdBackendWithinSessionClaim( if (durableStorage && backupOwner) { await persistNativeHarnessBackup({ db: input.db, storage: durableStorage, owner: backupOwner, scopeDigest: createHash("sha256").update(sessionScopeId).digest("hex"), root: pendingRoot, - signal: AbortSignal.timeout(300_000) }); + transferIdleTimeoutMs: NATIVE_HARNESS_UPLOAD_IDLE_TIMEOUT_MS }); } // Without durable storage the local publication below is the // save; its result is uncertain once it starts, so keep the diff --git a/server/src/storage/request-stats.ts b/server/src/storage/request-stats.ts index 30421fbab40..f60b1cc964e 100644 --- a/server/src/storage/request-stats.ts +++ b/server/src/storage/request-stats.ts @@ -36,6 +36,22 @@ export function collectStorageRequestStats(stats: StorageRequestStats, work: return current.run(stats, work); } +const progress = new AsyncLocalStorage<() => void>(); + +/** + * Runs `work` calling `onProgress` each time the store answers a request, + * takes a multipart part, or a resend starts its wait. A caller uses it to + * tell a slow transfer from a stalled one when the provider sends a file + * source itself and no stream chunk is visible. + */ +export function observeStorageProgress(onProgress: () => void, work: () => Promise): Promise { + return progress.run(onProgress, work); +} + +function noteProgress(): void { + try { progress.getStore()?.(); } catch { return; } +} + function metadataOf(value: unknown): { attempts?: unknown; totalRetryDelay?: unknown } | undefined { if (!value || typeof value !== "object") return undefined; const metadata = (value as { $metadata?: unknown }).$metadata; @@ -47,6 +63,7 @@ const count = (value: unknown, fallback: number) => /** Records one S3 request's outcome from its SDK response or error metadata. */ export function recordStorageRequest(outcome: unknown, failed: boolean): void { + noteProgress(); const stats = current.getStore(); if (!stats) return; const metadata = metadataOf(outcome); @@ -57,6 +74,7 @@ export function recordStorageRequest(outcome: unknown, failed: boolean): void { } export function recordStoragePart(): void { + noteProgress(); const stats = current.getStore(); if (stats) stats.parts += 1; } @@ -69,6 +87,7 @@ export function recordStorageThrottle(): void { /** Records a wait before resending a part. */ export function recordStorageBackoff(waitMs: number): void { + noteProgress(); const stats = current.getStore(); if (stats) stats.backoffMs += count(waitMs, 0); }