mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-20 09:50:48 +01:00
fix(cloud): migrate run evidence during online preparation (#22337)
Drive bounded run-log and provider trace migration pages during online durable-data preparation. Share the page owner and settled-native archive rules with background migration, preserve active-source and storage-integrity guards, and report fresh page results. Keep canceled transfers owned and tracked until the underlying operation settles. Retain page cursors across the preparation budget. Verified 52 focused tests, including embedded PostgreSQL and object storage, server typechecking, and all 58 successful PR checks. Greptile 5/5 with no unresolved review comments. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
264beefeff
commit
46dde54f52
9 files changed
+196
-46
No files matched your search
@@ -201,6 +201,14 @@ for one second. Existing references are the durable work inventory. Restarting
|
||||
reconstructs unfinished work from those references; already migrated rows are
|
||||
excluded. No new payload or job table is added.
|
||||
|
||||
The bounded online durable-data preparation pass also drives log and trace
|
||||
pages before its final eligibility check. It shares the background worker's page
|
||||
owner and settled-native archive rules. A busy owner is deferred rather than
|
||||
starting a competing transfer. Page cursors survive a spent preparation budget.
|
||||
The pass reports its current migration results, not a previous background sweep.
|
||||
Active writers and unproven sources remain blockers. An aborted transfer keeps
|
||||
its ownership and tracked work until the transfer actually settles.
|
||||
|
||||
Active writers, missing files, corrupt data, and storage failures leave history
|
||||
unresolved. Other pages continue. The server logs each blocked resource and a
|
||||
pass summary. Unresolved passes retry after five minutes, with backoff up to
|
||||
@@ -277,7 +285,9 @@ An archive preserves historical evidence; it does not certify a complete native
|
||||
transcript. The run's status, canonical events, and original completion metadata
|
||||
remain unchanged. Archive readers do not append missing native events, and
|
||||
writers cannot reopen or modify the archive. Its source cursor remains zero.
|
||||
The normal background importer retains its strict native coverage checks.
|
||||
The canonical importer retains its strict native coverage checks. Both the
|
||||
background worker and online preparation may delegate a settled native source
|
||||
with unproven coverage to the guarded read-only archive path.
|
||||
Rollbacks after enrollment require readers that support `s3_archive`.
|
||||
|
||||
An interrupted trace import keeps an `s3-import-v1:` reference: reads still use
|
||||
|
||||
@@ -2,7 +2,7 @@ import type { Db } from "@paperclipai/db";
|
||||
import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js";
|
||||
import { afterEach, expect, it, vi } from "vitest";
|
||||
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
|
||||
import { readRunEvidenceMigrationDeferrals, registerRunEvidenceMigrationWork } from "../services/run-evidence-migration-work.js";
|
||||
import { registerRunEvidenceMigrationWork } from "../services/run-evidence-migration-work.js";
|
||||
|
||||
const state = vi.hoisted(() => ({
|
||||
provider: "s3",
|
||||
@@ -143,10 +143,8 @@ it("archives a settled native log whose canonical coverage is unproven and repor
|
||||
apply: true, runId: "settled", kinds: ["log"], limit: 1, preserveSettledNativeFiles: true }));
|
||||
expect(state.migrate).toHaveBeenNthCalledWith(3, expect.anything(), expect.objectContaining({
|
||||
runId: "unsettled", preserveSettledNativeFiles: true }));
|
||||
expect(readRunEvidenceMigrationDeferrals()).toEqual({
|
||||
"run_logs:native_log_archive_source_unsettled": 1,
|
||||
"provider_traces:migration_failed": 1,
|
||||
});
|
||||
expect(state.warn).toHaveBeenCalledWith(expect.objectContaining({ id: "unsettled", reason: "native_log_archive_source_unsettled" }), expect.any(String));
|
||||
expect(state.warn).not.toHaveBeenCalledWith(expect.objectContaining({ id: "settled" }), expect.any(String));
|
||||
expect(state.probe!()).toBe("present");
|
||||
});
|
||||
it("retains startup work during drain and retries a failed sweep before becoming quiet", async () => {
|
||||
|
||||
@@ -8,6 +8,8 @@ import { describeEmbeddedPostgres, nativeRestartFixture } from "../__tests__/hel
|
||||
import { createMaintenanceS3Boundary } from "../__tests__/helpers/maintenance/s3-boundary.js";
|
||||
import { withMaintenanceProvider } from "../__tests__/helpers/maintenance/server-harness.js";
|
||||
import { migrateRunEvidence } from "./run-evidence-migration.js";
|
||||
import { migrateRunEvidencePage } from "./run-evidence-migration-page.js";
|
||||
import { createObjectRunLogStore } from "./object-run-log-store.js";
|
||||
import { getRunLogStore } from "./run-log-store.js";
|
||||
import { createObjectByteJournal } from "./object-byte-journal.js";
|
||||
import { readRecoverableRunLog } from "./native-runtime/native-run-log-recovery.js";
|
||||
@@ -87,6 +89,21 @@ describeEmbeddedPostgres("online native historical log preservation", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("clears a retained local log through the page shared by preparation and background work", async () => {
|
||||
await withSource(async f => {
|
||||
const result = await migrateRunEvidencePage(f.db, { kinds: ["log"], limit: 1 });
|
||||
expect(result.items).toMatchObject([{ id: f.runId, state: "migrated", historicalArchive: true }]);
|
||||
const [after] = await f.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId));
|
||||
expect(after.logStore).toBe("s3_archive");
|
||||
expect(after.status).toBe("failed");
|
||||
await rm(f.file);
|
||||
const reader = createObjectRunLogStore({ ...storage.resolveRunEvidenceStorage()!,
|
||||
assertWriterAuthority: async () => { throw new Error("archive is read-only"); } });
|
||||
expect((await readRecoverableRunLog(f.db, reader, { store: "s3_archive", logRef: after.logRef! })).content)
|
||||
.toBe(f.bytes.toString());
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves a committed successful run without changing its result or completeness metadata", async () => {
|
||||
await withSource(async f => {
|
||||
const [run] = await f.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId));
|
||||
|
||||
@@ -77,7 +77,7 @@ describe.skipIf(!support.supported)("online durable-data migration of local work
|
||||
migrateWorkspace: migrateRemoteWorkspace,
|
||||
nativeHarnessBackups: async () => undefined,
|
||||
nativeControllers: async () => undefined,
|
||||
runEvidenceDeferrals: () => ({}),
|
||||
runEvidencePage: async () => ({ items: [] }),
|
||||
eligibility: async () => ({ eligible: false, blockers: [] }),
|
||||
record: async () => undefined,
|
||||
}) as never;
|
||||
|
||||
@@ -39,7 +39,7 @@ function owners(overrides: Record<string, unknown> = {}) {
|
||||
for (const outcome of ["imported", "unchanged"]) input.onOutcome?.(outcome);
|
||||
return { inspected: 2, imported: 1, unchanged: 1, archivedFailed: 0, sourceAlreadyAbsent: 0, deferred: {} };
|
||||
},
|
||||
runEvidenceDeferrals: () => ({}),
|
||||
runEvidencePage: async (_db: Db, options: { kinds: string[] }) => { calls.push(options.kinds[0] === "log" ? "runLogs" : "providerTraces"); return { items: [] }; },
|
||||
eligibility: vi.fn(async () => ({ eligible: false, blockers: ["skills_require_migration", "volume_free_storage_flags_required"] })),
|
||||
record: vi.fn(async () => undefined),
|
||||
...overrides,
|
||||
@@ -54,7 +54,7 @@ describe("online durable-data migration pass", () => {
|
||||
const fixture = owners();
|
||||
const result = await runOnlineDurableDataMigration(db, { budgetMs: 60_000 }, fixture.owners);
|
||||
expect(fixture.calls).toEqual(["agentFiles", "skillObjects", "skillSnapshots", "skillVersions", "skillInstallations",
|
||||
"runtimeSkillAssets", "workspaceCompanies", "localWorkspaces", "nativeHarnessBackups", "nativeControllers"]);
|
||||
"runtimeSkillAssets", "runLogs", "providerTraces", "workspaceCompanies", "localWorkspaces", "nativeHarnessBackups", "nativeControllers"]);
|
||||
expect(result).toEqual({
|
||||
state: "completed",
|
||||
counts: { agentFiles: 1, skillObjects: 1, skillSnapshots: 1, skillInstallations: 1, runtimeSkillRuns: 1, nativeHarnessBackups: 1, nativeControllers: 1 },
|
||||
@@ -72,7 +72,10 @@ describe("online durable-data migration pass", () => {
|
||||
{ skillId: "s1", companyId: "c", status: "blocked", reason: "Saved version does not cover the installed inventory", code: "saved_version_incomplete" },
|
||||
{ skillId: "s2", companyId: "c", status: "blocked", reason: "network down" },
|
||||
{ skillId: "s3", companyId: "c", status: "linked_local" }] }) }),
|
||||
runEvidenceDeferrals: () => ({ "run_logs:native_log_archive_source_unsettled": 2 }),
|
||||
runEvidencePage: async (_db: Db, input: { kinds: string[] }) => ({ items: input.kinds[0] === "log" ? [
|
||||
{ kind: "log", id: "r1", state: "blocked", reason: "native_log_archive_source_unsettled" },
|
||||
{ kind: "log", id: "r2", state: "blocked", reason: "native_log_archive_source_unsettled" },
|
||||
] : [] }),
|
||||
});
|
||||
const result = await runOnlineDurableDataMigration(db, { budgetMs: 60_000 }, fixture.owners);
|
||||
expect(result.deferred).toMatchObject({ "skills:saved_version_incomplete": 1, "skills:blocked": 1, "skills:linked_local": 1,
|
||||
@@ -214,6 +217,50 @@ describe("online durable-data migration pass", () => {
|
||||
expect(agentFiles).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
describe("run evidence", () => {
|
||||
it("moves log and trace pages before checking final eligibility and reports only current blockers", async () => {
|
||||
let moved = false;
|
||||
const runEvidencePage = vi.fn(async (_db: Db, input: { kinds: string[]; afterRunId?: string }) => {
|
||||
if (input.kinds[0] === "trace") return { items: [{ kind: "trace", id: "t1", state: "migrated" }] };
|
||||
if (input.afterRunId) { moved = true; return { items: [] }; }
|
||||
return { items: [{ kind: "log", id: "r1", state: "migrated" }], nextRunId: "r1" };
|
||||
});
|
||||
const fixture = owners({ runEvidencePage, eligibility: async () => ({ eligible: moved, blockers: moved ? [] : ["run_logs_require_migration"] }) });
|
||||
const result = await runOnlineDurableDataMigration(db, { budgetMs: 60_000 }, fixture.owners);
|
||||
expect(result).toMatchObject({ eligible: true, blockers: [], counts: { runLogs: 1, providerTraces: 1 } });
|
||||
expect(runEvidencePage).toHaveBeenNthCalledWith(2, db, expect.objectContaining({ afterRunId: "r1", kinds: ["log"], limit: 1 }));
|
||||
expect(runEvidencePage).toHaveBeenNthCalledWith(3, db, expect.objectContaining({ kinds: ["trace"], signal: expect.any(AbortSignal) }));
|
||||
});
|
||||
|
||||
it("retains active-source and owner-busy blockers without hiding them behind a completed pass", async () => {
|
||||
const fixture = owners({ runEvidencePage: async (_db: Db, input: { kinds: string[] }) => input.kinds[0] === "trace"
|
||||
? { items: [], busy: true }
|
||||
: { items: [{ kind: "log", id: "active", state: "blocked", reason: "legacy_writer_not_settled" },
|
||||
{ kind: "log", id: "unknown", state: "blocked", reason: "unsafe /private/source secret" }] } });
|
||||
const result = await runOnlineDurableDataMigration(db, { budgetMs: 60_000 }, fixture.owners);
|
||||
expect(result.eligible).toBe(false);
|
||||
expect(result.deferred).toMatchObject({ "run_logs:legacy_writer_not_settled": 1, "run_logs:migration_failed": 1, "provider_traces:owner_busy": 1 });
|
||||
expect(JSON.stringify(result)).not.toContain("secret");
|
||||
});
|
||||
|
||||
it("resumes log paging from the saved cursor when the shared pass budget is spent", async () => {
|
||||
let clock = 0;
|
||||
const runEvidencePage = vi.fn(async (_db: Db, input: { kinds: string[]; afterRunId?: string }) => {
|
||||
if (input.kinds[0] === "log" && !input.afterRunId) {
|
||||
clock = 10;
|
||||
return { items: [{ kind: "log", id: "r1", state: "migrated" }], nextRunId: "r1" };
|
||||
}
|
||||
return { items: [] };
|
||||
});
|
||||
const fixture = owners({ runEvidencePage });
|
||||
expect(await runOnlineDurableDataMigration(db, { budgetMs: 10, now: () => clock }, fixture.owners))
|
||||
.toMatchObject({ state: "budget_exhausted", resumePhase: "run_logs", counts: { runLogs: 1 } });
|
||||
expect(await runOnlineDurableDataMigration(db, { budgetMs: 10, now: () => clock }, fixture.owners))
|
||||
.toMatchObject({ state: "completed" });
|
||||
expect(runEvidencePage).toHaveBeenNthCalledWith(2, db, expect.objectContaining({ kinds: ["log"], afterRunId: "r1" }));
|
||||
});
|
||||
});
|
||||
|
||||
describe("workspaces", () => {
|
||||
const workspace = (n: number) => ({ companyId: "c1", workspaceId: `w${n}`, cursor: `20:w${n}` });
|
||||
|
||||
@@ -229,7 +276,7 @@ describe("online durable-data migration pass", () => {
|
||||
migrateWorkspace: vi.fn(async () => { fixture.calls.push("migrateWorkspace"); return { alreadyMigrated: true, reasons: [] }; }),
|
||||
});
|
||||
const result = await runOnlineDurableDataMigration(db, { budgetMs: 60_000 }, fixture.owners);
|
||||
expect(fixture.calls.slice(fixture.calls.indexOf("runtimeSkillAssets"))).toEqual(["runtimeSkillAssets", "workspaceCompanies",
|
||||
expect(fixture.calls.slice(fixture.calls.indexOf("runtimeSkillAssets"))).toEqual(["runtimeSkillAssets", "runLogs", "providerTraces", "workspaceCompanies",
|
||||
"reconcile:c1", "reconcile:c2", "localWorkspaces", "inventoryWorkspace", "migrateWorkspace", "nativeHarnessBackups", "nativeControllers"]);
|
||||
expect(workspaceOperations).toHaveBeenCalledWith(db, { companyId: "c1", signal: expect.any(AbortSignal) });
|
||||
expect(result.counts).toMatchObject({ workspaceOperations: 2, workspaces: 1 });
|
||||
|
||||
@@ -17,7 +17,7 @@ import { prestageNativeControllers } from "./native-runtime/native-controller-mi
|
||||
import { listLocalWorkspaceCompanies, listLocalWorkspaces } from "./local-workspace-dependencies.js";
|
||||
import { reconcileOnlineWorkspaceOperations } from "./offline-workspace-operation-reconciliation.js";
|
||||
import { inventoryRemoteWorkspaceMigration, migrateRemoteWorkspace } from "./remote-workspace-migration.js";
|
||||
import { readRunEvidenceMigrationDeferrals } from "./run-evidence-migration-work.js";
|
||||
import { migrateRunEvidencePage } from "./run-evidence-migration-page.js";
|
||||
|
||||
export type OnlineDurableDataMigrationResult = {
|
||||
state: "completed" | "budget_exhausted" | "unsupported";
|
||||
@@ -48,7 +48,7 @@ type Owners = {
|
||||
signal: AbortSignal; sourceLockWaitMs: number }) => Promise<{ alreadyMigrated: boolean; reasons: string[] }>;
|
||||
nativeHarnessBackups: typeof prestageNativeHarnessBackups;
|
||||
nativeControllers: typeof prestageNativeControllers;
|
||||
runEvidenceDeferrals: () => Record<string, number>;
|
||||
runEvidencePage: typeof migrateRunEvidencePage;
|
||||
eligibility: (db: Db) => Promise<{ eligible: boolean; blockers: string[] }>;
|
||||
record: (db: Db, result: PassRecord) => Promise<void>;
|
||||
};
|
||||
@@ -77,13 +77,13 @@ const defaultOwners: Owners = {
|
||||
migrateWorkspace: migrateRemoteWorkspace,
|
||||
nativeHarnessBackups: prestageNativeHarnessBackups,
|
||||
nativeControllers: prestageNativeControllers,
|
||||
runEvidenceDeferrals: readRunEvidenceMigrationDeferrals,
|
||||
runEvidencePage: migrateRunEvidencePage,
|
||||
eligibility: readOnlineEphemeralEnrollmentEligibility,
|
||||
record: recordPass,
|
||||
};
|
||||
|
||||
const PHASES = ["agent_files", "skill_objects", "skill_snapshots", "skill_versions", "skill_installations",
|
||||
"runtime_skill_assets", "workspace_operations", "workspaces", "native_harness_backups", "native_controllers"] as const;
|
||||
"runtime_skill_assets", "run_logs", "provider_traces", "workspace_operations", "workspaces", "native_harness_backups", "native_controllers"] as const;
|
||||
type Phase = typeof PHASES[number];
|
||||
|
||||
let running: Promise<OnlineDurableDataMigrationResult> | null = null;
|
||||
@@ -135,7 +135,6 @@ async function migrate(db: Db, input: { budgetMs: number; now?: () => number },
|
||||
const deferred: Record<string, number> = {};
|
||||
const add = (bucket: Record<string, number>, key: string, by = 1) => { if (by) bucket[key] = (bucket[key] ?? 0) + by; };
|
||||
const finish = async (state: OnlineDurableDataMigrationResult["state"], reason?: string): Promise<OnlineDurableDataMigrationResult> => {
|
||||
if (state !== "unsupported") for (const [key, count] of Object.entries(owners.runEvidenceDeferrals())) add(deferred, key, count);
|
||||
const { eligible, blockers } = await owners.eligibility(db);
|
||||
const result: OnlineDurableDataMigrationResult = { state, ...(reason ? { reason } : {}),
|
||||
...(state === "budget_exhausted" ? { resumePhase: PHASES[progress.phase] } : {}), counts, deferred, eligible, blockers };
|
||||
@@ -188,6 +187,23 @@ async function migrate(db: Db, input: { budgetMs: number; now?: () => number },
|
||||
add(counts, "runtimeSkillRuns", result.results.filter(row => row.status === "preserved").length);
|
||||
return result.inspected < limit || !result.after ? null : result.after;
|
||||
}
|
||||
case "run_logs":
|
||||
case "provider_traces": {
|
||||
const logs = phase === "run_logs";
|
||||
const result = await untilAborted(owners.runEvidencePage(db, {
|
||||
kinds: [logs ? "log" : "trace"], limit, signal,
|
||||
...(logs ? { afterRunId: after } : { afterTraceId: after }),
|
||||
}), signal);
|
||||
if (result.busy) { add(deferred, `${phase}:owner_busy`); return null; }
|
||||
for (const item of result.items) {
|
||||
if (item.state === "migrated") add(counts, logs ? "runLogs" : "providerTraces");
|
||||
else if (item.state === "blocked") {
|
||||
const reason = item.reason && /^[a-z_]{1,60}$/.test(item.reason) ? item.reason : "migration_failed";
|
||||
add(deferred, `${phase}:${reason}`);
|
||||
}
|
||||
}
|
||||
return (logs ? result.nextRunId : result.nextTraceId) ?? null;
|
||||
}
|
||||
case "workspace_operations": {
|
||||
const companies = await owners.workspaceCompanies(db, { after, limit });
|
||||
for (const companyId of companies) {
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { afterEach, expect, it, vi } from "vitest";
|
||||
import { migrateRunEvidencePage } from "./run-evidence-migration-page.js";
|
||||
import { idleWorkSnapshot } from "./task-admission.js";
|
||||
|
||||
const migrate = vi.hoisted(() => vi.fn());
|
||||
vi.mock("./run-evidence-migration.js", () => ({ migrateRunEvidence: migrate }));
|
||||
afterEach(() => vi.resetAllMocks());
|
||||
|
||||
it("leaves active and unproven sources blocked and only retries settled native coverage through the archive owner", async () => {
|
||||
migrate.mockResolvedValueOnce({ items: [
|
||||
{ kind: "log", id: "active", state: "blocked", reason: "legacy_writer_not_settled" },
|
||||
{ kind: "log", id: "settled", state: "blocked", reason: "native_log_source_coverage_unproven" },
|
||||
{ kind: "log", id: "unproven", state: "blocked", reason: "legacy_length_or_digest_mismatch" },
|
||||
], nextRunId: "last" }).mockResolvedValueOnce({ items: [{ kind: "log", id: "settled", state: "migrated", historicalArchive: true }] });
|
||||
const db = {} as Db;
|
||||
const result = await migrateRunEvidencePage(db, { kinds: ["log"], limit: 3 });
|
||||
expect(migrate).toHaveBeenCalledTimes(2);
|
||||
expect(migrate).toHaveBeenLastCalledWith(db, expect.objectContaining({ apply: true, runId: "settled", preserveSettledNativeFiles: true }));
|
||||
expect(result).toMatchObject({ nextRunId: "last", items: [
|
||||
{ id: "active", state: "blocked" }, { id: "settled", state: "migrated" }, { id: "unproven", state: "blocked" },
|
||||
] });
|
||||
});
|
||||
|
||||
it("keeps ownership and idle work until an aborted upload actually settles", async () => {
|
||||
let release!: () => void;
|
||||
const gate = new Promise<void>(resolve => { release = resolve; });
|
||||
migrate.mockImplementationOnce(async () => { await gate; return { items: [] }; }).mockResolvedValue({ items: [] });
|
||||
const db = {} as Db, signal = new AbortController();
|
||||
const before = idleWorkSnapshot();
|
||||
const first = migrateRunEvidencePage(db, { kinds: ["log"], signal: signal.signal });
|
||||
signal.abort(new Error("deadline"));
|
||||
expect(await migrateRunEvidencePage(db, { kinds: ["log"] })).toEqual({ items: [], busy: true });
|
||||
expect(idleWorkSnapshot().active).toBeGreaterThan(before.active);
|
||||
expect(migrate).toHaveBeenCalledOnce();
|
||||
release();
|
||||
await expect(first).rejects.toThrow("deadline");
|
||||
expect(idleWorkSnapshot().active).toBe(before.active);
|
||||
expect(await migrateRunEvidencePage(db, { kinds: ["log"] })).toEqual({ items: [] });
|
||||
});
|
||||
|
||||
it("does not serialize different database owners and releases a failed owner", async () => {
|
||||
let release!: () => void;
|
||||
const gate = new Promise<void>(resolve => { release = resolve; });
|
||||
migrate.mockImplementationOnce(async () => { await gate; throw new Error("storage unavailable"); }).mockResolvedValue({ items: [] });
|
||||
const firstDb = {} as Db;
|
||||
const first = migrateRunEvidencePage(firstDb, { kinds: ["log"] });
|
||||
const settled = expect(first).rejects.toThrow("storage unavailable");
|
||||
expect(await migrateRunEvidencePage({} as Db, { kinds: ["trace"] })).toEqual({ items: [] });
|
||||
release(); await settled;
|
||||
expect(await migrateRunEvidencePage(firstDb, { kinds: ["log"] })).toEqual({ items: [] });
|
||||
});
|
||||
@@ -0,0 +1,36 @@
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { migrateRunEvidence, type RunEvidenceMigrationItem } from "./run-evidence-migration.js";
|
||||
import { beginIdleTrackedWork } from "./task-admission.js";
|
||||
|
||||
const owners = new WeakSet<Db>();
|
||||
const unprovenCoverage = new Set(["native_log_source_coverage_unproven", "native_run_log_source_integrity_error"]);
|
||||
type PageOptions = Pick<Parameters<typeof migrateRunEvidence>[1], "afterRunId" | "afterTraceId" | "limit" | "kinds" | "signal">;
|
||||
export type RunEvidenceMigrationPage = Awaited<ReturnType<typeof migrateRunEvidence>> & { busy?: boolean };
|
||||
|
||||
export async function migrateRunEvidencePage(db: Db, options: PageOptions): Promise<RunEvidenceMigrationPage> {
|
||||
options.signal?.throwIfAborted();
|
||||
if (owners.has(db)) return { items: [], busy: true };
|
||||
owners.add(db);
|
||||
const settled = beginIdleTrackedWork();
|
||||
try {
|
||||
const result = await migrateRunEvidence(db, { ...options, apply: true });
|
||||
options.signal?.throwIfAborted();
|
||||
const items: RunEvidenceMigrationItem[] = [];
|
||||
for (const item of result.items) {
|
||||
options.signal?.throwIfAborted();
|
||||
if (item.kind !== "log" || item.state !== "blocked" || !unprovenCoverage.has(item.reason ?? "")) {
|
||||
items.push(item);
|
||||
continue;
|
||||
}
|
||||
const archived = await migrateRunEvidence(db, { apply: true, runId: item.id, kinds: ["log"], limit: 1,
|
||||
preserveSettledNativeFiles: true, signal: options.signal });
|
||||
options.signal?.throwIfAborted();
|
||||
const current = archived.items.find(candidate => candidate.id === item.id);
|
||||
if (current) items.push(current);
|
||||
}
|
||||
return { ...result, items };
|
||||
} finally {
|
||||
owners.delete(db);
|
||||
settled();
|
||||
}
|
||||
}
|
||||
@@ -5,26 +5,7 @@ import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.
|
||||
import { DELIVERY_QUEUES } from "./delivery-work-notifications.js";
|
||||
import { registerDeadlineWork } from "./deadline-work.js";
|
||||
import { registerIdleLocalWorkProbe, type IdleLocalWork } from "./idle-local-work.js";
|
||||
import { migrateRunEvidence, type RunEvidenceMigrationItem } from "./run-evidence-migration.js";
|
||||
|
||||
const UNPROVEN_NATIVE_COVERAGE = new Set(["native_log_source_coverage_unproven", "native_run_log_source_integrity_error"]);
|
||||
let completedSweepDeferrals: Record<string, number> = {};
|
||||
|
||||
export function readRunEvidenceMigrationDeferrals(): Record<string, number> {
|
||||
return { ...completedSweepDeferrals };
|
||||
}
|
||||
|
||||
function deferralKey(item: RunEvidenceMigrationItem) {
|
||||
const code = item.reason && /^[a-z_]{1,60}$/.test(item.reason) ? item.reason : "migration_failed";
|
||||
return `${item.kind === "log" ? "run_logs" : "provider_traces"}:${code}`;
|
||||
}
|
||||
|
||||
async function archiveSettledNativeLog(db: Db, item: RunEvidenceMigrationItem, signal: AbortSignal) {
|
||||
if (item.kind !== "log" || item.state !== "blocked" || !UNPROVEN_NATIVE_COVERAGE.has(item.reason ?? "")) return item;
|
||||
const archived = await migrateRunEvidence(db, { apply: true, runId: item.id, kinds: ["log"], limit: 1,
|
||||
preserveSettledNativeFiles: true, signal });
|
||||
return archived.items.find(candidate => candidate.id === item.id) ?? null;
|
||||
}
|
||||
import { migrateRunEvidencePage } from "./run-evidence-migration-page.js";
|
||||
|
||||
export function registerRunEvidenceMigrationWork(
|
||||
db: Db,
|
||||
@@ -38,7 +19,6 @@ export function registerRunEvidenceMigrationWork(
|
||||
tracesDone = false,
|
||||
blocked = 0,
|
||||
migrated = 0;
|
||||
let deferrals: Record<string, number> = {};
|
||||
let retryMs = 5 * 60_000;
|
||||
let next: number | null = Date.now();
|
||||
let pending: IdleLocalWork = "unknown";
|
||||
@@ -49,8 +29,7 @@ export function registerRunEvidenceMigrationWork(
|
||||
continuationMs: 1_000,
|
||||
async run(signal) {
|
||||
if (next === null) return;
|
||||
const result = await migrateRunEvidence(db, {
|
||||
apply: true,
|
||||
const result = await migrateRunEvidencePage(db, {
|
||||
limit: 1,
|
||||
afterRunId: runCursor,
|
||||
afterTraceId: traceCursor,
|
||||
@@ -58,13 +37,10 @@ export function registerRunEvidenceMigrationWork(
|
||||
signal,
|
||||
});
|
||||
signal.throwIfAborted();
|
||||
for (const found of result.items) {
|
||||
const item = await archiveSettledNativeLog(db, found, signal);
|
||||
signal.throwIfAborted();
|
||||
if (!item) continue;
|
||||
if (result.busy) { pending = "present"; next = Date.now() + 1_000; return; }
|
||||
for (const item of result.items) {
|
||||
if (item.state === "blocked") {
|
||||
blocked++;
|
||||
deferrals[deferralKey(item)] = (deferrals[deferralKey(item)] ?? 0) + 1;
|
||||
logger.warn(
|
||||
{ kind: item.kind, id: item.id, reason: item.reason },
|
||||
"Run evidence migration deferred; source storage is still required",
|
||||
@@ -94,8 +70,6 @@ export function registerRunEvidenceMigrationWork(
|
||||
runCursor = traceCursor = undefined;
|
||||
logsDone = tracesDone = false;
|
||||
blocked = migrated = 0;
|
||||
completedSweepDeferrals = deferrals;
|
||||
deferrals = {};
|
||||
}
|
||||
},
|
||||
next: async () => next,
|
||||
|
||||
Reference in new issue
Block a user