diff --git a/doc/link-previews.md b/doc/link-previews.md new file mode 100644 index 00000000000..67d3f89184c --- /dev/null +++ b/doc/link-previews.md @@ -0,0 +1,65 @@ +# Rich link status caching + +Rich link metadata belongs to the link preview service. GitHub pull requests are +the first provider. Links retain their exact navigation URL and use a canonical +resource URL for cache identity. + +`link_previews` stores company, viewer, channel, resource kind, canonical URL, +free-form status, JSON metadata, check time and freshness expiry. It has no task +foreign key. The same resource on different tasks reuses a record. Channel, +resource kind, status and metadata are not GitHub enums; future provider adapters +can use their own values. This change does not register plugin resolvers. + +Personal provider access remains viewer-scoped. The unique index is +`(company_id, viewer_id, canonical_url)`. The empty viewer is reserved for local +trusted mode. Company/board authorization runs before cache reads. Neither cache +endpoint is accessible to agents. Personal results never enter `external_objects`. +A cache fill is derivative read data; it does not mutate tasks or produce activity +feed events. Cache HTTP responses use `private, no-store`. + +The browser registers mounted rich links after render. Up to 200 unique links +per active company view are eligible for automatic enrichment; further links +still work on hover/focus. One request reads up to 50 cached previews using one +indexed database select, without provider calls or task joins. The read-only GET +route also supports viewer-role members. Encoded request URLs stay below 6 KB. Duplicate links +share one browser query. Fresh browser results take precedence over older database +reads. Cache failures are quiet and do not block the task page. + +After document load, core query settlement and a 500ms quiet delay, browser idle +callbacks resolve stale or missing status with at most three requests in flight. +Hover/focus uses the same query and server path, so overlapping requests coalesce. +Queued links are removed on unmount and scope changes dispose the coordinator. +There is no recurring refresh, window-focus lookup or server background worker. +New links from subsequent page content get their own one-shot enrichment. + +The existing provider timeout, fixed-host lookup, managed connection selection, +public-access checks, negative cache window and process concurrency bound remain. +Provider results are stored before the successful lookup response completes when +the store is available within the shared five-second deadline. Stalled SQL keeps +its concurrency slot until it settles; the deadline still releases the response. +Expired process entries re-read durable storage before provider work. Both batch +reads and lookup fallback use check time to retain the newest known status; a +longer cache lifetime does not make an older status newer. When the durable read +fails too, provider-failure backoff stays local, so it cannot overwrite a saved +status that this process could not read. Database outages fall back to the +optional process cache/provider path. Successful +and unavailable results normally remain fresh for one minute; provider backoff +can extend that window up to one hour. Expired records retain last known status +for immediate display. An outage keeps that status and saves the negative +freshness window, so the page does not repeatedly retry. + +Each company/viewer scope retains at most 1,000 rows. A write removes rows beyond +that limit, keeping the newest check times, and rows whose freshness expired more +than 30 days before the write's check time. A transaction-scoped advisory lock +serializes writes for that scope across replicas, so concurrent URL misses cannot +exceed the cap. Cleanup makes no queries during page hydration and starts no +background worker. Inactive scopes remain bounded and are cleaned on their next +write; company deletion removes all their rows. Normal stale status remains +available until it ages out or the scope reaches its quota. Negative results use +the same cap and retention policy as known status. + +The single-link lookup route also limits each company/viewer to 600 requests per +minute per server process, using the existing in-memory request limiter. Excess +requests return 429 with Retry-After. Cached batch reads remain available during +that backoff and do not allocate stored rows. The durable quota is enforced across +replicas and remains effective when process-local request limits reset. diff --git a/packages/db/src/migrations/0489_real_sage.sql b/packages/db/src/migrations/0489_real_sage.sql new file mode 100644 index 00000000000..57c2b02e88a --- /dev/null +++ b/packages/db/src/migrations/0489_real_sage.sql @@ -0,0 +1,14 @@ +CREATE TABLE IF NOT EXISTS "link_previews" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "company_id" uuid NOT NULL CONSTRAINT "link_previews_company_id_companies_id_fk" REFERENCES "public"."companies"("id") ON DELETE cascade, + "viewer_id" text NOT NULL, + "channel" text NOT NULL, + "resource_kind" text NOT NULL, + "canonical_url" text NOT NULL, + "status" text, + "metadata" jsonb DEFAULT '{}'::jsonb NOT NULL, + "checked_at" timestamp with time zone NOT NULL, + "expires_at" timestamp with time zone NOT NULL +); +--> statement-breakpoint +CREATE UNIQUE INDEX IF NOT EXISTS "link_previews_scope_url_uq" ON "link_previews" USING btree ("company_id","viewer_id","canonical_url"); diff --git a/packages/db/src/migrations/meta/0484_snapshot.json b/packages/db/src/migrations/meta/0489_snapshot.json similarity index 99% rename from packages/db/src/migrations/meta/0484_snapshot.json rename to packages/db/src/migrations/meta/0489_snapshot.json index 1debf9ce54b..4eee50702b2 100644 --- a/packages/db/src/migrations/meta/0484_snapshot.json +++ b/packages/db/src/migrations/meta/0489_snapshot.json @@ -1,6 +1,6 @@ { - "id": "9389124c-379a-4b64-9c20-c7b37538e7c8", - "prevId": "ceaa38c4-681b-430e-9578-0854650d85b3", + "id": "dc61c7f3-cf3d-44c5-9296-86973e5f0925", + "prevId": "8f935a52-5f1e-4afe-8d13-c1770ca33968", "version": "7", "dialect": "postgresql", "tables": { @@ -6582,6 +6582,111 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.automatic_run_brakes": { + "name": "automatic_run_brakes", + "schema": "", + "columns": { + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'closed'" + }, + "reason": { + "name": "reason", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "state_changed_at": { + "name": "state_changed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "opened_at": { + "name": "opened_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "next_probe_at": { + "name": "next_probe_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "probe_run_id": { + "name": "probe_run_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "probe_successes": { + "name": "probe_successes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "release_window_started_at": { + "name": "release_window_started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "release_window_starts": { + "name": "release_window_starts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "next_creation_at": { + "name": "next_creation_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "automatic_run_brakes_company_id_companies_id_fk": { + "name": "automatic_run_brakes_company_id_companies_id_fk", + "tableFrom": "automatic_run_brakes", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.board_api_keys": { "name": "board_api_keys", "schema": "", @@ -36029,6 +36134,78 @@ "method": "btree", "with": {} }, + "heartbeat_runs_company_finished_idx": { + "name": "heartbeat_runs_company_finished_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "finished_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"heartbeat_runs\".\"finished_at\" is not null", + "concurrently": true, + "method": "btree", + "with": {} + }, + "heartbeat_runs_settled_process_identity_idx": { + "name": "heartbeat_runs_settled_process_identity_idx", + "columns": [ + { + "expression": "process_host_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"heartbeat_runs\".\"runtime_mode\" = 'native' and \"heartbeat_runs\".\"status\" in ('failed', 'cancelled', 'timed_out', 'interrupted')\n and \"heartbeat_runs\".\"finished_at\" is not null\n and (\"heartbeat_runs\".\"process_pid\" is not null or \"heartbeat_runs\".\"process_group_id\" is not null or \"heartbeat_runs\".\"process_started_at\" is not null)", + "concurrently": true, + "method": "btree", + "with": {} + }, + "heartbeat_runs_pending_api_stop_idx": { + "name": "heartbeat_runs_pending_api_stop_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"heartbeat_runs\".\"status\" = 'cancelled' and \"heartbeat_runs\".\"driver_kind\" = 'cursor_cloud_api'\n and (\"heartbeat_runs\".\"result_json\"->'executionCancellation'->>'state') = 'requested'", + "concurrently": true, + "method": "btree", + "with": {} + }, "heartbeat_runs_company_created_at_desc_idx": { "name": "heartbeat_runs_company_created_at_desc_idx", "columns": [ @@ -40722,6 +40899,131 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.issue_recovery_deferrals": { + "name": "issue_recovery_deferrals", + "schema": "", + "columns": { + "queue": { + "name": "queue", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "issue_id": { + "name": "issue_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "due_at": { + "name": "due_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "issue_recovery_deferrals_issue_id_issues_id_fk": { + "name": "issue_recovery_deferrals_issue_id_issues_id_fk", + "tableFrom": "issue_recovery_deferrals", + "tableTo": "issues", + "columnsFrom": [ + "issue_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "issue_recovery_deferrals_company_id_companies_id_fk": { + "name": "issue_recovery_deferrals_company_id_companies_id_fk", + "tableFrom": "issue_recovery_deferrals", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "issue_recovery_deferrals_pk": { + "name": "issue_recovery_deferrals_pk", + "columns": [ + "queue", + "issue_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.issue_recovery_full_passes": { + "name": "issue_recovery_full_passes", + "schema": "", + "columns": { + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "queue": { + "name": "queue", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "issue_recovery_full_passes_company_id_companies_id_fk": { + "name": "issue_recovery_full_passes_company_id_companies_id_fk", + "tableFrom": "issue_recovery_full_passes", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "issue_recovery_full_passes_pk": { + "name": "issue_recovery_full_passes_pk", + "columns": [ + "company_id", + "queue" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.issue_reference_mentions": { "name": "issue_reference_mentions", "schema": "", @@ -44779,6 +45081,123 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.link_previews": { + "name": "link_previews", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "viewer_id": { + "name": "viewer_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "channel": { + "name": "channel", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource_kind": { + "name": "resource_kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "canonical_url": { + "name": "canonical_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "checked_at": { + "name": "checked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "link_previews_scope_url_uq": { + "name": "link_previews_scope_url_uq", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "viewer_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "canonical_url", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "link_previews_company_id_companies_id_fk": { + "name": "link_previews_company_id_companies_id_fk", + "tableFrom": "link_previews", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.managed_agent_profiles": { "name": "managed_agent_profiles", "schema": "", diff --git a/packages/db/src/migrations/meta/_journal.json b/packages/db/src/migrations/meta/_journal.json index 464a6addffb..ee03bddd7a5 100644 --- a/packages/db/src/migrations/meta/_journal.json +++ b/packages/db/src/migrations/meta/_journal.json @@ -3410,6 +3410,13 @@ "when": 1794820062730, "tag": "0488_issue_recovery_full_passes", "breakpoints": true + }, + { + "idx": 489, + "version": "7", + "when": 1794841384470, + "tag": "0489_real_sage", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/db/src/schema/index.ts b/packages/db/src/schema/index.ts index 68878bc412f..f27b014644c 100644 --- a/packages/db/src/schema/index.ts +++ b/packages/db/src/schema/index.ts @@ -290,3 +290,4 @@ export { projectTaskExecutionPolicies, executionAdmissions, executionAdmissionPr export { budgetEvaluationWork } from "./budget_evaluation_work.js"; export { tailscaleConnections } from "./tailscale_connections.js"; export { nativeProviderArtifacts, nativeProviderArtifactReferences } from "./native_provider_artifacts.js"; +export { linkPreviews } from "./link_previews.js"; diff --git a/packages/db/src/schema/link_previews.ts b/packages/db/src/schema/link_previews.ts new file mode 100644 index 00000000000..c7b7ee334aa --- /dev/null +++ b/packages/db/src/schema/link_previews.ts @@ -0,0 +1,19 @@ +import { jsonb, pgTable, text, timestamp, uniqueIndex, uuid } from "drizzle-orm/pg-core"; +import { companies } from "./companies.js"; + +/** Optional enrichment, independent of tasks and of provider-specific status vocabularies. */ +export const linkPreviews = pgTable("link_previews", { + id: uuid("id").primaryKey().defaultRandom(), + companyId: uuid("company_id").notNull().references(() => companies.id, { onDelete: "cascade" }), + // Personal provider results must not be reused by another viewer. Empty is local-trusted. + viewerId: text("viewer_id").notNull(), + channel: text("channel").notNull(), + resourceKind: text("resource_kind").notNull(), + canonicalUrl: text("canonical_url").notNull(), + status: text("status"), + metadata: jsonb("metadata").$type>().notNull().default({}), + checkedAt: timestamp("checked_at", { withTimezone: true }).notNull(), + expiresAt: timestamp("expires_at", { withTimezone: true }).notNull(), +}, table => ({ + identity: uniqueIndex("link_previews_scope_url_uq").on(table.companyId, table.viewerId, table.canonicalUrl), +})); diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index efab7ad18d8..825867d65bb 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -2976,7 +2976,7 @@ export { TASK_IDENTITY_CHANGED_FIELD, issueActivityChangesTaskIdentity, } from "./issue-activity-identity.js"; -export { recognizeLinkPreview, type LinkPreviewTarget, type LinkPreviewState, type LinkPreview, type LinkPreviewResult } from "./link-preview.js"; +export { LINK_PREVIEW_BATCH_LIMIT, type LinkPreviewBatchResult, recognizeLinkPreview, type LinkPreviewTarget, type LinkPreviewState, type LinkPreview, type LinkPreviewResult } from "./link-preview.js"; export * from "./mcp-apps.js"; export * from "./tool-call-progress.js"; export * from "./mcp-skills.js"; diff --git a/packages/shared/src/link-preview.ts b/packages/shared/src/link-preview.ts index ffa5eb960a7..b7431d4c30a 100644 --- a/packages/shared/src/link-preview.ts +++ b/packages/shared/src/link-preview.ts @@ -18,10 +18,13 @@ export interface LinkPreview { export interface LinkPreviewResult { preview: LinkPreview | null; - /** Demand-driven cache lifetime; this does not schedule a refresh. */ + /** Remaining cache lifetime; zero means stale. This does not schedule a refresh. */ maxAgeMs: number; } +export const LINK_PREVIEW_BATCH_LIMIT = 50; +export interface LinkPreviewBatchResult { results: Record } + /** Recognition only. Never contacts a remote service or changes navigation. */ export function recognizeLinkPreview(href: string | null | undefined): LinkPreviewTarget | null { if (!href || href.length > 2048) return null; diff --git a/server/src/__tests__/issue-recovery-deadlines.test.ts b/server/src/__tests__/issue-recovery-deadlines.test.ts index f11f3f541f5..3543b5aa660 100644 --- a/server/src/__tests__/issue-recovery-deadlines.test.ts +++ b/server/src/__tests__/issue-recovery-deadlines.test.ts @@ -441,10 +441,19 @@ describeIssueRecovery("issue recovery production producers and deadlines", () => const [row] = await state.db.select().from(issueWatchdogs).where(eq(issueWatchdogs.id, watchdog.watchdog.id)); expect(row.watchdogIssueId).toBeTruthy(); reviewId = row.watchdogIssueId!; }, { timeout: 15_000 }); - await completed(reviewId!); + const review = await completed(reviewId!); expect(failures).toEqual([]); + // Ordinary assignment recovery can win the wake reason while the watchdog + // wake coalesces into that run. Verify its mandate and one dispatch instead. + expect(review.contextSnapshot).toMatchObject({ + watchdogId: watchdog.watchdog.id, + taskWatchdog: { + watchedIssueId: source.id, + stopFingerprint: expect.stringMatching(/^task_watchdog_stop:/), + }, + }); expect(await state.db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), - sql`${heartbeatRuns.contextSnapshot}->>'wakeReason' = 'task_watchdog_stopped_subtree'`))).toHaveLength(1); + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${reviewId}`))).toHaveLength(1); await finish(reviewId!); await taskWatchdogService(state.db).disableForIssue(f.companyId, source.id); if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); await quiet(); diff --git a/server/src/__tests__/link-preview-routes.test.ts b/server/src/__tests__/link-preview-routes.test.ts index a5026be77d8..41dc53a85a9 100644 --- a/server/src/__tests__/link-preview-routes.test.ts +++ b/server/src/__tests__/link-preview-routes.test.ts @@ -4,21 +4,44 @@ import { describe, expect, it, vi } from "vitest"; import type { Db } from "@paperclipai/db"; import { linkPreviewRoutes } from "../routes/link-previews.js"; import { HttpError } from "../errors.js"; +import { createInviteRateLimiter, type InviteRateLimiter } from "../services/invite-rate-limit.js"; -function app(actor: express.Request["actor"]) { +function app(actor: express.Request["actor"], limiter?: InviteRateLimiter) { const get = vi.fn(async () => ({ preview: null, maxAgeMs: 60_000 })); + const cached = vi.fn(async () => ({ results: {} })); const server = express(); + server.use(express.json()); server.use((req, _res, next) => { req.actor = actor; next(); }); - server.use(linkPreviewRoutes(null as unknown as Db, { get })); + server.use(linkPreviewRoutes(null as unknown as Db, { get, cached }, limiter)); server.use((error: HttpError, _req: express.Request, res: express.Response, _next: express.NextFunction) => { res.status(error.status ?? 500).json({ error: error.message }); }); - return { server, get }; + return { server, get, cached }; } const board = { type: "board", source: "session", userId: "alice", companyIds: ["company-a"] } as const; const path = "/companies/company-a/link-preview"; describe("link preview routes", () => { + it("limits lookups per viewer and company without blocking cached reads", async () => { + let now = 0; + const limiter = createInviteRateLimiter({ maxRequests: 2, windowMs: 1_000, now: () => now }); + const actor = { ...board, userId: "alice", companyIds: ["company-a", "company-b"] }; + const { server, get, cached } = app(actor, limiter); + const query = { url: "https://github.com/acme/web/pull/42" }; + for (let n = 0; n < 2; n++) expect((await request(server).get(path).query(query)).status).toBe(200); + const limited = await request(server).get(path).query(query); + expect(limited.status).toBe(429); + expect(limited.headers["retry-after"]).toBe("1"); + expect(get).toHaveBeenCalledTimes(2); + expect((await request(server).get("/companies/company-a/link-previews/cached").query(query)).status).toBe(200); + expect(cached).toHaveBeenCalledTimes(1); + expect((await request(server).get("/companies/company-b/link-preview").query(query)).status).toBe(200); + actor.userId = "bob"; + expect((await request(server).get(path).query(query)).status).toBe(200); + actor.userId = "alice"; + now = 1_001; + expect((await request(server).get(path).query(query)).status).toBe(200); + }); it("passes the authenticated viewer and disables shared HTTP caching", async () => { const { server, get } = app({ ...board, companyIds: [...board.companyIds] }); const response = await request(server).get(path).query({ url: "https://github.com/acme/web/pull/42" }); @@ -38,6 +61,25 @@ describe("link preview routes", () => { expect(get).not.toHaveBeenCalled(); } }); + it("reads a bounded batch only after viewer and company authorization", async () => { + const { server, cached, get } = app({ ...board, companyIds: [...board.companyIds] }); + const response = await request(server).get("/companies/company-a/link-previews/cached").query({ url: ["https://github.com/acme/web/pull/42"] }); + expect(response.status).toBe(200); + expect(response.headers["cache-control"]).toBe("private, no-store"); + expect(cached).toHaveBeenCalledWith({ companyId: "company-a", userId: "alice" }, ["https://github.com/acme/web/pull/42"]); + expect(get).not.toHaveBeenCalled(); + for (const body of [{}, { url: ["x".repeat(2049)] }, { url: Array(51).fill("a") }]) { + expect((await request(server).get("/companies/company-a/link-previews/cached").query(body)).status).toBe(400); + } + const denied = app({ ...board, companyIds: ["company-b"] }); + expect((await request(denied.server).get("/companies/company-a/link-previews/cached")).status).toBe(403); + expect(denied.cached).not.toHaveBeenCalled(); + }); + it("lets active viewer-role members read stored statuses", async () => { + const { server, cached } = app({ ...board, companyIds: [...board.companyIds], memberships: [{ companyId: "company-a", membershipRole: "viewer", status: "active" }] }); + expect((await request(server).get("/companies/company-a/link-previews/cached").query({ url: "https://github.com/acme/web/pull/42" })).status).toBe(200); + expect(cached).toHaveBeenCalledTimes(1); + }); it("bounds and validates the URL input", async () => { const { server, get } = app({ ...board, companyIds: [...board.companyIds] }); for (const query of [{}, { url: "x".repeat(2049) }, { url: ["a", "b"] }]) { diff --git a/server/src/__tests__/link-preview-store.integration.test.ts b/server/src/__tests__/link-preview-store.integration.test.ts new file mode 100644 index 00000000000..376ff11ac7a --- /dev/null +++ b/server/src/__tests__/link-preview-store.integration.test.ts @@ -0,0 +1,253 @@ +import { randomUUID } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import { companies, linkPreviews } from "@paperclipai/db"; +import { recognizeLinkPreview } from "@paperclipai/shared"; +import { sql } from "drizzle-orm"; +import { expect, it, vi } from "vitest"; +import { describeEmbeddedPostgres, useEmbeddedPostgres } from "./helpers/route-test-harness.js"; +import { createLinkPreviewStore } from "../services/link-preview-store.js"; +import { createLinkPreviewService } from "../services/link-previews.js"; + +const url = "https://github.com/acme/web/pull/42"; +const target = recognizeLinkPreview(url)!; +const preview = { target, title: "Durable PR", state: "merged" as const, author: "alice", updatedAt: null }; + +describeEmbeddedPostgres("durable link preview cache", () => { + const context = useEmbeddedPostgres("link-previews-"); + async function fixture() { + const companyId = randomUUID(); + await context.db.insert(companies).values({ id: companyId, name: "Link cache", issuePrefix: companyId }); + return { companyId, userId: "alice" }; + } + it("bounds retained rows across concurrent store instances and isolates their quotas", async () => { + const scope = await fixture(); + const other = { ...scope, userId: "bob" }; + const now = Date.now(); + const seed = Array.from({ length: 1_000 }, (_, n) => ({ + companyId: scope.companyId, viewerId: scope.userId, channel: "github", resourceKind: "pull_request", + canonicalUrl: url + n, status: null, metadata: {}, checkedAt: new Date(now - 1_000 + n), expiresAt: new Date(now + 60_000), + })); + await context.db.insert(linkPreviews).values([...seed, { ...seed[0], viewerId: other.userId }]); + const first = createLinkPreviewStore(context.db); + const second = createLinkPreviewStore(context.db); + await Promise.all(Array.from({ length: 8 }, (_, n) => (n % 2 ? first : second).write(scope, + recognizeLinkPreview(url + (1_000 + n))!, { result: { preview: null, maxAgeMs: 60_000 }, expiresAt: now + 60_000 }, now + n))); + const rows = await context.db.select().from(linkPreviews).where(sql`${linkPreviews.companyId} = ${scope.companyId}`); + expect(rows.filter(row => row.viewerId === scope.userId)).toHaveLength(1_000); + expect(rows.filter(row => row.viewerId === other.userId)).toHaveLength(1); + const saved = await first.read(scope, [recognizeLinkPreview(url + "0")!, recognizeLinkPreview(url + "1007")!]); + expect(saved.has(url + "0")).toBe(false); + expect(saved.has(url + "1007")).toBe(true); + }); + it("removes long-expired rows on writes and preserves recent stale status", async () => { + const scope = await fixture(); + const now = Date.now(); + const expired = now - 31 * 24 * 60 * 60_000; + await context.db.insert(linkPreviews).values([ + { companyId: scope.companyId, viewerId: scope.userId, channel: "github", resourceKind: "pull_request", canonicalUrl: url, + status: "merged", metadata: { title: preview.title }, checkedAt: new Date(expired - 60_000), expiresAt: new Date(expired) }, + { companyId: scope.companyId, viewerId: scope.userId, channel: "github", resourceKind: "pull_request", canonicalUrl: url + "1", + status: "merged", metadata: { title: preview.title }, checkedAt: new Date(now - 120_000), expiresAt: new Date(now - 60_000) }, + ]); + const store = createLinkPreviewStore(context.db); + await store.write(scope, recognizeLinkPreview(url + "2")!, { result: { preview: null, maxAgeMs: 60_000 }, expiresAt: now + 60_000 }, now); + const saved = await store.read(scope, [target, recognizeLinkPreview(url + "1")!, recognizeLinkPreview(url + "2")!]); + expect(saved.has(url)).toBe(false); + expect(saved.get(url + "1")?.result.preview?.state).toBe("merged"); + expect(saved.has(url + "2")).toBe(true); + }); + it.each(["failure", "success", "timeout", "stale timeout"])("keeps another replica's concurrent hydration after provider %s", async (settlement) => { + const scope = await fixture(); + const store = createLinkPreviewStore(context.db); + let now = 1_000; + const oldPreview = { ...preview, state: "open" as const }; + await store.write(scope, target, { result: { preview: oldPreview, maxAgeMs: 1_000 }, expiresAt: 2_000 }, now); + let finish!: (value: { preview: typeof oldPreview | null; maxAgeMs: number }) => void; + let started!: () => void; + const providerStarted = new Promise(done => { started = done; }); + const resolve = vi.fn(() => new Promise<{ preview: typeof oldPreview | null; maxAgeMs: number }>(done => { finish = done; started(); })); + const write = vi.fn(store.write); + const first = createLinkPreviewService([{ key: "github", resolve }], { store: { ...store, write }, now: () => now, timeoutMs: 1_000 }); + await first.cached(scope, [url]); + now = 3_000; + const pending = first.get(scope, url); + await providerStarted; + now = 4_000; + const expiresAt = settlement === "stale timeout" ? 3_500 : 34_000; + await store.write(scope, target, { result: { preview, maxAgeMs: 30_000 }, expiresAt }, now); + expect((await first.cached(scope, [url])).results[url].preview).toEqual(preview); + if (!settlement.includes("timeout")) finish({ preview: settlement === "success" ? oldPreview : null, maxAgeMs: 60_000 }); + expect(await pending).toEqual({ preview, maxAgeMs: Math.max(0, expiresAt - now) }); + // A second batch checks that the process did not retain the older status. + expect((await first.cached(scope, [url])).results[url].preview).toEqual(preview); + const saved = (await store.read(scope, [target])).get(url)!; + expect(saved.result.preview).toEqual(preview); + expect(saved.checkedAt).toBe(4_000); + expect(saved.expiresAt).toBe(expiresAt); + expect(write).not.toHaveBeenCalled(); + finish({ preview: null, maxAgeMs: 60_000 }); + }); + it.each(["open", "negative"])("rechecks expired %s memory before calling the provider", async (initialState) => { + const scope = await fixture(); + const store = createLinkPreviewStore(context.db); + let now = 1_000; + const resolve = vi.fn(async () => ({ preview: initialState === "open" ? { ...preview, state: "open" as const } : null, maxAgeMs: 60_000 })); + const first = createLinkPreviewService([{ key: "github", resolve }], { store, now: () => now }); + await first.get(scope, url); + now = 62_000; + const other = createLinkPreviewService([{ key: "github", resolve: async () => ({ preview, maxAgeMs: 60_000 }) }], { store, now: () => now }); + await other.get(scope, url); + resolve.mockRejectedValue(new Error("provider unavailable")); + + expect((await first.get(scope, url)).preview).toEqual(preview); + expect(resolve).toHaveBeenCalledTimes(1); + const saved = (await store.read(scope, [target])).get(url)!; + expect(saved.result.preview).toEqual(preview); + expect(saved.expiresAt).toBe(122_000); + }); + it.each(["failure", "timeout"])("retains the newest stale stored preview on provider %s", async (failure) => { + const scope = await fixture(); + const store = createLinkPreviewStore(context.db); + let now = 1_000; + let requests = 0; + const resolve = async (_target: import("@paperclipai/shared").LinkPreviewTarget, _scope: import("../services/link-previews.js").LinkPreviewScope, signal: AbortSignal) => { + if (++requests === 1) return { preview: { ...preview, state: "open" as const }, maxAgeMs: 60_000 }; + if (failure === "failure") throw new Error("provider unavailable"); + return new Promise((_done, reject) => signal.addEventListener("abort", () => reject(new Error("timeout")))); + }; + const first = createLinkPreviewService([{ key: "github", resolve }], { store, now: () => now, timeoutMs: 100 }); + await first.get(scope, url); + now = 62_000; + await store.write(scope, target, { result: { preview, maxAgeMs: 1_000 }, expiresAt: 63_000 }, now); + now = 64_000; + expect((await first.get(scope, url)).preview).toEqual(preview); + await vi.waitFor(async () => expect((await store.read(scope, [target])).get(url)?.expiresAt).toBe(124_000)); + expect((await store.read(scope, [target])).get(url)?.result.preview).toEqual(preview); + }); + it("compares check times rather than cache lifetimes when hydrating", async () => { + const scope = await fixture(); + const store = createLinkPreviewStore(context.db); + let now = 1_000; + await store.write(scope, target, { result: { preview: { ...preview, state: "open" }, maxAgeMs: 60_000 }, expiresAt: 61_000 }, now); + const service = createLinkPreviewService([{ key: "github", resolve: async () => ({ preview: null, maxAgeMs: 60_000 }) }], { store, now: () => now }); + await service.cached(scope, [url]); + await store.write(scope, target, { result: { preview, maxAgeMs: 1_000 }, expiresAt: 4_000 }, 3_000); + now = 5_000; + expect((await service.cached(scope, [url])).results[url]).toEqual({ preview, maxAgeMs: 0 }); + }); + it("keeps a failed durable read's fallback local and accepts recovered stored status", async () => { + const scope = await fixture(); + const store = createLinkPreviewStore(context.db); + const read = vi.fn(store.read); + let now = 1_000; + const resolve = vi.fn(async () => ({ preview: { ...preview, state: "open" as const }, maxAgeMs: 60_000 })); + const first = createLinkPreviewService([{ key: "github", resolve }], { store: { ...store, read }, now: () => now }); + await first.get(scope, url); + now = 62_000; + await store.write(scope, target, { result: { preview, maxAgeMs: 60_000 }, expiresAt: 122_000 }, now); + now = 64_000; + read.mockRejectedValueOnce(new Error("temporary database outage")); + resolve.mockRejectedValueOnce(new Error("provider unavailable")); + expect((await first.get(scope, url)).preview?.state).toBe("open"); + const saved = (await store.read(scope, [target])).get(url)!; + expect(saved.result.preview).toEqual(preview); + expect(saved.expiresAt).toBe(122_000); + expect((await first.cached(scope, [url])).results[url].preview).toEqual(preview); + }); + it.each(["missing", "failed"])("reads another replica's saved status after a %s batch read", async (initialRead) => { + const scope = await fixture(); + const store = createLinkPreviewStore(context.db); + const read = vi.fn(store.read); + if (initialRead === "failed") read.mockRejectedValueOnce(new Error("temporary database outage")); + const write = vi.fn(store.write); + const resolve = vi.fn(async () => { throw new Error("provider unavailable"); }); + const first = createLinkPreviewService([{ key: "github", resolve }], { store: { read, write } }); + expect(await first.cached(scope, [url])).toEqual({ results: {} }); + + const other = createLinkPreviewService([{ key: "github", resolve: async () => ({ preview, maxAgeMs: 60_000 }) }], { store }); + await other.get(scope, url); + expect((await first.get(scope, url)).preview).toEqual(preview); + expect(read).toHaveBeenCalledTimes(2); + expect(resolve).not.toHaveBeenCalled(); + expect(write).not.toHaveBeenCalled(); + expect((await store.read(scope, [target])).get(url)?.result.preview).toEqual(preview); + }); + it("survives service recreation, reuses canonical URLs across tasks, and isolates viewers/companies", async () => { + const scope = await fixture(); + const store = createLinkPreviewStore(context.db); + const resolve = vi.fn(async () => ({ preview, maxAgeMs: 60_000 })); + const make = () => createLinkPreviewService([{ key: "github", resolve }], { store }); + await make().get(scope, url); + const restored = await make().cached(scope, [url, "https://www.github.com/ACME/WEB/pull/42/files#review"]); + expect(Object.keys(restored.results)).toEqual([url]); + expect(restored.results[url].preview).toMatchObject({ title: preview.title, state: "merged", target: { canonicalUrl: url } }); + expect(restored.results[url].maxAgeMs).toBeGreaterThan(0); + await make().get(scope, url); + expect(resolve).toHaveBeenCalledTimes(1); + expect(await make().cached({ ...scope, userId: "bob" }, [url])).toEqual({ results: {} }); + expect(await make().cached(await fixture(), [url])).toEqual({ results: {} }); + }); + it("returns stale status immediately, refreshes on demand, and persists negative freshness", async () => { + const scope = await fixture(); + const store = createLinkPreviewStore(context.db); + let now = 1_000; + const resolve = vi.fn(async () => ({ preview, maxAgeMs: 60_000 })); + const make = () => createLinkPreviewService([{ key: "github", resolve }], { store, now: () => now }); + await make().get(scope, url); + now += 61_000; + expect((await make().cached(scope, [url])).results[url]).toEqual({ preview, maxAgeMs: 0 }); + expect(resolve).toHaveBeenCalledTimes(1); + resolve.mockRejectedValueOnce(new Error("outage")); + expect((await make().get(scope, url)).preview).toEqual(preview); + await make().get(scope, url); + expect(resolve).toHaveBeenCalledTimes(2); + const missing = url + "1"; + resolve.mockRejectedValueOnce(new Error("private")); + await make().get(scope, missing); + await make().get(scope, missing); + expect(resolve).toHaveBeenCalledTimes(3); + expect((await make().cached(scope, [missing])).results[missing].preview).toBeNull(); + }); + it("persists provider timeout freshness across service restarts without extending the response", async () => { + const scope = await fixture(); + const database = createLinkPreviewStore(context.db); + let saved!: () => void; + const persisted = new Promise(done => { saved = done; }); + const store = { ...database, write: async (...args: Parameters) => { await database.write(...args); saved(); } }; + const resolve = vi.fn((_target: import("@paperclipai/shared").LinkPreviewTarget, _scope: import("../services/link-previews.js").LinkPreviewScope, signal: AbortSignal) => new Promise((_done, reject) => { + signal.addEventListener("abort", () => reject(new Error("timeout"))); + })); + const first = createLinkPreviewService([{ key: "github", resolve }], { store, timeoutMs: 100 }); + expect((await first.get(scope, url)).preview).toBeNull(); + await persisted; + const restarted = createLinkPreviewService([{ key: "github", resolve }], { store }); + expect((await restarted.get(scope, url)).preview).toBeNull(); + expect(resolve).toHaveBeenCalledTimes(1); + expect((await restarted.cached(scope, [url])).results[url].maxAgeMs).toBeGreaterThan(0); + }); + it("reads a 50-link page in one indexed select and does not require task relationships", async () => { + const scope = await fixture(); + const store = createLinkPreviewStore(context.db); + const targets = Array.from({ length: 50 }, (_, n) => recognizeLinkPreview(url + n)!); + await Promise.all(targets.map(t => store.write(scope, t, { result: { preview: { ...preview, target: t }, maxAgeMs: 60_000 }, expiresAt: 61_000 }, 1_000))); + const select = vi.spyOn(context.db, "select"); + expect((await store.read(scope, targets)).size).toBe(50); + expect(select).toHaveBeenCalledTimes(1); + select.mockRestore(); + const plan = await context.db.execute(sql`EXPLAIN SELECT * FROM link_previews WHERE company_id = ${scope.companyId} AND viewer_id = 'alice' AND canonical_url IN (${sql.join(targets.map(t => sql`${t.canonicalUrl}`), sql`, `)})`); + expect(JSON.stringify(plan)).toContain("link_previews_scope_url_uq"); + }); + it("supports other channels/statuses and older writers cannot replace newer results", async () => { + const scope = await fixture(); + const store = createLinkPreviewStore(context.db); + const deployment = { ...target, provider: "vercel", kind: "deployment", canonicalUrl: "https://vercel.com/acme/build/123" }; + await context.db.insert(linkPreviews).values({ companyId: scope.companyId, viewerId: scope.userId, channel: deployment.provider, + resourceKind: deployment.kind, canonicalUrl: deployment.canonicalUrl, status: "deployed", metadata: { deploymentId: "123", region: "iad1" }, checkedAt: new Date(), expiresAt: new Date() }); + await store.write(scope, target, { result: { preview, maxAgeMs: 60_000 }, expiresAt: 65_000 }, 5_000); + await store.write(scope, target, { result: { preview: { ...preview, state: "open" }, maxAgeMs: 60_000 }, expiresAt: 61_000 }, 1_000); + expect((await store.read(scope, [target])).get(url)?.result.preview?.state).toBe("merged"); + const migration = await readFile(new URL("../../../packages/db/src/migrations/0489_real_sage.sql", import.meta.url), "utf8"); + for (const statement of migration.split("--> statement-breakpoint")) if (statement.trim()) await context.db.execute(sql.raw(statement)); + expect((await store.read(scope, [target])).get(url)?.result.preview?.state).toBe("merged"); + }); +}); diff --git a/server/src/__tests__/link-previews.test.ts b/server/src/__tests__/link-previews.test.ts index 5ef63cd6aef..6e4bfccfcc2 100644 --- a/server/src/__tests__/link-previews.test.ts +++ b/server/src/__tests__/link-previews.test.ts @@ -2,6 +2,8 @@ import { createServer } from "node:http"; import { once } from "node:events"; import { afterEach, describe, expect, it, vi } from "vitest"; import { createLinkPreviewService, type LinkPreviewProvider } from "../services/link-previews.js"; +import type { StoredLinkPreview } from "../services/link-preview-store.js"; +import { recognizeLinkPreview } from "@paperclipai/shared"; const scope = { companyId: "company-a", userId: "alice" }; const url = "https://github.com/acme/web/pull/42"; @@ -10,6 +12,51 @@ const provider = (resolve = vi.fn(async () => unavailable)): LinkPreviewProvider afterEach(() => vi.useRealTimers()); describe("demand-driven link preview cache", () => { + it.each(["failure", "success", "timeout", "write"])("keeps concurrent hydration newer than a pending provider %s", async (settlement) => { + vi.useFakeTimers(); + let now = 1_000; + const target = recognizeLinkPreview(url)!; + const oldPreview = { target, title: "Old status", state: "open" as const, author: null, updatedAt: null }; + const newPreview = { ...oldPreview, title: "New status", state: "merged" as const }; + let stored: StoredLinkPreview = { result: { preview: oldPreview, maxAgeMs: 1_000 }, expiresAt: 2_000, checkedAt: now }; + let finish!: (value: typeof unavailable | { preview: typeof oldPreview; maxAgeMs: number }) => void; + let finishWrite!: () => void; + const write = vi.fn(async () => { + if (settlement === "write") await new Promise(done => { finishWrite = done; }); + }); + const resolve = vi.fn(() => new Promise(done => { finish = done; })); + const service = createLinkPreviewService([{ key: "github", resolve }], { + now: () => now, timeoutMs: 20, + store: { read: async () => new Map([[url, stored]]), write }, + }); + await service.cached(scope, [url]); + now = 3_000; + const pending = service.get(scope, url); + await vi.advanceTimersByTimeAsync(0); + if (settlement === "write") { + finish({ preview: oldPreview, maxAgeMs: 60_000 }); + await vi.advanceTimersByTimeAsync(0); + expect(write).toHaveBeenCalledTimes(1); + } + now = 4_000; + stored = { result: { preview: newPreview, maxAgeMs: 30_000 }, expiresAt: 34_000, checkedAt: now }; + expect((await service.cached(scope, [url])).results[url].preview).toEqual(newPreview); + if (settlement === "write") finishWrite(); + else if (settlement === "timeout") await vi.advanceTimersByTimeAsync(21); + else finish(settlement === "failure" ? unavailable : { preview: oldPreview, maxAgeMs: 60_000 }); + expect(await pending).toEqual({ preview: newPreview, maxAgeMs: 30_000 }); + expect(await service.get(scope, url)).toEqual({ preview: newPreview, maxAgeMs: 30_000 }); + expect(resolve).toHaveBeenCalledTimes(1); + expect(write).toHaveBeenCalledTimes(settlement === "write" ? 1 : 0); + if (settlement === "timeout") { finish(unavailable); await vi.advanceTimersByTimeAsync(0); } + }); + it("omits absent records on repeated batch reads", async () => { + const service = createLinkPreviewService([provider()], { + store: { read: async () => new Map(), write: async () => {} }, + }); + expect(await service.cached(scope, [url])).toEqual({ results: {} }); + expect(await service.cached(scope, [url])).toEqual({ results: {} }); + }); it("does no work until requested and coalesces equivalent URLs", async () => { const resolve = vi.fn(async () => unavailable); const service = createLinkPreviewService([provider(resolve)]); @@ -86,6 +133,36 @@ describe("demand-driven link preview cache", () => { expect(resolve).toHaveBeenCalledTimes(2); finish(unavailable); }); + it("bounds a stalled database write and holds its concurrency slot until settlement", async () => { + vi.useFakeTimers(); + let finish!: () => void; + const write = vi.fn(async () => {}).mockImplementationOnce(() => new Promise(done => { finish = done; })); + const resolve = vi.fn(async (target: import("@paperclipai/shared").LinkPreviewTarget) => ({ preview: { target, title: "Known", state: "open" as const, author: null, updatedAt: null }, maxAgeMs: 60_000 })); + const service = createLinkPreviewService([{ key: "github", resolve }], { + store: { read: async () => new Map(), write }, timeoutMs: 20, maxConcurrent: 1, + }); + const first = service.get(scope, url); + await vi.advanceTimersByTimeAsync(21); + expect((await first).preview?.state).toBe("open"); + expect(await service.get(scope, url + "1")).toEqual(unavailable); + expect(resolve).toHaveBeenCalledTimes(1); + finish(); await vi.advanceTimersByTimeAsync(1); + await service.get(scope, url + "1"); + expect(resolve).toHaveBeenCalledTimes(2); + }); + it("keeps timeout persistence inside the concurrency budget after the provider aborts", async () => { + vi.useFakeTimers(); + let finishWrite!: () => void; + const write = vi.fn(async () => {}).mockImplementationOnce(() => new Promise(done => { finishWrite = done; })); + const resolve = vi.fn(async (_target: import("@paperclipai/shared").LinkPreviewTarget, _scope: import("../services/link-previews.js").LinkPreviewScope, signal: AbortSignal) => unavailable) + .mockImplementationOnce((_target, _scope, signal) => new Promise((_done, reject) => signal.addEventListener("abort", () => reject(new Error("timeout"))))); + const service = createLinkPreviewService([{ key: "github", resolve }], { store: { read: async () => new Map(), write }, timeoutMs: 20, maxConcurrent: 1 }); + const first = service.get(scope, url); + await vi.advanceTimersByTimeAsync(21); expect(await first).toEqual(unavailable); + await service.get(scope, url + "1"); expect(resolve).toHaveBeenCalledTimes(1); + finishWrite(); await vi.advanceTimersByTimeAsync(1); + await service.get(scope, url + "1"); expect(resolve).toHaveBeenCalledTimes(2); + }); it("aborts a real stalled HTTP read while an unrelated lookup remains responsive", async () => { const server = createServer((_req, res) => { res.writeHead(200, { "content-type": "application/json" }); res.write("{"); }); server.listen(0, "127.0.0.1"); diff --git a/server/src/__tests__/openapi-routes.test.ts b/server/src/__tests__/openapi-routes.test.ts index 399b53fc554..3a8e0157049 100644 --- a/server/src/__tests__/openapi-routes.test.ts +++ b/server/src/__tests__/openapi-routes.test.ts @@ -4,7 +4,7 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; import request from "supertest"; import { describe, expect, it } from "vitest"; -import { MUSE_ASSET_VERSION, MUSE_PUBLIC_ROUTES } from "@paperclipai/shared"; +import { LINK_PREVIEW_BATCH_LIMIT, MUSE_ASSET_VERSION, MUSE_PUBLIC_ROUTES } from "@paperclipai/shared"; import { COMPANY_IMPORT_TRANSFERS_ROUTE_PATH } from "@paperclipai/shared/company-import-transfer"; import { errorHandler } from "../middleware/index.js"; import { buildOpenApiSpec, openApiRoutes, applyPluginSettingsVisibility, buildOpenApiDocument } from "../routes/openapi.js"; @@ -338,6 +338,15 @@ function loadSpecRoutes() { } describe("openapi routes", () => { + it("describes cached link results with the lookup shape and shared batch bound", () => { + const paths = buildOpenApiSpec().paths; + const batch = paths["/api/companies/{companyId}/link-previews/cached"].get; + expect(batch.description).toContain(`at most ${LINK_PREVIEW_BATCH_LIMIT} URLs`); + const lookupResult = paths["/api/companies/{companyId}/link-preview"].get.responses[200].content["application/json"].schema; + expect(batch.responses[200].content["application/json"].schema.properties.results.additionalProperties).toEqual(lookupResult); + expect(lookupResult.properties).toHaveProperty("preview"); + expect(lookupResult.properties).toHaveProperty("maxAgeMs"); + }); it("tells agents that unpriced usage in cost reports is unknown cost, not zero", () => { const paths = buildOpenApiSpec().paths; for (const segment of ["summary", "by-agent", "by-agent-model", "by-project"]) { diff --git a/server/src/routes/link-previews.ts b/server/src/routes/link-previews.ts index bd631855b2f..5236a55f516 100644 --- a/server/src/routes/link-previews.ts +++ b/server/src/routes/link-previews.ts @@ -1,11 +1,15 @@ import { Router } from "express"; import type { Db } from "@paperclipai/db"; -import { badRequest } from "../errors.js"; +import { LINK_PREVIEW_BATCH_LIMIT } from "@paperclipai/shared"; +import { createLinkPreviewStore } from "../services/link-preview-store.js"; +import { badRequest, tooManyRequests } from "../errors.js"; +import { createInviteRateLimiter, type InviteRateLimiter } from "../services/invite-rate-limit.js"; import { assertBoard, assertCompanyAccess } from "./authz.js"; import { createLinkPreviewService } from "../services/link-previews.js"; import { createGitHubLinkPreviewProvider } from "../services/github-link-preview.js"; -export function linkPreviewRoutes(db: Db, service = createLinkPreviewService([createGitHubLinkPreviewProvider(db)])) { +export function linkPreviewRoutes(db: Db, service = createLinkPreviewService([createGitHubLinkPreviewProvider(db)], { store: createLinkPreviewStore(db) }), + limiter: InviteRateLimiter = createInviteRateLimiter({ maxRequests: 600 })) { const router = Router(); router.get("/companies/:companyId/link-preview", async (req, res) => { const companyId = req.params.companyId as string; @@ -15,7 +19,25 @@ export function linkPreviewRoutes(db: Db, service = createLinkPreviewService([cr if (typeof href !== "string" || href.length > 2048) throw badRequest("A link URL is required"); // Personal repository metadata must never be stored by a proxy or shared browser cache. res.set("Cache-Control", "private, no-store"); + const limit = limiter.consume(JSON.stringify([companyId, req.actor.userId ?? null])); + res.set("X-RateLimit-Limit", String(limit.limit)); + res.set("X-RateLimit-Remaining", String(limit.remaining)); + if (!limit.allowed) { + res.set("Retry-After", String(limit.retryAfterSeconds)); + throw tooManyRequests("Link preview rate limit exceeded"); + } res.json(await service.get({ companyId, userId: req.actor.userId ?? null }, href)); }); + // Read-only batch: page hydration never waits for a remote provider. + router.get("/companies/:companyId/link-previews/cached", async (req, res) => { + const companyId = req.params.companyId as string; + assertCompanyAccess(req, companyId); + assertBoard(req); + const urls = typeof req.query.url === "string" ? [req.query.url] : req.query.url; + if (!Array.isArray(urls) || !urls.length || urls.length > LINK_PREVIEW_BATCH_LIMIT + || urls.some(url => typeof url !== "string" || url.length > 2048)) throw badRequest("A bounded list of link URLs is required"); + res.set("Cache-Control", "private, no-store"); + res.json(await service.cached({ companyId, userId: req.actor.userId ?? null }, urls as string[])); + }); return router; } diff --git a/server/src/routes/openapi.ts b/server/src/routes/openapi.ts index 8762d89f512..101c9bf9d38 100644 --- a/server/src/routes/openapi.ts +++ b/server/src/routes/openapi.ts @@ -2,7 +2,7 @@ import { catalogTeamInstallSchema, catalogTeamPreviewSchema, updateWorkspacePref import { MUSE_ASSET_VERSION, museStopBoundarySchema, museDetectorCleanupAttestationSchema, createGrokInvitationSchema, configureGrokRoutineSchema, verifyGrokRoutineSchema, recoverGrokRoutineSchema, pairGrokBindingSchema, stopGrokWorkSchema, retryGrokDeliverySchema, updateFastResponseSchema } from "@paperclipai/shared"; import { slackTestMessageSchema, slackTestMessageResultSchema, slackManagerAuthorizationSchema, slackSetupOptionsSchema, slackManagedProvisionSchema, slackManagedResultSchema, slackRegistrationSchema, slackSetupActionSchema, slackInstallAuthorizationSchema, slackRegistrationStateSchema, slackAppConfigurationSchema, slackAvatarStateSchema, slackAccountStateSchema } from "@paperclipai/shared"; import { experimentalApiMetadata } from "./experimental-api-metadata.js"; -import { ATTENTION_SOURCE_KINDS } from "@paperclipai/shared"; +import { LINK_PREVIEW_BATCH_LIMIT, ATTENTION_SOURCE_KINDS } from "@paperclipai/shared"; import { agentCaseHandoffSchema, caseWorkflowDocumentTextSchema, doneCaseHandoffSchema, createWorkflowSchema, updateWorkflowSchema, listWorkflowsSchema, readWorkflowQuerySchema } from "@paperclipai/shared"; import { experimentalApiDocs, @@ -10459,26 +10459,41 @@ registry.registerPath({ responses: { 200: r.ok(), 401: r.unauthorized, 404: r.notFound }, }); +const linkPreviewResultSchema = z.object({ + preview: z.object({ + target: z.object({ provider: z.string(), kind: z.string(), canonicalUrl: z.string(), label: z.string() }), + title: z.string(), state: z.enum(["unknown", "open", "draft", "merged", "closed", "conflicts"]), + author: z.string().nullable(), updatedAt: z.string().nullable(), + }).nullable(), + maxAgeMs: z.number(), +}); + registry.registerPath({ method: "get", path: "/api/companies/{companyId}/link-preview", tags: ["companies"], summary: "Read optional link metadata for the current board user", - description: "Company access and board authentication are required. This read uses a viewer-scoped cache and returns a quiet null preview when provider access is unavailable. It does not change tasks or schedule background work.", + description: "Company access and board authentication are required. This read uses a viewer-scoped cache and returns a quiet null preview when provider access is unavailable. Each company/viewer can make 600 lookup requests per minute per server process. Excess requests return 429 with Retry-After; cached batch reads remain available. It does not change tasks or schedule background work.", request: { params: z.object({ companyId: z.string() }), query: z.object({ url: z.string().max(2048) }) }, responses: { - 200: r.ok(z.object({ - preview: z.object({ - target: z.object({ provider: z.string(), kind: z.string(), canonicalUrl: z.string(), label: z.string() }), - title: z.string(), state: z.enum(["unknown", "open", "draft", "merged", "closed", "conflicts"]), - author: z.string().nullable(), updatedAt: z.string().nullable(), - }).nullable(), - maxAgeMs: z.number(), - })), - 400: r.badRequest, 401: r.unauthorized, 403: r.forbidden, + 200: r.ok(linkPreviewResultSchema), + 400: r.badRequest, 401: r.unauthorized, 403: r.forbidden, 429: r.tooManyRequests, }, }); +registry.registerPath({ + method: "get", + path: "/api/companies/{companyId}/link-previews/cached", + tags: ["companies"], + summary: "Read a batch of stored link statuses without contacting providers", + description: `Board-only and company/viewer scoped. Reads at most ${LINK_PREVIEW_BATCH_LIMIT} URLs. Canonical links are reused across tasks. Stale records have maxAgeMs zero; absent records are omitted. This is a read-only cache projection.`, + request: { + params: z.object({ companyId: z.string() }), + query: z.object({ url: z.union([z.string().max(2048), z.array(z.string().max(2048)).min(1).max(LINK_PREVIEW_BATCH_LIMIT)]) }), + }, + responses: { 200: r.ok(z.object({ results: z.record(z.string(), linkPreviewResultSchema) })), 400: r.badRequest, 401: r.unauthorized, 403: r.forbidden }, +}); + registry.registerPath({ method: "get", path: "/api/issues/{id}/external-objects", diff --git a/server/src/services/link-preview-store.ts b/server/src/services/link-preview-store.ts new file mode 100644 index 00000000000..3bbe508f7f0 --- /dev/null +++ b/server/src/services/link-preview-store.ts @@ -0,0 +1,65 @@ +import { and, desc, eq, inArray, lt, lte, or, sql } from "drizzle-orm"; +import { linkPreviews, type Db } from "@paperclipai/db"; +import type { LinkPreviewResult, LinkPreviewTarget } from "@paperclipai/shared"; +import type { LinkPreviewScope } from "./link-previews.js"; + +export interface StoredLinkPreview { result: LinkPreviewResult; expiresAt: number; checkedAt: number } +export interface LinkPreviewStore { + read(scope: LinkPreviewScope, targets: readonly LinkPreviewTarget[]): Promise>; + write(scope: LinkPreviewScope, target: LinkPreviewTarget, entry: Omit, checkedAt: number): Promise; +} + +const MAX_ROWS_PER_VIEWER = 1_000; +const EXPIRED_RETENTION_MS = 30 * 24 * 60 * 60_000; + +export function createLinkPreviewStore(db: Db): LinkPreviewStore { + return { + async read(scope, targets) { + if (!targets.length) return new Map(); + // One covering identity-index lookup for the page, never a task join or a provider call. + const rows = await db.select().from(linkPreviews).where(and( + eq(linkPreviews.companyId, scope.companyId), eq(linkPreviews.viewerId, scope.userId ?? ""), + inArray(linkPreviews.canonicalUrl, targets.map(target => target.canonicalUrl)), + )); + const byUrl = new Map(targets.map(target => [target.canonicalUrl, target])); + return new Map(rows.flatMap(row => { + const target = byUrl.get(row.canonicalUrl); + if (!target || target.provider !== row.channel || target.kind !== row.resourceKind) return []; + const data = row.metadata; + const preview = row.status !== null && typeof data.title === "string" ? { + target, title: data.title, state: row.status as NonNullable["state"], + author: typeof data.author === "string" ? data.author : null, + updatedAt: typeof data.updatedAt === "string" ? data.updatedAt : null, + } : null; + return [[row.canonicalUrl, { result: { preview, maxAgeMs: 0 }, expiresAt: row.expiresAt.getTime(), checkedAt: row.checkedAt.getTime() }] as const]; + })); + }, + async write(scope, target, entry, checkedAt) { + const preview = entry.result.preview; + const values = { + companyId: scope.companyId, viewerId: scope.userId ?? "", channel: target.provider, + resourceKind: target.kind, canonicalUrl: target.canonicalUrl, status: preview?.state ?? null, + metadata: preview ? { title: preview.title, author: preview.author, updatedAt: preview.updatedAt } : {}, + checkedAt: new Date(checkedAt), expiresAt: new Date(entry.expiresAt), + }; + await db.transaction(async tx => { + // Serialize only this cache scope, including across server replicas. + // The row cap must hold when several distinct URLs are saved together. + const lockKey = JSON.stringify(["link-previews", values.companyId, values.viewerId]); + await tx.execute(sql`SELECT pg_advisory_xact_lock(hashtextextended(${lockKey}, 0))`); + await tx.insert(linkPreviews).values(values).onConflictDoUpdate({ + target: [linkPreviews.companyId, linkPreviews.viewerId, linkPreviews.canonicalUrl], + set: values, setWhere: lte(linkPreviews.checkedAt, values.checkedAt), + }); + const scopeCondition = and(eq(linkPreviews.companyId, values.companyId), eq(linkPreviews.viewerId, values.viewerId)); + // Cleanup runs on writes, without a worker or extra page-read queries. + // The identity index covers this scope, whose retained size is bounded. + await tx.delete(linkPreviews).where(and(scopeCondition, or( + lt(linkPreviews.expiresAt, new Date(checkedAt - EXPIRED_RETENTION_MS)), + inArray(linkPreviews.id, tx.select({ id: linkPreviews.id }).from(linkPreviews).where(scopeCondition) + .orderBy(desc(linkPreviews.checkedAt), desc(linkPreviews.id)).offset(MAX_ROWS_PER_VIEWER)), + ))); + }); + }, + }; +} diff --git a/server/src/services/link-previews.ts b/server/src/services/link-previews.ts index 29041b51ce9..ae8bf43300d 100644 --- a/server/src/services/link-previews.ts +++ b/server/src/services/link-previews.ts @@ -1,4 +1,5 @@ import type { LinkPreviewResult, LinkPreviewTarget } from "@paperclipai/shared"; +import type { LinkPreviewStore, StoredLinkPreview } from "./link-preview-store.js"; import { recognizeLinkPreview } from "@paperclipai/shared"; export interface LinkPreviewScope { companyId: string; userId: string | null } @@ -9,22 +10,49 @@ export interface LinkPreviewProvider { const UNAVAILABLE: LinkPreviewResult = { preview: null, maxAgeMs: 60_000 }; -/** Request-driven and process-local. No DB writes, sweep, timers while idle, or cross-viewer reuse. */ +/** Request-driven cache: durable across restarts, isolated by viewer, with no idle worker. */ export function createLinkPreviewService(providers: readonly LinkPreviewProvider[], options: { - now?: () => number; timeoutMs?: number; maxEntries?: number; maxConcurrent?: number; + store?: LinkPreviewStore; now?: () => number; timeoutMs?: number; maxEntries?: number; maxConcurrent?: number; } = {}) { const now = options.now ?? Date.now; - const cache = new Map(); + const cache = new Map(); const inFlight = new Map>(); let activeOperations = 0; const byKey = new Map(providers.map(provider => [provider.key, provider])); + function remember(key: string, entry: StoredLinkPreview) { + if ((cache.get(key)?.checkedAt ?? -Infinity) > entry.checkedAt) return; + cache.delete(key); + while (cache.size >= (options.maxEntries ?? 1_000)) cache.delete(cache.keys().next().value!); + cache.set(key, entry); + } + const keyOf = (scope: LinkPreviewScope, url: string) => JSON.stringify([scope.companyId, scope.userId, url]); + const remaining = (entry: StoredLinkPreview) => ({ ...entry.result, maxAgeMs: Math.max(0, entry.expiresAt - now()) }); + const newest = (memory: StoredLinkPreview | undefined, stored: StoredLinkPreview | undefined) => + stored && (!memory || stored.checkedAt >= memory.checkedAt) ? stored : memory; + return { + async cached(scope: LinkPreviewScope, hrefs: readonly string[]) { + const targets = [...new Map(hrefs.map(recognizeLinkPreview).filter(target => target && byKey.has(target.provider)) + .map(target => [target!.canonicalUrl, target!])).values()]; + const stored = await options.store?.read(scope, targets).catch(() => new Map()); + const results: Record = {}; + for (const target of targets) { + const key = keyOf(scope, target.canonicalUrl); + const memory = cache.get(key); + const entry = newest(memory, stored?.get(target.canonicalUrl)); + if (entry) { remember(key, entry); results[target.canonicalUrl] = remaining(entry); } + // A batch miss is not a provider result. A later lookup must still + // check durable storage, which another replica may have populated. + else cache.delete(key); + } + return { results }; + }, async get(scope: LinkPreviewScope, href: string): Promise { const target = recognizeLinkPreview(href); const provider = target && byKey.get(target.provider); if (!target || !provider) return UNAVAILABLE; - const key = JSON.stringify([scope.companyId, scope.userId, target.canonicalUrl]); + const key = keyOf(scope, target.canonicalUrl); const cached = cache.get(key); if (cached && cached.expiresAt > now()) { return { ...cached.result, maxAgeMs: cached.expiresAt - now() }; @@ -35,20 +63,74 @@ export function createLinkPreviewService(providers: readonly LinkPreviewProvider const controller = new AbortController(); let timer: ReturnType; + let previous = cached; + const checkedAt = now(); + let providerStarted = false; + let storeWriteStarted = false; + let durableReadSucceeded = !options.store; + let timeoutWrite: Promise | undefined; + const fallback = (): LinkPreviewResult => ({ preview: newest(previous, cache.get(key))?.result.preview ?? null, maxAgeMs: 60_000 }); const deadline = new Promise(resolve => { - timer = setTimeout(() => { controller.abort(); resolve(UNAVAILABLE); }, options.timeoutMs ?? 5_000); + timer = setTimeout(() => { + const current = cache.get(key); + if (current && current.checkedAt > checkedAt) { + controller.abort(); + resolve(remaining(current)); + return; + } + const result = current && current.expiresAt > now() ? remaining(current) : fallback(); + const entry = { result, expiresAt: now() + result.maxAgeMs, + checkedAt: durableReadSucceeded || storeWriteStarted ? checkedAt : previous?.checkedAt ?? 0 }; + remember(key, entry); + // Publish timeout freshness without delaying the caller. The lookup + // keeps its slot until both the provider and this write settle. + if (options.store && durableReadSucceeded && providerStarted && !storeWriteStarted) { + timeoutWrite = Promise.resolve().then(() => options.store!.write(scope, target, entry, checkedAt)).catch(() => {}); + } + controller.abort(); + resolve(result); + }, options.timeoutMs ?? 5_000); }); - // Keep an underlying operation in the concurrency budget until it settles, - // even if a credential backend ignores cancellation after our deadline. + // All underlying work, including uncancellable SQL, retains its slot until + // settlement. The caller's deadline does not release the resource budget. activeOperations++; - const operation = Promise.resolve().then(() => provider.resolve(target, scope, controller.signal)) - .catch(() => UNAVAILABLE).finally(() => { activeOperations--; }); - const request = Promise.race([operation, deadline]).then(result => { + const operation = Promise.resolve().then(async () => { + if (options.store) { + // Fresh memory returned above. Expired memory is only a fallback: + // another replica may have saved a newer result, even with a shorter TTL. + const stored = await options.store.read(scope, [target]).catch(() => null); + durableReadSucceeded = stored !== null; + previous = newest(newest(previous, cache.get(key)), stored?.get(target.canonicalUrl)); + if (!controller.signal.aborted && previous) remember(key, previous); + } + if (controller.signal.aborted) return fallback(); + if (previous && previous.expiresAt > now()) { remember(key, previous); return remaining(previous); } + providerStarted = true; + const result = await provider.resolve(target, scope, controller.signal).catch(() => UNAVAILABLE); + if (controller.signal.aborted) return fallback(); + // A page batch can hydrate another replica's newer status while the + // provider is pending. That result wins over this older lookup. + previous = newest(previous, cache.get(key)); + if (previous && previous.checkedAt > checkedAt) return remaining(previous); const bounded = { ...result, maxAgeMs: Math.max(1_000, Math.min(result.maxAgeMs, 60 * 60_000)) }; - cache.delete(key); - while (cache.size >= (options.maxEntries ?? 1_000)) cache.delete(cache.keys().next().value!); - cache.set(key, { result: bounded, expiresAt: now() + bounded.maxAgeMs }); - return bounded; + if (!bounded.preview && previous?.result.preview) bounded.preview = previous.result.preview; + const entry = { result: bounded, expiresAt: now() + bounded.maxAgeMs, + checkedAt: durableReadSucceeded || result.preview ? checkedAt : previous?.checkedAt ?? 0 }; + remember(key, entry); + // A failed durable read cannot establish which saved status a provider + // failure would replace. Keep that backoff local until storage recovers. + if (options.store && (durableReadSucceeded || result.preview)) { + storeWriteStarted = true; + await options.store.write(scope, target, entry, checkedAt).catch(() => {}); + } + return remaining(entry); + }).catch(fallback).finally(async () => { await timeoutWrite; activeOperations--; }); + const request = Promise.race([operation, deadline]).then(result => { + const current = cache.get(key); + // Hydration can also finish while the operation awaits its SQL write. + if (current && current.checkedAt > checkedAt) return remaining(current); + if (!current || current.expiresAt <= now()) remember(key, { result, expiresAt: now() + result.maxAgeMs, checkedAt }); + return result; }).finally(() => { clearTimeout(timer!); if (inFlight.get(key) === request) inFlight.delete(key); diff --git a/tests/runner-e2e/source-sdk-public-inputs.json b/tests/runner-e2e/source-sdk-public-inputs.json index e738504dd06..6d024d3da2b 100644 --- a/tests/runner-e2e/source-sdk-public-inputs.json +++ b/tests/runner-e2e/source-sdk-public-inputs.json @@ -268,7 +268,7 @@ "packages/shared/src/home-paths.ts": "fa4f1c440653b76b3dbb7a51026d557287deb45d", "packages/shared/src/humanize-connection.test.ts": "ba9e90a44c63452b0bf9f733adb90241921e8eca", "packages/shared/src/humanize-connection.ts": "fbd414476a0ef7106a7c80d8572208781b091699", - "packages/shared/src/index.ts": "efab7ad18d89a60ddd3f961e35060b68424ef40b", + "packages/shared/src/index.ts": "825867d65bbb0783a2294e1838231e68669a8fc3", "packages/shared/src/issue-activity-identity.test.ts": "3dc294640f54c2be67943e3e0149cf4eff2c95ca", "packages/shared/src/issue-activity-identity.ts": "1b0f07ec3fe0506aaaf80a4fa9893f3baf8f0075", "packages/shared/src/issue-attribution.test.ts": "84563685e7f3c48029243cf1cc24a3a09ee3cde9", @@ -282,7 +282,7 @@ "packages/shared/src/issue-write-denial.test.ts": "e21b93aa79e6635c019ac338eff954ec618a98ed", "packages/shared/src/issue-write-denial.ts": "e6b4d72a83f6e4e76e85e34a80f88d1d69b7dfe4", "packages/shared/src/link-preview.test.ts": "136041c928343ad6272f43701b39cf95be255f3e", - "packages/shared/src/link-preview.ts": "ffa5eb960a7e4bcfa914b4d6029c9fc50f17e027", + "packages/shared/src/link-preview.ts": "b7431d4c30a12067e80e6806728b3ae7c7541f8f", "packages/shared/src/markdown-work-products.test.ts": "7293d14ad45ec2b11b5deaab4ce070c36f7c9010", "packages/shared/src/markdown-work-products.ts": "16a88cfb717f325c247a516e78b47d53871b52ed", "packages/shared/src/mcp-apps.test.ts": "848f3247bdc98af398daa144b0e44991a3f352b3", diff --git a/tests/storybook-visual/link-previews.spec.ts b/tests/storybook-visual/link-previews.spec.ts index 7d21fb31252..4e70349f802 100644 --- a/tests/storybook-visual/link-previews.spec.ts +++ b/tests/storybook-visual/link-previews.spec.ts @@ -167,3 +167,54 @@ for (const available of [true, false]) { await expect(links.nth(1)).toHaveAttribute("data-link-state", available ? "open" : "unknown"); }); } + +test("page load restores stored status and refreshes stale data on idle without hover", async ({ page }, testInfo) => { + const errors: string[] = []; + page.on("pageerror", error => errors.push(error.message)); + await page.addInitScript(() => { + const realFetch = window.fetch.bind(window); + let activeFetch = realFetch; + Object.defineProperty(window, "fetch", { + configurable: true, get: () => activeFetch, + set: (fixtureFetch: typeof fetch) => { + activeFetch = (input, init) => { + const url = input instanceof Request ? input.url : String(input); + return url.includes("/link-preview?") || url.includes("/link-previews/cached") ? realFetch(input, init) : fixtureFetch(input, init); + }; + }, + }); + }); + const canonicalUrl = "https://github.com/acme/web/pull/241"; + const preview = { target: { provider: "github", kind: "pull_request", canonicalUrl, label: "acme/web#241" }, + title: "Status from the database", state: "merged", author: "octocat", updatedAt: null }; + let cached = 0, resolved = 0; + let release!: () => void; + const refresh = new Promise(done => { release = done; }); + await page.route("**/api/companies/company-storybook/link-previews/cached?*", async route => { + cached++; + expect(new URL(route.request().url()).searchParams.getAll("url")).toEqual([canonicalUrl]); + await route.fulfill({ json: { results: { [canonicalUrl]: { preview, maxAgeMs: 0 } } } }); + }); + await page.route("**/api/companies/company-storybook/link-preview?*", async route => { + resolved++; + await refresh; + await route.fulfill({ json: { preview: { ...preview, state: "closed" }, maxAgeMs: 60_000 } }); + }); + await page.goto("/iframe.html?id=tasks-link-previews--page-load&viewMode=story&globals=theme:dark"); + await finished(page); + const links = page.locator("a[data-link-state]"); + await expect(links).toHaveCount(2); + await expect(links.first()).toHaveAttribute("data-link-state", "merged"); + await expect(links.last()).toHaveAttribute("data-link-state", "merged"); + await expect(page.getByRole("tooltip")).toHaveCount(0); + await expect.poll(() => resolved).toBe(1); + expect(cached).toBe(1); + release(); + await expect(links.first()).toHaveAttribute("data-link-state", "closed"); + await expect(links.last()).toHaveAttribute("data-link-state", "closed"); + await links.first().focus(); + await expect(page.getByRole("tooltip")).toBeVisible(); + expect(resolved).toBe(1); + await page.screenshot({ path: testInfo.outputPath("pr-status-from-database-after-idle.png"), fullPage: true }); + expect(errors).toEqual([]); +}); diff --git a/ui/src/components/LinkPreview.tsx b/ui/src/components/LinkPreview.tsx index e2c12e40c5f..d505edc10b7 100644 --- a/ui/src/components/LinkPreview.tsx +++ b/ui/src/components/LinkPreview.tsx @@ -1,12 +1,11 @@ -import { useId, useMemo, useRef, useState, type ReactElement } from "react"; +import { useEffect, useId, useMemo, useRef, useState, type ReactElement } from "react"; import { useQuery } from "@tanstack/react-query"; import { PreviewCard } from "@base-ui/react/preview-card"; import { GitMerge, GitPullRequest, GitPullRequestClosed, GitPullRequestDraft, TriangleAlert } from "lucide-react"; import type { LinkPreview as Preview, LinkPreviewResult, LinkPreviewState, LinkPreviewTarget } from "@paperclipai/shared"; import { useLinkPreviewContext, type LinkPreviewContextValue } from "@/context/LinkPreviewContext"; import { timeAgo } from "@/lib/timeAgo"; -import { retryTransientOnly } from "@/lib/query-client"; -import { queryKeys } from "@/lib/queryKeys"; +import { linkPreviewQueryOptions } from "@/lib/link-preview-query"; export const linkPreviewStateLabel: Record = { unknown: "Status unknown", open: "Open", merged: "Merged", conflicts: "Merge conflicts", draft: "Draft", closed: "Closed", @@ -18,13 +17,13 @@ export const linkPreviewStateIcon = { function PreviewCardContents({ target, preview, loading }: { target: LinkPreviewTarget; preview?: Preview | null; loading: boolean }) { const state = preview?.state ?? "unknown"; - const Icon = linkPreviewStateIcon[state]; + const Icon = linkPreviewStateIcon[state] ?? linkPreviewStateIcon.unknown; const updated = preview?.updatedAt && Number.isFinite(Date.parse(preview.updatedAt)) ? preview.updatedAt : null; return
{preview ? <>

{preview.title}

@@ -50,17 +49,8 @@ function ConnectedLinkPreview({ context, target, children }: { getBoundingClientRect: () => triggerRef.current?.getClientRects()[0] ?? new DOMRect(), get contextElement() { return triggerRef.current ?? undefined; }, }), []); - const query = useQuery({ - queryKey: queryKeys.linkPreview(context.companyId, context.userId, target.canonicalUrl), - // Optional enrichment: a failed read is a cached absence, not a global error. - queryFn: async (): Promise => { - try { return await context.load(target); } catch { return { preview: null, maxAgeMs: 60_000 }; } - }, - enabled: open, - staleTime: query => query.state.data?.maxAgeMs ?? 60_000, - gcTime: 5 * 60_000, - retry: retryTransientOnly(0), refetchOnMount: false, refetchOnWindowFocus: false, refetchOnReconnect: false, - }); + useEffect(() => context.register?.(target), [context, target.canonicalUrl]); + const query = useQuery({ ...linkPreviewQueryOptions(context, target), enabled: open }); return setOpen(true)} onBlur={() => setOpen(false)} diff --git a/ui/src/components/MarkdownUrlLink.tsx b/ui/src/components/MarkdownUrlLink.tsx index 43316bb1d80..e8787de22bf 100644 --- a/ui/src/components/MarkdownUrlLink.tsx +++ b/ui/src/components/MarkdownUrlLink.tsx @@ -57,7 +57,7 @@ export function MarkdownUrlLink({ href, children, title, style, knownState, exte const label = github?.label ? compactUrlLabel(children, href, github.label) : children; const target = recognizeLinkPreview(href); const render = (state: LinkPreviewState = knownState ?? "unknown", describedBy?: string, preview?: Preview | null) => { - const StateIcon = target ? linkPreviewStateIcon[state] : Icon; + const StateIcon = target ? linkPreviewStateIcon[state] ?? linkPreviewStateIcon.unknown : Icon; const leadingIcon = github ?