diff --git a/doc/plans/2026-04-24-external-object-reference-backfill.md b/doc/plans/2026-04-24-external-object-reference-backfill.md new file mode 100644 index 00000000000..02b8e86730c --- /dev/null +++ b/doc/plans/2026-04-24-external-object-reference-backfill.md @@ -0,0 +1,28 @@ +# External Object Reference Backfill + +## Purpose + +Backfill existing issue titles, descriptions, comments, and documents into `external_object_mentions` after the Phase 3 detector registry and mention sync service exist. + +## Command Shape + +Add a command parallel to the internal issue-reference backfill: + +```sh +pnpm external-objects:backfill +pnpm external-objects:backfill -- --company +pnpm external-objects:backfill -- --dry-run +``` + +## Required Behavior + +- Scan issue title and description sources, issue comments, and issue documents through the same external-object mention sync service used by write hooks. +- Preserve company boundaries: every scan query, mention upsert, object upsert, and dry-run summary must include `company_id`. +- Use shared URL extraction and canonicalization helpers so raw URLs, URL userinfo, query strings, and fragments are never persisted. +- Replace mentions per source instead of appending, using the same source key shape as live write sync. +- Report counts by company, source kind, provider key, object type, skipped userinfo URLs, and unresolved URLs. +- Default to a non-interactive safe run; `--dry-run` must avoid writes while still reporting what would change. + +## Phase 3 Hook + +The command should be implemented only after the detector registry can classify URLs and the service can upsert mentions/placeholders consistently. Until then, Phase 2 owns the schema and shared helpers that make the command safe. diff --git a/packages/db/src/external-objects-schema.test.ts b/packages/db/src/external-objects-schema.test.ts new file mode 100644 index 00000000000..897a98d7269 --- /dev/null +++ b/packages/db/src/external-objects-schema.test.ts @@ -0,0 +1,47 @@ +import { getTableConfig } from "drizzle-orm/pg-core"; +import { describe, expect, it } from "vitest"; +import { externalObjectMentions } from "./schema/external_object_mentions.js"; +import { externalObjects } from "./schema/external_objects.js"; + +function indexColumns(table: Parameters[0], indexName: string): string[] { + const index = getTableConfig(table).indexes.find((candidate) => candidate.config.name === indexName); + if (!index) return []; + return index.config.columns.map((column) => (column as { name: string }).name); +} + +describe("external object reference schema", () => { + it("scopes external object uniqueness by company", () => { + expect(indexColumns(externalObjects, "external_objects_company_external_id_uq")).toEqual([ + "company_id", + "provider_key", + "object_type", + "external_id", + ]); + expect(indexColumns(externalObjects, "external_objects_company_identity_uq")).toEqual([ + "company_id", + "provider_key", + "object_type", + "canonical_identity_hash", + ]); + }); + + it("indexes status, refresh scheduling, source issue, and object lookups by company", () => { + expect(indexColumns(externalObjects, "external_objects_company_provider_status_idx")).toEqual([ + "company_id", + "provider_key", + "status_category", + ]); + expect(indexColumns(externalObjects, "external_objects_company_refresh_idx")).toEqual([ + "company_id", + "next_refresh_at", + ]); + expect(indexColumns(externalObjectMentions, "external_object_mentions_company_source_issue_idx")).toEqual([ + "company_id", + "source_issue_id", + ]); + expect(indexColumns(externalObjectMentions, "external_object_mentions_company_object_idx")).toEqual([ + "company_id", + "object_id", + ]); + }); +}); diff --git a/packages/db/src/migrations/0106_external_object_references.sql b/packages/db/src/migrations/0106_external_object_references.sql new file mode 100644 index 00000000000..070a36b4529 --- /dev/null +++ b/packages/db/src/migrations/0106_external_object_references.sql @@ -0,0 +1,106 @@ +CREATE TABLE IF NOT EXISTS "external_objects" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "company_id" uuid NOT NULL, + "provider_key" text NOT NULL, + "plugin_id" uuid, + "object_type" text NOT NULL, + "external_id" text NOT NULL, + "sanitized_canonical_url" text, + "canonical_identity_hash" text, + "display_title" text, + "status_key" text, + "status_label" text, + "status_category" text DEFAULT 'unknown' NOT NULL, + "status_tone" text DEFAULT 'neutral' NOT NULL, + "liveness" text DEFAULT 'unknown' NOT NULL, + "is_terminal" boolean DEFAULT false NOT NULL, + "data" jsonb DEFAULT '{}'::jsonb NOT NULL, + "remote_version" text, + "etag" text, + "last_resolved_at" timestamp with time zone, + "last_changed_at" timestamp with time zone, + "last_error_at" timestamp with time zone, + "next_refresh_at" timestamp with time zone, + "last_error_code" text, + "last_error_message" text, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + "updated_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +CREATE TABLE IF NOT EXISTS "external_object_mentions" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "company_id" uuid NOT NULL, + "source_issue_id" uuid NOT NULL, + "source_kind" text NOT NULL, + "source_record_id" uuid, + "document_key" text, + "property_key" text, + "matched_text_redacted" text, + "sanitized_display_url" text, + "canonical_identity_hash" text, + "canonical_identity" jsonb, + "object_id" uuid, + "provider_key" text, + "detector_key" text, + "object_type" text, + "confidence" text DEFAULT 'exact' NOT NULL, + "created_by_plugin_id" uuid, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + "updated_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +DO $$ BEGIN + ALTER TABLE "external_objects" ADD CONSTRAINT "external_objects_company_id_companies_id_fk" FOREIGN KEY ("company_id") REFERENCES "public"."companies"("id") ON DELETE cascade ON UPDATE no action; +EXCEPTION + WHEN duplicate_object THEN null; +END $$; +--> statement-breakpoint +DO $$ BEGIN + ALTER TABLE "external_objects" ADD CONSTRAINT "external_objects_plugin_id_plugins_id_fk" FOREIGN KEY ("plugin_id") REFERENCES "public"."plugins"("id") ON DELETE set null ON UPDATE no action; +EXCEPTION + WHEN duplicate_object THEN null; +END $$; +--> statement-breakpoint +DO $$ BEGIN + ALTER TABLE "external_object_mentions" ADD CONSTRAINT "external_object_mentions_company_id_companies_id_fk" FOREIGN KEY ("company_id") REFERENCES "public"."companies"("id") ON DELETE cascade ON UPDATE no action; +EXCEPTION + WHEN duplicate_object THEN null; +END $$; +--> statement-breakpoint +DO $$ BEGIN + ALTER TABLE "external_object_mentions" ADD CONSTRAINT "external_object_mentions_source_issue_id_issues_id_fk" FOREIGN KEY ("source_issue_id") REFERENCES "public"."issues"("id") ON DELETE cascade ON UPDATE no action; +EXCEPTION + WHEN duplicate_object THEN null; +END $$; +--> statement-breakpoint +DO $$ BEGIN + ALTER TABLE "external_object_mentions" ADD CONSTRAINT "external_object_mentions_object_id_external_objects_id_fk" FOREIGN KEY ("object_id") REFERENCES "public"."external_objects"("id") ON DELETE set null ON UPDATE no action; +EXCEPTION + WHEN duplicate_object THEN null; +END $$; +--> statement-breakpoint +DO $$ BEGIN + ALTER TABLE "external_object_mentions" ADD CONSTRAINT "external_object_mentions_created_by_plugin_id_plugins_id_fk" FOREIGN KEY ("created_by_plugin_id") REFERENCES "public"."plugins"("id") ON DELETE set null ON UPDATE no action; +EXCEPTION + WHEN duplicate_object THEN null; +END $$; +--> statement-breakpoint +CREATE INDEX IF NOT EXISTS "external_objects_company_provider_object_idx" ON "external_objects" USING btree ("company_id","provider_key","object_type"); +--> statement-breakpoint +CREATE INDEX IF NOT EXISTS "external_objects_company_provider_status_idx" ON "external_objects" USING btree ("company_id","provider_key","status_category"); +--> statement-breakpoint +CREATE INDEX IF NOT EXISTS "external_objects_company_refresh_idx" ON "external_objects" USING btree ("company_id","next_refresh_at"); +--> statement-breakpoint +CREATE UNIQUE INDEX IF NOT EXISTS "external_objects_company_external_id_uq" ON "external_objects" USING btree ("company_id","provider_key","object_type","external_id"); +--> statement-breakpoint +CREATE UNIQUE INDEX IF NOT EXISTS "external_objects_company_identity_uq" ON "external_objects" USING btree ("company_id","provider_key","object_type","canonical_identity_hash"); +--> statement-breakpoint +CREATE INDEX IF NOT EXISTS "external_object_mentions_company_source_issue_idx" ON "external_object_mentions" USING btree ("company_id","source_issue_id"); +--> statement-breakpoint +CREATE INDEX IF NOT EXISTS "external_object_mentions_company_object_idx" ON "external_object_mentions" USING btree ("company_id","object_id"); +--> statement-breakpoint +CREATE INDEX IF NOT EXISTS "external_object_mentions_company_provider_idx" ON "external_object_mentions" USING btree ("company_id","provider_key","object_type"); +--> statement-breakpoint +CREATE UNIQUE INDEX IF NOT EXISTS "external_object_mentions_company_source_record_uq" ON "external_object_mentions" USING btree ("company_id","source_issue_id","source_kind","source_record_id","document_key","property_key","canonical_identity_hash") WHERE "source_record_id" is not null and "canonical_identity_hash" is not null; +--> statement-breakpoint +CREATE UNIQUE INDEX IF NOT EXISTS "external_object_mentions_company_source_null_record_uq" ON "external_object_mentions" USING btree ("company_id","source_issue_id","source_kind","document_key","property_key","canonical_identity_hash") WHERE "source_record_id" is null and "canonical_identity_hash" is not null; diff --git a/packages/db/src/migrations/meta/_journal.json b/packages/db/src/migrations/meta/_journal.json index ae94c3645b3..4e818013a0c 100644 --- a/packages/db/src/migrations/meta/_journal.json +++ b/packages/db/src/migrations/meta/_journal.json @@ -743,6 +743,13 @@ "when": 1781902000000, "tag": "0105_instance_scoped_environments", "breakpoints": true + }, + { + "idx": 106, + "version": "7", + "when": 1781902100000, + "tag": "0106_external_object_references", + "breakpoints": true } ] } diff --git a/packages/db/src/schema/external_object_mentions.ts b/packages/db/src/schema/external_object_mentions.ts new file mode 100644 index 00000000000..95cb8e9c80a --- /dev/null +++ b/packages/db/src/schema/external_object_mentions.ts @@ -0,0 +1,61 @@ +import { sql } from "drizzle-orm"; +import { index, jsonb, pgTable, text, timestamp, uniqueIndex, uuid } from "drizzle-orm/pg-core"; +import type { ExternalObjectMentionConfidence, ExternalObjectMentionSourceKind } from "@paperclipai/shared"; +import { companies } from "./companies.js"; +import { externalObjects } from "./external_objects.js"; +import { issues } from "./issues.js"; +import { plugins } from "./plugins.js"; + +export const externalObjectMentions = pgTable( + "external_object_mentions", + { + id: uuid("id").primaryKey().defaultRandom(), + companyId: uuid("company_id").notNull().references(() => companies.id, { onDelete: "cascade" }), + sourceIssueId: uuid("source_issue_id").notNull().references(() => issues.id, { onDelete: "cascade" }), + sourceKind: text("source_kind").$type().notNull(), + sourceRecordId: uuid("source_record_id"), + documentKey: text("document_key"), + propertyKey: text("property_key"), + matchedTextRedacted: text("matched_text_redacted"), + sanitizedDisplayUrl: text("sanitized_display_url"), + canonicalIdentityHash: text("canonical_identity_hash"), + canonicalIdentity: jsonb("canonical_identity").$type>(), + objectId: uuid("object_id").references(() => externalObjects.id, { onDelete: "set null" }), + providerKey: text("provider_key"), + detectorKey: text("detector_key"), + objectType: text("object_type"), + confidence: text("confidence").$type().notNull().default("exact"), + createdByPluginId: uuid("created_by_plugin_id").references(() => plugins.id, { onDelete: "set null" }), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), + updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), + }, + (table) => ({ + companySourceIssueIdx: index("external_object_mentions_company_source_issue_idx").on( + table.companyId, + table.sourceIssueId, + ), + companyObjectIdx: index("external_object_mentions_company_object_idx").on(table.companyId, table.objectId), + companyProviderIdx: index("external_object_mentions_company_provider_idx").on( + table.companyId, + table.providerKey, + table.objectType, + ), + companySourceMentionWithRecordUq: uniqueIndex("external_object_mentions_company_source_record_uq").on( + table.companyId, + table.sourceIssueId, + table.sourceKind, + table.sourceRecordId, + table.documentKey, + table.propertyKey, + table.canonicalIdentityHash, + ).where(sql`${table.sourceRecordId} is not null and ${table.canonicalIdentityHash} is not null`), + companySourceMentionWithoutRecordUq: uniqueIndex("external_object_mentions_company_source_null_record_uq").on( + table.companyId, + table.sourceIssueId, + table.sourceKind, + table.documentKey, + table.propertyKey, + table.canonicalIdentityHash, + ).where(sql`${table.sourceRecordId} is null and ${table.canonicalIdentityHash} is not null`), + }), +); diff --git a/packages/db/src/schema/external_objects.ts b/packages/db/src/schema/external_objects.ts new file mode 100644 index 00000000000..d3028adef9e --- /dev/null +++ b/packages/db/src/schema/external_objects.ts @@ -0,0 +1,74 @@ +import { + boolean, + index, + jsonb, + pgTable, + text, + timestamp, + uniqueIndex, + uuid, +} from "drizzle-orm/pg-core"; +import type { + ExternalObjectLivenessState, + ExternalObjectStatusCategory, + ExternalObjectStatusTone, +} from "@paperclipai/shared"; +import { companies } from "./companies.js"; +import { plugins } from "./plugins.js"; + +export const externalObjects = pgTable( + "external_objects", + { + id: uuid("id").primaryKey().defaultRandom(), + companyId: uuid("company_id").notNull().references(() => companies.id, { onDelete: "cascade" }), + providerKey: text("provider_key").notNull(), + pluginId: uuid("plugin_id").references(() => plugins.id, { onDelete: "set null" }), + objectType: text("object_type").notNull(), + externalId: text("external_id").notNull(), + sanitizedCanonicalUrl: text("sanitized_canonical_url"), + canonicalIdentityHash: text("canonical_identity_hash"), + displayTitle: text("display_title"), + statusKey: text("status_key"), + statusLabel: text("status_label"), + statusCategory: text("status_category").$type().notNull().default("unknown"), + statusTone: text("status_tone").$type().notNull().default("neutral"), + liveness: text("liveness").$type().notNull().default("unknown"), + isTerminal: boolean("is_terminal").notNull().default(false), + data: jsonb("data").$type>().notNull().default({}), + remoteVersion: text("remote_version"), + etag: text("etag"), + lastResolvedAt: timestamp("last_resolved_at", { withTimezone: true }), + lastChangedAt: timestamp("last_changed_at", { withTimezone: true }), + lastErrorAt: timestamp("last_error_at", { withTimezone: true }), + nextRefreshAt: timestamp("next_refresh_at", { withTimezone: true }), + lastErrorCode: text("last_error_code"), + lastErrorMessage: text("last_error_message"), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), + updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), + }, + (table) => ({ + companyProviderObjectIdx: index("external_objects_company_provider_object_idx").on( + table.companyId, + table.providerKey, + table.objectType, + ), + companyProviderStatusIdx: index("external_objects_company_provider_status_idx").on( + table.companyId, + table.providerKey, + table.statusCategory, + ), + companyRefreshIdx: index("external_objects_company_refresh_idx").on(table.companyId, table.nextRefreshAt), + companyExternalIdUq: uniqueIndex("external_objects_company_external_id_uq").on( + table.companyId, + table.providerKey, + table.objectType, + table.externalId, + ), + companyCanonicalIdentityUq: uniqueIndex("external_objects_company_identity_uq").on( + table.companyId, + table.providerKey, + table.objectType, + table.canonicalIdentityHash, + ), + }), +); diff --git a/packages/db/src/schema/index.ts b/packages/db/src/schema/index.ts index cda88aedbef..5b2d9e1a496 100644 --- a/packages/db/src/schema/index.ts +++ b/packages/db/src/schema/index.ts @@ -36,6 +36,8 @@ export { issueWatchdogs } from "./issue_watchdogs.js"; export { issuePlanDecompositions } from "./issue_plan_decompositions.js"; export { issueRecoveryActions } from "./issue_recovery_actions.js"; export { issueReferenceMentions } from "./issue_reference_mentions.js"; +export { externalObjects } from "./external_objects.js"; +export { externalObjectMentions } from "./external_object_mentions.js"; export { issueRelations } from "./issue_relations.js"; export { routines, routineRevisions, routineTriggers, routineRuns } from "./routines.js"; export { issueWorkProducts } from "./issue_work_products.js"; diff --git a/packages/shared/src/constants.ts b/packages/shared/src/constants.ts index 5ec46818888..e1db31cc979 100644 --- a/packages/shared/src/constants.ts +++ b/packages/shared/src/constants.ts @@ -358,6 +358,53 @@ export const DOCUMENT_ANNOTATION_ANCHOR_CONFIDENCES = [ export type DocumentAnnotationAnchorConfidence = (typeof DOCUMENT_ANNOTATION_ANCHOR_CONFIDENCES)[number]; +export const EXTERNAL_OBJECT_STATUS_CATEGORIES = [ + "unknown", + "open", + "waiting", + "running", + "succeeded", + "failed", + "blocked", + "closed", + "archived", + "auth_required", + "unreachable", +] as const; +export type ExternalObjectStatusCategory = (typeof EXTERNAL_OBJECT_STATUS_CATEGORIES)[number]; + +export const EXTERNAL_OBJECT_STATUS_TONES = [ + "neutral", + "info", + "success", + "warning", + "danger", + "muted", +] as const; +export type ExternalObjectStatusTone = (typeof EXTERNAL_OBJECT_STATUS_TONES)[number]; + +export const EXTERNAL_OBJECT_LIVENESS_STATES = [ + "unknown", + "fresh", + "stale", + "auth_required", + "unreachable", +] as const; +export type ExternalObjectLivenessState = (typeof EXTERNAL_OBJECT_LIVENESS_STATES)[number]; + +export const EXTERNAL_OBJECT_MENTION_SOURCE_KINDS = [ + "title", + "description", + "comment", + "document", + "property", + "plugin", +] as const; +export type ExternalObjectMentionSourceKind = (typeof EXTERNAL_OBJECT_MENTION_SOURCE_KINDS)[number]; + +export const EXTERNAL_OBJECT_MENTION_CONFIDENCES = ["exact", "likely", "possible"] as const; +export type ExternalObjectMentionConfidence = (typeof EXTERNAL_OBJECT_MENTION_CONFIDENCES)[number]; + export const ISSUE_EXECUTION_POLICY_MODES = ["normal", "auto"] as const; export type IssueExecutionPolicyMode = (typeof ISSUE_EXECUTION_POLICY_MODES)[number]; diff --git a/packages/shared/src/external-objects.test.ts b/packages/shared/src/external-objects.test.ts new file mode 100644 index 00000000000..0c0d5457087 --- /dev/null +++ b/packages/shared/src/external-objects.test.ts @@ -0,0 +1,126 @@ +import { describe, expect, it } from "vitest"; +import { + buildExternalObjectMentionSourceKey, + buildExternalObjectScopedIdentityKey, + canonicalizeExternalObjectUrl, + extractExternalObjectCanonicalUrls, + findExternalObjectUrlMatches, + formatExternalObjectMentionSourceLabel, +} from "./external-objects.js"; +import { externalObjectProviderKeySchema, externalObjectTypeSchema } from "./validators/external-object.js"; + +describe("external object references", () => { + it("extracts external urls without changing internal issue reference behavior", () => { + expect( + findExternalObjectUrlMatches( + "See PAP-1, /issues/PAP-2, https://paperclip.ing/PAP/issues/PAP-3, and https://github.com/acme/app/pull/4.", + ), + ).toEqual([{ index: 70, length: 34, matchedText: "https://github.com/acme/app/pull/4" }]); + }); + + it("ignores urls inside inline and fenced code", () => { + const markdown = [ + "Use https://github.com/acme/app/pull/1 here.", + "`https://github.com/acme/app/pull/2` should not count.", + "```", + "https://github.com/acme/app/pull/3", + "```", + ].join("\n"); + + expect(findExternalObjectUrlMatches(markdown).map((match) => match.matchedText)).toEqual([ + "https://github.com/acme/app/pull/1", + ]); + }); + + it("canonicalizes urls by stripping query and fragment by default", () => { + expect(canonicalizeExternalObjectUrl("HTTPS://GitHub.com/acme/app/pull/1?token=secret#discussion")).toMatchObject({ + sanitizedCanonicalUrl: "https://github.com/acme/app/pull/1", + sanitizedDisplayUrl: "https://github.com/acme/app/pull/1", + redactedMatchedText: "https://github.com/acme/app/pull/1", + canonicalIdentity: { + scheme: "https", + host: "github.com", + path: "/acme/app/pull/1", + }, + }); + }); + + it("rejects urls with userinfo", () => { + expect(canonicalizeExternalObjectUrl("https://token:secret@github.com/acme/app/pull/1")).toBeNull(); + }); + + it("hashes provider-required query identity values without storing plaintext", () => { + const first = canonicalizeExternalObjectUrl("https://deploy.test/run?id=secret-run&token=drop", { + identityQueryParams: ["id"], + }); + const second = canonicalizeExternalObjectUrl("https://deploy.test/run?id=secret-run&token=other", { + identityQueryParams: ["id"], + }); + + expect(first?.sanitizedCanonicalUrl).toBe("https://deploy.test/run"); + expect(first?.canonicalIdentity.queryParamHashes?.id).toHaveLength(64); + expect(first?.canonicalIdentity.queryParamHashes?.id).not.toContain("secret-run"); + expect(second?.canonicalIdentityHash).toBe(first?.canonicalIdentityHash); + }); + + it("dedupes extracted canonical urls by canonical identity", () => { + expect( + extractExternalObjectCanonicalUrls( + "https://github.com/acme/app/pull/1?token=a and https://github.com/acme/app/pull/1#discussion", + ).map((entry) => entry.sanitizedCanonicalUrl), + ).toEqual(["https://github.com/acme/app/pull/1"]); + }); + + it("includes company id in scoped object identity keys", () => { + const base = { + providerKey: "github", + objectType: "pull_request", + canonicalIdentityHash: "hash", + }; + + expect(buildExternalObjectScopedIdentityKey({ companyId: "company-a", ...base })).not.toBe( + buildExternalObjectScopedIdentityKey({ companyId: "company-b", ...base }), + ); + }); + + it("builds source keys for replacing mentions from the same source", () => { + const oldMentionSource = buildExternalObjectMentionSourceKey({ + companyId: "company-a", + sourceIssueId: "issue-1", + sourceKind: "comment", + sourceRecordId: "comment-1", + }); + const newMentionSource = buildExternalObjectMentionSourceKey({ + companyId: "company-a", + sourceIssueId: "issue-1", + sourceKind: "comment", + sourceRecordId: "comment-1", + }); + const anotherCompanySource = buildExternalObjectMentionSourceKey({ + companyId: "company-b", + sourceIssueId: "issue-1", + sourceKind: "comment", + sourceRecordId: "comment-1", + }); + + expect(newMentionSource).toBe(oldMentionSource); + expect(anotherCompanySource).not.toBe(oldMentionSource); + }); + + it("formats stable source labels", () => { + expect(formatExternalObjectMentionSourceLabel({ sourceKind: "title" })).toBe("Title"); + expect(formatExternalObjectMentionSourceLabel({ sourceKind: "document", documentKey: "plan" })).toBe( + "Document: plan", + ); + expect(formatExternalObjectMentionSourceLabel({ sourceKind: "property", propertyKey: "pr" })).toBe( + "Property: pr", + ); + }); + + it("validates provider keys and object types", () => { + expect(externalObjectProviderKeySchema.parse("github.enterprise")).toBe("github.enterprise"); + expect(externalObjectTypeSchema.parse("pull_request")).toBe("pull_request"); + expect(externalObjectProviderKeySchema.safeParse("GitHub").success).toBe(false); + expect(externalObjectTypeSchema.safeParse("pull-request").success).toBe(false); + }); +}); diff --git a/packages/shared/src/external-objects.ts b/packages/shared/src/external-objects.ts new file mode 100644 index 00000000000..443ab43033d --- /dev/null +++ b/packages/shared/src/external-objects.ts @@ -0,0 +1,262 @@ +import { createHash } from "node:crypto"; +import type { ExternalObjectMentionSourceKind } from "./constants.js"; +import { parseIssueReferenceHref } from "./issue-references.js"; + +export interface ExternalObjectUrlMatch { + index: number; + length: number; + matchedText: string; +} + +export interface ExternalObjectCanonicalIdentity { + scheme: "http" | "https"; + host: string; + path: string; + queryParamHashes?: Record; +} + +export interface ExternalObjectUrlCanonicalizationOptions { + identityQueryParams?: readonly string[]; +} + +export interface ExternalObjectCanonicalUrl { + sanitizedCanonicalUrl: string; + sanitizedDisplayUrl: string; + canonicalIdentity: ExternalObjectCanonicalIdentity; + canonicalIdentityHash: string; + redactedMatchedText: string; +} + +export interface ExternalObjectMentionSource { + companyId?: string; + sourceIssueId?: string; + sourceKind: ExternalObjectMentionSourceKind; + sourceRecordId?: string | null; + documentKey?: string | null; + propertyKey?: string | null; +} + +const EXTERNAL_URL_TOKEN_RE = /https?:\/\/[^\s<>()]+/gi; + +function preserveNewlinesAsWhitespace(value: string) { + return value.replace(/[^\n]/g, " "); +} + +function stripMarkdownCode(markdown: string): string { + if (!markdown) return ""; + + let output = ""; + let index = 0; + + while (index < markdown.length) { + const remaining = markdown.slice(index); + const fenceMatch = /^(?:```+|~~~+)/.exec(remaining); + const atLineStart = index === 0 || markdown[index - 1] === "\n"; + + if (atLineStart && fenceMatch) { + const fence = fenceMatch[0]!; + const blockStart = index; + index += fence.length; + while (index < markdown.length && markdown[index] !== "\n") index += 1; + if (index < markdown.length) index += 1; + + while (index < markdown.length) { + const lineStart = index === 0 || markdown[index - 1] === "\n"; + if (lineStart && markdown.startsWith(fence, index)) { + index += fence.length; + while (index < markdown.length && markdown[index] !== "\n") index += 1; + if (index < markdown.length) index += 1; + break; + } + index += 1; + } + + output += preserveNewlinesAsWhitespace(markdown.slice(blockStart, index)); + continue; + } + + if (markdown[index] === "`") { + let tickCount = 1; + while (index + tickCount < markdown.length && markdown[index + tickCount] === "`") { + tickCount += 1; + } + const fence = "`".repeat(tickCount); + const inlineStart = index; + index += tickCount; + const closeIndex = markdown.indexOf(fence, index); + if (closeIndex === -1) { + output += markdown.slice(inlineStart, inlineStart + tickCount); + index = inlineStart + tickCount; + continue; + } + index = closeIndex + tickCount; + output += preserveNewlinesAsWhitespace(markdown.slice(inlineStart, index)); + continue; + } + + output += markdown[index]!; + index += 1; + } + + return output; +} + +function trimTrailingPunctuation(token: string): string { + let trimmed = token; + while (trimmed.length > 0) { + const last = trimmed[trimmed.length - 1]!; + if (!".,!?;:".includes(last) && last !== ")" && last !== "]") break; + + if ( + (last === ")" && (trimmed.match(/\(/g)?.length ?? 0) >= (trimmed.match(/\)/g)?.length ?? 0)) + || (last === "]" && (trimmed.match(/\[/g)?.length ?? 0) >= (trimmed.match(/\]/g)?.length ?? 0)) + ) { + break; + } + trimmed = trimmed.slice(0, -1); + } + return trimmed; +} + +function sha256Hex(value: string): string { + return createHash("sha256").update(value).digest("hex"); +} + +function stableStringify(value: unknown): string { + if (Array.isArray(value)) { + return `[${value.map((entry) => stableStringify(entry)).join(",")}]`; + } + if (value && typeof value === "object") { + const record = value as Record; + return `{${Object.keys(record) + .sort() + .map((key) => `${JSON.stringify(key)}:${stableStringify(record[key])}`) + .join(",")}}`; + } + return JSON.stringify(value); +} + +function normalizePathname(pathname: string): string { + return pathname || "/"; +} + +export function findExternalObjectUrlMatches(markdown: string): ExternalObjectUrlMatch[] { + if (!markdown) return []; + + const scrubbed = stripMarkdownCode(markdown); + const matches: ExternalObjectUrlMatch[] = []; + let match: RegExpExecArray | null; + const re = new RegExp(EXTERNAL_URL_TOKEN_RE); + + while ((match = re.exec(scrubbed)) !== null) { + const matchedText = trimTrailingPunctuation(match[0]); + if (!matchedText || parseIssueReferenceHref(matchedText)) continue; + + matches.push({ + index: match.index, + length: matchedText.length, + matchedText, + }); + } + + return matches; +} + +export function canonicalizeExternalObjectUrl( + value: string, + options: ExternalObjectUrlCanonicalizationOptions = {}, +): ExternalObjectCanonicalUrl | null { + let url: URL; + try { + url = new URL(value.trim()); + } catch { + return null; + } + + if (url.protocol !== "http:" && url.protocol !== "https:") return null; + if (url.username || url.password) return null; + + const scheme = url.protocol === "https:" ? "https" : "http"; + const path = normalizePathname(url.pathname); + const sanitizedCanonicalUrl = `${scheme}://${url.host.toLowerCase()}${path}`; + const identityQueryParams = new Set(options.identityQueryParams ?? []); + const queryParamHashes: Record = {}; + + for (const key of [...identityQueryParams].sort()) { + const values = url.searchParams.getAll(key); + if (values.length === 0) continue; + queryParamHashes[key] = sha256Hex(values.join("\u0000")); + } + + const canonicalIdentity: ExternalObjectCanonicalIdentity = { + scheme, + host: url.host.toLowerCase(), + path, + ...(Object.keys(queryParamHashes).length > 0 ? { queryParamHashes } : {}), + }; + + return { + sanitizedCanonicalUrl, + sanitizedDisplayUrl: sanitizedCanonicalUrl, + canonicalIdentity, + canonicalIdentityHash: sha256Hex(stableStringify(canonicalIdentity)), + redactedMatchedText: sanitizedCanonicalUrl, + }; +} + +export function extractExternalObjectCanonicalUrls( + markdown: string, + options: ExternalObjectUrlCanonicalizationOptions = {}, +): ExternalObjectCanonicalUrl[] { + const seen = new Set(); + const ordered: ExternalObjectCanonicalUrl[] = []; + + for (const match of findExternalObjectUrlMatches(markdown)) { + const canonical = canonicalizeExternalObjectUrl(match.matchedText, options); + if (!canonical || seen.has(canonical.canonicalIdentityHash)) continue; + seen.add(canonical.canonicalIdentityHash); + ordered.push(canonical); + } + + return ordered; +} + +export function buildExternalObjectScopedIdentityKey(args: { + companyId: string; + providerKey: string; + objectType: string; + canonicalIdentityHash: string; +}): string { + return [args.companyId, args.providerKey, args.objectType, args.canonicalIdentityHash].join(":"); +} + +export function buildExternalObjectMentionSourceKey(source: Required> & ExternalObjectMentionSource): string { + return [ + source.companyId, + source.sourceIssueId, + source.sourceKind, + source.sourceRecordId ?? "", + source.documentKey ?? "", + source.propertyKey ?? "", + ].join(":"); +} + +export function formatExternalObjectMentionSourceLabel(source: ExternalObjectMentionSource): string { + switch (source.sourceKind) { + case "title": + return "Title"; + case "description": + return "Description"; + case "comment": + return "Comment"; + case "document": + return source.documentKey ? `Document: ${source.documentKey}` : "Document"; + case "property": + return source.propertyKey ? `Property: ${source.propertyKey}` : "Property"; + case "plugin": + return "Plugin"; + } +} diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 319f484f9f8..2ecec888d1f 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -92,6 +92,11 @@ export { DOCUMENT_ANNOTATION_THREAD_STATUSES, DOCUMENT_ANNOTATION_ANCHOR_STATES, DOCUMENT_ANNOTATION_ANCHOR_CONFIDENCES, + EXTERNAL_OBJECT_STATUS_CATEGORIES, + EXTERNAL_OBJECT_STATUS_TONES, + EXTERNAL_OBJECT_LIVENESS_STATES, + EXTERNAL_OBJECT_MENTION_SOURCE_KINDS, + EXTERNAL_OBJECT_MENTION_CONFIDENCES, ISSUE_EXECUTION_POLICY_MODES, ISSUE_EXECUTION_STAGE_TYPES, ISSUE_MONITOR_SCHEDULED_BY, @@ -217,6 +222,11 @@ export { type DocumentAnnotationThreadStatus, type DocumentAnnotationAnchorState, type DocumentAnnotationAnchorConfidence, + type ExternalObjectStatusCategory, + type ExternalObjectStatusTone, + type ExternalObjectLivenessState, + type ExternalObjectMentionSourceKind, + type ExternalObjectMentionConfidence, type IssueExecutionPolicyMode, type IssueExecutionStageType, type IssueMonitorScheduledBy, @@ -522,6 +532,8 @@ export type { DocumentTextRange, UpdateDocumentAnnotationThreadRequest, AttachmentArtifactWorkProductMetadata, + ExternalObject, + ExternalObjectMention, Issue, IssueAssigneeAdapterOverrides, IssueBlockerAttention, @@ -829,6 +841,20 @@ export { type VerifyDocumentAnchorSelectorResult, } from "./document-anchors.js"; +export { + buildExternalObjectScopedIdentityKey, + buildExternalObjectMentionSourceKey, + canonicalizeExternalObjectUrl, + extractExternalObjectCanonicalUrls, + findExternalObjectUrlMatches, + formatExternalObjectMentionSourceLabel, + type ExternalObjectCanonicalIdentity, + type ExternalObjectCanonicalUrl, + type ExternalObjectMentionSource, + type ExternalObjectUrlCanonicalizationOptions, + type ExternalObjectUrlMatch, +} from "./external-objects.js"; + export { sidebarOrderPreferenceSchema, upsertSidebarOrderPreferenceSchema, @@ -924,11 +950,24 @@ export { feedbackTargetTypeSchema, feedbackTraceStatusSchema, feedbackVoteValueSchema, + externalObjectStatusCategorySchema, + externalObjectStatusToneSchema, + externalObjectLivenessStateSchema, + externalObjectMentionSourceKindSchema, + externalObjectMentionConfidenceSchema, + externalObjectProviderKeySchema, + externalObjectTypeSchema, + externalObjectCanonicalIdentitySchema, + externalObjectMentionSourceSchema, upsertIssueFeedbackVoteSchema, type CreateCompany, type UpdateCompany, type UpdateCompanyBranding, type UpsertIssueFeedbackVote, + type ExternalObjectCanonicalIdentityInput, + type ExternalObjectMentionSourceInput, + type ExternalObjectProviderKeyInput, + type ExternalObjectTypeInput, environmentDriverSchema, environmentStatusSchema, environmentLeaseStatusSchema, diff --git a/packages/shared/src/types/external-object.ts b/packages/shared/src/types/external-object.ts new file mode 100644 index 00000000000..3c785079a7b --- /dev/null +++ b/packages/shared/src/types/external-object.ts @@ -0,0 +1,58 @@ +import type { + ExternalObjectLivenessState, + ExternalObjectMentionConfidence, + ExternalObjectMentionSourceKind, + ExternalObjectStatusCategory, + ExternalObjectStatusTone, +} from "../constants.js"; + +export interface ExternalObject { + id: string; + companyId: string; + providerKey: string; + pluginId: string | null; + objectType: string; + externalId: string; + sanitizedCanonicalUrl: string | null; + canonicalIdentityHash: string | null; + displayTitle: string | null; + statusKey: string | null; + statusLabel: string | null; + statusCategory: ExternalObjectStatusCategory; + statusTone: ExternalObjectStatusTone; + liveness: ExternalObjectLivenessState; + isTerminal: boolean; + data: Record; + remoteVersion: string | null; + etag: string | null; + lastResolvedAt: string | null; + lastChangedAt: string | null; + lastErrorAt: string | null; + nextRefreshAt: string | null; + lastErrorCode: string | null; + lastErrorMessage: string | null; + createdAt: string; + updatedAt: string; +} + +export interface ExternalObjectMention { + id: string; + companyId: string; + sourceIssueId: string; + sourceKind: ExternalObjectMentionSourceKind; + sourceRecordId: string | null; + documentKey: string | null; + propertyKey: string | null; + matchedTextRedacted: string | null; + sanitizedDisplayUrl: string | null; + canonicalIdentityHash: string | null; + canonicalIdentity: Record | null; + objectId: string | null; + providerKey: string | null; + detectorKey: string | null; + objectType: string | null; + confidence: ExternalObjectMentionConfidence; + createdByPluginId: string | null; + createdAt: string; + updatedAt: string; +} diff --git a/packages/shared/src/types/index.ts b/packages/shared/src/types/index.ts index 5354d5ea4f8..ac3d2d3e21e 100644 --- a/packages/shared/src/types/index.ts +++ b/packages/shared/src/types/index.ts @@ -265,6 +265,11 @@ export type { CompanyArtifactSource, CompanyArtifactsResponse, } from "./artifact.js"; + +export type { + ExternalObject, + ExternalObjectMention, +} from "./external-object.js"; export type { Issue, IssueWorkMode, diff --git a/packages/shared/src/validators/external-object.ts b/packages/shared/src/validators/external-object.ts new file mode 100644 index 00000000000..33ae0ce4900 --- /dev/null +++ b/packages/shared/src/validators/external-object.ts @@ -0,0 +1,38 @@ +import { z } from "zod"; +import { + EXTERNAL_OBJECT_LIVENESS_STATES, + EXTERNAL_OBJECT_MENTION_CONFIDENCES, + EXTERNAL_OBJECT_MENTION_SOURCE_KINDS, + EXTERNAL_OBJECT_STATUS_CATEGORIES, + EXTERNAL_OBJECT_STATUS_TONES, +} from "../constants.js"; + +export const externalObjectStatusCategorySchema = z.enum(EXTERNAL_OBJECT_STATUS_CATEGORIES); +export const externalObjectStatusToneSchema = z.enum(EXTERNAL_OBJECT_STATUS_TONES); +export const externalObjectLivenessStateSchema = z.enum(EXTERNAL_OBJECT_LIVENESS_STATES); +export const externalObjectMentionSourceKindSchema = z.enum(EXTERNAL_OBJECT_MENTION_SOURCE_KINDS); +export const externalObjectMentionConfidenceSchema = z.enum(EXTERNAL_OBJECT_MENTION_CONFIDENCES); +export const externalObjectProviderKeySchema = z.string().trim().min(1).max(80).regex(/^[a-z][a-z0-9_.-]*$/); +export const externalObjectTypeSchema = z.string().trim().min(1).max(80).regex(/^[a-z][a-z0-9_]*$/); + +export const externalObjectCanonicalIdentitySchema = z + .object({ + scheme: z.enum(["http", "https"]), + host: z.string().trim().min(1), + path: z.string().trim().min(1), + queryParamHashes: z.record(z.string().regex(/^[a-f0-9]{64}$/)).optional(), + }) + .strict(); + +export const externalObjectMentionSourceSchema = z + .object({ + sourceKind: externalObjectMentionSourceKindSchema, + documentKey: z.string().trim().min(1).optional().nullable(), + propertyKey: z.string().trim().min(1).optional().nullable(), + }) + .strict(); + +export type ExternalObjectCanonicalIdentityInput = z.infer; +export type ExternalObjectMentionSourceInput = z.infer; +export type ExternalObjectProviderKeyInput = z.infer; +export type ExternalObjectTypeInput = z.infer; diff --git a/packages/shared/src/validators/index.ts b/packages/shared/src/validators/index.ts index 6323a7dc880..b04e1533d78 100644 --- a/packages/shared/src/validators/index.ts +++ b/packages/shared/src/validators/index.ts @@ -59,6 +59,22 @@ export { updateResourceMembershipSchema, type UpdateResourceMembership, } from "./resource-memberships.js"; + +export { + externalObjectStatusCategorySchema, + externalObjectStatusToneSchema, + externalObjectLivenessStateSchema, + externalObjectMentionSourceKindSchema, + externalObjectMentionConfidenceSchema, + externalObjectProviderKeySchema, + externalObjectTypeSchema, + externalObjectCanonicalIdentitySchema, + externalObjectMentionSourceSchema, + type ExternalObjectCanonicalIdentityInput, + type ExternalObjectMentionSourceInput, + type ExternalObjectProviderKeyInput, + type ExternalObjectTypeInput, +} from "./external-object.js"; export { companySkillSourceTypeSchema, companySkillTrustLevelSchema,