Add GitHub external object provider

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-06-23 00:35:04 +00:00
1 parent d880cb1a9a
commit b66cc65035
3 files changed
+653 -3

No files matched your search

@@ -23,6 +23,7 @@ import {
import { canonicalizeExternalObjectUrl } from "@paperclipai/shared";
import type { PaperclipPluginManifestV1 } from "@paperclipai/shared";
import type { PluginWorkerManager } from "../services/plugin-worker-manager.js";
import { createGitHubExternalObjectProvider } from "../services/github-external-object-provider.js";
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
@@ -123,6 +124,223 @@ describe("external object registries", () => {
});
});
describe("GitHub external object provider", () => {
function githubObject(path: string, objectType: "pull_request" | "issue") {
const canonical = canonicalizeExternalObjectUrl(`https://github.com/acme/app/${path}`);
if (!canonical) throw new Error("expected canonical url");
return {
id: randomUUID(),
companyId: "company-1",
providerKey: "github",
objectType,
externalId: `acme/app#${path}`,
sanitizedCanonicalUrl: canonical.sanitizedCanonicalUrl,
canonicalIdentityHash: canonical.canonicalIdentityHash,
displayTitle: "acme/app#42",
statusKey: null,
statusLabel: null,
statusCategory: "unknown",
statusTone: "neutral",
liveness: "unknown",
isTerminal: false,
data: {},
remoteVersion: null,
etag: null,
} as any;
}
function response(body: Record<string, unknown>, init: ResponseInit = {}) {
return new Response(JSON.stringify(body), {
status: 200,
headers: { "content-type": "application/json", etag: '"etag-1"', ...(init.headers ?? {}) },
...init,
});
}
it("detects GitHub pull request and issue URLs before the generic fallback", async () => {
const provider = createGitHubExternalObjectProvider({} as any, { tokenProvider: null });
const pr = canonicalizeExternalObjectUrl("https://github.com/Acme/App/pull/42?token=secret#discussion");
const issue = canonicalizeExternalObjectUrl("https://github.com/Acme/App/issues/7");
const other = canonicalizeExternalObjectUrl("https://example.com/Acme/App/pull/42");
if (!pr || !issue || !other) throw new Error("expected canonical urls");
const detections = await provider.detector.detect({
companyId: "company-1",
urls: [pr, issue, other],
sourceContext: {
companyId: "company-1",
sourceIssueId: "issue-1",
sourceKind: "description",
sourceRecordId: null,
documentKey: null,
propertyKey: null,
},
});
expect(detections).toEqual([
expect.objectContaining({
providerKey: "github",
objectType: "pull_request",
externalId: "acme/app#pull/42",
displayTitle: "Acme/App#42",
}),
expect.objectContaining({
providerKey: "github",
objectType: "issue",
externalId: "acme/app#issues/7",
displayTitle: "Acme/App#7",
}),
]);
expect(JSON.stringify(detections)).not.toContain("secret");
});
it.each([
[
"open",
{ state: "open", draft: false, merged: false, title: "Ship it", updated_at: "2026-04-24T01:02:03Z" },
{ statusKey: "open", statusLabel: "Open", statusCategory: "open", statusTone: "info", isTerminal: false },
],
[
"draft",
{ state: "open", draft: true, merged: false, title: "WIP", updated_at: "2026-04-24T01:02:03Z" },
{ statusKey: "draft", statusLabel: "Draft", statusCategory: "waiting", statusTone: "warning", isTerminal: false },
],
[
"closed",
{ state: "closed", draft: false, merged: false, title: "Closed PR", updated_at: "2026-04-24T01:02:03Z" },
{ statusKey: "closed", statusLabel: "Closed", statusCategory: "closed", statusTone: "muted", isTerminal: true },
],
[
"merged",
{ state: "closed", draft: false, merged: true, title: "Merged PR", updated_at: "2026-04-24T01:02:03Z" },
{ statusKey: "merged", statusLabel: "Merged", statusCategory: "succeeded", statusTone: "success", isTerminal: true },
],
])("resolves a %s pull request snapshot", async (_name, body, expected) => {
const fetch = vi.fn(async () => response(body));
const provider = createGitHubExternalObjectProvider({} as any, { fetch, tokenProvider: null });
const resolver = provider.resolvers.find((entry) => entry.objectType === "pull_request")!;
const result = await resolver.resolve({
companyId: "company-1",
object: githubObject("pull/42", "pull_request"),
});
expect(fetch).toHaveBeenCalledWith(
"https://api.github.com/repos/acme/app/pulls/42",
expect.objectContaining({
headers: expect.not.objectContaining({ authorization: expect.any(String) }),
}),
);
expect(result).toEqual({
ok: true,
snapshot: expect.objectContaining({
...expected,
displayTitle: expect.stringContaining(String(body.title)),
remoteVersion: "2026-04-24T01:02:03Z",
etag: '"etag-1"',
data: expect.objectContaining({
provider: "github",
owner: "acme",
repo: "app",
number: 42,
}),
}),
});
expect(JSON.stringify(result)).not.toContain("authorization");
});
it.each([
[
"open",
{ state: "open", title: "Issue", state_reason: null, updated_at: "2026-04-24T01:02:03Z" },
{ statusKey: "open", statusLabel: "Open", statusCategory: "open", statusTone: "info", isTerminal: false },
],
[
"closed",
{ state: "closed", title: "Issue", state_reason: "completed", updated_at: "2026-04-24T01:02:03Z" },
{ statusKey: "closed_completed", statusLabel: "Closed: completed", statusCategory: "closed", statusTone: "muted", isTerminal: true },
],
])("resolves a %s issue snapshot", async (_name, body, expected) => {
const fetch = vi.fn(async () => response(body));
const provider = createGitHubExternalObjectProvider({} as any, { fetch, tokenProvider: null });
const resolver = provider.resolvers.find((entry) => entry.objectType === "issue")!;
const result = await resolver.resolve({
companyId: "company-1",
object: githubObject("issues/42", "issue"),
});
expect(fetch).toHaveBeenCalledWith("https://api.github.com/repos/acme/app/issues/42", expect.any(Object));
expect(result).toEqual({
ok: true,
snapshot: expect.objectContaining({
...expected,
data: expect.objectContaining({
provider: "github",
owner: "acme",
repo: "app",
number: 42,
}),
}),
});
});
it("uses a configured token without storing it in the resolved snapshot", async () => {
const fetch = vi.fn(async (_url: string, init?: RequestInit) => {
expect(init?.headers).toEqual(expect.objectContaining({ authorization: "Bearer ghp_secret" }));
return response({ state: "open", draft: false, merged: false, title: "Private PR" });
});
const provider = createGitHubExternalObjectProvider({} as any, {
fetch,
tokenProvider: async () => "ghp_secret",
});
const resolver = provider.resolvers.find((entry) => entry.objectType === "pull_request")!;
const result = await resolver.resolve({
companyId: "company-1",
object: githubObject("pull/42", "pull_request"),
});
expect(result.ok).toBe(true);
expect(JSON.stringify(result)).not.toContain("ghp_secret");
});
it.each([
[
"auth-required",
new Response("", { status: 401 }),
{ ok: false, liveness: "auth_required", errorCode: "github_auth_required" },
],
[
"rate-limit",
new Response("", {
status: 403,
headers: { "x-ratelimit-remaining": "0", "x-ratelimit-reset": String(Math.floor(Date.now() / 1000) + 120) },
}),
{ ok: false, liveness: "unreachable", errorCode: "github_rate_limited" },
],
[
"not-found",
new Response("", { status: 404, headers: { etag: '"missing"' } }),
{ ok: true, snapshot: expect.objectContaining({ statusKey: "not_found", statusCategory: "archived", statusTone: "muted" }) },
],
])("maps %s responses to provider-safe results", async (_name, githubResponse, expected) => {
const provider = createGitHubExternalObjectProvider({} as any, {
fetch: async () => githubResponse,
tokenProvider: null,
});
const resolver = provider.resolvers.find((entry) => entry.objectType === "pull_request")!;
const result = await resolver.resolve({
companyId: "company-1",
object: githubObject("pull/42", "pull_request"),
});
expect(result).toEqual(expect.objectContaining(expected));
expect(JSON.stringify(result)).not.toContain("http");
});
});
describeEmbeddedPostgres("externalObjectService", () => {
let db!: ReturnType<typeof createDb>;
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
@@ -179,8 +397,9 @@ describeEmbeddedPostgres("externalObjectService", () => {
expect(objectRows).toHaveLength(1);
expect(objectRows[0]).toMatchObject({
companyId,
providerKey: "url",
objectType: "link",
providerKey: "github",
objectType: "pull_request",
externalId: "acme/app#pull/42",
sanitizedCanonicalUrl: "https://github.com/acme/app/pull/42",
liveness: "unknown",
statusCategory: "unknown",
+8 -1
View File
@@ -16,6 +16,7 @@ import type { PluginExternalObjectRecordSnapshot, PluginExternalObjectResolveRes
import { notFound } from "../errors.js";
import { logger } from "../middleware/logger.js";
import { logActivity, type LogActivityInput } from "./activity-log.js";
import { createGitHubExternalObjectProvider, type GitHubExternalObjectProviderOptions } from "./github-external-object-provider.js";
import { publishLiveEvent } from "./live-events.js";
import type { PluginWorkerManager } from "./plugin-worker-manager.js";
@@ -343,16 +344,22 @@ export function externalObjectService(
detectors?: ExternalObjectDetector[];
resolvers?: ExternalObjectResolver[];
pluginWorkerManager?: PluginWorkerManager;
github?: GitHubExternalObjectProviderOptions | false;
} = {},
) {
const githubProvider = opts.github === false ? null : createGitHubExternalObjectProvider(db, opts.github);
const pluginProviderDetector = opts.pluginWorkerManager
? createPluginProviderDetector(db, opts.pluginWorkerManager)
: null;
const detectorRegistry = createExternalObjectDetectorRegistry([
...(pluginProviderDetector ? [pluginProviderDetector] : []),
...(opts.detectors ?? []),
...(githubProvider ? [githubProvider.detector] : []),
]);
const resolverRegistry = createExternalObjectResolverRegistry([
...(opts.resolvers ?? []),
...(githubProvider?.resolvers ?? []),
]);
const resolverRegistry = createExternalObjectResolverRegistry(opts.resolvers ?? []);
async function issueById(issueId: string, dbOrTx: any = db) {
return dbOrTx
@@ -0,0 +1,424 @@
import type { Db } from "@paperclipai/db";
import type { ExternalObjectCanonicalUrl } from "@paperclipai/shared";
import { ghFetch, gitHubApiBase } from "./github-fetch.js";
import { secretService } from "./secrets.js";
import type {
ExternalObjectDetection,
ExternalObjectDetector,
ExternalObjectResolver,
ExternalObjectResolverSnapshot,
ExternalObjectResolveResult,
} from "./external-objects.js";
type FetchLike = (url: string, init?: RequestInit) => Promise<Response>;
export interface GitHubExternalObjectProviderOptions {
fetch?: FetchLike;
tokenProvider?: (companyId: string) => Promise<string | null> | string | null;
secretNames?: readonly string[];
}
interface GitHubObjectIdentity {
host: string;
owner: string;
repo: string;
number: number;
objectType: "pull_request" | "issue";
pathKind: "pull" | "issues";
}
const DEFAULT_GITHUB_TOKEN_SECRET_NAMES = ["GITHUB_TOKEN", "GH_TOKEN", "PAPERCLIP_GITHUB_TOKEN"] as const;
const GITHUB_OBJECT_TTL_SECONDS = 300;
function isGitHubHost(host: string) {
const h = host.toLowerCase();
return h === "github.com" || h === "www.github.com";
}
function asRecord(value: unknown): Record<string, unknown> | null {
return value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : null;
}
function asString(value: unknown) {
return typeof value === "string" ? value : null;
}
function asBoolean(value: unknown) {
return typeof value === "boolean" ? value : null;
}
function asNestedString(record: Record<string, unknown>, key: string, nestedKey: string) {
const nested = asRecord(record[key]);
return nested ? asString(nested[nestedKey]) : null;
}
function parseGitHubCanonicalUrl(canonical: ExternalObjectCanonicalUrl): GitHubObjectIdentity | null {
if (canonical.canonicalIdentity.scheme !== "https") return null;
const host = canonical.canonicalIdentity.host.toLowerCase();
if (!isGitHubHost(host)) return null;
const parts = canonical.canonicalIdentity.path.split("/").filter(Boolean);
if (parts.length !== 4) return null;
const [owner, repo, kind, rawNumber] = parts;
if (!owner || !repo || !kind || !rawNumber) return null;
if (!/^[A-Za-z0-9_.-]+$/.test(owner) || !/^[A-Za-z0-9_.-]+$/.test(repo)) return null;
if (!/^[1-9][0-9]*$/.test(rawNumber)) return null;
if (kind !== "pull" && kind !== "issues") return null;
return {
host: host === "www.github.com" ? "github.com" : host,
owner,
repo,
number: Number(rawNumber),
pathKind: kind,
objectType: kind === "pull" ? "pull_request" : "issue",
};
}
function parseGitHubObject(object: { externalId: string; sanitizedCanonicalUrl: string | null }): GitHubObjectIdentity | null {
const match = /^([^/]+)\/([^/]+)#(pull|issues)\/([1-9][0-9]*)$/.exec(object.externalId);
if (!match) return null;
let host = "github.com";
if (object.sanitizedCanonicalUrl) {
try {
const url = new URL(object.sanitizedCanonicalUrl);
if (isGitHubHost(url.hostname)) host = url.hostname === "www.github.com" ? "github.com" : url.hostname;
} catch {
return null;
}
}
return {
host,
owner: match[1]!,
repo: match[2]!,
pathKind: match[3] as "pull" | "issues",
number: Number(match[4]),
objectType: match[3] === "pull" ? "pull_request" : "issue",
};
}
function externalIdFor(identity: GitHubObjectIdentity) {
return `${identity.owner.toLowerCase()}/${identity.repo.toLowerCase()}#${identity.pathKind}/${identity.number}`;
}
function displayTitleFor(identity: GitHubObjectIdentity) {
return `${identity.owner}/${identity.repo}#${identity.number}`;
}
function retryAfterSeconds(response: Response) {
const retryAfter = response.headers.get("retry-after");
if (retryAfter && /^[0-9]+$/.test(retryAfter)) return Number(retryAfter);
const reset = response.headers.get("x-ratelimit-reset");
if (reset && /^[0-9]+$/.test(reset)) {
return Math.max(1, Number(reset) - Math.floor(Date.now() / 1000));
}
return 300;
}
function failureFromGitHubResponse(response: Response): ExternalObjectResolveResult | null {
if (response.status === 401) {
return {
ok: false,
liveness: "auth_required",
errorCode: "github_auth_required",
errorMessage: "GitHub authentication is required to refresh this object.",
retryAfterSeconds: retryAfterSeconds(response),
};
}
if (response.status === 403) {
const rateLimitRemaining = response.headers.get("x-ratelimit-remaining");
if (rateLimitRemaining === "0") {
return {
ok: false,
liveness: "unreachable",
errorCode: "github_rate_limited",
errorMessage: "GitHub rate limit reached while refreshing this object.",
retryAfterSeconds: retryAfterSeconds(response),
};
}
return {
ok: false,
liveness: "auth_required",
errorCode: "github_forbidden",
errorMessage: "GitHub rejected the configured credentials for this object.",
retryAfterSeconds: retryAfterSeconds(response),
};
}
if (response.status === 429 || response.status >= 500) {
return {
ok: false,
liveness: "unreachable",
errorCode: response.status === 429 ? "github_rate_limited" : "github_unreachable",
errorMessage: `GitHub returned HTTP ${response.status} while refreshing this object.`,
retryAfterSeconds: retryAfterSeconds(response),
};
}
return null;
}
function notFoundSnapshot(identity: GitHubObjectIdentity, etag: string | null): ExternalObjectResolverSnapshot {
return {
displayTitle: displayTitleFor(identity),
statusKey: "not_found",
statusLabel: "Not found",
statusCategory: "archived",
statusTone: "muted",
isTerminal: true,
etag,
ttlSeconds: GITHUB_OBJECT_TTL_SECONDS,
data: {
provider: "github",
owner: identity.owner,
repo: identity.repo,
number: identity.number,
notFound: true,
},
};
}
function pullRequestSnapshot(identity: GitHubObjectIdentity, body: Record<string, unknown>, etag: string | null): ExternalObjectResolverSnapshot {
const title = asString(body.title);
const state = asString(body.state) ?? "unknown";
const draft = asBoolean(body.draft) ?? false;
const merged = (asBoolean(body.merged) ?? false) || Boolean(asString(body.merged_at));
const authorLogin = asNestedString(body, "user", "login");
const headRef = asNestedString(body, "head", "ref");
const baseRef = asNestedString(body, "base", "ref");
const reviewDecision = asString(body.review_decision);
let statusKey = state;
let statusLabel = state === "open" ? "Open" : state === "closed" ? "Closed" : "Unknown";
let statusCategory: ExternalObjectResolverSnapshot["statusCategory"] = state === "open" ? "open" : "unknown";
let statusTone: ExternalObjectResolverSnapshot["statusTone"] = state === "open" ? "info" : "neutral";
let isTerminal = false;
if (merged) {
statusKey = "merged";
statusLabel = "Merged";
statusCategory = "succeeded";
statusTone = "success";
isTerminal = true;
} else if (state === "closed") {
statusKey = "closed";
statusLabel = "Closed";
statusCategory = "closed";
statusTone = "muted";
isTerminal = true;
} else if (draft) {
statusKey = "draft";
statusLabel = "Draft";
statusCategory = "waiting";
statusTone = "warning";
}
return {
displayTitle: title ? `${displayTitleFor(identity)}: ${title}` : displayTitleFor(identity),
statusKey,
statusLabel,
statusCategory,
statusTone,
isTerminal,
remoteVersion: asString(body.updated_at),
etag,
ttlSeconds: GITHUB_OBJECT_TTL_SECONDS,
data: {
provider: "github",
owner: identity.owner,
repo: identity.repo,
number: identity.number,
state,
merged,
draft,
...(authorLogin ? { authorLogin } : {}),
...(headRef ? { headRef } : {}),
...(baseRef ? { baseRef } : {}),
...(reviewDecision ? { reviewDecision } : {}),
},
};
}
function issueSnapshot(identity: GitHubObjectIdentity, body: Record<string, unknown>, etag: string | null): ExternalObjectResolverSnapshot {
const title = asString(body.title);
const state = asString(body.state) ?? "unknown";
const stateReason = asString(body.state_reason);
const authorLogin = asNestedString(body, "user", "login");
const statusKey = state === "closed" && stateReason ? `closed_${stateReason}` : state;
const statusLabel = state === "closed"
? stateReason
? `Closed: ${stateReason.replace(/_/g, " ")}`
: "Closed"
: state === "open"
? "Open"
: "Unknown";
return {
displayTitle: title ? `${displayTitleFor(identity)}: ${title}` : displayTitleFor(identity),
statusKey,
statusLabel,
statusCategory: state === "open" ? "open" : state === "closed" ? "closed" : "unknown",
statusTone: state === "open" ? "info" : state === "closed" ? "muted" : "neutral",
isTerminal: state === "closed",
remoteVersion: asString(body.updated_at),
etag,
ttlSeconds: GITHUB_OBJECT_TTL_SECONDS,
data: {
provider: "github",
owner: identity.owner,
repo: identity.repo,
number: identity.number,
state,
...(stateReason ? { stateReason } : {}),
...(authorLogin ? { authorLogin } : {}),
},
};
}
async function safeJson(response: Response) {
try {
return asRecord(await response.json());
} catch {
return null;
}
}
async function defaultTokenProvider(db: Db, companyId: string, secretNames: readonly string[]) {
const secrets = secretService(db);
for (const secretName of secretNames) {
const secret = await secrets.getByName(companyId, secretName);
if (!secret) continue;
const token = await secrets.resolveSecretValue(companyId, secret.id, "latest");
const trimmed = token.trim();
if (trimmed) return trimmed;
}
return null;
}
export function createGitHubExternalObjectProvider(
db: Db,
opts: GitHubExternalObjectProviderOptions = {},
): { detector: ExternalObjectDetector; resolvers: ExternalObjectResolver[] } {
const fetchImpl = opts.fetch ?? ghFetch;
const secretNames = opts.secretNames ?? DEFAULT_GITHUB_TOKEN_SECRET_NAMES;
const tokenProvider = Object.prototype.hasOwnProperty.call(opts, "tokenProvider") && opts.tokenProvider !== undefined
? opts.tokenProvider
: ((companyId: string) => defaultTokenProvider(db, companyId, secretNames));
const detector: ExternalObjectDetector = {
key: "github",
detect({ urls }): ExternalObjectDetection[] {
return urls.flatMap((canonical) => {
const identity = parseGitHubCanonicalUrl(canonical);
if (!identity) return [];
return [{
canonical,
detectorKey: "github",
providerKey: "github",
objectType: identity.objectType,
externalId: externalIdFor(identity),
displayTitle: displayTitleFor(identity),
confidence: "exact",
}];
});
},
};
function resolver(objectType: GitHubObjectIdentity["objectType"]): ExternalObjectResolver {
return {
providerKey: "github",
objectType,
async resolve({ companyId, object }) {
const identity = parseGitHubObject(object);
if (!identity || identity.objectType !== objectType) {
return {
ok: false,
liveness: "unreachable",
errorCode: "github_invalid_identity",
errorMessage: "GitHub object identity is invalid.",
retryAfterSeconds: GITHUB_OBJECT_TTL_SECONDS,
};
}
let token: string | null = null;
try {
token = typeof tokenProvider === "function" ? await tokenProvider(companyId) : tokenProvider;
} catch {
return {
ok: false,
liveness: "auth_required",
errorCode: "github_token_unavailable",
errorMessage: "Configured GitHub credentials could not be resolved.",
retryAfterSeconds: GITHUB_OBJECT_TTL_SECONDS,
};
}
token = token?.trim() || null;
const headers: Record<string, string> = {
accept: "application/vnd.github+json",
"user-agent": "paperclip-external-object-resolver",
"x-github-api-version": "2022-11-28",
};
if (token) headers.authorization = `Bearer ${token}`;
const apiKind = objectType === "pull_request" ? "pulls" : "issues";
const url = `${gitHubApiBase(identity.host)}/repos/${encodeURIComponent(identity.owner)}/${encodeURIComponent(identity.repo)}/${apiKind}/${identity.number}`;
let response: Response;
try {
response = await fetchImpl(url, { headers });
} catch {
return {
ok: false,
liveness: "unreachable",
errorCode: "github_fetch_failed",
errorMessage: "GitHub could not be reached while refreshing this object.",
retryAfterSeconds: GITHUB_OBJECT_TTL_SECONDS,
};
}
const etag = response.headers.get("etag");
if (response.status === 404) {
return { ok: true, snapshot: notFoundSnapshot(identity, etag) };
}
const failure = failureFromGitHubResponse(response);
if (failure) return failure;
if (!response.ok) {
return {
ok: false,
liveness: "unreachable",
errorCode: "github_unexpected_response",
errorMessage: `GitHub returned HTTP ${response.status} while refreshing this object.`,
retryAfterSeconds: GITHUB_OBJECT_TTL_SECONDS,
};
}
const body = await safeJson(response);
if (!body) {
return {
ok: false,
liveness: "unreachable",
errorCode: "github_invalid_response",
errorMessage: "GitHub returned an invalid object response.",
retryAfterSeconds: GITHUB_OBJECT_TTL_SECONDS,
};
}
return {
ok: true,
snapshot: objectType === "pull_request"
? pullRequestSnapshot(identity, body, etag)
: issueSnapshot(identity, body, etag),
};
},
};
}
return {
detector,
resolvers: [resolver("pull_request"), resolver("issue")],
};
}