mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-25 02:26:12 +02:00
Add external object reference services
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed
1 parent
d4dc12bd01
commit
abca7f0ef3
12 files changed
+1298
No files matched your search
@@ -724,6 +724,7 @@ export const LIVE_EVENT_TYPES = [
|
||||
"heartbeat.run.log",
|
||||
"agent.status",
|
||||
"activity.logged",
|
||||
"external_object.updated",
|
||||
"plugin.ui.updated",
|
||||
"plugin.worker.crashed",
|
||||
"plugin.worker.restarted",
|
||||
|
||||
@@ -534,6 +534,9 @@ export type {
|
||||
AttachmentArtifactWorkProductMetadata,
|
||||
ExternalObject,
|
||||
ExternalObjectMention,
|
||||
ExternalObjectMentionGroup,
|
||||
ExternalObjectSummary,
|
||||
ExternalObjectSummaryItem,
|
||||
Issue,
|
||||
IssueAssigneeAdapterOverrides,
|
||||
IssueBlockerAttention,
|
||||
|
||||
@@ -56,3 +56,32 @@ export interface ExternalObjectMention {
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
export interface ExternalObjectMentionGroup {
|
||||
object: ExternalObject | null;
|
||||
mentions: ExternalObjectMention[];
|
||||
mentionCount: number;
|
||||
sourceLabels: string[];
|
||||
}
|
||||
|
||||
export interface ExternalObjectSummaryItem {
|
||||
id: string;
|
||||
providerKey: string;
|
||||
objectType: string;
|
||||
displayTitle: string | null;
|
||||
statusCategory: ExternalObjectStatusCategory;
|
||||
statusTone: ExternalObjectStatusTone;
|
||||
liveness: ExternalObjectLivenessState;
|
||||
isTerminal: boolean;
|
||||
}
|
||||
|
||||
export interface ExternalObjectSummary {
|
||||
total: number;
|
||||
byStatusCategory: Record<string, number>;
|
||||
byLiveness: Record<string, number>;
|
||||
highestSeverity: ExternalObjectStatusTone;
|
||||
staleCount: number;
|
||||
authRequiredCount: number;
|
||||
unreachableCount: number;
|
||||
objects: ExternalObjectSummaryItem[];
|
||||
}
|
||||
@@ -269,6 +269,9 @@ export type {
|
||||
export type {
|
||||
ExternalObject,
|
||||
ExternalObjectMention,
|
||||
ExternalObjectMentionGroup,
|
||||
ExternalObjectSummary,
|
||||
ExternalObjectSummaryItem,
|
||||
} from "./external-object.js";
|
||||
export type {
|
||||
Issue,
|
||||
|
||||
@@ -0,0 +1,199 @@
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const issueId = "11111111-1111-4111-8111-111111111111";
|
||||
const companyId = "22222222-2222-4222-8222-222222222222";
|
||||
const ownerAgentId = "33333333-3333-4333-8333-333333333333";
|
||||
const peerAgentId = "44444444-4444-4444-8444-444444444444";
|
||||
const ownerRunId = "55555555-5555-4555-8555-555555555555";
|
||||
|
||||
const mockIssueService = vi.hoisted(() => ({
|
||||
assertCheckoutOwner: vi.fn(),
|
||||
getById: vi.fn(),
|
||||
}));
|
||||
|
||||
const mockAccessService = vi.hoisted(() => ({
|
||||
hasPermission: vi.fn(),
|
||||
}));
|
||||
|
||||
const mockAgentService = vi.hoisted(() => ({
|
||||
list: vi.fn(),
|
||||
}));
|
||||
|
||||
const mockExternalObjectsService = vi.hoisted(() => ({
|
||||
getIssueSummary: vi.fn(),
|
||||
listForIssue: vi.fn(),
|
||||
refreshIssueObjects: vi.fn(),
|
||||
}));
|
||||
|
||||
function registerRouteMocks() {
|
||||
vi.doMock("../services/external-objects.js", () => ({
|
||||
externalObjectService: () => mockExternalObjectsService,
|
||||
}));
|
||||
|
||||
vi.doMock("../services/index.js", () => ({
|
||||
accessService: () => mockAccessService,
|
||||
agentService: () => mockAgentService,
|
||||
documentService: () => ({}),
|
||||
executionWorkspaceService: () => ({}),
|
||||
feedbackService: () => ({}),
|
||||
goalService: () => ({}),
|
||||
heartbeatService: () => ({
|
||||
wakeup: vi.fn(async () => undefined),
|
||||
reportRunActivity: vi.fn(async () => undefined),
|
||||
getRun: vi.fn(async () => null),
|
||||
getActiveRunForAgent: vi.fn(async () => null),
|
||||
cancelRun: vi.fn(async () => null),
|
||||
}),
|
||||
instanceSettingsService: () => ({}),
|
||||
issueApprovalService: () => ({}),
|
||||
issueReferenceService: () => ({
|
||||
listIssueReferenceSummary: async () => ({ outbound: [], inbound: [] }),
|
||||
}),
|
||||
issueService: () => mockIssueService,
|
||||
issueThreadInteractionService: () => ({}),
|
||||
logActivity: vi.fn(async () => undefined),
|
||||
projectService: () => ({}),
|
||||
routineService: () => ({}),
|
||||
workProductService: () => ({}),
|
||||
}));
|
||||
}
|
||||
|
||||
function makeIssue(overrides: Record<string, unknown> = {}) {
|
||||
return {
|
||||
id: issueId,
|
||||
companyId,
|
||||
status: "in_progress",
|
||||
priority: "medium",
|
||||
projectId: null,
|
||||
goalId: null,
|
||||
parentId: null,
|
||||
assigneeAgentId: ownerAgentId,
|
||||
assigneeUserId: null,
|
||||
identifier: "PAP-2265",
|
||||
title: "External object routes",
|
||||
executionWorkspaceId: null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
async function createApp(actor: Express.Request["actor"]) {
|
||||
const [{ errorHandler }, { issueRoutes }] = await Promise.all([
|
||||
vi.importActual<typeof import("../middleware/index.js")>("../middleware/index.js"),
|
||||
vi.importActual<typeof import("../routes/issues.js")>("../routes/issues.js"),
|
||||
]);
|
||||
const app = express();
|
||||
app.use(express.json());
|
||||
app.use((req, _res, next) => {
|
||||
req.actor = actor;
|
||||
next();
|
||||
});
|
||||
app.use("/api", issueRoutes({} as any, { provider: "local_disk" } as any));
|
||||
app.use(errorHandler);
|
||||
return app;
|
||||
}
|
||||
|
||||
function boardActor(): Express.Request["actor"] {
|
||||
return {
|
||||
type: "board",
|
||||
userId: "board-user",
|
||||
userName: null,
|
||||
userEmail: null,
|
||||
companyIds: [companyId],
|
||||
memberships: [],
|
||||
isInstanceAdmin: false,
|
||||
source: "local_implicit",
|
||||
};
|
||||
}
|
||||
|
||||
function ownerActor(): Express.Request["actor"] {
|
||||
return {
|
||||
type: "agent",
|
||||
agentId: ownerAgentId,
|
||||
companyId,
|
||||
keyId: "key-1",
|
||||
runId: ownerRunId,
|
||||
source: "agent_key",
|
||||
};
|
||||
}
|
||||
|
||||
function peerActor(): Express.Request["actor"] {
|
||||
return {
|
||||
type: "agent",
|
||||
agentId: peerAgentId,
|
||||
companyId,
|
||||
keyId: "key-2",
|
||||
runId: "66666666-6666-4666-8666-666666666666",
|
||||
source: "agent_key",
|
||||
};
|
||||
}
|
||||
|
||||
describe("external object routes", () => {
|
||||
beforeEach(() => {
|
||||
vi.resetModules();
|
||||
vi.doUnmock("../routes/issues.js");
|
||||
vi.doUnmock("../services/index.js");
|
||||
vi.doUnmock("../services/external-objects.js");
|
||||
registerRouteMocks();
|
||||
vi.resetAllMocks();
|
||||
mockIssueService.getById.mockResolvedValue(makeIssue());
|
||||
mockIssueService.assertCheckoutOwner.mockResolvedValue({ adoptedFromRunId: null });
|
||||
mockAccessService.hasPermission.mockResolvedValue(false);
|
||||
mockAgentService.list.mockResolvedValue([
|
||||
{ id: ownerAgentId, companyId, reportsTo: null, permissions: { canCreateAgents: false } },
|
||||
{ id: peerAgentId, companyId, reportsTo: null, permissions: { canCreateAgents: false } },
|
||||
]);
|
||||
mockExternalObjectsService.getIssueSummary.mockResolvedValue({ total: 1, objects: [] });
|
||||
mockExternalObjectsService.listForIssue.mockResolvedValue([]);
|
||||
mockExternalObjectsService.refreshIssueObjects.mockResolvedValue([
|
||||
{ object: { id: "77777777-7777-4777-8777-777777777777" }, refreshed: false, reason: "no_resolver" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("enforces company access on read routes", async () => {
|
||||
const app = await createApp({ ...ownerActor(), companyId: "other-company" });
|
||||
|
||||
const res = await request(app).get(`/api/issues/${issueId}/external-object-summary`);
|
||||
|
||||
expect(res.status).toBe(403);
|
||||
expect(mockExternalObjectsService.getIssueSummary).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("allows board users to read issue external object summaries", async () => {
|
||||
const app = await createApp(boardActor());
|
||||
|
||||
const res = await request(app).get(`/api/issues/${issueId}/external-object-summary`);
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.total).toBe(1);
|
||||
expect(mockExternalObjectsService.getIssueSummary).toHaveBeenCalledWith(issueId);
|
||||
});
|
||||
|
||||
it("requires active checkout ownership for agent manual refresh", async () => {
|
||||
const app = await createApp(peerActor());
|
||||
|
||||
const res = await request(app)
|
||||
.post(`/api/issues/${issueId}/external-objects/refresh`)
|
||||
.send({});
|
||||
|
||||
expect(res.status).toBe(409);
|
||||
expect(res.body.error).toBe("Issue is checked out by another agent");
|
||||
expect(mockExternalObjectsService.refreshIssueObjects).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("allows the checked-out agent to request manual refresh", async () => {
|
||||
const app = await createApp(ownerActor());
|
||||
|
||||
const res = await request(app)
|
||||
.post(`/api/issues/${issueId}/external-objects/refresh`)
|
||||
.send({});
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(mockIssueService.assertCheckoutOwner).toHaveBeenCalledWith(issueId, ownerAgentId, ownerRunId);
|
||||
expect(mockExternalObjectsService.refreshIssueObjects).toHaveBeenCalledWith(issueId, expect.objectContaining({
|
||||
companyId,
|
||||
actor: expect.objectContaining({ actorType: "agent", actorId: ownerAgentId }),
|
||||
}));
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,255 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
companies,
|
||||
createDb,
|
||||
externalObjectMentions,
|
||||
externalObjects,
|
||||
issueComments,
|
||||
issues,
|
||||
} from "@paperclipai/db";
|
||||
import {
|
||||
getEmbeddedPostgresTestSupport,
|
||||
startEmbeddedPostgresTestDatabase,
|
||||
} from "./helpers/embedded-postgres.js";
|
||||
import {
|
||||
createExternalObjectDetectorRegistry,
|
||||
createExternalObjectResolverRegistry,
|
||||
externalObjectService,
|
||||
type ExternalObjectResolver,
|
||||
} from "../services/external-objects.js";
|
||||
import { canonicalizeExternalObjectUrl } from "@paperclipai/shared";
|
||||
|
||||
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
|
||||
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
|
||||
|
||||
if (!embeddedPostgresSupport.supported) {
|
||||
console.warn(
|
||||
`Skipping embedded Postgres external object tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`,
|
||||
);
|
||||
}
|
||||
|
||||
describe("external object registries", () => {
|
||||
it("lets provider detectors claim urls before the generic fallback", async () => {
|
||||
const canonical = canonicalizeExternalObjectUrl("https://github.com/acme/app/pull/42");
|
||||
if (!canonical) throw new Error("expected canonical url");
|
||||
const registry = createExternalObjectDetectorRegistry([
|
||||
{
|
||||
key: "github",
|
||||
detect: ({ urls }) =>
|
||||
urls.map((url) => ({
|
||||
canonical: url,
|
||||
detectorKey: "github",
|
||||
providerKey: "github",
|
||||
objectType: "pull_request",
|
||||
externalId: "acme/app#42",
|
||||
confidence: "exact",
|
||||
})),
|
||||
},
|
||||
]);
|
||||
|
||||
const detections = await registry.detect({
|
||||
companyId: "company-1",
|
||||
urls: [canonical],
|
||||
sourceContext: {
|
||||
companyId: "company-1",
|
||||
sourceIssueId: "issue-1",
|
||||
sourceKind: "description",
|
||||
sourceRecordId: null,
|
||||
documentKey: null,
|
||||
propertyKey: null,
|
||||
},
|
||||
});
|
||||
|
||||
expect(detections).toHaveLength(1);
|
||||
expect(detections[0]).toMatchObject({
|
||||
providerKey: "github",
|
||||
objectType: "pull_request",
|
||||
externalId: "acme/app#42",
|
||||
});
|
||||
});
|
||||
|
||||
it("falls back to generic url objects when no provider detector claims a url", async () => {
|
||||
const canonical = canonicalizeExternalObjectUrl("https://example.com/path?token=secret#frag");
|
||||
if (!canonical) throw new Error("expected canonical url");
|
||||
const registry = createExternalObjectDetectorRegistry([]);
|
||||
|
||||
const detections = await registry.detect({
|
||||
companyId: "company-1",
|
||||
urls: [canonical],
|
||||
sourceContext: {
|
||||
companyId: "company-1",
|
||||
sourceIssueId: "issue-1",
|
||||
sourceKind: "description",
|
||||
sourceRecordId: null,
|
||||
documentKey: null,
|
||||
propertyKey: null,
|
||||
},
|
||||
});
|
||||
|
||||
expect(detections[0]).toMatchObject({
|
||||
providerKey: "url",
|
||||
objectType: "link",
|
||||
externalId: canonical.canonicalIdentityHash,
|
||||
displayTitle: "https://example.com/path",
|
||||
});
|
||||
});
|
||||
|
||||
it("matches resolvers by provider and optional object type", () => {
|
||||
const fallbackResolver: ExternalObjectResolver = {
|
||||
providerKey: "github",
|
||||
resolve: async () => ({
|
||||
ok: true,
|
||||
snapshot: { statusCategory: "unknown", statusTone: "neutral" },
|
||||
}),
|
||||
};
|
||||
const pullRequestResolver: ExternalObjectResolver = {
|
||||
providerKey: "github",
|
||||
objectType: "pull_request",
|
||||
resolve: async () => ({
|
||||
ok: true,
|
||||
snapshot: { statusCategory: "open", statusTone: "info" },
|
||||
}),
|
||||
};
|
||||
const registry = createExternalObjectResolverRegistry([pullRequestResolver, fallbackResolver]);
|
||||
|
||||
expect(registry.find({ providerKey: "github", objectType: "pull_request" })).toBe(pullRequestResolver);
|
||||
expect(registry.find({ providerKey: "github", objectType: "issue" })).toBe(fallbackResolver);
|
||||
expect(registry.find({ providerKey: "linear", objectType: "issue" })).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describeEmbeddedPostgres("externalObjectService", () => {
|
||||
let db!: ReturnType<typeof createDb>;
|
||||
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
|
||||
|
||||
beforeAll(async () => {
|
||||
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-external-objects-");
|
||||
db = createDb(tempDb.connectionString);
|
||||
}, 20_000);
|
||||
|
||||
afterEach(async () => {
|
||||
await db.delete(externalObjectMentions);
|
||||
await db.delete(externalObjects);
|
||||
await db.delete(issueComments);
|
||||
await db.delete(issues);
|
||||
await db.delete(companies);
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await tempDb?.cleanup();
|
||||
});
|
||||
|
||||
async function createIssue(companyId = randomUUID()) {
|
||||
const issueId = randomUUID();
|
||||
await db.insert(companies).values({
|
||||
id: companyId,
|
||||
name: "Paperclip",
|
||||
issuePrefix: `E${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`,
|
||||
requireBoardApprovalForNewAgents: false,
|
||||
});
|
||||
await db.insert(issues).values({
|
||||
id: issueId,
|
||||
companyId,
|
||||
identifier: "PAP-2265",
|
||||
title: "External refs",
|
||||
description: "Track https://github.com/acme/app/pull/42?token=secret#discussion twice https://github.com/acme/app/pull/42.",
|
||||
status: "todo",
|
||||
priority: "medium",
|
||||
});
|
||||
return { companyId, issueId };
|
||||
}
|
||||
|
||||
it("syncs sanitized, deduped mentions without storing secret-bearing urls", async () => {
|
||||
const { companyId, issueId } = await createIssue();
|
||||
const svc = externalObjectService(db);
|
||||
|
||||
await svc.syncIssue(issueId);
|
||||
|
||||
const [objectRows, mentionRows] = await Promise.all([
|
||||
db.select().from(externalObjects),
|
||||
db.select().from(externalObjectMentions),
|
||||
]);
|
||||
expect(objectRows).toHaveLength(1);
|
||||
expect(objectRows[0]).toMatchObject({
|
||||
companyId,
|
||||
providerKey: "url",
|
||||
objectType: "link",
|
||||
sanitizedCanonicalUrl: "https://github.com/acme/app/pull/42",
|
||||
liveness: "unknown",
|
||||
statusCategory: "unknown",
|
||||
});
|
||||
expect(JSON.stringify(objectRows[0])).not.toContain("secret");
|
||||
expect(mentionRows).toHaveLength(1);
|
||||
expect(mentionRows[0]).toMatchObject({
|
||||
companyId,
|
||||
sourceIssueId: issueId,
|
||||
sourceKind: "description",
|
||||
sanitizedDisplayUrl: "https://github.com/acme/app/pull/42",
|
||||
matchedTextRedacted: "https://github.com/acme/app/pull/42",
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves last-known status when resolver reports auth and unreachable failures", async () => {
|
||||
const { companyId, issueId } = await createIssue();
|
||||
const resolver: ExternalObjectResolver = {
|
||||
providerKey: "url",
|
||||
objectType: "link",
|
||||
resolve: vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
snapshot: {
|
||||
statusCategory: "open",
|
||||
statusTone: "info",
|
||||
statusKey: "open",
|
||||
statusLabel: "Open",
|
||||
ttlSeconds: 1,
|
||||
},
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
ok: false,
|
||||
liveness: "auth_required",
|
||||
errorCode: "auth_required",
|
||||
errorMessage: "token failed for https://github.com/acme/app/pull/42?token=secret",
|
||||
retryAfterSeconds: 60,
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
ok: false,
|
||||
liveness: "unreachable",
|
||||
errorCode: "network",
|
||||
errorMessage: "GET https://github.com/acme/app/pull/42 failed",
|
||||
retryAfterSeconds: 60,
|
||||
}),
|
||||
};
|
||||
const svc = externalObjectService(db, { resolvers: [resolver] });
|
||||
await svc.syncIssue(issueId);
|
||||
const object = await db.select().from(externalObjects).then((rows) => rows[0]!);
|
||||
|
||||
await svc.refreshObject(object.id, { companyId, force: true });
|
||||
await svc.refreshObject(object.id, { companyId, force: true });
|
||||
await svc.refreshObject(object.id, { companyId, force: true });
|
||||
|
||||
const updated = await db.select().from(externalObjects).then((rows) => rows[0]!);
|
||||
expect(updated.statusCategory).toBe("open");
|
||||
expect(updated.statusLabel).toBe("Open");
|
||||
expect(updated.liveness).toBe("unreachable");
|
||||
expect(updated.lastErrorMessage).toContain("[redacted-url]");
|
||||
expect(updated.lastErrorMessage).not.toContain("secret");
|
||||
});
|
||||
|
||||
it("keeps external object identities company-scoped for duplicate urls", async () => {
|
||||
const first = await createIssue();
|
||||
const second = await createIssue();
|
||||
const svc = externalObjectService(db);
|
||||
|
||||
await svc.syncIssue(first.issueId);
|
||||
await svc.syncIssue(second.issueId);
|
||||
|
||||
const objectRows = await db.select().from(externalObjects);
|
||||
expect(objectRows).toHaveLength(2);
|
||||
expect(new Set(objectRows.map((row) => row.companyId))).toEqual(new Set([first.companyId, second.companyId]));
|
||||
expect(new Set(objectRows.map((row) => row.canonicalIdentityHash))).toHaveSize(1);
|
||||
});
|
||||
});
|
||||
@@ -147,11 +147,15 @@ import {
|
||||
resolveCoreTrustPreset,
|
||||
type TrustPresetResolution,
|
||||
} from "../services/trust-preset-resolver.js";
|
||||
import { externalObjectService } from "../services/external-objects.js";
|
||||
|
||||
const MAX_ISSUE_COMMENT_LIMIT = 500;
|
||||
const updateIssueRouteSchema = updateIssueSchema.extend({
|
||||
interrupt: z.boolean().optional(),
|
||||
});
|
||||
const refreshExternalObjectsSchema = z.object({
|
||||
objectIds: z.array(z.string().uuid()).max(50).optional(),
|
||||
}).strict();
|
||||
|
||||
const promoteLowTrustOutputSchema = z.object({
|
||||
sourceArtifactKind: z.enum(["comment", "document", "work_product", "issue"]),
|
||||
@@ -1104,6 +1108,7 @@ export function issueRoutes(
|
||||
? heartbeat.wakeup
|
||||
: opts.taskWatchdogEnqueueWakeup ?? undefined,
|
||||
}) ?? noopTaskWatchdogService();
|
||||
const externalObjectsSvc = externalObjectService(db);
|
||||
const routinesSvc = routineService(db, {
|
||||
pluginWorkerManager: opts.pluginWorkerManager,
|
||||
});
|
||||
@@ -3705,6 +3710,62 @@ export function issueRoutes(
|
||||
res.json(workProducts);
|
||||
});
|
||||
|
||||
router.get("/issues/:id/external-objects", async (req, res) => {
|
||||
const id = req.params.id as string;
|
||||
const issue = await svc.getById(id);
|
||||
if (!issue) {
|
||||
res.status(404).json({ error: "Issue not found" });
|
||||
return;
|
||||
}
|
||||
assertCompanyAccess(req, issue.companyId);
|
||||
const objects = await externalObjectsSvc.listForIssue(issue.id);
|
||||
res.json(objects);
|
||||
});
|
||||
|
||||
router.get("/issues/:id/external-object-summary", async (req, res) => {
|
||||
const id = req.params.id as string;
|
||||
const issue = await svc.getById(id);
|
||||
if (!issue) {
|
||||
res.status(404).json({ error: "Issue not found" });
|
||||
return;
|
||||
}
|
||||
assertCompanyAccess(req, issue.companyId);
|
||||
const summary = await externalObjectsSvc.getIssueSummary(issue.id);
|
||||
res.json(summary);
|
||||
});
|
||||
|
||||
router.post("/issues/:id/external-objects/refresh", validate(refreshExternalObjectsSchema), async (req, res) => {
|
||||
const id = req.params.id as string;
|
||||
const issue = await svc.getById(id);
|
||||
if (!issue) {
|
||||
res.status(404).json({ error: "Issue not found" });
|
||||
return;
|
||||
}
|
||||
assertCompanyAccess(req, issue.companyId);
|
||||
if (!(await assertAgentIssueMutationAllowed(req, res, issue))) return;
|
||||
const actor = getActorInfo(req);
|
||||
const results = await externalObjectsSvc.refreshIssueObjects(issue.id, {
|
||||
companyId: issue.companyId,
|
||||
objectIds: req.body.objectIds,
|
||||
actor,
|
||||
});
|
||||
await logActivity(db, {
|
||||
companyId: issue.companyId,
|
||||
actorType: actor.actorType,
|
||||
actorId: actor.actorId,
|
||||
agentId: actor.agentId,
|
||||
runId: actor.runId,
|
||||
action: "external_object.refresh_requested",
|
||||
entityType: "issue",
|
||||
entityId: issue.id,
|
||||
details: {
|
||||
issueId: issue.id,
|
||||
objectIds: results.map((result) => result.object.id),
|
||||
},
|
||||
});
|
||||
res.json({ refreshed: results });
|
||||
});
|
||||
|
||||
router.get("/issues/:id/documents", async (req, res) => {
|
||||
const id = req.params.id as string;
|
||||
const issue = await svc.getById(id);
|
||||
@@ -3991,6 +4052,7 @@ export function issueRoutes(
|
||||
const redirectedFromLockedDocument =
|
||||
"redirectedFromLockedDocument" in result ? result.redirectedFromLockedDocument : null;
|
||||
await issueReferencesSvc.syncDocument(doc.id);
|
||||
await externalObjectsSvc.syncDocumentSafely(doc.id);
|
||||
const referenceSummaryAfter = await issueReferencesSvc.listIssueReferenceSummary(issue.id);
|
||||
const referenceDiff = issueReferencesSvc.diffIssueReferenceSummary(referenceSummaryBefore, referenceSummaryAfter);
|
||||
const remappedAnnotations = result.created
|
||||
@@ -4220,6 +4282,7 @@ export function issueRoutes(
|
||||
});
|
||||
await issueReferencesSvc.syncDocument(result.document.id);
|
||||
const referenceSummaryAfter = await issueReferencesSvc.listIssueReferenceSummary(issue.id);
|
||||
await externalObjectsSvc.syncDocumentSafely(result.document.id);
|
||||
const referenceDiff = issueReferencesSvc.diffIssueReferenceSummary(referenceSummaryBefore, referenceSummaryAfter);
|
||||
const remappedAnnotations = await documentAnnotationsSvc.remapOpenThreadsForDocument({
|
||||
issueId: issue.id,
|
||||
@@ -4333,6 +4396,7 @@ export function issueRoutes(
|
||||
}
|
||||
await issueReferencesSvc.deleteDocumentSource(removed.id);
|
||||
const referenceSummaryAfter = await issueReferencesSvc.listIssueReferenceSummary(issue.id);
|
||||
if (removed) await externalObjectsSvc.syncDocumentSafely(removed.id);
|
||||
const referenceDiff = issueReferencesSvc.diffIssueReferenceSummary(referenceSummaryBefore, referenceSummaryAfter);
|
||||
const actor = getActorInfo(req);
|
||||
await logActivity(db, {
|
||||
@@ -5000,6 +5064,7 @@ export function issueRoutes(
|
||||
watchdogActorRunId: actor.runId,
|
||||
});
|
||||
await issueReferencesSvc.syncIssue(issue.id);
|
||||
await externalObjectsSvc.syncIssueSafely(issue.id);
|
||||
const referenceSummary = await issueReferencesSvc.listIssueReferenceSummary(issue.id);
|
||||
const referenceDiff = issueReferencesSvc.diffIssueReferenceSummary(
|
||||
issueReferencesSvc.emptySummary(),
|
||||
@@ -5165,6 +5230,7 @@ export function issueRoutes(
|
||||
actorUserId: actor.actorType === "user" ? actor.actorId : null,
|
||||
watchdogActorRunId: actor.runId,
|
||||
});
|
||||
await externalObjectsSvc.syncIssueSafely(issue.id);
|
||||
|
||||
await logActivity(db, {
|
||||
companyId: parent.companyId,
|
||||
@@ -5933,6 +5999,7 @@ export function issueRoutes(
|
||||
|
||||
if (titleOrDescriptionChanged) {
|
||||
await issueReferencesSvc.syncIssue(issue.id);
|
||||
await externalObjectsSvc.syncIssueSafely(issue.id);
|
||||
}
|
||||
const updateReferenceSummaryAfter = titleOrDescriptionChanged
|
||||
? await issueReferencesSvc.listIssueReferenceSummary(issue.id)
|
||||
@@ -6238,6 +6305,7 @@ export function issueRoutes(
|
||||
sourceTrust: await sourceTrustForActorWrite(issue, actor),
|
||||
});
|
||||
await issueReferencesSvc.syncComment(comment.id);
|
||||
await externalObjectsSvc.syncCommentSafely(comment.id);
|
||||
const commentReferenceSummaryAfter = await issueReferencesSvc.listIssueReferenceSummary(issue.id);
|
||||
const commentReferenceDiff = issueReferencesSvc.diffIssueReferenceSummary(
|
||||
commentReferenceSummaryBefore,
|
||||
@@ -7677,6 +7745,7 @@ export function issueRoutes(
|
||||
}
|
||||
|
||||
await issueReferencesSvc.syncComment(comment.id);
|
||||
await externalObjectsSvc.syncCommentSafely(comment.id);
|
||||
const commentReferenceSummaryAfter = await issueReferencesSvc.listIssueReferenceSummary(currentIssue.id);
|
||||
const commentReferenceDiff = issueReferencesSvc.diffIssueReferenceSummary(
|
||||
commentReferenceSummaryBefore,
|
||||
|
||||
@@ -15,6 +15,7 @@ import { trackProjectCreated } from "@paperclipai/shared/telemetry";
|
||||
import { validate } from "../middleware/validate.js";
|
||||
import { accessService, projectService, logActivity, workspaceOperationService } from "../services/index.js";
|
||||
import { conflict, forbidden } from "../errors.js";
|
||||
import { externalObjectService } from "../services/external-objects.js";
|
||||
import { assertCompanyAccess, getActorInfo } from "./authz.js";
|
||||
import {
|
||||
buildWorkspaceRuntimeDesiredStatePatch,
|
||||
@@ -44,6 +45,7 @@ export function projectRoutes(db: Db) {
|
||||
const access = accessService(db);
|
||||
const secretsSvc = secretService(db);
|
||||
const workspaceOperations = workspaceOperationService(db);
|
||||
const externalObjectsSvc = externalObjectService(db);
|
||||
const strictSecretsMode = process.env.PAPERCLIP_SECRETS_STRICT_MODE === "true";
|
||||
const environmentsSvc = environmentService(db);
|
||||
|
||||
@@ -139,6 +141,18 @@ export function projectRoutes(db: Db) {
|
||||
res.json(project);
|
||||
});
|
||||
|
||||
router.get("/projects/:id/external-object-summary", async (req, res) => {
|
||||
const id = req.params.id as string;
|
||||
const project = await svc.getById(id);
|
||||
if (!project) {
|
||||
res.status(404).json({ error: "Project not found" });
|
||||
return;
|
||||
}
|
||||
assertCompanyAccess(req, project.companyId);
|
||||
const summary = await externalObjectsSvc.getProjectSummary(project.id);
|
||||
res.json(summary);
|
||||
});
|
||||
|
||||
router.post("/companies/:companyId/projects", validate(createProjectSchema), async (req, res) => {
|
||||
const companyId = req.params.companyId as string;
|
||||
assertCompanyAccess(req, companyId);
|
||||
|
||||
@@ -0,0 +1,702 @@
|
||||
import { and, asc, eq, inArray, isNull, lte } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { documents, externalObjectMentions, externalObjects, issueComments, issueDocuments, issues } from "@paperclipai/db";
|
||||
import {
|
||||
extractExternalObjectCanonicalUrls,
|
||||
formatExternalObjectMentionSourceLabel,
|
||||
type ExternalObjectCanonicalUrl,
|
||||
type ExternalObjectLivenessState,
|
||||
type ExternalObjectMentionConfidence,
|
||||
type ExternalObjectMentionSourceKind,
|
||||
type ExternalObjectStatusCategory,
|
||||
type ExternalObjectStatusTone,
|
||||
} from "@paperclipai/shared";
|
||||
import { notFound } from "../errors.js";
|
||||
import { logger } from "../middleware/logger.js";
|
||||
import { logActivity, type LogActivityInput } from "./activity-log.js";
|
||||
import { publishLiveEvent } from "./live-events.js";
|
||||
|
||||
export interface ExternalObjectSourceContext {
|
||||
companyId: string;
|
||||
sourceIssueId: string;
|
||||
sourceKind: ExternalObjectMentionSourceKind;
|
||||
sourceRecordId: string | null;
|
||||
documentKey: string | null;
|
||||
propertyKey: string | null;
|
||||
}
|
||||
|
||||
export interface ExternalObjectDetection {
|
||||
canonical: ExternalObjectCanonicalUrl;
|
||||
detectorKey: string;
|
||||
providerKey: string;
|
||||
objectType: string;
|
||||
externalId: string;
|
||||
displayTitle?: string | null;
|
||||
confidence?: ExternalObjectMentionConfidence;
|
||||
pluginId?: string | null;
|
||||
}
|
||||
|
||||
export interface ExternalObjectDetector {
|
||||
key: string;
|
||||
detect(input: {
|
||||
companyId: string;
|
||||
urls: ExternalObjectCanonicalUrl[];
|
||||
sourceContext: ExternalObjectSourceContext;
|
||||
}): Promise<ExternalObjectDetection[]> | ExternalObjectDetection[];
|
||||
}
|
||||
|
||||
export interface ExternalObjectResolverSnapshot {
|
||||
displayTitle?: string | null;
|
||||
statusKey?: string | null;
|
||||
statusLabel?: string | null;
|
||||
statusCategory: ExternalObjectStatusCategory;
|
||||
statusTone: ExternalObjectStatusTone;
|
||||
isTerminal?: boolean;
|
||||
data?: Record<string, unknown>;
|
||||
remoteVersion?: string | null;
|
||||
etag?: string | null;
|
||||
ttlSeconds?: number;
|
||||
}
|
||||
|
||||
export type ExternalObjectResolveResult =
|
||||
| { ok: true; snapshot: ExternalObjectResolverSnapshot }
|
||||
| {
|
||||
ok: false;
|
||||
liveness: Extract<ExternalObjectLivenessState, "auth_required" | "unreachable">;
|
||||
errorCode: string;
|
||||
errorMessage?: string | null;
|
||||
retryAfterSeconds?: number;
|
||||
};
|
||||
|
||||
export interface ExternalObjectResolver {
|
||||
providerKey: string;
|
||||
objectType?: string;
|
||||
resolve(input: {
|
||||
companyId: string;
|
||||
object: ExternalObjectRecord;
|
||||
}): Promise<ExternalObjectResolveResult>;
|
||||
}
|
||||
|
||||
type ExternalObjectRecord = typeof externalObjects.$inferSelect;
|
||||
type ExternalObjectMentionRecord = typeof externalObjectMentions.$inferSelect;
|
||||
|
||||
const DEFAULT_REFRESH_TTL_SECONDS = 300;
|
||||
const DEFAULT_RETRY_AFTER_SECONDS = 300;
|
||||
|
||||
function sourceWhere(input: ExternalObjectSourceContext) {
|
||||
const conditions = [
|
||||
eq(externalObjectMentions.companyId, input.companyId),
|
||||
eq(externalObjectMentions.sourceIssueId, input.sourceIssueId),
|
||||
eq(externalObjectMentions.sourceKind, input.sourceKind),
|
||||
];
|
||||
if (input.sourceRecordId) {
|
||||
conditions.push(eq(externalObjectMentions.sourceRecordId, input.sourceRecordId));
|
||||
} else {
|
||||
conditions.push(isNull(externalObjectMentions.sourceRecordId));
|
||||
}
|
||||
if (input.documentKey) {
|
||||
conditions.push(eq(externalObjectMentions.documentKey, input.documentKey));
|
||||
} else {
|
||||
conditions.push(isNull(externalObjectMentions.documentKey));
|
||||
}
|
||||
if (input.propertyKey) {
|
||||
conditions.push(eq(externalObjectMentions.propertyKey, input.propertyKey));
|
||||
} else {
|
||||
conditions.push(isNull(externalObjectMentions.propertyKey));
|
||||
}
|
||||
return and(...conditions);
|
||||
}
|
||||
|
||||
function addSeconds(date: Date, seconds: number) {
|
||||
return new Date(date.getTime() + Math.max(1, seconds) * 1000);
|
||||
}
|
||||
|
||||
function visibleLiveness(object: ExternalObjectRecord, now = new Date()): ExternalObjectLivenessState {
|
||||
if (object.liveness === "fresh" && object.nextRefreshAt && object.nextRefreshAt <= now) {
|
||||
return "stale";
|
||||
}
|
||||
return object.liveness;
|
||||
}
|
||||
|
||||
function objectChanged(before: ExternalObjectRecord, after: ExternalObjectRecord) {
|
||||
return (
|
||||
before.statusKey !== after.statusKey ||
|
||||
before.statusLabel !== after.statusLabel ||
|
||||
before.statusCategory !== after.statusCategory ||
|
||||
before.statusTone !== after.statusTone ||
|
||||
before.isTerminal !== after.isTerminal
|
||||
);
|
||||
}
|
||||
|
||||
function sanitizeErrorMessage(message: string | null | undefined) {
|
||||
if (!message) return null;
|
||||
return message
|
||||
.replace(/https?:\/\/[^\s<>()]+/gi, "[redacted-url]")
|
||||
.replace(/\b(?:token|key|secret|authorization|bearer)=\S+/gi, "$1=[redacted]");
|
||||
}
|
||||
|
||||
function genericUrlDetector(): ExternalObjectDetector {
|
||||
return {
|
||||
key: "generic-url",
|
||||
detect({ urls }) {
|
||||
return urls.map((canonical) => ({
|
||||
canonical,
|
||||
detectorKey: "generic-url",
|
||||
providerKey: "url",
|
||||
objectType: "link",
|
||||
externalId: canonical.canonicalIdentityHash,
|
||||
displayTitle: canonical.sanitizedDisplayUrl,
|
||||
confidence: "possible",
|
||||
}));
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function createExternalObjectDetectorRegistry(detectors: ExternalObjectDetector[] = []) {
|
||||
const entries = [...detectors, genericUrlDetector()];
|
||||
|
||||
async function detect(input: {
|
||||
companyId: string;
|
||||
urls: ExternalObjectCanonicalUrl[];
|
||||
sourceContext: ExternalObjectSourceContext;
|
||||
}) {
|
||||
const claimed = new Set<string>();
|
||||
const detections: ExternalObjectDetection[] = [];
|
||||
for (const detector of entries) {
|
||||
const remaining = input.urls.filter((url) => !claimed.has(url.canonicalIdentityHash));
|
||||
if (remaining.length === 0) break;
|
||||
try {
|
||||
const detected = await detector.detect({ ...input, urls: remaining });
|
||||
for (const detection of detected) {
|
||||
if (claimed.has(detection.canonical.canonicalIdentityHash)) continue;
|
||||
claimed.add(detection.canonical.canonicalIdentityHash);
|
||||
detections.push({ ...detection, detectorKey: detection.detectorKey || detector.key });
|
||||
}
|
||||
} catch (err) {
|
||||
logger.warn({ err, detectorKey: detector.key }, "external object detector failed");
|
||||
}
|
||||
}
|
||||
return detections;
|
||||
}
|
||||
|
||||
return { detect };
|
||||
}
|
||||
|
||||
export function createExternalObjectResolverRegistry(resolvers: ExternalObjectResolver[] = []) {
|
||||
function find(object: Pick<ExternalObjectRecord, "providerKey" | "objectType">) {
|
||||
return resolvers.find(
|
||||
(resolver) =>
|
||||
resolver.providerKey === object.providerKey &&
|
||||
(!resolver.objectType || resolver.objectType === object.objectType),
|
||||
) ?? null;
|
||||
}
|
||||
return { find };
|
||||
}
|
||||
|
||||
export function externalObjectService(
|
||||
db: Db,
|
||||
opts: {
|
||||
detectors?: ExternalObjectDetector[];
|
||||
resolvers?: ExternalObjectResolver[];
|
||||
} = {},
|
||||
) {
|
||||
const detectorRegistry = createExternalObjectDetectorRegistry(opts.detectors ?? []);
|
||||
const resolverRegistry = createExternalObjectResolverRegistry(opts.resolvers ?? []);
|
||||
|
||||
async function issueById(issueId: string, dbOrTx: any = db) {
|
||||
return dbOrTx
|
||||
.select({
|
||||
id: issues.id,
|
||||
companyId: issues.companyId,
|
||||
title: issues.title,
|
||||
description: issues.description,
|
||||
})
|
||||
.from(issues)
|
||||
.where(eq(issues.id, issueId))
|
||||
.then((rows: Array<{ id: string; companyId: string; title: string; description: string | null }>) => rows[0] ?? null);
|
||||
}
|
||||
|
||||
async function upsertObjectFromDetection(
|
||||
companyId: string,
|
||||
detection: ExternalObjectDetection,
|
||||
dbOrTx: any,
|
||||
): Promise<ExternalObjectRecord> {
|
||||
const now = new Date();
|
||||
const canonical = detection.canonical;
|
||||
const values = {
|
||||
companyId,
|
||||
providerKey: detection.providerKey,
|
||||
pluginId: detection.pluginId ?? null,
|
||||
objectType: detection.objectType,
|
||||
externalId: detection.externalId,
|
||||
sanitizedCanonicalUrl: canonical.sanitizedCanonicalUrl,
|
||||
canonicalIdentityHash: canonical.canonicalIdentityHash,
|
||||
displayTitle: detection.displayTitle ?? canonical.sanitizedDisplayUrl,
|
||||
updatedAt: now,
|
||||
};
|
||||
const inserted = await dbOrTx
|
||||
.insert(externalObjects)
|
||||
.values(values)
|
||||
.onConflictDoUpdate({
|
||||
target: [
|
||||
externalObjects.companyId,
|
||||
externalObjects.providerKey,
|
||||
externalObjects.objectType,
|
||||
externalObjects.externalId,
|
||||
],
|
||||
set: {
|
||||
sanitizedCanonicalUrl: values.sanitizedCanonicalUrl,
|
||||
canonicalIdentityHash: values.canonicalIdentityHash,
|
||||
displayTitle: values.displayTitle,
|
||||
updatedAt: now,
|
||||
},
|
||||
})
|
||||
.returning();
|
||||
return inserted[0]!;
|
||||
}
|
||||
|
||||
async function replaceSourceMentions(
|
||||
input: ExternalObjectSourceContext & { text: string | null | undefined },
|
||||
dbOrTx: any = db,
|
||||
) {
|
||||
const urls = extractExternalObjectCanonicalUrls(input.text ?? "");
|
||||
await dbOrTx.delete(externalObjectMentions).where(sourceWhere(input));
|
||||
if (urls.length === 0) return;
|
||||
|
||||
const detections = await detectorRegistry.detect({
|
||||
companyId: input.companyId,
|
||||
urls,
|
||||
sourceContext: input,
|
||||
});
|
||||
if (detections.length === 0) return;
|
||||
|
||||
const seen = new Set<string>();
|
||||
const values: Array<typeof externalObjectMentions.$inferInsert> = [];
|
||||
for (const detection of detections) {
|
||||
const canonicalHash = detection.canonical.canonicalIdentityHash;
|
||||
const sourceKey = `${detection.providerKey}:${detection.objectType}:${canonicalHash}`;
|
||||
if (seen.has(sourceKey)) continue;
|
||||
seen.add(sourceKey);
|
||||
const object = await upsertObjectFromDetection(input.companyId, detection, dbOrTx);
|
||||
values.push({
|
||||
companyId: input.companyId,
|
||||
sourceIssueId: input.sourceIssueId,
|
||||
sourceKind: input.sourceKind,
|
||||
sourceRecordId: input.sourceRecordId,
|
||||
documentKey: input.documentKey,
|
||||
propertyKey: input.propertyKey,
|
||||
matchedTextRedacted: detection.canonical.redactedMatchedText,
|
||||
sanitizedDisplayUrl: detection.canonical.sanitizedDisplayUrl,
|
||||
canonicalIdentityHash: canonicalHash,
|
||||
canonicalIdentity: detection.canonical.canonicalIdentity as unknown as Record<string, unknown>,
|
||||
objectId: object.id,
|
||||
providerKey: detection.providerKey,
|
||||
detectorKey: detection.detectorKey,
|
||||
objectType: detection.objectType,
|
||||
confidence: detection.confidence ?? "exact",
|
||||
createdByPluginId: detection.pluginId ?? null,
|
||||
});
|
||||
}
|
||||
if (values.length > 0) {
|
||||
await dbOrTx.insert(externalObjectMentions).values(values);
|
||||
}
|
||||
}
|
||||
|
||||
async function syncIssue(issueId: string, dbOrTx: any = db) {
|
||||
const runSync = async (tx: any) => {
|
||||
const issue = await issueById(issueId, tx);
|
||||
if (!issue) throw notFound("Issue not found");
|
||||
await replaceSourceMentions({
|
||||
companyId: issue.companyId,
|
||||
sourceIssueId: issue.id,
|
||||
sourceKind: "title",
|
||||
sourceRecordId: null,
|
||||
documentKey: null,
|
||||
propertyKey: null,
|
||||
text: issue.title,
|
||||
}, tx);
|
||||
await replaceSourceMentions({
|
||||
companyId: issue.companyId,
|
||||
sourceIssueId: issue.id,
|
||||
sourceKind: "description",
|
||||
sourceRecordId: null,
|
||||
documentKey: null,
|
||||
propertyKey: null,
|
||||
text: issue.description,
|
||||
}, tx);
|
||||
};
|
||||
return dbOrTx === db ? db.transaction(runSync) : runSync(dbOrTx);
|
||||
}
|
||||
|
||||
async function syncComment(commentId: string, dbOrTx: any = db) {
|
||||
const comment = await dbOrTx
|
||||
.select({
|
||||
id: issueComments.id,
|
||||
companyId: issueComments.companyId,
|
||||
issueId: issueComments.issueId,
|
||||
body: issueComments.body,
|
||||
})
|
||||
.from(issueComments)
|
||||
.where(eq(issueComments.id, commentId))
|
||||
.then((rows: Array<{ id: string; companyId: string; issueId: string; body: string }>) => rows[0] ?? null);
|
||||
if (!comment) throw notFound("Issue comment not found");
|
||||
await replaceSourceMentions({
|
||||
companyId: comment.companyId,
|
||||
sourceIssueId: comment.issueId,
|
||||
sourceKind: "comment",
|
||||
sourceRecordId: comment.id,
|
||||
documentKey: null,
|
||||
propertyKey: null,
|
||||
text: comment.body,
|
||||
}, dbOrTx);
|
||||
}
|
||||
|
||||
async function syncDocument(documentId: string, dbOrTx: any = db) {
|
||||
const document = await dbOrTx
|
||||
.select({
|
||||
documentId: documents.id,
|
||||
companyId: documents.companyId,
|
||||
issueId: issueDocuments.issueId,
|
||||
key: issueDocuments.key,
|
||||
body: documents.latestBody,
|
||||
})
|
||||
.from(issueDocuments)
|
||||
.innerJoin(documents, eq(issueDocuments.documentId, documents.id))
|
||||
.where(eq(documents.id, documentId))
|
||||
.then((rows: Array<{ documentId: string; companyId: string; issueId: string; key: string; body: string }>) => rows[0] ?? null);
|
||||
if (!document) {
|
||||
await dbOrTx
|
||||
.delete(externalObjectMentions)
|
||||
.where(and(eq(externalObjectMentions.sourceKind, "document"), eq(externalObjectMentions.sourceRecordId, documentId)));
|
||||
return;
|
||||
}
|
||||
await replaceSourceMentions({
|
||||
companyId: document.companyId,
|
||||
sourceIssueId: document.issueId,
|
||||
sourceKind: "document",
|
||||
sourceRecordId: document.documentId,
|
||||
documentKey: document.key,
|
||||
propertyKey: null,
|
||||
text: document.body,
|
||||
}, dbOrTx);
|
||||
}
|
||||
|
||||
async function safeSync(label: string, fn: () => Promise<void>) {
|
||||
try {
|
||||
await fn();
|
||||
} catch (err) {
|
||||
logger.warn({ err }, `external object ${label} sync failed`);
|
||||
}
|
||||
}
|
||||
|
||||
async function syncIssueSafely(issueId: string) {
|
||||
await safeSync("issue", () => syncIssue(issueId));
|
||||
}
|
||||
|
||||
async function syncCommentSafely(commentId: string) {
|
||||
await safeSync("comment", () => syncComment(commentId));
|
||||
}
|
||||
|
||||
async function syncDocumentSafely(documentId: string) {
|
||||
await safeSync("document", () => syncDocument(documentId));
|
||||
}
|
||||
|
||||
function toObjectPayload(object: ExternalObjectRecord, now = new Date()) {
|
||||
return {
|
||||
...object,
|
||||
liveness: visibleLiveness(object, now),
|
||||
};
|
||||
}
|
||||
|
||||
async function listForIssue(issueId: string) {
|
||||
const issue = await issueById(issueId);
|
||||
if (!issue) throw notFound("Issue not found");
|
||||
const rows = await db
|
||||
.select({
|
||||
mention: externalObjectMentions,
|
||||
object: externalObjects,
|
||||
})
|
||||
.from(externalObjectMentions)
|
||||
.leftJoin(externalObjects, eq(externalObjectMentions.objectId, externalObjects.id))
|
||||
.where(and(
|
||||
eq(externalObjectMentions.companyId, issue.companyId),
|
||||
eq(externalObjectMentions.sourceIssueId, issue.id),
|
||||
))
|
||||
.orderBy(asc(externalObjectMentions.sourceKind), asc(externalObjectMentions.createdAt));
|
||||
const now = new Date();
|
||||
const grouped = new Map<string, {
|
||||
object: ReturnType<typeof toObjectPayload> | null;
|
||||
mentions: ExternalObjectMentionRecord[];
|
||||
mentionCount: number;
|
||||
sourceLabels: string[];
|
||||
}>();
|
||||
for (const row of rows) {
|
||||
const key = row.object?.id ?? `mention:${row.mention.id}`;
|
||||
const existing = grouped.get(key) ?? {
|
||||
object: row.object ? toObjectPayload(row.object, now) : null,
|
||||
mentions: [],
|
||||
mentionCount: 0,
|
||||
sourceLabels: [],
|
||||
};
|
||||
existing.mentions.push(row.mention);
|
||||
existing.mentionCount += 1;
|
||||
const label = formatExternalObjectMentionSourceLabel({
|
||||
sourceKind: row.mention.sourceKind,
|
||||
documentKey: row.mention.documentKey,
|
||||
propertyKey: row.mention.propertyKey,
|
||||
});
|
||||
if (!existing.sourceLabels.includes(label)) existing.sourceLabels.push(label);
|
||||
grouped.set(key, existing);
|
||||
}
|
||||
return [...grouped.values()];
|
||||
}
|
||||
|
||||
function summarizeObjects(objects: Array<ReturnType<typeof toObjectPayload>>) {
|
||||
const byStatusCategory: Record<string, number> = {};
|
||||
const byLiveness: Record<string, number> = {};
|
||||
let highestSeverity: ExternalObjectStatusTone = "neutral";
|
||||
const severityRank: Record<ExternalObjectStatusTone, number> = {
|
||||
neutral: 0,
|
||||
muted: 0,
|
||||
success: 1,
|
||||
info: 2,
|
||||
warning: 3,
|
||||
danger: 4,
|
||||
};
|
||||
for (const object of objects) {
|
||||
byStatusCategory[object.statusCategory] = (byStatusCategory[object.statusCategory] ?? 0) + 1;
|
||||
byLiveness[object.liveness] = (byLiveness[object.liveness] ?? 0) + 1;
|
||||
const livenessTone = object.liveness === "auth_required" || object.liveness === "unreachable"
|
||||
? "danger"
|
||||
: object.liveness === "stale"
|
||||
? "warning"
|
||||
: object.statusTone;
|
||||
if (severityRank[livenessTone] > severityRank[highestSeverity]) highestSeverity = livenessTone;
|
||||
}
|
||||
return {
|
||||
total: objects.length,
|
||||
byStatusCategory,
|
||||
byLiveness,
|
||||
highestSeverity,
|
||||
staleCount: byLiveness.stale ?? 0,
|
||||
authRequiredCount: byLiveness.auth_required ?? 0,
|
||||
unreachableCount: byLiveness.unreachable ?? 0,
|
||||
};
|
||||
}
|
||||
|
||||
async function getIssueSummary(issueId: string) {
|
||||
const groups = await listForIssue(issueId);
|
||||
const objects = groups.flatMap((group) => (group.object ? [group.object] : []));
|
||||
return {
|
||||
...summarizeObjects(objects),
|
||||
objects: objects.map((object) => ({
|
||||
id: object.id,
|
||||
providerKey: object.providerKey,
|
||||
objectType: object.objectType,
|
||||
displayTitle: object.displayTitle,
|
||||
statusCategory: object.statusCategory,
|
||||
statusTone: object.statusTone,
|
||||
liveness: object.liveness,
|
||||
isTerminal: object.isTerminal,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
async function getProjectSummary(projectId: string) {
|
||||
const projectIssues = await db
|
||||
.select({ id: issues.id, companyId: issues.companyId })
|
||||
.from(issues)
|
||||
.where(and(eq(issues.projectId, projectId), inArray(issues.status, ["todo", "in_progress", "in_review", "blocked"])));
|
||||
if (projectIssues.length === 0) return { ...summarizeObjects([]), objects: [] };
|
||||
const companyIds = new Set(projectIssues.map((issue) => issue.companyId));
|
||||
if (companyIds.size !== 1) return { ...summarizeObjects([]), objects: [] };
|
||||
const issueIds = projectIssues.map((issue) => issue.id);
|
||||
const rows = await db
|
||||
.select({ object: externalObjects })
|
||||
.from(externalObjectMentions)
|
||||
.innerJoin(externalObjects, eq(externalObjectMentions.objectId, externalObjects.id))
|
||||
.where(and(
|
||||
eq(externalObjectMentions.companyId, projectIssues[0]!.companyId),
|
||||
inArray(externalObjectMentions.sourceIssueId, issueIds),
|
||||
));
|
||||
const now = new Date();
|
||||
const objectsById = new Map<string, ReturnType<typeof toObjectPayload>>();
|
||||
for (const row of rows) objectsById.set(row.object.id, toObjectPayload(row.object, now));
|
||||
const objects = [...objectsById.values()];
|
||||
return {
|
||||
...summarizeObjects(objects),
|
||||
objects: objects.slice(0, 25).map((object) => ({
|
||||
id: object.id,
|
||||
providerKey: object.providerKey,
|
||||
objectType: object.objectType,
|
||||
displayTitle: object.displayTitle,
|
||||
statusCategory: object.statusCategory,
|
||||
statusTone: object.statusTone,
|
||||
liveness: object.liveness,
|
||||
isTerminal: object.isTerminal,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
async function refreshObject(
|
||||
objectId: string,
|
||||
input: {
|
||||
companyId: string;
|
||||
actor?: Pick<LogActivityInput, "actorType" | "actorId" | "agentId" | "runId">;
|
||||
force?: boolean;
|
||||
now?: Date;
|
||||
},
|
||||
) {
|
||||
const now = input.now ?? new Date();
|
||||
const object = await db
|
||||
.select()
|
||||
.from(externalObjects)
|
||||
.where(and(eq(externalObjects.id, objectId), eq(externalObjects.companyId, input.companyId)))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
if (!object) throw notFound("External object not found");
|
||||
if (!input.force && object.nextRefreshAt && object.nextRefreshAt > now) {
|
||||
return { object: toObjectPayload(object, now), refreshed: false, reason: "backoff" as const };
|
||||
}
|
||||
|
||||
const resolver = resolverRegistry.find(object);
|
||||
if (!resolver) {
|
||||
const [updated] = await db
|
||||
.update(externalObjects)
|
||||
.set({
|
||||
liveness: visibleLiveness(object, now) === "fresh" ? "stale" : object.liveness,
|
||||
nextRefreshAt: addSeconds(now, DEFAULT_RETRY_AFTER_SECONDS),
|
||||
updatedAt: now,
|
||||
})
|
||||
.where(and(eq(externalObjects.id, object.id), eq(externalObjects.companyId, object.companyId)))
|
||||
.returning();
|
||||
return { object: toObjectPayload(updated ?? object, now), refreshed: false, reason: "no_resolver" as const };
|
||||
}
|
||||
|
||||
const result = await resolver.resolve({ companyId: object.companyId, object });
|
||||
if (!result.ok) {
|
||||
const [updated] = await db
|
||||
.update(externalObjects)
|
||||
.set({
|
||||
liveness: result.liveness,
|
||||
lastErrorAt: now,
|
||||
lastErrorCode: result.errorCode,
|
||||
lastErrorMessage: sanitizeErrorMessage(result.errorMessage),
|
||||
nextRefreshAt: addSeconds(now, result.retryAfterSeconds ?? DEFAULT_RETRY_AFTER_SECONDS),
|
||||
updatedAt: now,
|
||||
})
|
||||
.where(and(eq(externalObjects.id, object.id), eq(externalObjects.companyId, object.companyId)))
|
||||
.returning();
|
||||
publishLiveEvent({
|
||||
companyId: object.companyId,
|
||||
type: "external_object.updated",
|
||||
payload: { objectId: object.id, liveness: result.liveness },
|
||||
});
|
||||
return { object: toObjectPayload(updated ?? object, now), refreshed: true, reason: result.liveness };
|
||||
}
|
||||
|
||||
const snapshot = result.snapshot;
|
||||
const patch = {
|
||||
displayTitle: snapshot.displayTitle ?? object.displayTitle,
|
||||
statusKey: snapshot.statusKey ?? object.statusKey,
|
||||
statusLabel: snapshot.statusLabel ?? object.statusLabel,
|
||||
statusCategory: snapshot.statusCategory,
|
||||
statusTone: snapshot.statusTone,
|
||||
isTerminal: snapshot.isTerminal ?? object.isTerminal,
|
||||
data: snapshot.data ?? object.data,
|
||||
remoteVersion: snapshot.remoteVersion ?? object.remoteVersion,
|
||||
etag: snapshot.etag ?? object.etag,
|
||||
liveness: "fresh" as ExternalObjectLivenessState,
|
||||
lastResolvedAt: now,
|
||||
lastErrorAt: null,
|
||||
lastErrorCode: null,
|
||||
lastErrorMessage: null,
|
||||
nextRefreshAt: addSeconds(now, snapshot.ttlSeconds ?? DEFAULT_REFRESH_TTL_SECONDS),
|
||||
updatedAt: now,
|
||||
};
|
||||
const [updated] = await db
|
||||
.update(externalObjects)
|
||||
.set({
|
||||
...patch,
|
||||
lastChangedAt: objectChanged(object, { ...object, ...patch }) ? now : object.lastChangedAt,
|
||||
})
|
||||
.where(and(eq(externalObjects.id, object.id), eq(externalObjects.companyId, object.companyId)))
|
||||
.returning();
|
||||
const next = updated ?? object;
|
||||
if (objectChanged(object, next) && input.actor) {
|
||||
await logActivity(db, {
|
||||
companyId: object.companyId,
|
||||
actorType: input.actor.actorType,
|
||||
actorId: input.actor.actorId,
|
||||
agentId: input.actor.agentId,
|
||||
runId: input.actor.runId,
|
||||
action: "external_object.status_changed",
|
||||
entityType: "external_object",
|
||||
entityId: object.id,
|
||||
details: {
|
||||
providerKey: object.providerKey,
|
||||
objectType: object.objectType,
|
||||
statusCategory: next.statusCategory,
|
||||
statusLabel: next.statusLabel,
|
||||
_previous: {
|
||||
statusCategory: object.statusCategory,
|
||||
statusLabel: object.statusLabel,
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
publishLiveEvent({
|
||||
companyId: object.companyId,
|
||||
type: "external_object.updated",
|
||||
payload: { objectId: object.id, statusCategory: next.statusCategory, liveness: next.liveness },
|
||||
});
|
||||
return { object: toObjectPayload(next, now), refreshed: true, reason: "resolved" as const };
|
||||
}
|
||||
|
||||
async function refreshIssueObjects(issueId: string, input: {
|
||||
companyId: string;
|
||||
objectIds?: string[];
|
||||
actor?: Pick<LogActivityInput, "actorType" | "actorId" | "agentId" | "runId">;
|
||||
}) {
|
||||
const groups = await listForIssue(issueId);
|
||||
const objectIds = groups
|
||||
.flatMap((group) => (group.object ? [group.object.id] : []))
|
||||
.filter((id) => !input.objectIds || input.objectIds.includes(id));
|
||||
const results = [];
|
||||
for (const objectId of objectIds) {
|
||||
results.push(await refreshObject(objectId, { companyId: input.companyId, actor: input.actor }));
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
async function refreshDueObjects(companyId: string, limit = 50, now = new Date()) {
|
||||
const due = await db
|
||||
.select({ id: externalObjects.id })
|
||||
.from(externalObjects)
|
||||
.where(and(eq(externalObjects.companyId, companyId), lte(externalObjects.nextRefreshAt, now)))
|
||||
.limit(limit);
|
||||
const results = [];
|
||||
for (const row of due) {
|
||||
results.push(await refreshObject(row.id, {
|
||||
companyId,
|
||||
actor: { actorType: "system", actorId: "external-object-resolver", agentId: null, runId: null },
|
||||
now,
|
||||
}));
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
return {
|
||||
syncIssue,
|
||||
syncComment,
|
||||
syncDocument,
|
||||
syncIssueSafely,
|
||||
syncCommentSafely,
|
||||
syncDocumentSafely,
|
||||
listForIssue,
|
||||
getIssueSummary,
|
||||
getProjectSummary,
|
||||
refreshObject,
|
||||
refreshIssueObjects,
|
||||
refreshDueObjects,
|
||||
};
|
||||
}
|
||||
@@ -33,6 +33,15 @@ export {
|
||||
resolveTaskWatchdogMutationScope,
|
||||
taskWatchdogScopeAllowsIssueMutation,
|
||||
} from "./task-watchdog-scope.js";
|
||||
export {
|
||||
createExternalObjectDetectorRegistry,
|
||||
createExternalObjectResolverRegistry,
|
||||
externalObjectService,
|
||||
type ExternalObjectDetector,
|
||||
type ExternalObjectResolver,
|
||||
type ExternalObjectResolveResult,
|
||||
type ExternalObjectResolverSnapshot,
|
||||
} from "./external-objects.js";
|
||||
export { goalService } from "./goals.js";
|
||||
export { activityService, type ActivityFilters } from "./activity.js";
|
||||
export { approvalService } from "./approvals.js";
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
import type { ExternalObjectMentionGroup, ExternalObjectSummary } from "@paperclipai/shared";
|
||||
import { api } from "./client";
|
||||
|
||||
export const externalObjectsApi = {
|
||||
listForIssue: (issueId: string) =>
|
||||
api.get<ExternalObjectMentionGroup[]>(`/issues/${issueId}/external-objects`),
|
||||
getIssueSummary: (issueId: string) =>
|
||||
api.get<ExternalObjectSummary>(`/issues/${issueId}/external-object-summary`),
|
||||
refreshIssueObjects: (issueId: string, data?: { objectIds?: string[] }) =>
|
||||
api.post<{ refreshed: unknown[] }>(`/issues/${issueId}/external-objects/refresh`, data ?? {}),
|
||||
getProjectSummary: (projectId: string) =>
|
||||
api.get<ExternalObjectSummary>(`/projects/${projectId}/external-object-summary`),
|
||||
};
|
||||
@@ -6,6 +6,7 @@ export { companiesApi } from "./companies";
|
||||
export { agentsApi } from "./agents";
|
||||
export { projectsApi } from "./projects";
|
||||
export { issuesApi } from "./issues";
|
||||
export { externalObjectsApi } from "./externalObjects";
|
||||
export { routinesApi } from "./routines";
|
||||
export { goalsApi } from "./goals";
|
||||
export { approvalsApi } from "./approvals";
|
||||
|
||||
Reference in new issue
Block a user