import { randomUUID } from "node:crypto"; import { and, desc, eq, inArray, isNull, lte, ne, or, sql } from "drizzle-orm"; import { z } from "zod"; import { type Db, browserUseSessions as sessions, browserUseRuns as runs, browserUseBrowsers as browsers, browserUseSettings as settings, connectionGrants, connectionGrantMembers, companyMemberships, companySecrets, toolInvocations, toolConnections, issues, heartbeatRuns, agents, costEvents, } from "@paperclipai/db"; import { BROWSER_USE_IDLE_MS, type BrowserUseControl, type BrowserUseSettings, type BrowserUseViewportRequest, type TaskBrowser, } from "@paperclipai/shared"; import { browserUseClient, browserUseCostCap, BrowserUseError, browserUseTerminal, browserUseViewerUrl, isBrowserUseConnection, sanitizeBrowserUse, type BrowserUseRequest, } from "./browser-use-client.js"; import { secretService } from "./secrets.js"; import { budgetService, type BudgetServiceHooks } from "./budgets.js"; import { issueTreeControlService } from "./issue-tree-control.js"; import { accessService } from "./access.js"; import { createCostEventInTransaction } from "./costs.js"; import { createFinanceEventInTransaction } from "./finance.js"; import { withAccountingTransaction } from "./accounting-transaction.js"; import { logActivity } from "./activity-log.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { browserUseViewports } from "./browser-use-viewport.js"; import { forbidden, notFound, conflict } from "../errors.js"; type Session = typeof sessions.$inferSelect; type Binding = { companyId: string; agentId: string | null; runId: string | null; issueId: string | null; approvedInvocationId?: string; }; type Grant = typeof connectionGrants.$inferSelect; const argsSchema = z .object({ task: z.string().min(1).max(30000).optional(), sessionId: z.string().uuid().optional(), profileId: z.string().uuid().optional(), maxCostUsd: z.number().positive().finite().optional(), }) .strict(); const terminalSessions = ["closed", "failed"]; const liveSessions = ["starting", "running", "idle", "stopping"]; const requestMarker = (invocationId: string) => `\n\n[Paperclip request: ${invocationId}]`; export function browserUseService( db: Db, request?: BrowserUseRequest, budgetHooks: BudgetServiceHooks = {}, authorizeSession?: ( session: Session, run: typeof runs.$inferSelect, ) => Promise, ) { const budgets = budgetService(db, budgetHooks); const secrets = secretService(db); async function credentials( s: Pick, cleanup = false, ) { const [row] = await db .select({ grant: connectionGrants, connection: toolConnections }) .from(connectionGrants) .innerJoin( toolConnections, eq(toolConnections.id, connectionGrants.connectionId), ) .where( and( eq(connectionGrants.id, s.grantId), eq(connectionGrants.companyId, s.companyId), eq(toolConnections.companyId, s.companyId), eq(toolConnections.id, s.connectionId), ), ) .limit(1); if (!row || !isBrowserUseConnection(row.connection)) throw forbidden("Browser connection is unavailable."); if ( !cleanup && (row.grant.status !== "active" || row.connection.status !== "active" || !row.connection.enabled) ) throw forbidden("Browser access was revoked."); const ref = row.grant.credentialSecretRefs.find( (r) => r.configPath === "credentials.apiKey" || r.configPath.toLowerCase() === "headers.x-browser-use-api-key", ); if (!ref) throw forbidden("Reconnect Browser Use to restore its API key."); if (row.grant.kind === "user") { const [secret] = await db .select() .from(companySecrets) .where( and( eq(companySecrets.id, ref.secretId), eq(companySecrets.companyId, s.companyId), ), ) .limit(1); if ( !row.grant.subjectUserId || secret?.scope !== "user" || secret.ownerUserId !== row.grant.subjectUserId || !secret.userSecretDefinitionId ) { throw forbidden("Personal browser credential ownership is invalid."); } } const key = await secrets.resolveSecretValue( s.companyId, ref.secretId, ref.versionSelector ?? "latest", { // The exact persisted grant is the authority, as in the gateway's // grant resolver. Cleanup must survive removal of runtime bindings. allowUserSecretScope: row.grant.kind === "user", accessContext: { consumerType: "tool_connection", consumerId: s.connectionId, configPath: ref.configPath, responsibleUserId: row.grant.subjectUserId, actorType: "system", actorId: cleanup ? "browser-use-cloud-cleanup" : "browser-use-cloud", }, }, ); return { ...row, client: browserUseClient( { "X-Browser-Use-API-Key": key }, request, s.connectionId, ), }; } async function owned(binding: Binding, grant: Grant, id: string) { const [s] = await db .select() .from(sessions) .where( and( eq(sessions.id, id), eq(sessions.companyId, binding.companyId), eq(sessions.issueId, binding.issueId!), eq(sessions.agentId, binding.agentId!), eq(sessions.grantId, grant.id), eq(sessions.connectionId, grant.connectionId), ), ) .limit(1); if (!s) throw notFound( "Browser conversation not found for this task and credential.", ); return s; } async function assertBinding(binding: Binding, approved = false) { if (!binding.issueId || !binding.agentId || !binding.runId) throw forbidden( "Browser tasks require an active Paperclip task and agent run. Use browser_profiles for a connection test.", ); const [issue] = await db .select() .from(issues) .where( and( eq(issues.id, binding.issueId), eq(issues.companyId, binding.companyId), ), ) .limit(1); const [run] = await db .select() .from(heartbeatRuns) .where( and( eq(heartbeatRuns.id, binding.runId), eq(heartbeatRuns.companyId, binding.companyId), eq(heartbeatRuns.agentId, binding.agentId), ), ) .limit(1); if ( !issue || issue.assigneeAgentId !== binding.agentId || !run || (!approved && run.status !== "running") ) throw forbidden( "The browser's owning task or agent run is no longer active.", ); if ( await issueTreeControlService(db).getActivePauseHoldGate( binding.companyId, issue.id, ) ) throw forbidden("This task is paused."); const blocked = await budgets.getInvocationBlock( binding.companyId, binding.agentId, { issueId: issue.id, projectId: issue.projectId }, ); if (blocked) throw forbidden(blocked.reason); return issue; } async function getSettings( companyId: string, grantId: string, ): Promise { const [s] = await db .select() .from(settings) .where( and(eq(settings.grantId, grantId), eq(settings.companyId, companyId)), ) .limit(1); return { allowedProfileIds: s?.allowedProfileIds ?? [], maxCostUsd: s?.maxCostUsd ? Number(s.maxCostUsd) : null, }; } async function latest(s: Session) { return ( await db .select() .from(runs) .where(and(eq(runs.companyId, s.companyId), eq(runs.sessionId, s.id))) .orderBy(desc(runs.createdAt)) .limit(1) )[0]; } async function snapshot(s: Session) { const run = await latest(s); const ownedBrowsers = await db .select({ id: browsers.id, status: browsers.status }) .from(browsers) .where( and(eq(browsers.companyId, s.companyId), eq(browsers.sessionId, s.id)), ); return { sessionId: s.id, status: s.status, runStatus: run?.status ?? null, progress: run?.progress ?? null, result: run?.result ?? null, costCents: run?.accountedCents ?? 0, browsers: ownedBrowsers, error: s.error, }; } async function signal( s: Session, action: string, actorId = "browser-use-cloud", actorType: "system" | "user" = "system", ) { await logActivity(db, { companyId: s.companyId, actorType, actorId, action, entityType: "issue", entityId: s.issueId, details: { browserSessionId: s.id }, }); } async function execute( binding: Binding, grant: Grant, invocationId: string, name: string, value: unknown, ) { const args = argsSchema.parse(value ?? {}); const config = await getSettings(binding.companyId, grant.id); const scope = { companyId: binding.companyId, connectionId: grant.connectionId, grantId: grant.id, }; const { client } = await credentials(scope); if (name === "browser_profiles") { // A safe Test action that creates no paid work and exposes no unapproved profile. if (!config.allowedProfileIds.length) return { profiles: [] }; const profiles: Array<{ id: string; name?: string | null }> = []; for (let page = 1; page <= 100; page++) { const list = await client.profiles(page); profiles.push( ...list.items.filter((p) => config.allowedProfileIds.includes(p.id)), ); if (page * 100 >= list.totalItems) break; } return { profiles: sanitizeBrowserUse(profiles) }; } let approved = false; if (binding.approvedInvocationId === invocationId) { const [invocation] = await db .select() .from(toolInvocations) .where( and( eq(toolInvocations.id, invocationId), eq(toolInvocations.companyId, binding.companyId), eq(toolInvocations.agentId, binding.agentId!), eq(toolInvocations.runId, binding.runId!), eq(toolInvocations.issueId, binding.issueId!), eq(toolInvocations.connectionId, grant.connectionId), eq(toolInvocations.status, "executing"), eq(toolInvocations.approvalState, "approved"), ), ); approved = Boolean(invocation); } await assertBinding(binding, approved); if (name === "browser_sessions") { const rows = await db .select() .from(sessions) .where( and( eq(sessions.companyId, binding.companyId), eq(sessions.issueId, binding.issueId!), eq(sessions.agentId, binding.agentId!), eq(sessions.grantId, grant.id), ), ); return { sessions: await Promise.all(rows.map(snapshot)) }; } if (name !== "browser_start") { if (!args.sessionId) throw conflict("Choose a browser conversation from this task."); const s = await owned(binding, grant, args.sessionId); if (name === "browser_status") { // The signed approval dispatch owns the run until the resumed task // observes it. Thereafter normal parent-run cancellation applies. if (!approved) await db .update(runs) .set({ heartbeatRunId: binding.runId!, detachedUntil: null }) .where( and( eq(runs.sessionId, s.id), eq(runs.companyId, s.companyId), sql`${runs.detachedUntil} is not null`, ), ); return snapshot(s); } if (name === "browser_cancel" || name === "browser_end") { await control(s, name === "browser_cancel" ? "cancel" : "end"); return snapshot( (await db.select().from(sessions).where(eq(sessions.id, s.id)))[0], ); } if (name !== "browser_continue") throw notFound("Unknown Browser Use action."); } if (!args.task) throw conflict("A browser task is required."); if ( args.profileId && (!config.allowedProfileIds.includes(args.profileId) || name !== "browser_start") ) throw forbidden("This profile is not allowed for this credential."); const existing = await db .select() .from(runs) .where( and( eq(runs.companyId, binding.companyId), eq(runs.invocationId, invocationId), ), ) .limit(1); if (existing[0]) return snapshot(await owned(binding, grant, existing[0].sessionId)); const issue = await assertBinding(binding, approved); const { policies } = await budgets.overview(binding.companyId); const caps = policies .filter( (p) => p.isActive && p.hardStopEnabled && p.metric === "billed_cents" && p.amount > 0 && (p.scopeType === "company" || (p.scopeType === "agent" && p.scopeId === binding.agentId) || (p.scopeType === "project" && p.scopeId === issue.projectId)), ) .map((p) => p.remainingAmount / 100); const maxCostUsd = browserUseCostCap( config.maxCostUsd, args.maxCostUsd, ...caps, ); const sid = args.sessionId; const s = await db.transaction(async (tx) => { await notifyDeliveryWork(tx, DELIVERY_QUEUES.browser); let row: Session; if (sid) { await owned(binding, grant, sid); const [updated] = await tx .update(sessions) .set({ status: "starting", nextPollAt: new Date(), idleDeadline: null, updatedAt: new Date(), }) .where( and( eq(sessions.id, sid), eq(sessions.companyId, binding.companyId), eq(sessions.status, "idle"), isNull(sessions.stopRequested), or( isNull(sessions.leaseUntil), lte(sessions.leaseUntil, new Date()), ), ), ) .returning(); if (!updated) throw conflict("This browser conversation is busy or has ended."); row = updated; } else { [row] = await tx .insert(sessions) .values({ ...scope, issueId: binding.issueId!, agentId: binding.agentId!, status: "starting", }) .returning(); } await tx.insert(runs).values({ companyId: binding.companyId, sessionId: row.id, heartbeatRunId: binding.runId!, invocationId, detachedUntil: approved ? new Date(Date.now() + BROWSER_USE_IDLE_MS) : null, }); return row; }); let result: Awaited>; try { result = await client.start({ // The v4 API has no create idempotency key. An opaque marker lets a // read-only recovery scan identify this exact request after a lost reply. task: args.task + requestMarker(invocationId), ...(s.providerSessionId ? { sessionId: s.providerSessionId } : {}), browserSettings: { record: false, // Live-verified v4 option; currently omitted from provider OpenAPI. allowResizing: true, ...(args.profileId ? { profileId: args.profileId } : {}), }, ...(maxCostUsd === undefined ? {} : { maxCostUsd }), }); await db.transaction(async (tx) => { await notifyDeliveryWork(tx, DELIVERY_QUEUES.browser); await tx .update(runs) .set({ providerRunId: result.id, status: result.status }) .where(eq(runs.invocationId, invocationId)); await tx .update(sessions) .set({ providerSessionId: result.sessionId, status: "running", error: null, nextPollAt: new Date(), updatedAt: new Date(), }) .where(eq(sessions.id, s.id)); }); } catch (error) { // A lost create response can mean paid work exists. Never submit it again. const rejected = error instanceof BrowserUseError && error.requestRejected; const message = rejected ? error.message : "Browser run creation could not be confirmed. Paperclip is locating and stopping possible provider work. Do not start another run yet."; await db.transaction(async tx => { await notifyDeliveryWork(tx, DELIVERY_QUEUES.browser); const [current] = await tx.select().from(sessions).where(eq(sessions.id, s.id)).for("update"); // A restart must not leave a drained rejection attached to a starting // session: that combination has neither recovery work nor an idle timer. await tx.update(runs) .set({ status: rejected ? "failed" : "unknown", eventsDrained: rejected ? 1 : 0 }) .where(eq(runs.invocationId, invocationId)); const stopping = !rejected || current.stopRequested === "end"; await tx.update(sessions).set({ status: stopping ? "stopping" : (s.providerSessionId ? "idle" : "failed"), stopRequested: stopping ? "end" : null, idleDeadline: rejected && s.providerSessionId ? new Date(Date.now() + BROWSER_USE_IDLE_MS) : null, error: message, nextPollAt: new Date(), updatedAt: new Date(), }).where(eq(sessions.id, s.id)); }); throw new BrowserUseError( error instanceof BrowserUseError ? error.status : 502, message, ); } // An audit failure must not overwrite a successfully persisted paid run. await signal(s, "browser.started"); return { sessionId: s.id, status: result.status, instruction: "Poll browser_status until terminal. The human's Browser tab opens when the browser is ready.", }; } async function control( s: Session, action: BrowserUseControl, userId?: string, ) { if (action === "keep_open") { const updated = await db .update(sessions) .set({ idleDeadline: new Date(Date.now() + BROWSER_USE_IDLE_MS), updatedAt: new Date(), }) .where( and( eq(sessions.id, s.id), eq(sessions.companyId, s.companyId), eq(sessions.status, "idle"), isNull(sessions.stopRequested), or( isNull(sessions.leaseUntil), lte(sessions.leaseUntil, new Date()), ), ), ) .returning(); if (!updated.length) throw conflict("Only an idle browser can be kept open."); } else { if (terminalSessions.includes(s.status)) return; await db.transaction(async tx => { await notifyDeliveryWork(tx, DELIVERY_QUEUES.browser); await tx .update(sessions) .set({ stopRequested: action, ...(action === "end" ? { status: "stopping" } : {}), nextPollAt: new Date(), updatedAt: new Date(), }) .where( and( eq(sessions.id, s.id), action === "cancel" ? or( isNull(sessions.stopRequested), ne(sessions.stopRequested, "end"), ) : undefined, ), ); }); } await signal( s, `browser.${action}_requested`, userId, userId ? "user" : "system", ); } async function humanGrant( companyId: string, grantId: string, userId: string, cleanup = false, ) { const [grant] = await db .select() .from(connectionGrants) .where( and( eq(connectionGrants.id, grantId), eq(connectionGrants.companyId, companyId), ), ) .limit(1); if (!grant || (!cleanup && grant.status !== "active")) throw forbidden("Browser credential access is unavailable."); const [membership] = await db .select({ id: companyMemberships.id, role: companyMemberships.membershipRole, }) .from(companyMemberships) .where( and( eq(companyMemberships.companyId, companyId), eq(companyMemberships.principalType, "user"), eq(companyMemberships.principalId, userId), eq(companyMemberships.status, "active"), ), ) .limit(1); if ((!membership || membership.role === "viewer") && userId !== "board") throw forbidden("Company membership is no longer active."); if (grant.kind === "agent" && grant.createdByUserId === userId) return grant; if (grant.kind === "user" && grant.subjectUserId === userId) return grant; if (grant.kind !== "organization") throw forbidden("This browser belongs to another credential."); const members = await db .select() .from(connectionGrantMembers) .where( and( eq(connectionGrantMembers.companyId, companyId), eq(connectionGrantMembers.grantId, grantId), ), ); if ( members.length && !members.some((m) => m.subjectType === "user" && m.subjectId === userId) ) throw forbidden("This browser credential is not shared with you."); return grant; } async function list( companyId: string, issueId: string, userId: string, ): Promise { const rows = await db .select({ s: sessions, b: browsers }) .from(sessions) .leftJoin(browsers, eq(browsers.sessionId, sessions.id)) .where( and(eq(sessions.companyId, companyId), eq(sessions.issueId, issueId)), ) .orderBy(sessions.createdAt); const out: TaskBrowser[] = []; for (const { s, b } of rows) { try { await humanGrant(companyId, s.grantId, userId); } catch { continue; } const run = await latest(s); out.push({ id: b?.id ?? s.id, sessionId: s.id, issueId, status: b?.status === "stopped" ? "closed" : !b && !terminalSessions.includes(s.status) && s.status !== "stopping" ? "starting" : (s.status as TaskBrowser["status"]), runStatus: run?.status ?? null, progress: run?.progress ?? null, costCents: run?.accountedCents ?? 0, idleDeadline: s.idleDeadline?.toISOString() ?? null, expiresAt: b?.expiresAt?.toISOString() ?? null, error: s.error, createdAt: (b?.createdAt ?? s.createdAt).toISOString(), }); } return out; } async function humanSession( companyId: string, issueId: string, browserId: string, userId: string, ) { const [b] = await db .select() .from(browsers) .where(and(eq(browsers.id, browserId), eq(browsers.companyId, companyId))) .limit(1); const [s] = await db .select() .from(sessions) .where( and( eq(sessions.id, b?.sessionId ?? browserId), eq(sessions.companyId, companyId), eq(sessions.issueId, issueId), ), ) .limit(1); if (!s) throw notFound("Browser not found."); await humanGrant(companyId, s.grantId, userId); return { s, b }; } async function viewer( companyId: string, issueId: string, browserId: string, userId: string, viewerId?: string, ) { const { s, b } = await humanSession(companyId, issueId, browserId, userId); if ( !b || b.status !== "active" || terminalSessions.includes(s.status) || s.stopRequested === "end" ) throw conflict("This browser is not available. It may have expired."); const { client } = await credentials(s); const result = await client.browser(b.providerBrowserId); if ( result.status !== "active" || result.agentSessionId !== s.providerSessionId ) throw conflict("This browser is no longer active."); const viewportState = await browserUseViewports.current( b.providerBrowserId, `${userId}:${viewerId ?? ""}`, ); return { url: browserUseViewerUrl(result.liveUrl), viewport: viewportState.preset, viewportState, }; } async function presence( companyId: string, issueId: string, browserId: string, userId: string, ) { const { s, b } = await humanSession(companyId, issueId, browserId, userId); if ( !b || b.status !== "active" || !["running", "idle"].includes(s.status) || s.stopRequested === "end" ) throw conflict("This browser is no longer active."); // Presence only renews existing idle browsers. It never starts paid work, // overrides a stop/revocation, or extends the provider's absolute lifetime. if (s.status !== "idle") return { accepted: true }; const now = Date.now(); const deadline = new Date( Math.min(now + BROWSER_USE_IDLE_MS, b.expiresAt?.getTime() ?? Infinity), ); if ( s.idleDeadline && s.idleDeadline.getTime() >= deadline.getTime() - 60_000 ) return { accepted: true }; const updated = await db .update(sessions) .set({ idleDeadline: deadline, updatedAt: new Date(now) }) .where( and( eq(sessions.id, s.id), eq(sessions.companyId, companyId), eq(sessions.status, "idle"), isNull(sessions.stopRequested), // The reconciler holds this lease while deciding whether to close. or( isNull(sessions.leaseUntil), lte(sessions.leaseUntil, new Date(now)), ), ), ) .returning(); if (updated.length) await signal(s, "browser.viewing", userId, "user"); return { accepted: updated.length > 0 }; } async function resize( companyId: string, issueId: string, browserId: string, userId: string, request: BrowserUseViewportRequest, ) { const { s, b } = await humanSession(companyId, issueId, browserId, userId); if ( !b || b.status !== "active" || !["running", "idle"].includes(s.status) || s.stopRequested === "end" ) throw conflict("This browser is not available. It may have expired."); const { client } = await credentials(s); const detail = await client.browser(b.providerBrowserId); if ( detail.status !== "active" || detail.agentSessionId !== s.providerSessionId ) throw conflict("This browser is no longer active."); if (!detail.cdpUrl) throw conflict( "Browser resizing is unavailable. Reconnect the view and try again.", ); const result = await browserUseViewports.resize( b.providerBrowserId, detail.cdpUrl, request, detail.timeoutAt, `${userId}:${request.viewerId ?? ""}`, ); if (result.applied) await logActivity(db, { companyId, actorType: "user", actorId: userId, action: "browser.resized", entityType: "issue", entityId: issueId, details: { browserSessionId: s.id, browserId, preset: result.preset, width: result.width, height: result.height, }, }); return result; } async function releaseViewport( companyId: string, issueId: string, browserId: string, userId: string, viewerId: string, ) { const { b } = await humanSession(companyId, issueId, browserId, userId); if (b) await browserUseViewports.releaseLease( b.providerBrowserId, `${userId}:${viewerId}`, ); return { accepted: true }; } async function discoverBrowsers( s: Session, client: ReturnType, ) { if (!s.providerSessionId) return; for (let page = 1; page <= 100; page++) { const result = await client.browsers(s.providerSessionId, page); for (const item of result.items) { // List is scoped by session server-side. Check full detail too before // registering an instance; never infer ownership from an event URL. const detail = await client.browser(item.id); if (detail.agentSessionId !== s.providerSessionId) continue; if (detail.status === "stopped") await browserUseViewports.release(item.id); const rows = await db .insert(browsers) .values({ companyId: s.companyId, sessionId: s.id, providerBrowserId: item.id, status: detail.status, expiresAt: detail.timeoutAt ? new Date(detail.timeoutAt) : null, }) .onConflictDoNothing() .returning(); if (rows.length) await signal(s, "browser.ready"); await db .update(browsers) .set({ status: detail.status }) .where( and( eq(browsers.companyId, s.companyId), eq(browsers.sessionId, s.id), eq(browsers.providerBrowserId, item.id), ), ); } if (page * 100 >= result.totalItems) break; } } async function account( s: Session, run: typeof runs.$inferSelect, summary: Awaited< ReturnType["summary"]> >, ) { const cents = Math.max(0, Math.round(Number(summary.totalCostUsd) * 100)); if (!Number.isSafeInteger(cents)) throw new BrowserUseError(502, "Browser Use returned invalid usage."); const event = await withAccountingTransaction(db, s.companyId, async (tx, publications) => { const [issue] = await tx .select({ projectId: issues.projectId }) .from(issues) .where(and(eq(issues.id, s.issueId), eq(issues.companyId, s.companyId))); const [current] = await tx .select() .from(runs) .where(eq(runs.id, run.id)) .for("update"); if (cents <= current.accountedCents) return null; const e = await createCostEventInTransaction(tx, s.companyId, { agentId: s.agentId, issueId: s.issueId, projectId: issue?.projectId, heartbeatRunId: run.heartbeatRunId, provider: "browser-use-cloud", biller: "browser-use-cloud", billingType: "metered_api", model: summary.model, costCents: cents - current.accountedCents, occurredAt: new Date(), billingCode: `browser-use-cloud:${run.id}`, idempotencyKey: `browser-use-cloud:${run.id}:${cents}`, }, publications); await createFinanceEventInTransaction(tx, publications, s.companyId, { agentId: e.agentId, issueId: e.issueId, projectId: e.projectId, heartbeatRunId: e.heartbeatRunId, costEventId: e.id, billingCode: e.billingCode, eventKind: "inference_charge", direction: "debit", biller: "browser-use-cloud", provider: "browser-use-cloud", model: e.model, amountCents: e.costCents, currency: "USD", description: "Browser Use agent run", occurredAt: e.occurredAt, }); await tx .update(runs) .set({ accountedCents: cents }) .where(eq(runs.id, run.id)); return e; }); // Re-evaluate on retries too: accounting may have committed immediately // before a crash interrupted budget enforcement. const lastEvent = event ?? ( await db .select() .from(costEvents) .where( and( eq(costEvents.companyId, s.companyId), eq(costEvents.billingCode, `browser-use-cloud:${run.id}`), ), ) .orderBy(desc(costEvents.createdAt)) .limit(1) )[0]; if (lastEvent) await budgets.evaluateCostEvent(lastEvent); } async function reconcile(s: Session) { const { grant, connection, client } = await credentials(s, true); let end = s.stopRequested === "end" || grant.status !== "active" || !connection.enabled || connection.status !== "active" || (s.idleDeadline !== null && s.idleDeadline.getTime() <= Date.now()); const [issue] = await db .select() .from(issues) .where(and(eq(issues.id, s.issueId), eq(issues.companyId, s.companyId))); const [agent] = await db .select() .from(agents) .where(and(eq(agents.id, s.agentId), eq(agents.companyId, s.companyId))); if ( !issue || issue.assigneeAgentId !== s.agentId || issue.status === "cancelled" || agent?.status === "paused" || agent?.status === "terminated" ) end = true; if ( await issueTreeControlService(db).getActivePauseHoldGate( s.companyId, s.issueId, ) ) end = true; if ( await budgets.getInvocationBlock(s.companyId, s.agentId, { issueId: s.issueId, projectId: issue?.projectId, }) ) end = true; const allRuns = await db .select() .from(runs) .where(and(eq(runs.sessionId, s.id), eq(runs.companyId, s.companyId))) .orderBy(desc(runs.createdAt)); if ( allRuns[0] && authorizeSession && !(await authorizeSession(s, allRuns[0])) ) end = true; let active = false; let pendingAccounting = false; let discovered = false; for (const run of allRuns) { if (!run.providerRunId) { if (run.eventsDrained) continue; // A definitive rejection made no paid run. if (run.status === "creating" && Date.now() - run.createdAt.getTime() <= 60_000) { active = true; continue; } end = true; await db.update(sessions).set({ status: "stopping", stopRequested: "end" }) .where(eq(sessions.id, s.id)); let cursor = run.recoveryCursor; for (let page = 0; page < 5 && !run.providerRunId; page++) { const listed = await client.listRuns(s.providerSessionId, cursor); const matches = listed.runs.filter(candidate => candidate.task.endsWith(requestMarker(run.invocationId)) && (!s.providerSessionId || candidate.sessionId === s.providerSessionId)); if (matches.length > 1) throw new BrowserUseError(502, "Browser recovery found ambiguous provider work."); const found = matches[0]; if (found) { await db.transaction(async tx => { await tx.update(runs).set({ providerRunId: found.id, status: found.status, recoveryCursor: null }) .where(eq(runs.id, run.id)); await tx.update(sessions).set({ providerSessionId: found.sessionId }) .where(eq(sessions.id, s.id)); }); run.providerRunId = found.id; run.status = found.status; s.providerSessionId = found.sessionId; break; } if (listed.hasMore && (!listed.nextCursor || listed.nextCursor === cursor)) throw new BrowserUseError(502, "Browser recovery cursor did not advance."); cursor = listed.hasMore ? listed.nextCursor! : null; await db.update(runs).set({ status: "unknown", recoveryCursor: cursor }).where(eq(runs.id, run.id)); if (!listed.hasMore) break; } if (!run.providerRunId) { // Absence from a list cannot prove a timed-out POST was rejected. // Keep cleanup pending and retain its credential for later discovery. active = true; continue; } } const [parent] = await db .select({ status: heartbeatRuns.status }) .from(heartbeatRuns) .where(eq(heartbeatRuns.id, run.heartbeatRunId)); if ( run.id === allRuns[0].id && parent && ["failed", "cancelled", "timed_out"].includes(parent.status) ) end = true; if (browserUseTerminal(run.status) && run.eventsDrained) continue; let state = await client.status(run.providerRunId); if ( run.id === allRuns[0].id && !browserUseTerminal(state.status) && (issue?.status === "done" || (parent?.status !== "running" && (!run.detachedUntil || run.detachedUntil.getTime() <= Date.now()))) ) end = true; if ( (end || s.stopRequested === "cancel") && !browserUseTerminal(state.status) ) { await client.cancel(run.providerRunId); state = await client.status(run.providerRunId); } active ||= !browserUseTerminal(state.status); let after = run.eventCursor; let drained = false; // Bounded pages yield fairly across sessions. Cursor commits on every page. for (let page = 0; page < 5; page++) { const events = await client.events(run.providerRunId, after); const fresh = events.events.filter((e) => e.id > after); if (!discovered && fresh.some((e) => e.type === "browser.ready")) { await discoverBrowsers(s, client); discovered = true; } const cursor = Math.max( after, events.nextAfter ?? 0, ...fresh.map((e) => e.id), ); const progress = fresh.at(-1)?.type ?? run.progress; await db .update(runs) .set({ eventCursor: cursor, progress: progress ? String(sanitizeBrowserUse(progress)).slice(0, 200) : null, }) .where(eq(runs.id, run.id)); if (!events.hasMore) { drained = true; break; } if (cursor <= after) throw new BrowserUseError( 502, "Browser Use event cursor did not advance.", ); after = cursor; } await db .update(runs) .set({ status: state.status }) .where(eq(runs.id, run.id)); pendingAccounting ||= browserUseTerminal(state.status) && !drained; if (browserUseTerminal(state.status) && drained) { const summary = await client.summary(run.providerRunId); await account(s, run, summary); await db .update(runs) .set({ result: sanitizeBrowserUse({ result: summary.result, output: summary.output, }), eventsDrained: 1, }) .where(eq(runs.id, run.id)); await signal(s, "browser.run_finished"); } } if (!discovered) await discoverBrowsers(s, client); if (end) { await db .update(sessions) .set({ status: "stopping", stopRequested: "end" }) .where(eq(sessions.id, s.id)); const live = await db .select() .from(browsers) .where( and(eq(browsers.sessionId, s.id), eq(browsers.status, "active")), ); for (const b of live) { await client.stop(b.providerBrowserId); const confirmed = await client.browser(b.providerBrowserId); if (confirmed.status !== "stopped") throw new BrowserUseError( 502, "Browser shutdown is not confirmed. Paperclip will retry.", ); await db .update(browsers) .set({ status: "stopped" }) .where(eq(browsers.id, b.id)); } // A cancelling run can spawn another browser before exiting; keep sweeping. if (!active && !pendingAccounting) { await db .update(sessions) .set({ status: "closed", error: null, stopRequested: null }) .where(eq(sessions.id, s.id)); await signal(s, "browser.closed"); } } else if ( !active && allRuns.length && allRuns.every((r) => r.providerRunId || r.eventsDrained) ) { await db .update(sessions) .set({ status: "idle", idleDeadline: s.idleDeadline ?? new Date(Date.now() + BROWSER_USE_IDLE_MS), stopRequested: null, error: null, }) .where( and( eq(sessions.id, s.id), ne(sessions.status, "starting"), or( isNull(sessions.stopRequested), eq(sessions.stopRequested, "cancel"), ), eq(sessions.updatedAt, s.updatedAt), ), ); } } // Wait for a concurrent sweep rather than reporting completion while its // lease is still held. A filtered cleanup must still inspect its own scope. let sweeping: Promise | null = null; async function sweep(connectionId?: string): Promise { if (sweeping) { await sweeping; return sweep(connectionId); } sweeping = reconcileDue(connectionId); try { await sweeping; } finally { sweeping = null; } } async function reconcileDue(connectionId?: string) { const due = await db .select() .from(sessions) .where( and( connectionId ? eq(sessions.connectionId, connectionId) : undefined, inArray(sessions.status, liveSessions), lte(sessions.nextPollAt, new Date()), or( isNull(sessions.leaseUntil), lte(sessions.leaseUntil, new Date()), ), ), ) .limit(30); for (const s of due) { const token = randomUUID(); const lease = await db .update(sessions) .set({ leaseToken: token, leaseUntil: new Date(Date.now() + 180_000), }) .where( and( eq(sessions.id, s.id), or( isNull(sessions.leaseUntil), lte(sessions.leaseUntil, new Date()), ), ), ) .returning(); if (!lease.length) continue; // Renew while paginating provider events or stopping multiple browsers. const renewal = setInterval(() => { void db .update(sessions) .set({ leaseUntil: new Date(Date.now() + 180_000) }) .where(and(eq(sessions.id, s.id), eq(sessions.leaseToken, token))) .catch(() => {}); }, 30_000); renewal.unref?.(); let delay = s.status === "idle" ? 15000 : 3000; try { await reconcile(s); } catch (e) { delay = Math.max( 10000, e instanceof BrowserUseError ? e.retryAfterMs : 0, ); await db .update(sessions) .set({ error: s.stopRequested === "end" ? "Browser shutdown is not confirmed. Paperclip will retry." : "Browser synchronization failed. Paperclip will retry.", }) .where(eq(sessions.id, s.id)); } finally { clearInterval(renewal); const release = (tx: Pick) => tx.update(sessions).set({ leaseToken: null, leaseUntil: null, nextPollAt: new Date(Date.now() + delay + Math.floor(Math.random() * 500)), }).where(and(eq(sessions.id, s.id), eq(sessions.leaseToken, token))); if (connectionId) { // Credential-removal sweeps can race the app worker. Wake it when // their lease ends so it does not wait for the old lease expiry. await db.transaction(async tx => { await notifyDeliveryWork(tx, DELIVERY_QUEUES.browser); await release(tx); }); } else { await release(db); } } } } async function nextSweepAt(): Promise { const [row] = await db.select({ // An overdue row owned by another sweep cannot run before its lease ends. at: sql`min(greatest(${sessions.nextPollAt}, ${sessions.leaseUntil}))`, }).from(sessions).where(inArray(sessions.status, liveSessions)); return row?.at == null ? null : new Date(row.at).getTime(); } return { execute, sweep, nextSweepAt, list, viewer, resize, humanGrant, humanSession, releaseViewport, presence, control, getSettings, async stopBeforeCredentialRemoval(companyId: string, connectionId: string) { const scope = and( eq(sessions.companyId, companyId), eq(sessions.connectionId, connectionId), inArray(sessions.status, liveSessions), ); await db.transaction(async tx => { await notifyDeliveryWork(tx, DELIVERY_QUEUES.browser); await tx .update(sessions) .set({ stopRequested: "end", nextPollAt: new Date(), updatedAt: new Date(), }) .where(scope); }); await sweep(connectionId); const [pending] = await db .select({ id: sessions.id }) .from(sessions) .where(scope) .limit(1); if (pending) throw conflict( "Browser access is disabled. Shutdown is still pending; retry removal after the browsers stop. Paperclip retains the key only for cleanup until then.", ); }, async availableProfiles( companyId: string, grantId: string, userId: string, ) { const grant = await humanGrant(companyId, grantId, userId); const { client } = await credentials({ companyId, grantId, connectionId: grant.connectionId, }); const profiles: Array<{ id: string; name: string | null }> = []; for (let page = 1; page <= 100; page++) { const result = await client.profiles(page); profiles.push( ...result.items.map((p) => ({ id: p.id, name: p.name ? String(sanitizeBrowserUse(p.name)) : null, })), ); if (page * 100 >= result.totalItems) break; } return profiles; }, async saveSettings( companyId: string, grantId: string, userId: string, value: BrowserUseSettings, ) { const grant = await humanGrant(companyId, grantId, userId); const managesShared = grant.kind === "organization" && (userId === "board" || (await accessService(db).hasPermission( companyId, "user", userId, "tools:manage_connections", ))); if ( !managesShared && grant.createdByUserId !== userId && grant.subjectUserId !== userId ) throw forbidden( "Only the credential owner or a shared connection manager can change browser settings.", ); const { client } = await credentials({ companyId, grantId, connectionId: grant.connectionId, }); if (value.allowedProfileIds.length) { const available = new Set(); for (let page = 1; page <= 100; page++) { const list = await client.profiles(page); for (const p of list.items) available.add(p.id); if (page * 100 >= list.totalItems) break; } if (value.allowedProfileIds.some((id) => !available.has(id))) throw forbidden( "A selected profile is not available to this credential.", ); } const row = { companyId, grantId, allowedProfileIds: value.allowedProfileIds, maxCostUsd: value.maxCostUsd?.toString() ?? null, }; await db .insert(settings) .values(row) .onConflictDoUpdate({ target: settings.grantId, set: row }); await logActivity(db, { companyId, actorType: "user", actorId: userId, action: "browser.settings_updated", entityType: "tool_connection", entityId: grant.connectionId, details: { grantId }, }); return value; }, }; }