diff --git a/server/src/__tests__/ai-connections.test.ts b/server/src/__tests__/ai-connections.test.ts index c72e1dff56b..ae747a42c69 100644 --- a/server/src/__tests__/ai-connections.test.ts +++ b/server/src/__tests__/ai-connections.test.ts @@ -17,7 +17,7 @@ import { mkdtemp, realpath, rm, access, readFile, writeFile, stat } from "node:f import os from "node:os"; import path from "node:path"; import { and, eq, sql } from "drizzle-orm"; -import { createDb, type Db, companies, agents, agentTaskSessions, heartbeatRuns, companyMemberships, connectionGrants, toolApplications, connectionGrantDelegations, connectionGrantMembers, toolConnections, toolConnectionInstalls, aiConnectionDefaults, aiProviderDefaults, adapterAuthSessions, environments, plugins, issues, issueThreadInteractions, issueRecoveryActions, connectionIntentDeliveries, agentWakeupRequests, companySecrets, companySecretVersions, userSecretDefinitions, principalPermissionGrants } from "@paperclipai/db"; +import { createDb, type Db, companies, agents, agentTaskSessions, heartbeatRuns, companyMemberships, connectionGrants, toolApplications, connectionGrantDelegations, connectionGrantMembers, toolConnections, toolConnectionInstalls, aiConnectionDefaults, aiProviderDefaults, adapterAuthSessions, environments, plugins, issues, issueThreadInteractions, issueRecoveryActions, connectionIntentDeliveries, agentWakeupRequests, companySecrets, companySecretVersions, secretAccessEvents, userSecretDefinitions, principalPermissionGrants } from "@paperclipai/db"; import { startEmbeddedPostgresTestDatabase } from "@paperclipai/db/test-embedded-postgres"; import { fetchCompanyQuotaWindows } from "../services/quota-windows.js"; import { quotaCredentialRecoveryPath, quotaCredentialRecovery, quotaCredentialHash } from "../services/quota-credential-recovery.js"; @@ -717,8 +717,10 @@ describe("managed AI connections", () => { try { const first = await fetchCompanyQuotaWindows(db, companyId, owner); const [after] = await db.select().from(companySecrets).where(eq(companySecrets.id, secretId)); - expect(after.lastResolvedAt).not.toBeNull(); - expect(after.updatedAt.getTime()).toBeGreaterThan(before.updatedAt.getTime()); + // A read is recorded as an access event, never written to the row that + // admissions share-lock to fence rotation. + expect(after.updatedAt.getTime()).toBe(before.updatedAt.getTime()); + expect(await db.select().from(secretAccessEvents).where(and(eq(secretAccessEvents.secretId, secretId), eq(secretAccessEvents.outcome, "success")))).not.toHaveLength(0); expect(first[0].ok).toBe(true); expect(await fetchCompanyQuotaWindows(db, companyId, owner)).toEqual(first); expect(fetchSpy).toHaveBeenCalledTimes(1); diff --git a/server/src/__tests__/antigravity-ai-runtime.test.ts b/server/src/__tests__/antigravity-ai-runtime.test.ts index 7c81ddc1926..3d766eaa66f 100644 --- a/server/src/__tests__/antigravity-ai-runtime.test.ts +++ b/server/src/__tests__/antigravity-ai-runtime.test.ts @@ -35,7 +35,7 @@ describe("Antigravity private managed AI home", () => { const version = [{ epoch: 1, version: 1 }]; const db = { select: (fields: Record) => ({ from: () => ({ where: () => Object.assign(Promise.resolve("refs" in fields ? [{ refs: account.refs, status: "active" }] : version), - { limit: async () => version }), + { limit: async () => version, for: async () => version }), }) }) } as unknown as Db; const input = { companyId: "44444444-4444-4444-8444-444444444444", agentId: "55555555-5555-4555-8555-555555555555", responsibleUserId: "alice", diff --git a/server/src/__tests__/antigravity-managed-subscription.test.ts b/server/src/__tests__/antigravity-managed-subscription.test.ts index 0a60321d15e..3da53671f3c 100644 --- a/server/src/__tests__/antigravity-managed-subscription.test.ts +++ b/server/src/__tests__/antigravity-managed-subscription.test.ts @@ -123,7 +123,8 @@ describe("managed Antigravity subscription projection and writeback", () => { await expect(access(join(provider, "auth.json"))).rejects.toMatchObject({ code: "ENOENT" }); await writeFile(token, credential("resolved-project")); await runtime.cleanup(); - expect(account.locks).toEqual(["grant:update", "secret:update"]); + // Preparation reads the secret version under a share lock before and after resolving it. + expect(account.locks).toEqual(["secret:share", "secret:share", "grant:update", "secret:update"]); expect(account.rotation).toHaveBeenCalledWith(account.refs[0].secretId, { value: credential("resolved-project"), preserveAiSessionEpoch: true }, { userId: "alice" }); await expect(access(runtime.home!)).rejects.toMatchObject({ code: "ENOENT" }); @@ -216,7 +217,7 @@ describe("managed Antigravity subscription projection and writeback", () => { } else { await expect(runtime.cleanup()).rejects.toThrow("The refreshed Google account does not match the selected connection."); expect(account.rotation).not.toHaveBeenCalled(); - expect(account.locks).toEqual([]); + expect(account.locks).toEqual(["secret:share", "secret:share"]); expect(account.journal).toMatchObject({ provider: "google", google: { verified: false }, value: credential("resolved-project", "fixture-rotated-refresh") }); expect(await readFile(token, "utf8")).toBe(credential("resolved-project", "fixture-rotated-refresh")); } diff --git a/server/src/__tests__/decision-models.test.ts b/server/src/__tests__/decision-models.test.ts index 92299eb18b6..a5890fdb8b8 100644 --- a/server/src/__tests__/decision-models.test.ts +++ b/server/src/__tests__/decision-models.test.ts @@ -405,6 +405,54 @@ describe("company decision service", () => { expect((await first).status).toBe("succeeded"); expect((await second).status).toBe("succeeded"); expect(await db.select().from(costEvents).where(eq(costEvents.companyId, f.companyId))).toHaveLength(2); }); + it("keeps an admission's budget observation when another decision reads the same credential mid-admission", async () => { + const f = await fixture(); + await db.insert(budgetPolicies).values({ companyId: f.companyId, scopeType: "company", scopeId: f.companyId, + windowKind: "calendar_month_utc", amount: 1, reservationCents: "1", notifyEnabled: false }); + let unlock!: () => void, locked!: (pid: number) => void, finishProviders!: () => void; + const lockGate = new Promise(resolve => { unlock = resolve; }); + const acquired = new Promise(resolve => { locked = resolve; }); + const providerGate = new Promise(resolve => { finishProviders = resolve; }); + const holding = db.transaction(async tx => { + await tx.select().from(companyDecisionModels).where(eq(companyDecisionModels.companyId, f.companyId)).for("no key update"); + const [row] = await tx.execute<{ pid: number }>(sql`select pg_backend_pid() as pid`); + locked(row.pid); await lockGate; + }); + const pid = await acquired; + const waitingAdmissions = (count: number) => vi.waitFor(async () => { + const [row] = await db.execute<{ count: number }>(sql`select count(*)::int as count from pg_stat_activity where ${pid} = any(pg_blocking_pids(pid))`); + expect(row.count).toBe(count); + }, { timeout: 10_000 }); + let providersEntered = 0, bothEntered!: () => void; + const bothProvidersEntered = new Promise(resolve => { bothEntered = resolve; }); + f.provider.mockImplementation(async () => { + if (++providersEntered === 2) bothEntered(); + await providerGate; return structuredClone(outcome); + }); + // Order the two admissions so the second one's credential read happens + // after the first admission took its budget observation. Reading a + // credential must not invalidate that observation. + const first = f.service.decide(f.context, DECISION_TEST_REQUEST); + let second: ReturnType | undefined; + try { + await waitingAdmissions(1); + second = f.service.decide(f.context, DECISION_TEST_REQUEST); + await waitingAdmissions(2); + // An obligation committed after both observations. An admission that + // keeps its observation ignores it; one that restarts is denied by it. + await db.insert(budgetReservations).values({ companyId: f.companyId, decisionInvocationId: randomUUID(), amountCents: "1" }); + unlock(); await holding; + // Before the providers finish, a decision settles only if admission denied it. + const settledEarly = (name: string, pending: typeof first) => pending.then(result => `${name} settled before its provider call: ${JSON.stringify(result)}`); + expect(await Promise.race([bothProvidersEntered.then(() => "both admitted"), settledEarly("first", first), settledEarly("second", second)])).toBe("both admitted"); + expect(f.provider).toHaveBeenCalledTimes(2); + const holds = await db.select().from(budgetReservations).where(and(eq(budgetReservations.companyId, f.companyId), sql`${budgetReservations.decisionInvocationId} in (select id from decision_invocations)`)); + expect(holds).toHaveLength(2); + expect(holds.every(row => row.state === "held" && row.amountCents === "1.0000000")).toBe(true); + } finally { unlock(); finishProviders(); await holding; await Promise.allSettled([first, second]); } + expect((await first).status).toBe("succeeded"); expect((await second!).status).toBe("succeeded"); + expect(await db.select().from(costEvents).where(eq(costEvents.companyId, f.companyId))).toHaveLength(2); + }); it("denies new work when an already committed reservation consumes the observed budget", async () => { const f = await fixture(); await db.insert(budgetPolicies).values({ companyId: f.companyId, scopeType: "company", scopeId: f.companyId, windowKind: "calendar_month_utc", amount: 1, reservationCents: "1", notifyEnabled: false }); diff --git a/server/src/__tests__/secrets-service.test.ts b/server/src/__tests__/secrets-service.test.ts index 7af2791881e..77125a82c0e 100644 --- a/server/src/__tests__/secrets-service.test.ts +++ b/server/src/__tests__/secrets-service.test.ts @@ -2660,6 +2660,37 @@ describeEmbeddedPostgres("secretService", () => { ).rejects.toThrow(/not bound/i); }); + it("records a read as \"Last used\" without writing the secret row that admissions lock", async () => { + const companyId = await seedCompany(); + const svc = secretService(db); + const secret = await svc.create(companyId, { + name: `last-used-${randomUUID()}`, + provider: "local_encrypted", + value: "runtime-secret", + }); + await svc.createBinding({ companyId, secretId: secret.id, targetType: "system", targetId: "system", configPath: "env.API_KEY" }); + const [before] = await db.select().from(companySecrets).where(eq(companySecrets.id, secret.id)); + expect((await svc.list(companyId)).find((row) => row.id === secret.id)?.lastResolvedAt).toBeNull(); + + await svc.resolveSecretValue(companyId, secret.id, "latest", { consumerType: "system", consumerId: "system", configPath: "env.API_KEY" }); + + // The row keeps its version: a concurrent REPEATABLE READ admission that + // share-locks it to fence rotation is not aborted by this read. + const [after] = await db.select().from(companySecrets).where(eq(companySecrets.id, secret.id)); + expect(after).toEqual(before); + const listed = (await svc.list(companyId)).find((row) => row.id === secret.id); + expect(listed?.lastResolvedAt).toBeInstanceOf(Date); + expect(listed!.lastResolvedAt!.getTime()).toBeGreaterThanOrEqual(before!.createdAt.getTime()); + + // A read made without a consumer context also counts as use. + const unattributed = await svc.create(companyId, { name: `last-used-unattributed-${randomUUID()}`, provider: "local_encrypted", value: "runtime-secret" }); + const [unattributedBefore] = await db.select().from(companySecrets).where(eq(companySecrets.id, unattributed.id)); + await svc.resolveSecretValue(companyId, unattributed.id, "latest"); + const [unattributedAfter] = await db.select().from(companySecrets).where(eq(companySecrets.id, unattributed.id)); + expect(unattributedAfter).toEqual(unattributedBefore); + expect((await svc.list(companyId)).find((row) => row.id === unattributed.id)?.lastResolvedAt).toBeInstanceOf(Date); + }); + it("preserves the original resolution error when failure access logging fails", async () => { const companyId = await seedCompany(); const svc = secretService(db); diff --git a/server/src/services/ai-connection-runtime.ts b/server/src/services/ai-connection-runtime.ts index 03ccb040d50..9ac8f911e05 100644 --- a/server/src/services/ai-connection-runtime.ts +++ b/server/src/services/ai-connection-runtime.ts @@ -284,8 +284,11 @@ export async function prepareManagedAiRuntime( .from(connectionGrants).where(and(eq(connectionGrants.companyId, input.companyId), eq(connectionGrants.id, selection.grant.id))); const currentRef = currentGrant?.refs.find(ref => ref.configPath === "ai.credential"); if (currentGrant?.status !== "active" || currentRef?.secretId !== credentialRef.secretId || currentRef.versionSelector !== credentialRef.versionSelector) return undefined; + // The share lock waits behind a rotating writer, so the read after + // resolution sees a concurrent rotation instead of the pre-rotation + // version. A lock takes no new row version and aborts no admission. return (await db.select({ epoch: companySecrets.aiSessionEpoch, version: companySecrets.latestVersion }) - .from(companySecrets).where(and(eq(companySecrets.companyId, input.companyId), eq(companySecrets.id, credentialRef.secretId))).limit(1))[0]; + .from(companySecrets).where(and(eq(companySecrets.companyId, input.companyId), eq(companySecrets.id, credentialRef.secretId))).for("share"))[0]; }; const { value, freshness } = await (async () => { if (noAuth) return { value: "", freshness: undefined }; diff --git a/server/src/services/ai-connections.ts b/server/src/services/ai-connections.ts index 25b812ffc3f..5805ea231c4 100644 --- a/server/src/services/ai-connections.ts +++ b/server/src/services/ai-connections.ts @@ -615,11 +615,12 @@ export function aiConnectionService(db: Db) { } else { value = await secrets.resolveSecretValue(row.connection.companyId, secret.id, "latest", context); } - // Resolution records lastResolvedAt and can wait behind a rotating writer. - // Re-read after that wait instead of handing a run the pre-rotation token. + // Wait behind a rotating writer (the share lock), then re-read instead of + // handing a run the pre-rotation token. A lock takes no new row version, + // so it never aborts a concurrent REPEATABLE READ admission. const [latest] = await db.select().from(companySecrets).where(and( eq(companySecrets.id, secret.id), eq(companySecrets.companyId, row.connection.companyId), - )); + )).for("share"); if (!latest || latest.status !== "active") throw unprocessable("Reconnect this AI account"); if (latest.latestVersion !== secret.latestVersion) { if (retry >= 2) throw unprocessable("AI credentials are changing. Retry this execution.", { code: "ai_connection_busy" }); diff --git a/server/src/services/secrets.ts b/server/src/services/secrets.ts index 65f88cd18c2..6c0d8d44e59 100644 --- a/server/src/services/secrets.ts +++ b/server/src/services/secrets.ts @@ -88,6 +88,12 @@ import { withAccountHomeSecretMutationLock, } from "@paperclipai/adapter-codex-local/server"; +function latestDate(a: Date | null, b: Date | null): Date | null { + if (!a) return b; + if (!b) return a; + return a.getTime() >= b.getTime() ? a : b; +} + const ENV_KEY_RE = /^[A-Za-z_][A-Za-z0-9_]*$/; const AGENT_ACCESS_CONFIG_PATH_PREFIX = "access."; // System consumer id for a durable orphan-sandbox teardown. The cleanup sweep @@ -642,6 +648,8 @@ type SecretConsumerContext = { allowedBindingIds?: string[] | null; }; +const UNATTRIBUTED_SECRET_READ: SecretConsumerContext = { consumerType: "system", consumerId: "system", actorType: "system" }; + type SecretBindingContext = Omit & { consumerType: SecretBindingTargetType; }; @@ -1365,6 +1373,19 @@ export function secretService(db: Db | DbTransaction) { }; } + /** Newest successful resolution per secret, read backwards on (secret_id, created_at). */ + async function lastSuccessfulResolutions(secretIds: string[]): Promise> { + if (secretIds.length === 0) return new Map(); + const rows = await db.execute<{ secret_id: string; used_at: string | Date | null }>(sql` + select ids.secret_id, ( + select e.created_at from ${secretAccessEvents} e + where e.secret_id = ids.secret_id and e.outcome = 'success' + order by e.created_at desc limit 1 + ) as used_at + from unnest(array[${sql.join(secretIds.map((id) => sql`${id}`), sql`, `)}]::uuid[]) as ids(secret_id)`); + return new Map(rows.flatMap((row) => row.used_at ? [[row.secret_id, new Date(row.used_at)] as const] : [])); + } + async function resolveSecretValueInternal( companyId: string, secretId: string, @@ -1428,12 +1449,12 @@ export function secretService(db: Db | DbTransaction) { if (isTailscaleNodeState(value) && systemPurpose !== "tailscale_network") { throw forbidden("Network node state requires its server-only consumer", { code: "secret_system_only" }); } + // The access event is the usage record; "Last used" is derived from it + // in list(). A read must not write the secret row: admission paths + // (decisions, fast responses, browser use, prompt-injection checks) + // share-lock it under REPEATABLE READ to fence rotation, and a new row + // version would abort them with a serialization failure. if (!readOnlyEnvironmentInspection) await Promise.all([ - db - .update(companySecrets) - .set({ lastResolvedAt: new Date(), updatedAt: new Date() }) - .where(eq(companySecrets.id, secret.id)) - .catch(() => undefined), recordAccessEvent({ companyId, secretId: secret.id, @@ -1441,7 +1462,8 @@ export function secretService(db: Db | DbTransaction) { secretScope: secret.scope, version: resolvedVersion, provider: providerId, - context: accessContext, + // A read without a consumer still counts as "Last used". + context: accessContext ?? UNATTRIBUTED_SECRET_READ, credentialOwnerUserId: secret.ownerUserId ?? null, credentialSubjectType: secret.scope === "user" ? "user" : null, credentialSubjectId: secret.ownerUserId ?? null, @@ -3873,8 +3895,10 @@ export function secretService(db: Db | DbTransaction) { .groupBy(companySecretBindings.secretId), ]); const countsBySecretId = new Map(referenceCounts.map((row) => [row.secretId, row.count])); + const lastUsedBySecretId = await lastSuccessfulResolutions(secrets.map((secret) => secret.id)); return secrets.map((secret) => ({ ...secret, + lastResolvedAt: latestDate(secret.lastResolvedAt, lastUsedBySecretId.get(secret.id) ?? null), referenceCount: countsBySecretId.get(secret.id) ?? 0, })); },