diff --git a/docs/deploy/secrets.md b/docs/deploy/secrets.md index 38230795789..73a32aeaf17 100644 --- a/docs/deploy/secrets.md +++ b/docs/deploy/secrets.md @@ -27,6 +27,16 @@ you bind to an agent as exposed to that agent. Limit blast radius with bindings provider supports them, and rotation when an agent transcript or downstream system might have captured a value. +Paperclip masks the resolved value of each secret-bound environment variable in +the run output that it stores: the run log, the run excerpts, and the run +transcript events. This applies to local and native runs. The mask is +`***REDACTED***`. Paperclip masks only exact text: the full value, the value +without outer whitespace, its JSON-escaped form, and each line of a multi-line +value. Paperclip does not mask a value or line that has fewer than 8 +characters. This mask prevents accidental leaks. It is not a security boundary: +an agent can change a value (for example, encode it in base64) before it prints +it. + ## Using Secrets In Runs Creating a company secret does not automatically create an environment variable. diff --git a/server/src/__tests__/run-output-secret-env-redaction.test.ts b/server/src/__tests__/run-output-secret-env-redaction.test.ts new file mode 100644 index 00000000000..2afe2f13c32 --- /dev/null +++ b/server/src/__tests__/run-output-secret-env-redaction.test.ts @@ -0,0 +1,94 @@ +import { createHash, randomUUID } from "node:crypto"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { eq } from "drizzle-orm"; +import { agents, companies, createDb, type Db } from "@paperclipai/db"; +import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { drainHeartbeatRunsToQuiescence } from "./helpers/drain-heartbeat-runs.js"; +import { heartbeatService } from "../services/heartbeat.js"; +import { secretService } from "../services/secrets.js"; + +describe("secret-bound environment values in a local run", () => { + let db: Db; + let database: Awaited>; + let directory: string; + let companyId: string; + + beforeAll(async () => { + directory = await mkdtemp(join(tmpdir(), "paperclip-secret-env-output-")); + vi.stubEnv("PAPERCLIP_HOME", directory); + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", ""); + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY_FILE", join(directory, "master.key")); + database = await startEmbeddedPostgresTestDatabase("paperclip-secret-env-output-"); + db = createDb(database.connectionString); + companyId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Secret output test", issuePrefix: "SEO", + defaultResponsibleUserId: "secret-output-owner" }); + }); + + afterAll(async () => { + await database?.cleanup(); + vi.unstubAllEnvs(); + await rm(directory, { recursive: true, force: true }); + }); + + it("masks the latest resolved secret value in each run's stored log and run row and leaves plain and short values", async () => { + const [agent] = await db.insert(agents).values({ companyId, name: randomUUID(), role: "engineer" }).returning(); + const svc = secretService(db); + const secretValue = `sk_test_synthetic_${randomUUID()}`; + const shortValue = "pin42"; + const plainValue = "https://plain.example.test/api"; + const [secret, shortSecret] = await Promise.all([ + svc.create(companyId, { name: `stripe-${randomUUID()}`, provider: "local_encrypted", value: secretValue }), + svc.create(companyId, { name: `pin-${randomUUID()}`, provider: "local_encrypted", value: shortValue }), + ]); + const env = { + STRIPE_KEY: { type: "secret_ref", secretId: secret.id, version: "latest" }, + SHORT_PIN: { type: "secret_ref", secretId: shortSecret.id, version: "latest" }, + PLAIN_URL: plainValue, + }; + await svc.syncEnvBindingsForTarget(companyId, { targetType: "agent", targetId: agent!.id }, env); + await db.update(agents).set({ adapterType: "process", adapterConfig: { + command: process.execPath, cwd: directory, + args: ["-e", [ + "const v=process.env.STRIPE_KEY;", + "process.stdout.write('STRIPE_KEY='+v.slice(0,12));", + "setTimeout(()=>{process.stdout.write(v.slice(12)+'\\n');", + "console.log(JSON.stringify({env:{STRIPE_KEY:v}}));", + "console.log('SHORT_PIN='+process.env.SHORT_PIN);", + "console.log('PLAIN_URL='+process.env.PLAIN_URL);", + "console.log('DIGEST='+require('node:crypto').createHash('sha256').update(v).digest('hex'));", + "console.error('stderr '+v);},30);", + ].join("")], + env, + } }).where(eq(agents.id, agent!.id)); + const heartbeat = heartbeatService(db); + try { + for (const value of [secretValue, `sk_test_rotated_${randomUUID()}`]) { + if (value !== secretValue) await svc.rotate(secret.id, { value }, { userId: "secret-output-owner" }); + const run = await heartbeat.invoke(agent!.id, "on_demand", {}, "manual"); + await drainHeartbeatRunsToQuiescence(db, heartbeat); + const finished = await heartbeat.getRun(run!.id); + expect(finished?.status, JSON.stringify(finished)).toBe("succeeded"); + const log = (await heartbeat.readLog(run!.id)).content; + const events = await heartbeat.listEvents(run!.id); + for (const stored of [log, JSON.stringify(finished), JSON.stringify(events)]) { + expect(stored).not.toContain(value); + expect(stored).not.toContain(value.slice(12)); + } + const stdout = log.trim().split("\n").map(line => JSON.parse(line) as { stream: string; chunk: string }) + .filter(line => line.stream === "stdout").map(line => line.chunk).join(""); + expect(stdout).toContain("STRIPE_KEY=***REDACTED***\n"); + expect(stdout).toContain(JSON.stringify({ env: { STRIPE_KEY: "***REDACTED***" } })); + expect(stdout).toContain(`SHORT_PIN=${shortValue}`); + expect(stdout).toContain(`PLAIN_URL=${plainValue}`); + expect(stdout).toContain(`DIGEST=${createHash("sha256").update(value).digest("hex")}`); + expect(finished?.stderrExcerpt).toContain("stderr ***REDACTED***"); + } + } finally { + await drainHeartbeatRunsToQuiescence(db, heartbeat); + } + }, 60_000); +}); diff --git a/server/src/modules/wake-queue/adapters/postgres.test.ts b/server/src/modules/wake-queue/adapters/postgres.test.ts index aac74f1f3bc..09976ee2e64 100644 --- a/server/src/modules/wake-queue/adapters/postgres.test.ts +++ b/server/src/modules/wake-queue/adapters/postgres.test.ts @@ -1557,6 +1557,25 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => { } finally { unsubscribe(); } }); + it("keeps the run's stored redaction history when a coalesced wake replaces its context", async () => { + const companyId = await seedCompany(); + const agentId = await seedAgent({ companyId }); + const history = { + paperclipSecretRedactions: [{ fingerprintSha256: "a".repeat(64), material: { scheme: "synthetic" } }], + paperclipOutputSecretRedactions: [{ fingerprintSha256: "b".repeat(64), material: { scheme: "synthetic" } }], + }; + const runId = await seedRun({ companyId, agentId, status: "running", contextSnapshot: { taskKey: "issue-1", ...history } }); + await db.transaction(async (tx) => createWakeAdmissionWriter().coalesceIntoActiveExecutionRun( + createAdmissionTransactionScope(companyId, tx as unknown as Db), { + companyId, agentId, activeExecutionRunId: runId, activeExecutionRunStatus: "running", + mergedContextSnapshot: { taskKey: "issue-1", commentId: "fresh-input" }, + source: "on_demand", triggerDetail: null, payload: null, + requestedByActorType: "user", requestedByActorId: null, idempotencyKey: null, + })); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)))[0].contextSnapshot) + .toEqual({ taskKey: "issue-1", commentId: "fresh-input", ...history }); + }); + // Review test (b), the coalesce write: a foreign company affects no row // on the active execution run either. it("refuses to coalesce into a run for a company that does not own it, and leaves the run and the wake table untouched", async () => { diff --git a/server/src/modules/wake-queue/adapters/postgres.ts b/server/src/modules/wake-queue/adapters/postgres.ts index 4b4e05ad30d..7e92e667918 100644 --- a/server/src/modules/wake-queue/adapters/postgres.ts +++ b/server/src/modules/wake-queue/adapters/postgres.ts @@ -10,6 +10,7 @@ import { notifyChatRunMilestones } from "../../../services/chat-run-milestone-no import { DELIVERY_QUEUES, notifyDeliveryWork } from "../../../services/delivery-work-notifications.js"; import { isAcknowledgedNativeReassignmentStop, isAcknowledgedNativeStop } from "../../../services/acknowledged-native-stop.js"; import { readRunCancellation } from "../../../services/run-cancellation.js"; +import { withRunRedactionHistory } from "../../../services/run-secret-redaction.js"; import { isCompletedOnboardingHandoffWake } from "../../../services/chat-completion-delivery.js"; import { instanceSettingsService } from "../../../services/instance-settings.js"; import { currentConversationCommentCondition } from "../../../services/agent-conversations.js"; @@ -1108,7 +1109,7 @@ export function createWakeAdmissionWriter(): WakeAdmissionWriter { eq(agentWakeupRequests.agentId, input.agentId), eq(agentWakeupRequests.runId, current.id), eq(agentWakeupRequests.status, primary.status))); const mergedRun = await tx .update(heartbeatRuns) - .set({ contextSnapshot: input.mergedContextSnapshot, updatedAt: now }) + .set({ contextSnapshot: withRunRedactionHistory(input.mergedContextSnapshot, current.contextSnapshot), updatedAt: now }) .where( and( eq(heartbeatRuns.id, input.activeExecutionRunId), diff --git a/server/src/services/agent-identity-redaction.test.ts b/server/src/services/agent-identity-redaction.test.ts new file mode 100644 index 00000000000..10b84e87a39 --- /dev/null +++ b/server/src/services/agent-identity-redaction.test.ts @@ -0,0 +1,138 @@ +import { describe, expect, it } from "vitest"; +import { createAgentIdentityRedactor, secretEnvironmentValues, secretOutputForms } from "./agent-identity-redaction.js"; + +const REDACTED = "***REDACTED***"; + +function streamed(redactor: ReturnType, chunks: string[]) { + return chunks.map((chunk) => redactor.chunk("stdout", chunk)).join("") + redactor.finish("stdout"); +} + +describe("secret environment values", () => { + it("selects only secret-bound values and leaves plain values alone", () => { + expect(secretEnvironmentValues({ + STRIPE_KEY: "sk_test_synthetic_0123456789", + PLAIN_URL: "https://plain.example.test/api", + EMPTY: "", + NUMBER: 12345678, + }, new Set(["STRIPE_KEY", "EMPTY", "NUMBER", "MISSING"]))).toEqual(["sk_test_synthetic_0123456789"]); + expect(secretEnvironmentValues(null, ["ANY"])).toEqual([]); + }); + + it("skips forms shorter than the minimum length and whitespace-only values", () => { + expect(secretOutputForms(["abc1234", "x", " ", " abc "])).toEqual([]); + expect(secretOutputForms(["abcd1234"])).toEqual(["abcd1234"]); + }); + + it("adds trimmed, JSON-escaped, and per-line forms of a multi-line value", () => { + const value = "-----BEGIN SYNTHETIC KEY-----\nline-one-synthetic\r\nline-two \"quoted\"\n-----END SYNTHETIC KEY-----\n"; + const forms = secretOutputForms([value]); + expect(forms).toContain(value); + expect(forms).toContain(value.trim()); + expect(forms).toContain(JSON.stringify(value).slice(1, -1)); + expect(forms).toContain("line-one-synthetic"); + expect(forms).toContain("line-two \"quoted\""); + expect(forms).toContain("line-two \\\"quoted\\\""); + expect(forms).not.toContain("-----BEGIN SYNTHETIC KEY-----"); + expect(forms).toEqual([...forms].sort((a, b) => b.length - a.length)); + }); +}); + +describe("agent output redactor with secret values", () => { + it("masks a value that contains another secret without leaking either", () => { + const outer = "outer-synthetic-secret-inner-value"; + const inner = "inner-value"; + const redactor = createAgentIdentityRedactor(undefined, [outer, inner]); + expect(redactor.redact(`env: OUTER=${outer} INNER=${inner}`)).toBe(`env: OUTER=${REDACTED} INNER=${REDACTED}`); + }); + + it("holds a possible longer secret before it masks a shorter secret at the same position", () => { + const redactor = createAgentIdentityRedactor(undefined, ["abcdefgh", "abcdefgh1234567890"]); + expect(streamed(redactor, ["KEY=abcdefgh", "1234567890\n"])).toBe(`KEY=${REDACTED}\n`); + expect(streamed(redactor, ["KEY=abcdefgh", "-other\n"])).toBe(`KEY=${REDACTED}-other\n`); + expect(streamed(redactor, ["A=abcdefgh B=abcdefgh123", "4567890 C=abcdefgh"])).toBe(`A=${REDACTED} B=${REDACTED} C=${REDACTED}`); + }); + + it("masks a secret split across streamed chunks and keeps plain output", () => { + const secret = "sk_test_synthetic_streamed_value"; + const redactor = createAgentIdentityRedactor(undefined, [secret]); + expect(streamed(redactor, [`PLAIN=visible KEY=${secret.slice(0, 10)}`, `${secret.slice(10)}\ndone\n`])) + .toBe(`PLAIN=visible KEY=${REDACTED}\ndone\n`); + for (let split = 1; split < secret.length; split++) { + expect(streamed(redactor, [`x${secret.slice(0, split)}`, `${secret.slice(split)}y`])).toBe(`x${REDACTED}y`); + } + }); + + it("masks each line when a multi-line value is printed without its line breaks", () => { + const value = "first-synthetic-line\nsecond-synthetic-line"; + const redactor = createAgentIdentityRedactor(undefined, [value]); + expect(redactor.redact("echo: first-synthetic-line second-synthetic-line")).toBe(`echo: ${REDACTED} ${REDACTED}`); + expect(redactor.redact(JSON.stringify({ output: value }))).toBe(JSON.stringify({ output: REDACTED })); + }); + + it("keeps protocol fields unchanged when a secret equals a protocol word", () => { + const redactor = createAgentIdentityRedactor(undefined, ["completed", "succeeded", "commandExecution", "item-secret-id"]); + const event = { + schema: "paperclip.prp.event.v1", eventType: "turn.completed", itemId: "item-secret-id", + payload: { kind: "commandExecution", status: "completed", text: "the command completed", output: "commandExecution done" }, + terminal: { schema: "paperclip.prp.terminal.v1", turnTerminalState: "completed", runTerminalState: "succeeded" }, + }; + expect(redactor.redact(event)).toEqual({ + ...event, + payload: { kind: "commandExecution", status: "completed", text: `the command ${REDACTED}`, output: `${REDACTED} done` }, + }); + }); + + it("keeps delta protocol fields while it masks streamed output", () => { + const redactor = createAgentIdentityRedactor(undefined, ["commandExecution"]); + const first = redactor.delta("turn:item", { itemId: "item", kind: "commandExecution", text: "run commandExec" }, "item"); + expect(first).toEqual({ itemId: "item", kind: "commandExecution", text: "run " }); + const settled = redactor.settleDeltas("turn:item", { itemId: "item" }, false); + expect(settled).toEqual({ itemId: "item", outputTails: [{ itemId: "item", payload: { itemId: "item", kind: "commandExecution", text: REDACTED } }] }); + }); + + it("masks the rest of a secret that a new redactor receives after a restart", () => { + const secret = "abcdefgh1234567890"; + const before = createAgentIdentityRedactor(undefined, [secret], { maskResumedSuffixes: true }); + expect(before.chunk("item", "echo abcdefgh")).toBe("echo "); + const after = createAgentIdentityRedactor(undefined, [secret], { maskResumedSuffixes: true }); + expect(after.chunk("item", "1234567890 done") + after.finish("item")).toBe(`${REDACTED} done`); + expect(after.chunk("item", " 1234567890")).toBe(" 1234567890"); + const inside = createAgentIdentityRedactor(undefined, [secret], { maskResumedSuffixes: true }); + expect(inside.chunk("item", "3456") + inside.chunk("item", "7890 next") + inside.finish("item")).toBe(`${REDACTED}${REDACTED} next`); + const pem = createAgentIdentityRedactor(undefined, ["-----BEGIN SYNTHETIC-----\nbody-line-synthetic-0123456789\n-----END SYNTHETIC-----"], + { maskResumedSuffixes: true }); + expect(pem.chunk("item", "body-line-syn") + pem.chunk("item", "thetic-0123456789 x") + pem.finish("item")).toBe(`${REDACTED} x`); + const fresh = createAgentIdentityRedactor(undefined, [secret], { maskResumedSuffixes: true }); + expect(fresh.chunk("item", "hello 7890") + fresh.finish("item")).toBe("hello 7890"); + const overlapping = createAgentIdentityRedactor(undefined, ["old-prefix-XYZabcdefgh", secret], { maskResumedSuffixes: true }); + expect(overlapping.chunk("item", "XYZabcdefgh1234567890 done") + overlapping.finish("item")).toBe(`${REDACTED} done`); + const heldInside = createAgentIdentityRedactor(undefined, ["old-prefix-XYZabcdefgh", secret], { maskResumedSuffixes: true }); + expect(heldInside.chunk("item", "XYZabcdefgh123") + heldInside.chunk("item", "4567890 done") + heldInside.finish("item")) + .toBe(`${REDACTED}${REDACTED} done`); + const diverging = createAgentIdentityRedactor(undefined, ["old-prefix-XYZabcdefgh", secret], { maskResumedSuffixes: true }); + expect(diverging.chunk("item", "XYZabcdefgh123") + diverging.chunk("item", " other") + diverging.finish("item")) + .not.toContain("XYZabcdefgh"); + const ending = createAgentIdentityRedactor(undefined, ["old-prefix-XYZabcdef", "abcdefghij"], { maskResumedSuffixes: true }); + expect(ending.chunk("item", "XYZabcdef") + ending.finish("item")).toBe(`${REDACTED}${REDACTED}`); + const continuing = createAgentIdentityRedactor(undefined, ["old-prefix-XYZabcdef", "abcdefghij"], { maskResumedSuffixes: true }); + expect(continuing.chunk("item", "XYZabcdef") + continuing.chunk("item", "ghij done") + continuing.finish("item")) + .toBe(`${REDACTED}${REDACTED} done`); + const shortCarry = createAgentIdentityRedactor(undefined, ["old-prefix-XYZabc", "abcdefghij"], { maskResumedSuffixes: true }); + expect(shortCarry.chunk("item", "XYZabc") + shortCarry.chunk("item", "x tail") + shortCarry.finish("item")) + .toBe(`${REDACTED}${REDACTED}x tail`); + const bytes = createAgentIdentityRedactor(undefined, [secret], { maskResumedSuffixes: true }); + expect(["1", "2", "3", "4", "567890 done"].map((piece) => bytes.chunk("item", piece)).join("") + bytes.finish("item")) + .toBe(`${REDACTED}${REDACTED} done`); + const shortItem = createAgentIdentityRedactor(undefined, [secret], { maskResumedSuffixes: true }); + expect(shortItem.chunk("item", "12") + shortItem.finish("item")).toBe("12"); + const local = createAgentIdentityRedactor(undefined, [secret]); + expect(local.chunk("item", "1234567890 done") + local.finish("item")).toBe("1234567890 done"); + }); + + it("adds the secrets of a later attempt without dropping earlier ones", () => { + const redactor = createAgentIdentityRedactor(undefined, ["sk_test_original_value"]); + redactor.addSecretValues(["sk_test_rotated_value", "sk_test_original_value"]); + expect(redactor.secretSources).toEqual(["sk_test_original_value", "sk_test_rotated_value"]); + expect(redactor.redact("a=sk_test_original_value b=sk_test_rotated_value")).toBe(`a=${REDACTED} b=${REDACTED}`); + }); +}); diff --git a/server/src/services/agent-identity-redaction.ts b/server/src/services/agent-identity-redaction.ts index 6bcd1e7d32a..5a3b8f6ecdc 100644 --- a/server/src/services/agent-identity-redaction.ts +++ b/server/src/services/agent-identity-redaction.ts @@ -1,40 +1,138 @@ import { REDACTED_EVENT_VALUE } from "../redaction.js"; import { redactRegisteredSecretValues } from "./run-secret-redaction.js"; +export const MIN_SECRET_OUTPUT_VALUE_LENGTH = 8; +const MIN_RESUMED_SUFFIX_LENGTH = 4; + +const PROTOCOL_KEY = /^(?:id|ids|ref|refs|schema|type|kind|status|state|outcome|phase|stream|channel|role|level|color|transport|operation|namespace|provider|model|biller|disposition|priority|mode|method)$|(?:Id|Ids|Ref|Refs|Type|Kind|State|Status|Code|Disposition|Revision|Version|Mode|Schema|Digest|Sha256|At)$|_(?:id|ids|type|kind|state|status|code)$/; + +export function secretEnvironmentValues(env: unknown, secretKeys: Iterable): string[] { + if (!env || typeof env !== "object" || Array.isArray(env)) return []; + const record = env as Record; + return [...new Set([...secretKeys].map(key => record[key]).filter((value): value is string => typeof value === "string" && value.length > 0))]; +} + +export function secretOutputForms(secrets: readonly string[]): string[] { + const forms = new Set(); + for (const value of secrets) { + const lines = /[\r\n]/.test(value) ? value.split(/\r?\n|\r/).map(line => line.trim()).filter(line => !line.startsWith("-----")) : []; + for (const form of [value, value.trim(), ...lines]) { + if (form.trim().length < MIN_SECRET_OUTPUT_VALUE_LENGTH) continue; + forms.add(form); + forms.add(JSON.stringify(form).slice(1, -1)); + } + } + return [...forms].sort((a, b) => b.length - a.length); +} + +function replaceValues(text: string, values: readonly string[]): string { + return values.reduce((result, value) => result.split(value).join(REDACTED_EVENT_VALUE), text); +} + +function resumedSuffixLength(text: string, candidates: readonly string[]): number { + let longest = 0; + for (const value of candidates) { + for (let size = value.length - 1; size >= MIN_RESUMED_SUFFIX_LENGTH && size > longest; size--) { + const suffix = value.slice(-size); + if (text.startsWith(suffix)) { longest = size; break; } + if (text.length >= MIN_RESUMED_SUFFIX_LENGTH && text.length > longest && suffix.startsWith(text)) { longest = text.length; break; } + } + } + return longest; +} + +function redactOutputSecretValues(input: T, values: readonly string[], key?: string): T { + if (values.length === 0 || (key !== undefined && PROTOCOL_KEY.test(key))) return input; + if (typeof input === "string") return replaceValues(input, values) as T; + if (Array.isArray(input)) return input.map((entry) => redactOutputSecretValues(entry, values, key)) as T; + if (!input || typeof input !== "object" || input instanceof Date) return input; + return Object.fromEntries(Object.entries(input).map(([entryKey, entry]) => + [entryKey, redactOutputSecretValues(entry, values, entryKey)])) as T; +} + /** Keep private material in memory only; do not copy it into run snapshots. */ -export function createAgentIdentityRedactor(privateKeyPem?: string) { +export function createAgentIdentityRedactor(privateKeyPem?: string, secrets: readonly string[] = [], + options: { maskResumedSuffixes?: boolean } = {}) { const values = privateKeyPem ? [...new Set([ privateKeyPem, privateKeyPem.trim(), JSON.stringify(privateKeyPem).slice(1, -1), ...privateKeyPem.split(/\r?\n/).filter(line => line && !line.startsWith("-----")), ])].sort((a, b) => b.length - a.length) : []; + const secretSources: string[] = []; + const secretValues: string[] = []; const pending = new Map(); + const seenStreams = new Set(); + const maskedPending = new Map(); const deltaStreams = new Map; paths: string[]; }>(); const redactor = { values, - redact(value: T): T { return redactRegisteredSecretValues(value, values); }, + secretSources, + secretValues, + addSecretValues(secrets: readonly string[]) { + secretSources.splice(0, secretSources.length, ...new Set([...secretSources, ...secrets.filter(Boolean)])); + secretValues.splice(0, secretValues.length, ...secretOutputForms(secretSources)); + }, + redact(value: T, key?: string): T { + return redactOutputSecretValues(redactRegisteredSecretValues(value, values), secretValues, key); + }, // Hold only a suffix that could begin a secret. Separate buffers prevent // interleaved stdout/stderr from defeating matching across output chunks. chunk(stream: string, chunk: string): string { - let text = (pending.get(stream) ?? "") + chunk; - for (const value of values) text = text.split(value).join(REDACTED_EVENT_VALUE); - let held = 0; - for (const value of values) { - for (let size = Math.min(value.length - 1, text.length); size > held; size--) { - if (text.endsWith(value.slice(0, size))) { held = size; break; } - } + const text = (pending.get(stream) ?? "") + chunk; + const candidates = [...values, ...secretValues].sort((a, b) => b.length - a.length); + const resumes = options.maskResumedSuffixes === true && !seenStreams.has(stream); + const resumed = Math.max(resumes ? resumedSuffixLength(text, candidates) : 0, maskedPending.get(stream) ?? 0); + if (resumes && resumed === 0 && text.length > 0 && text.length < MIN_RESUMED_SUFFIX_LENGTH + && candidates.some((value) => value.indexOf(text, 1) !== -1)) { + pending.set(stream, text); + return ""; } - if (held) pending.set(stream, text.slice(-held)); + const masked: Array<[number, number]> = resumed > 0 ? [[0, resumed]] : []; + let end = text.length; + for (let index = 0; index < text.length;) { + let matched = 0; + let partial = false; + for (const value of candidates) { + if (value.charCodeAt(0) !== text.charCodeAt(index)) continue; + if (text.startsWith(value, index)) { matched = value.length; break; } + if (text.length - index < value.length && value.startsWith(text.slice(index))) { partial = true; break; } + } + if (partial) { end = index; break; } + if (matched) { + masked.push([index, index + matched]); + index += matched; + } else index++; + } + if (end < text.length) pending.set(stream, text.slice(end)); else pending.delete(stream); - return held ? text.slice(0, -held) : text; + if (end < Math.min(resumed, text.length)) maskedPending.set(stream, Math.min(resumed, text.length) - end); + else maskedPending.delete(stream); + if (resumes && end > resumed) seenStreams.add(stream); + const merged: Array<[number, number]> = []; + for (const [from, to] of masked.sort((left, right) => left[0] - right[0])) { + const last = merged.at(-1); + if (last && from < last[1]) last[1] = Math.max(last[1], to); + else merged.push([from, to]); + } + let output = ""; + let cursor = 0; + for (const [from, to] of merged) { + if (from >= end) break; + output += text.slice(cursor, from) + REDACTED_EVENT_VALUE; + cursor = Math.min(to, end); + } + return output + text.slice(cursor, end); }, // An interrupted partial secret must not be flushed as plaintext. finish(stream: string): string { const held = pending.get(stream); + const heldMasked = maskedPending.has(stream); pending.delete(stream); + maskedPending.delete(stream); + if (held && heldMasked) return REDACTED_EVENT_VALUE; // Ed25519 PEM/PKCS#8 values begin with fixed structural bytes. Preserve // ordinary endings such as a dash; longer interrupted fragments stay hidden. return held ? (held.length < 8 ? held : REDACTED_EVENT_VALUE) : ""; @@ -48,7 +146,7 @@ export function createAgentIdentityRedactor(privateKeyPem?: string) { stream = JSON.stringify([stream, payload.itemId, payload.providerItemId, payload.channel, payload.stream]); const visit = (entry: unknown, path: string, field: string): unknown => { if (typeof entry === "string") { - if (!/^(text|delta|output|patch)$/.test(field)) return redactor.redact(entry); + if (!/^(text|delta|output|patch)$/.test(field)) return redactor.redact(entry, field || undefined); const key = `${stream}:${path}`; const output = redactor.chunk(key, entry); if (pending.has(key)) paths.push(key); @@ -66,7 +164,7 @@ export function createAgentIdentityRedactor(privateKeyPem?: string) { } if (paths.length) deltaStreams.set(stream, { scope, itemId, paths, payload: Object.fromEntries(["itemId", "providerItemId", "kind", "channel", "stream"] - .filter(key => typeof payload[key] === "string").map(key => [key, redactor.redact(payload[key])])), + .filter(key => typeof payload[key] === "string").map(key => [key, redactor.redact(payload[key], key)])), }); else deltaStreams.delete(stream); return result; @@ -91,5 +189,6 @@ export function createAgentIdentityRedactor(privateKeyPem?: string) { ] } : payload; }, }; + redactor.addSecretValues(secrets); return redactor; } diff --git a/server/src/services/heartbeat/execution-configuration.ts b/server/src/services/heartbeat/execution-configuration.ts index 7943a5c2bb4..ecc99e2d841 100644 --- a/server/src/services/heartbeat/execution-configuration.ts +++ b/server/src/services/heartbeat/execution-configuration.ts @@ -13,7 +13,7 @@ import { readPaperclipSkillSyncPreference, resolvePaperclipDesiredSkillNames, re import { parseObject } from "../../adapters/utils.js"; import { logger } from "../../middleware/logger.js"; import { AI_CONNECTION_BUSY_RETRY_REASON, extractWakeCommentIds, isNonAssigneeWorkspaceBusyRetry } from "../../modules/run-dispatch/index.js"; -import { createAgentIdentityRedactor } from "../agent-identity-redaction.js"; +import { createAgentIdentityRedactor, secretEnvironmentValues } from "../agent-identity-redaction.js"; import { agentIdentityService, supportsManagedAgentIdentity } from "../agent-identity.js"; import { readAiConnectionConfigurationFailure, readAiCredentialAccessFailure } from "../ai-connection-configuration-failure.js"; import { AI_AUTH_ENV_KEYS, isAiConnectionBusy, prepareManagedAiRuntime, stripAiAuthBindings } from "../ai-connection-runtime.js"; @@ -224,6 +224,7 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hear trustPreset, })); onFailureSecretsResolved(() => collectRunFailureSecretValues(resolvedConfig.env, secretKeys)); + identityRedactor.addSecretValues(secretEnvironmentValues(resolvedConfig.env, secretKeys)); if (aiBinding) { try { managedAiRuntime = await time("configuration_ai_runtime", () => prepareManagedAiRuntime(db, { companyId: agent.companyId, agentId: agent.id, responsibleUserId, adapterType: agent.adapterType, binding: aiBinding, config: resolvedConfig })); diff --git a/server/src/services/heartbeat/run-output.test.ts b/server/src/services/heartbeat/run-output.test.ts index 26becfaf358..2a65a7da867 100644 --- a/server/src/services/heartbeat/run-output.test.ts +++ b/server/src/services/heartbeat/run-output.test.ts @@ -10,7 +10,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { createDurableRunLogStore } from "../run-log-store.js"; import type { StorageProvider } from "../../storage/types.js"; import { createHeartbeatRunOutput, isNewRunLog } from "./run-output.js"; -import { createAgentIdentityRedactor } from "../agent-identity-redaction.js"; +import { createAgentIdentityRedactor, secretEnvironmentValues } from "../agent-identity-redaction.js"; import { formatNativeRunLogEvent } from "../native-runtime/native-run-log-export.js"; import { generateKeyPairSync } from "node:crypto"; import { reportStorageOutageWaitsTo, runInOwnStorageScope } from "../../storage/storage-unavailable.js"; @@ -398,3 +398,51 @@ describe("object-backed diagnostic recording", () => { expect(writes.filter((write) => write.lastOutputSeq !== undefined)).toHaveLength(2); }); }); + +describe("secret-bound environment values in local run output", () => { + it("masks resolved secret values in the stored log, excerpt, live event, and adapter events", async () => { + const { publishLiveEvent } = await import("../live-events.js"); + vi.mocked(publishLiveEvent).mockClear(); + const secret = "sk_test_synthetic_local_0123456789"; + const shortSecret = "pin42"; + const plain = "https://plain.example.test/api"; + const env = { STRIPE_KEY: secret, SHORT_PIN: shortSecret, PLAIN_URL: plain }; + const secretKeys = new Set(["STRIPE_KEY", "SHORT_PIN"]); + const row = { ...newRunRow }; + const chunks: string[] = []; + const events = vi.fn(async (_run: unknown, _event: { payload?: Record }) => undefined); + const output = createHeartbeatRunOutput(createRunRowDb(row), { + run: row as Run, + runId: row.id!, + issueId: null, + runLogStore: { + begin: async () => ({ store: "local_file", logRef: "company/agent/run.ndjson" }), + append: async (_handle, event) => { + chunks.push(event.chunk); + return Buffer.byteLength(event.chunk); + }, + enqueue: undefined, + flush: async () => undefined, + }, + identityRedactor: createAgentIdentityRedactor(undefined, secretEnvironmentValues(env, secretKeys)), + secretKeys, + appendRunEvent: events as never, + getCurrentUserRedactionOptions: async () => ({ enabled: false }), + }); + const handlers = await output.begin(row as Run); + await handlers.onLog("stdout", `STRIPE_KEY=${secret.slice(0, 12)}`); + await handlers.onLog("stdout", `${secret.slice(12)}\nSHORT_PIN=${shortSecret}\nPLAIN_URL=${plain}\n`); + await handlers.onAdapterEvent({ eventType: "tool.output", stream: "stdout", level: "info", message: `echo ${secret}`, + payload: { output: `${secret}\n` } }); + await handlers.onAdapterMeta({ adapterType: "process", command: "synthetic", commandArgs: ["--token", secret], + env: { ...env } } as never); + const stored = chunks.join(""); + expect(stored).toBe(`STRIPE_KEY=***REDACTED***\nSHORT_PIN=${shortSecret}\nPLAIN_URL=${plain}\n`); + expect(output.stdoutExcerpt).not.toContain(secret.slice(12)); + expect(JSON.stringify(vi.mocked(publishLiveEvent).mock.calls)).not.toContain(secret.slice(12)); + const persistedEvents = JSON.stringify(events.mock.calls.map(([, event]) => event)); + expect(persistedEvents).not.toContain(secret); + expect(persistedEvents).toContain("***REDACTED***"); + expect(persistedEvents).toContain(plain); + }); +}); diff --git a/server/src/services/heartbeat/runtime-execution.test.ts b/server/src/services/heartbeat/runtime-execution.test.ts index 9c9b821a707..252ce2c57ed 100644 --- a/server/src/services/heartbeat/runtime-execution.test.ts +++ b/server/src/services/heartbeat/runtime-execution.test.ts @@ -24,6 +24,7 @@ import { createAdapterExecutionControl } from "../adapter-execution-control.js"; import { executionWorkspaceService } from "../execution-workspaces.js"; import { workspaceOperationService } from "../workspace-operations.js"; import { instanceSettingsService } from "../instance-settings.js"; +import { createRunOutputSecretStore, createRunSecretRedactionRegistry } from "../run-secret-redaction.js"; import { executePaperclipNativeSession, NativeCancellationPendingRecoveryError, NativeControllerDetachedForRestartError, nativeSessionFailureSourceCode, nativeSessionFailureDisposition } from "../native-runtime/native-session-executor.js"; import { NativeRunnerOwnershipUnverifiedError } from "../native-runtime/native-runner-ownership.js"; import { NativeWorkspaceFinalizationBusyError, NativeWorkspaceFinalizationOwnershipLostError } from "../native-runtime/native-workspace-finalization-ownership.js"; @@ -421,6 +422,26 @@ describe.skipIf(!support.supported)("heartbeat runtime execution boundary", () = expect(acquired).not.toHaveBeenCalled(); }); + it("passes current and earlier attempts' secrets and observability values to native dispatch", async () => { + const { input } = await fixture(true); + const store = createRunOutputSecretStore(db); + await store.registerAll(input.run.companyId, input.run.id, ["sk_test_synthetic_before_rotation"]); + input.output.identityRedactor = createAgentIdentityRedactor(undefined, ["sk_test_synthetic_after_rotation", "1", "completed"]); + input.config.observability = { environment: { OTEL_EXPORTER_OTLP_LOGS_HEADERS: "authorization=synthetic-observability-token" } } as never; + await executeHeartbeatRuntime(db, input); + expect(executePaperclipNativeSession).toHaveBeenCalledWith(expect.objectContaining({ + outputSecrets: expect.arrayContaining(["sk_test_synthetic_before_rotation", "sk_test_synthetic_after_rotation", + "synthetic-observability-token"]), + })); + expect((await store.valuesForRun(input.run.companyId, input.run.id)).sort()) + .toEqual(["completed", "sk_test_synthetic_after_rotation", "sk_test_synthetic_before_rotation"]); + expect(await createRunSecretRedactionRegistry(db).redactionValuesForRun(input.run.companyId, input.run.id)).toEqual([]); + const response = { id: input.run.id, eventType: "turn.completed", contextSnapshot: (await read(input.run.id)).contextSnapshot }; + expect(await createRunSecretRedactionRegistry(db).redactForRun(input.run.companyId, input.run.id, response)) + .toEqual({ ...response, contextSnapshot: expect.not.objectContaining({ paperclipOutputSecretRedactions: expect.anything() }) }); + expect(input.output.identityRedactor.redact("old sk_test_synthetic_before_rotation")).toBe("old ***REDACTED***"); + }); + it.each([false, true])("preserves the dispatch gate's early return and cleanup (native: %s)", async native => { const { input, adapter } = await fixture(native); input.control.dispatchResolvedInteractionContinuationWithAtomicGate = async () => ({ dispatched: false }); @@ -481,6 +502,17 @@ describe.skipIf(!support.supported)("heartbeat runtime execution boundary", () = expectCleanup(input); }); + it.each([false, true])("masks a bound secret in the failed workspace record and keeps the original error (native: %s)", async native => { + const { input, adapter } = await fixture(native); + input.output.identityRedactor = createAgentIdentityRedactor(undefined, ["sk_test_synthetic_failure_value"]); + const error = new Error("provider failed near sk_test_synthetic_failure_value"); + if (native) vi.mocked(executePaperclipNativeSession).mockRejectedValueOnce(error); + else adapter.execute.mockRejectedValueOnce(error); + await expect(executeHeartbeatRuntime(db, input)).rejects.toBe(error); + expect(error.message).toBe("provider failed near sk_test_synthetic_failure_value"); + expect(await finalizations()).toMatchObject([{ status: "failed", metadata: { errorMessage: "provider failed near ***REDACTED***" } }]); + }); + it.each([ [new NativeCancellationPendingRecoveryError(), null], [new NativeControllerDetachedForRestartError(), "onNativeSessionResumeScheduled"], diff --git a/server/src/services/heartbeat/runtime-execution.ts b/server/src/services/heartbeat/runtime-execution.ts index da895e24d19..6dd198dcbc6 100644 --- a/server/src/services/heartbeat/runtime-execution.ts +++ b/server/src/services/heartbeat/runtime-execution.ts @@ -36,6 +36,7 @@ import { } from "./workspaces.js"; import { recordLegacyWorkspaceRestoreFailure } from "../legacy-execution-recovery.js"; import { configuredEnvironmentProjection } from "../../vendor/paperclip-runner/index.js"; +import { observabilitySecretValues } from "@paperclipai/adapter-utils/agent-observability"; import { buildAgentIdentityEnv } from "@paperclipai/adapter-utils/server-utils"; import { externalObjectService } from "../external-objects.js"; import { isAgentDirectoryCopy } from "../agent-directory-working-copies.js"; @@ -96,7 +97,8 @@ import type { createHeartbeatRunState } from "./run-state.js"; import type { createHeartbeatLifecycle } from "./run-lifecycle.js"; import type { createHeartbeatRunControl } from "./run-control.js"; import type { createHeartbeatQueue } from "./queue.js"; -import type { createAgentIdentityRedactor } from "../agent-identity-redaction.js"; +import { secretOutputForms, type createAgentIdentityRedactor } from "../agent-identity-redaction.js"; +import { createRunOutputSecretStore } from "../run-secret-redaction.js"; import type { createAdapterExecutionControl } from "../adapter-execution-control.js"; import type { executionWorkspaceService } from "../execution-workspaces.js"; import type { workspaceOperationService } from "../workspace-operations.js"; @@ -641,6 +643,10 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe nativeDispatchAtMs, }), ); + const outputSecretStore = createRunOutputSecretStore(db); + await outputSecretStore.registerAll(run.companyId, run.id, + identityRedactor.secretSources.filter((value) => secretOutputForms([value]).length > 0)); + identityRedactor.addSecretValues(await outputSecretStore.valuesForRun(run.companyId, run.id)); const stopCleanup = runOptions.nativeRestartRecovery?.kind === "reattach_remote_runner" && await readClaimedNativeStopRecovery(db, run, runOptions.nativeLeaseOwner); const guardedDispatch = @@ -715,6 +721,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe managedAiAccountIdentity: managedAiRuntime?.sessionIdentity, managedAiCredentialWriteback: managedAiRuntime?.credentialWriteback, managedAiCredentialHome: managedAiRuntime ? String((managedAiRuntime.config.env as Record).CODEX_HOME) : undefined, + outputSecrets: [...identityRedactor.secretSources, ...observabilitySecretValues(observability)], dotWorkspaceRoot: nativeExecution.provider.kind === "openai_dot" && resolvedConfig.dotWorkspaceAccess === true ? executionWorkspace.cwd : undefined, runnerEnvironment: { ...configuredEnvironmentProjection(configuredTaskEnvironment, secretKeys ?? []), @@ -1161,7 +1168,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe }, runGoalControlRequestId, adapterErr instanceof Error - ? adapterErr.message + ? identityRedactor.redact(adapterErr.message) : "session_goal_control_failed", ).catch(() => undefined); } @@ -1222,10 +1229,11 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe await recordWorkspaceFinalize(settleAcceptedResult ? "succeeded" : "failed", settleAcceptedResult ? { runnerSettlementDiagnostic: postResultFailure } : { - errorMessage: + errorMessage: identityRedactor.redact( adapterErr instanceof Error ? adapterErr.message : String(adapterErr), + ), ...(postResultFailure ? { failureDiagnostic: postResultFailure } : {}), }); } catch (recordErr) { diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index f14159ccfce..04339beaeaf 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -302,7 +302,7 @@ const state = vi.hoisted(() => ({ vi.mock("../../runtime-api.js", async importOriginal => ({ ...await importOriginal(), resolveLocalRuntimeApiUrl: state.localRuntimeApiUrl })); -const accountingEvents = vi.hoisted(() => ({ committed: undefined as undefined | ((event: PrpEvent) => Promise), duplicate: undefined as undefined | ((event: PrpEvent) => Promise) })); +const accountingEvents = vi.hoisted(() => ({ committed: undefined as undefined | ((event: PrpEvent) => Promise), duplicate: undefined as undefined | ((event: PrpEvent) => Promise), secrets: undefined as undefined | readonly string[] })); vi.mock("./paperclip-control-plane-port.js", async importOriginal => { const original = await importOriginal(); return { ...original, PaperclipControlPlanePort: class extends original.PaperclipControlPlanePort { @@ -310,6 +310,7 @@ vi.mock("./paperclip-control-plane-port.js", async importOriginal => { super(...args); accountingEvents.committed = args[2]?.onCommittedEvent ? event => args[2]!.onCommittedEvent!(event, 1) : undefined; accountingEvents.duplicate = args[2]?.onDuplicateEvent ? event => args[2]!.onDuplicateEvent!(event, 1) : undefined; + accountingEvents.secrets = args[2]?.secrets; } } }; }); @@ -15542,6 +15543,53 @@ describe("native session bounded recovery", () => { } }); + it("redacts secret-bound environment values from native failure writes and logs and gives them to the control plane port", async () => { + const secret = "sk_test_synthetic_native_failure_value"; + const failure = new Error(`native provider failed: ${secret}`); + const updates: Array<{ table: unknown; values: Record }> = []; + const logs: string[] = []; + accountingEvents.secrets = undefined; + state.execute.mockReset().mockRejectedValueOnce(failure); + state.upsertRecoveryAction.mockReset().mockResolvedValue({}); + const service = vi.spyOn(issueServiceModule, "issueService").mockReturnValue({ + update: vi.fn(async () => ({ status: "blocked", statusVersion: 1 })), + } as unknown as ReturnType); + try { + await expect(executePaperclipNativeSession({ + db: leaseDb(execution, {}, {}, updates), execution, runnerInstanceId: "runner", + runnerEnvironment: { STRIPE_KEY: secret, PLAIN_URL: "https://plain.example.test/api" }, + outputSecrets: [secret], + onLog: async (_stream, text) => { logs.push(text); }, + })).rejects.toBe(failure); + expect(accountingEvents.secrets).toEqual([secret]); + for (const output of [inspect(updates.map(entry => entry.values), { depth: null }), logs.join("")]) { + expect(output).not.toContain(secret); + } + expect(inspect(updates.map(entry => entry.values), { depth: null })).toContain("***REDACTED***"); + expect(failure.message).toBe(`native provider failed: ${secret}`); + } finally { + service.mockRestore(); + } + }); + + it("classifies a native failure from its original text when a bound secret equals words in it", async () => { + const failure = new Error("runner_local_connect_failed: PRP command session.open timed out"); + state.execute.mockReset().mockRejectedValueOnce(failure); + state.upsertRecoveryAction.mockReset().mockResolvedValue({}); + const service = vi.spyOn(issueServiceModule, "issueService").mockReturnValue({ + update: vi.fn(async () => ({ status: "blocked", statusVersion: 1 })), + } as unknown as ReturnType); + try { + await expect(executePaperclipNativeSession({ + db: leaseDb(execution, {}, {}, []), execution, runnerInstanceId: "runner", + outputSecrets: ["timed out", "session.open"], + })).rejects.toBe(failure); + expect(nativeSessionFailureSourceCode(failure)).toBe("provider_session_open_timeout"); + } finally { + service.mockRestore(); + } + }); + it("persists plain cleanup recovery without an automatic cleanup wake", async () => { const updates: Array<{ table: unknown; values: Record }> = []; diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 0878e6b9128..a43f59d77d5 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -10365,6 +10365,7 @@ export async function executePaperclipNativeSession(input: { dotWorkspaceRoot?: string; observability?: ProviderObservabilityLaunch; runnerEnvironment?: NodeJS.ProcessEnv; + outputSecrets?: readonly string[]; /** Private grant materialization; never a user-configured host path. */ managedAiCredentialHome?: string; managedAiCredentialIdentity?: string; @@ -10789,9 +10790,13 @@ async function executePaperclipNativeSessionAttempt( if (input.useRunnerd) await timeController("warm_restore", () => restoreWarmRunnerdController( replayPortablePreparation ? { ...input, restartRecovery: undefined } : input)); const identityRedactor = createAgentIdentityRedactor(input.runnerEnvironment?.PAPERCLIP_AGENT_PRIVATE_KEY); + const outputRedactor = createAgentIdentityRedactor(input.runnerEnvironment?.PAPERCLIP_AGENT_PRIVATE_KEY, input.outputSecrets); const redactIdentityText = (text: string) => redactSensitiveText( identityRedactor.chunk("diagnostic", text) + identityRedactor.finish("diagnostic"), ); + const redactOutputText = (text: string) => redactSensitiveText( + outputRedactor.chunk("diagnostic", text) + outputRedactor.finish("diagnostic"), + ); if ( input.execution.provider.kind !== "cursor_cloud" && input.execution.provider.kind !== "codex" && @@ -11583,6 +11588,7 @@ async function executePaperclipNativeSessionAttempt( }, { privateKeyPem: input.runnerEnvironment?.PAPERCLIP_AGENT_PRIVATE_KEY, + secrets: input.outputSecrets, assertControllerActive, assertRunAdmissionActive, taskSessionPublication: input.taskSessionPublication, @@ -12481,7 +12487,7 @@ async function executePaperclipNativeSessionAttempt( } }, onPostCompletionEnrichmentFailure: async ({ stage, error }) => { - const detail = redactIdentityText( + const detail = redactOutputText( error instanceof Error ? error.message : String(error), ).slice(-4_096); await input.onLog?.( @@ -12492,7 +12498,7 @@ async function executePaperclipNativeSessionAttempt( onSessionQuarantined: async (reason) => { await input.onLog?.( "stderr", - `[paperclip-runner] warm native session quarantined: ${redactIdentityText(reason).slice(-1_000)}\n`, + `[paperclip-runner] warm native session quarantined: ${redactOutputText(reason).slice(-1_000)}\n`, ); }, onContinuityBreak: async (continuity) => { @@ -12874,7 +12880,7 @@ async function executePaperclipNativeSessionAttempt( ), { companyId: input.execution.binding.companyId }); const failedAtMs = Date.now(); - const executionFailureMessage = redactIdentityText( + const executionFailureMessage = redactOutputText( error instanceof Error ? error.message : String(error), ).slice(-4_096); if (!taskSettleScope) { @@ -13093,10 +13099,10 @@ async function executePaperclipNativeSessionAttempt( const { exhausted } = recoveryProjection; const integrityFailure = sourceFailureCode === "native_event_replay_conflict"; - const message = redactIdentityText( + const message = redactOutputText( error instanceof Error ? error.message : String(error), ).slice(0, 2_000); - const sanitizedStderrTail = redactIdentityText(message).slice(-4_096); + const sanitizedStderrTail = redactOutputText(message).slice(-4_096); const remoteRunnerRecovery = readRemoteRunnerRecoveryError(error); // Set inside the transaction only when the write below genuinely // transitions the run into "failed". Read after the transaction @@ -13137,7 +13143,7 @@ async function executePaperclipNativeSessionAttempt( recoveryState: phase === "retryable_failure" ? "resuming_session" : "blocked", failureCode, - failureDetail: identityRedactor.redact({ + failureDetail: outputRedactor.redact({ message, ...(providerFailure ? { providerFailure } : {}), ...(remoteRunnerRecovery ? { remoteRunnerRecovery } : {}), diff --git a/server/src/services/native-runtime/paperclip-control-plane-port.test.ts b/server/src/services/native-runtime/paperclip-control-plane-port.test.ts index 7509476e6f9..385960d8cf0 100644 --- a/server/src/services/native-runtime/paperclip-control-plane-port.test.ts +++ b/server/src/services/native-runtime/paperclip-control-plane-port.test.ts @@ -52,6 +52,7 @@ import { import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; import { startRecoveryPostgres } from "../../__tests__/helpers/recovery-postgres.js"; import { PaperclipControlPlanePort } from "./paperclip-control-plane-port.js"; +import { secretEnvironmentValues } from "../agent-identity-redaction.js"; import { nativeSha256 } from "./canonical.js"; import { NativeRunCoordinatorStore } from "./native-run-coordinator-store.js"; import { finalizeNativeRun, repairCommittedNativeChatResponse } from "./native-run-finalizer.js"; @@ -458,6 +459,7 @@ describe("PaperclipControlPlanePort conformance", () => { }); async function terminalObserverFixture(options: { + secrets?: readonly string[]; onCommittedEvent?: (event: PrpEvent) => Promise; onDuplicateEvent?: (event: PrpEvent) => Promise; assertControllerActive?: () => void; @@ -480,6 +482,86 @@ describe("PaperclipControlPlanePort conformance", () => { return { identity, runnerId, binding, port, event, snapshot, result }; } + it("masks the rest of a secret that arrives after a controller restart between deltas", async () => { + const identity = { ...CONTROL_PLANE_CONFORMANCE_OPEN.identity, runId: randomUUID(), sessionId: randomUUID() }; + const runnerId = randomUUID(); + const secret = "abcdefgh1234567890"; + await db.insert(heartbeatRuns).values({ + id: identity.runId, companyId: identity.companyId, agentId: identity.agentId, + status: "running", runtimeMode: "native", nativeIssueId: identity.issueId, + nativeSessionId: identity.sessionId, runnerInstanceId: runnerId, + completionContractId: contractId, completionContractSha256: contractSha, + contextSnapshot: { issueId: identity.issueId }, + }); + const binding = { ...identity, completionContractId: contractId, completionContractSha256: contractSha, + sourceInstanceId: runnerId, controlPlaneSourceInstanceId: `restart-${identity.runId}` }; + const delta = (sourceSeq: number, text: string): PrpEvent => ({ + schema: "paperclip.prp.event.v1", sourceEventId: `${runnerId}:${sourceSeq}`, sourceSeq, + sourceInstanceId: runnerId, sourceKind: "runner", runId: identity.runId, + normalizedSessionId: identity.sessionId, turnId: "restart-turn", itemId: "restart-item", + eventType: "item.delta", schemaVersion: 1, priority: 1, emittedAt: new Date().toISOString(), + payload: { itemId: "restart-item", kind: "commandExecution", text, update: { delta: text } }, + }); + const before = new PaperclipControlPlanePort(db, binding, { secrets: [secret] }); + await before.openRun({ identity, backendKind: "mock", sourceInstanceId: runnerId }); + await before.appendEvent(delta(1, `echo ${secret.slice(0, 8)}`)); + const after = new PaperclipControlPlanePort(db, binding, { secrets: [secret] }); + await after.appendEvent(delta(2, `${secret.slice(8)} done`)); + const persisted = await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, identity.runId)); + expect(JSON.stringify(persisted)).not.toContain(secret.slice(8)); + expect(JSON.stringify(persisted)).not.toContain(secret.slice(0, 8)); + expect(JSON.stringify(persisted)).toContain('"text":"echo "'); + expect(JSON.stringify(persisted)).toContain('"text":"***REDACTED*** done"'); + }); + + it("keeps a held fragment of a resumed secret masked when the turn ends", async () => { + const identity = { ...CONTROL_PLANE_CONFORMANCE_OPEN.identity, runId: randomUUID(), sessionId: randomUUID() }; + const runnerId = randomUUID(); + await db.insert(heartbeatRuns).values({ + id: identity.runId, companyId: identity.companyId, agentId: identity.agentId, + status: "running", runtimeMode: "native", nativeIssueId: identity.issueId, + nativeSessionId: identity.sessionId, runnerInstanceId: runnerId, + completionContractId: contractId, completionContractSha256: contractSha, + contextSnapshot: { issueId: identity.issueId }, + }); + const observed: PrpEvent[] = []; + const port = new PaperclipControlPlanePort(db, { ...identity, completionContractId: contractId, completionContractSha256: contractSha, + sourceInstanceId: runnerId, controlPlaneSourceInstanceId: `ending-${identity.runId}` }, + { secrets: ["old-prefix-XYZabcdef", "abcdefghij"], onCommittedEvent: async event => { observed.push(event); } }); + await port.openRun({ identity, backendKind: "mock", sourceInstanceId: runnerId }); + const delta: PrpEvent = { schema: "paperclip.prp.event.v1", sourceEventId: `${runnerId}:1`, sourceSeq: 1, + sourceInstanceId: runnerId, sourceKind: "runner", runId: identity.runId, normalizedSessionId: identity.sessionId, + turnId: "ending-turn", itemId: "ending-item", eventType: "item.delta", schemaVersion: 1, priority: 1, + emittedAt: new Date().toISOString(), payload: { itemId: "ending-item", kind: "commandExecution", text: "XYZabcdef" } }; + const terminal: PrpEvent = { schema: "paperclip.prp.event.v1", sourceEventId: `${runnerId}:2`, sourceSeq: 2, + sourceInstanceId: runnerId, sourceKind: "runner", runId: identity.runId, normalizedSessionId: identity.sessionId, + turnId: "ending-turn", eventType: "turn.completed", schemaVersion: 1, priority: 0, + emittedAt: new Date().toISOString(), payload: { status: "completed" } }; + await port.appendEvent(delta); + await port.appendEvent(terminal); + const persisted = await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, identity.runId)); + for (const value of [persisted, observed]) { + expect(JSON.stringify(value)).not.toContain("abcdef"); + expect(JSON.stringify(value)).not.toContain("XYZ"); + } + expect(observed.at(-1)?.payload.outputTails).toEqual([{ itemId: "ending-item", + payload: { itemId: "ending-item", kind: "commandExecution", text: "***REDACTED***" } }]); + }); + + it("completes a native run when a bound secret equals a protocol word", async () => { + const observed: PrpEvent[] = []; + const f = await terminalObserverFixture({ secrets: ["interrupted", "completed", "succeeded", "satisfied"], + async onCommittedEvent(event) { observed.push(event); } }); + await f.port.appendEvent(f.event); + await f.port.checkpointSession(f.snapshot); + await f.port.completeRun(f.result); + expect(observed.map(event => [event.eventType, event.payload.status])).toEqual([["turn.interrupted", "interrupted"]]); + const [result] = await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, f.identity.runId)); + expect(JSON.stringify(result)).toContain('"turnTerminalState":"completed"'); + expect(JSON.stringify(result)).toContain('"runTerminalState":"succeeded"'); + expect(JSON.stringify(result)).toContain('"status":"satisfied"'); + }); + it.each(["default", "deferred"] as const)("keeps %s terminal observation owned before checkpoint and result acceptance", async mode => { let release!: () => void; const gate = new Promise(resolve => { release = resolve; }); @@ -1190,6 +1272,62 @@ describe("PaperclipControlPlanePort conformance", () => { } }); + it("masks secret-bound environment values in persisted native events and leaves plain values", async () => { + const identity = { ...CONTROL_PLANE_CONFORMANCE_OPEN.identity, runId: randomUUID(), sessionId: randomUUID() }; + const runnerId = randomUUID(); + const secret = `sk_test_synthetic_${randomUUID()}`; + const plain = "https://plain.example.test/api"; + await db.insert(heartbeatRuns).values({ + id: identity.runId, companyId: identity.companyId, agentId: identity.agentId, + status: "running", runtimeMode: "native", nativeIssueId: identity.issueId, + nativeSessionId: identity.sessionId, runnerInstanceId: runnerId, + completionContractId: contractId, completionContractSha256: contractSha, + contextSnapshot: { issueId: identity.issueId }, + }); + const observed: PrpEvent[] = []; + const port = new PaperclipControlPlanePort(db, { + ...identity, completionContractId: contractId, completionContractSha256: contractSha, + sourceInstanceId: runnerId, controlPlaneSourceInstanceId: `secret-env-${identity.runId}`, + }, { + secrets: secretEnvironmentValues({ STRIPE_KEY: secret, SHORT_PIN: "pin42", PLAIN_URL: plain }, + ["STRIPE_KEY", "SHORT_PIN"]), + onCommittedEvent: async event => { observed.push(event); }, + }); + await port.openRun({ identity, backendKind: "mock", sourceInstanceId: runnerId }); + const envOutput = `STRIPE_KEY=${secret}\nSHORT_PIN=pin42\nPLAIN_URL=${plain}\n`; + await port.appendEvent({ + schema: "paperclip.prp.event.v1", sourceEventId: `${runnerId}:1`, sourceSeq: 1, + sourceInstanceId: runnerId, sourceKind: "runner", runId: identity.runId, + normalizedSessionId: identity.sessionId, eventType: "tool.execution.started", schemaVersion: 1, + priority: 0, emittedAt: new Date().toISOString(), payload: { + schema: "paperclip.tool.execution.v1", executionId: "secret-env-output", transport: "builtin", + operation: "execute", name: "env", target: null, namespace: null, readOnly: true, + status: "running", durationMs: null, exitCode: null, progress: null, + output: envOutput, outputBytes: Buffer.byteLength(envOutput), outputTruncated: false, + outputDigest: `sha256:${"a".repeat(64)}`, + }, + }); + let sourceSeq = 2; + for (const chunk of [`echo ${secret.slice(0, 15)}`, `${secret.slice(15)} done`]) { + await port.appendEvent({ + schema: "paperclip.prp.event.v1", sourceEventId: `${runnerId}:${sourceSeq}`, sourceSeq: sourceSeq++, + sourceInstanceId: runnerId, sourceKind: "runner", runId: identity.runId, + normalizedSessionId: identity.sessionId, turnId: "secret-turn", itemId: "secret-item", + eventType: "item.delta", schemaVersion: 1, priority: 1, emittedAt: new Date().toISOString(), + payload: { itemId: "secret-item", kind: "commandExecution", text: chunk, update: { delta: chunk } }, + }); + } + const persistedEvents = await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, identity.runId)); + for (const value of [persistedEvents, observed]) { + expect(JSON.stringify(value)).not.toContain(secret.slice(15)); + expect(JSON.stringify(value)).toContain("***REDACTED***"); + expect(JSON.stringify(value)).toContain(plain); + expect(JSON.stringify(value)).toContain("SHORT_PIN=pin42"); + } + const deltas = observed.filter(event => event.payload.itemId === "secret-item").map(event => event.payload.text); + expect(deltas.join("")).toBe("echo ***REDACTED*** done"); + }); + it("runs the unchanged package conformance suite against Paperclip persistence", async () => { const identity = CONTROL_PLANE_CONFORMANCE_OPEN.identity; const committedEventIds: string[] = []; diff --git a/server/src/services/native-runtime/paperclip-control-plane-port.ts b/server/src/services/native-runtime/paperclip-control-plane-port.ts index 679bbf4cb2a..c02b79b6b4b 100644 --- a/server/src/services/native-runtime/paperclip-control-plane-port.ts +++ b/server/src/services/native-runtime/paperclip-control-plane-port.ts @@ -143,6 +143,7 @@ export class PaperclipControlPlanePort implements ControlPlanePort { options: { /** Runtime-only secret; never part of the persisted binding. */ privateKeyPem?: string; + secrets?: readonly string[]; /** Synchronous revocation fence for a controller handing off on restart. */ assertControllerActive?: () => void; /** Read-only admission may finish an existing startup solely for restart detachment. */ @@ -162,7 +163,7 @@ export class PaperclipControlPlanePort implements ControlPlanePort { this.#db = db; this.#assertControllerActive = options.assertControllerActive; this.#assertRunAdmissionActive = options.assertRunAdmissionActive ?? options.assertControllerActive; - this.#identityRedactor = createAgentIdentityRedactor(options.privateKeyPem); + this.#identityRedactor = createAgentIdentityRedactor(options.privateKeyPem, options.secrets, { maskResumedSuffixes: true }); this.#binding = structuredClone(binding); this.#onCommittedEvent = options.onCommittedEvent; this.#onEventPersistence = options.onEventPersistence; @@ -416,7 +417,7 @@ export class PaperclipControlPlanePort implements ControlPlanePort { )) throw new Error("native_deferred_observer_terminal_required"); const terminalItem = /^item\.(completed|failed|cancelled)$/.test(event.eventType); const terminalTurn = /^(turn|session)\.(completed|failed|cancelled|interrupted|closed)$/.test(event.eventType); - if ((event.eventType === "item.delta" || terminalItem || terminalTurn) && this.#identityRedactor.values.length > 0) { + if ((event.eventType === "item.delta" || terminalItem || terminalTurn) && (this.#identityRedactor.values.length > 0 || this.#identityRedactor.secretValues.length > 0)) { const key = `${event.sourceInstanceId}:${event.sourceEventId}`; const cached = this.#redactedDeltas.get(key); const originalSha = nativeSha256(validated.event); diff --git a/server/src/services/run-secret-redaction.ts b/server/src/services/run-secret-redaction.ts index 69a6df6df93..1cea93cb01c 100644 --- a/server/src/services/run-secret-redaction.ts +++ b/server/src/services/run-secret-redaction.ts @@ -22,6 +22,7 @@ function withoutSqlRunError(value: T): T { } const REGISTRY_KEY = "paperclipSecretRedactions"; +const OUTPUT_SECRET_KEY = "paperclipOutputSecretRedactions"; // Project only the registry: run contexts can contain megabytes of prompt data. const registrySnapshot = sql`jsonb_build_object('paperclipSecretRedactions', ${heartbeatRuns.contextSnapshot} -> 'paperclipSecretRedactions')`; @@ -40,9 +41,9 @@ function asRecord(value: unknown): Record | null { : null; } -function registryEntries(contextSnapshot: unknown): RegistryEntry[] { +function registryEntries(contextSnapshot: unknown, key = REGISTRY_KEY): RegistryEntry[] { const context = asRecord(contextSnapshot); - const raw = context?.[REGISTRY_KEY]; + const raw = context?.[key]; if (!Array.isArray(raw)) return []; return raw.flatMap((value) => { const entry = asRecord(value); @@ -70,13 +71,67 @@ export function redactRegisteredSecretValues(input: T, values: string[], cred if (!record) return input; return Object.fromEntries( Object.entries(record) - .filter(([key]) => key !== REGISTRY_KEY && key !== "nativeControllerRecovery" && key !== "nativeTaskSessionPublication" && key !== "legacyProcessSettlement") + .filter(([key]) => key !== REGISTRY_KEY && key !== OUTPUT_SECRET_KEY && key !== "nativeControllerRecovery" && key !== "nativeTaskSessionPublication" && key !== "legacyProcessSettlement") .map(([key, value]) => [key, key === "credentialAccess" ? (credentialConnectionName ? { connectionName: redactText(credentialConnectionName, values) } : {}) : redactRegisteredSecretValues(value, values, credentialConnectionName)]), ) as T; } +async function registerEncryptedValues(db: Db, key: string, companyId: string, runId: string, values: readonly string[]) { + const provider = getSecretProvider("local_encrypted"); + const fingerprints = new Map(values.filter(Boolean).map((value) => + [createHash("sha256").update(value).digest("hex"), value] as const)); + if (fingerprints.size === 0) return; + await db.transaction(async (tx) => { + const row = await tx.select({ contextSnapshot: heartbeatRuns.contextSnapshot }) + .from(heartbeatRuns) + .where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, runId))) + .for("update") + .then((rows) => rows[0] ?? null); + if (!row) throw new Error("Heartbeat run redaction registration failed"); + const registered = new Set(registryEntries(row.contextSnapshot, key).map((entry) => entry.fingerprintSha256)); + const added: RegistryEntry[] = []; + for (const [fingerprintSha256, value] of fingerprints) { + if (registered.has(fingerprintSha256)) continue; + const prepared = await provider.createSecret({ value }); + added.push({ fingerprintSha256, material: prepared.material }); + } + if (added.length === 0) return; + const contextSnapshot = asRecord(row.contextSnapshot) ?? {}; + const currentEntries = Array.isArray(contextSnapshot[key]) + ? contextSnapshot[key] + : []; + await tx.update(heartbeatRuns) + .set({ + contextSnapshot: { ...contextSnapshot, [key]: [...currentEntries, ...added] }, + updatedAt: new Date(), + }) + .where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, runId))); + }); +} + +export function withRunRedactionHistory>(next: T, current: unknown): T { + const locked = asRecord(current) ?? {}; + const preserved = [REGISTRY_KEY, OUTPUT_SECRET_KEY].filter((key) => Array.isArray(locked[key])); + return preserved.length === 0 ? next : { ...next, ...Object.fromEntries(preserved.map((key) => [key, locked[key]])) }; +} + +export function createRunOutputSecretStore(db: Db) { + const provider = getSecretProvider("local_encrypted"); + const snapshot = sql`jsonb_build_object('paperclipOutputSecretRedactions', ${heartbeatRuns.contextSnapshot} -> 'paperclipOutputSecretRedactions')`; + return { + registerAll: (companyId: string, runId: string, values: readonly string[]) => + registerEncryptedValues(db, OUTPUT_SECRET_KEY, companyId, runId, values), + valuesForRun: async (companyId: string, runId: string) => { + const rows = await db.select({ contextSnapshot: snapshot }).from(heartbeatRuns) + .where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, runId))); + return Promise.all(rows.flatMap((row) => registryEntries(row.contextSnapshot, OUTPUT_SECRET_KEY)) + .map((entry) => provider.resolveVersion({ material: entry.material, externalRef: null }))); + }, + }; +} + export function createRunSecretRedactionRegistry(db: Db) { const provider = getSecretProvider("local_encrypted"); @@ -116,32 +171,8 @@ export function createRunSecretRedactionRegistry(db: Db) { redactionValuesForRun: async (companyId: string, runId: string) => valuesForRuns(await db .select({ contextSnapshot: registrySnapshot }).from(heartbeatRuns) .where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, runId)))), - register: async (companyId: string, runId: string, value: string) => { - const fingerprintSha256 = createHash("sha256").update(value).digest("hex"); - await db.transaction(async (tx) => { - const row = await tx.select({ contextSnapshot: heartbeatRuns.contextSnapshot }) - .from(heartbeatRuns) - .where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, runId))) - .for("update") - .then((rows) => rows[0] ?? null); - if (!row) throw new Error("Heartbeat run redaction registration failed"); - if (registryEntries(row.contextSnapshot).some((entry) => entry.fingerprintSha256 === fingerprintSha256)) { - return; - } - const prepared = await provider.createSecret({ value }); - const entry: RegistryEntry = { fingerprintSha256, material: prepared.material }; - const contextSnapshot = asRecord(row.contextSnapshot) ?? {}; - const currentEntries = Array.isArray(contextSnapshot[REGISTRY_KEY]) - ? contextSnapshot[REGISTRY_KEY] - : []; - await tx.update(heartbeatRuns) - .set({ - contextSnapshot: { ...contextSnapshot, [REGISTRY_KEY]: [...currentEntries, entry] }, - updatedAt: new Date(), - }) - .where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, runId))); - }); - }, + register: (companyId: string, runId: string, value: string) => + registerEncryptedValues(db, REGISTRY_KEY, companyId, runId, [value]), redactForRuns: async (companyId: string, runs: T[], viewerUserId: string | null = null): Promise => { if (runs.length === 0) return []; const rows = await db.select({ id: heartbeatRuns.id, contextSnapshot: registrySnapshot, credentialConnectionName: credentialAccessConnectionNameSql(viewerUserId) })