feat: add activity log retention controls

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-07-17 02:11:32 +00:00
1 parent 5d42382df4
commit de6c4258d3
16 files changed
+388 -8

No files matched your search

+19
View File
@@ -644,6 +644,25 @@ pnpm dev
If you set `DATABASE_URL`, the server will use that instead of embedded PostgreSQL.
## Activity Log Retention
Activity log rows are kept forever by default. Board users can set a company-specific
retention window through the company update API:
```json
{ "activityLogRetentionDays": 30 }
```
Set `activityLogRetentionDays` to `null` to restore keep-forever behavior. Accepted
retention values are whole days from 1 through 36500. The server runs one sweep at
startup and then hourly, deleting only rows strictly older than the configured window
in bounded, company-scoped batches.
The `activity_log.details` payload is capped at 64 KiB of serialized UTF-8 JSON.
Oversized details do not reject the activity write; they are replaced with a marker
containing `_paperclipTruncated`, `_paperclipOriginalBytes`,
`_paperclipMaxBytes`, and a bounded `_paperclipPreview` string.
## Automatic DB Backups
Paperclip can run automatic logical database backups on a timer. These backups cover
+3
View File
@@ -140,6 +140,7 @@ Human auth tables (`users`, `sessions`, and provider-specific auth artifacts) ar
- `budget_monthly_cents` int not null default 0
- `spent_monthly_cents` int not null default 0
- `attachment_max_bytes` int not null
- `activity_log_retention_days` int null (null means keep forever)
- `require_board_approval_for_new_agents` boolean not null default false
- feedback sharing consent fields
- branding fields such as `brand_color`
@@ -325,6 +326,8 @@ Invariant: each event must attach to agent and company; rollups are aggregation,
- `details` jsonb null
- `created_at` timestamptz not null default now()
Activity details are capped at 64 KiB of serialized UTF-8 JSON at write time. Oversized payloads are replaced by a bounded marker containing the original byte count and a JSON preview. A scheduled maintenance sweep deletes rows older than each company's optional retention policy without crossing company boundaries; null retention keeps rows forever.
## 7.12 `project_memberships` + `agent_memberships`
Per-user project/agent membership is personal visibility state for board users. It only controls whether a resource appears in the current user's sidebar; it must not grant or revoke access to all-pages, detail pages, selectors, assignment flows, search, or existing permissions.
@@ -0,0 +1,2 @@
ALTER TABLE "companies"
ADD COLUMN IF NOT EXISTS "activity_log_retention_days" integer;
@@ -1233,6 +1233,13 @@
"when": 1784241826832,
"tag": "0177_activity_log_responsible_user",
"breakpoints": true
},
{
"idx": 178,
"version": "7",
"when": 1784241827832,
"tag": "0178_activity_log_retention",
"breakpoints": true
}
]
}
+1
View File
@@ -16,6 +16,7 @@ export const companies = pgTable(
attachmentMaxBytes: integer("attachment_max_bytes")
.notNull()
.default(10 * 1024 * 1024),
activityLogRetentionDays: integer("activity_log_retention_days"),
defaultResponsibleUserId: text("default_responsible_user_id"),
requireBoardApprovalForNewAgents: boolean("require_board_approval_for_new_agents")
.notNull()
@@ -37,6 +37,7 @@ export interface CompanyPortabilityCompanyManifestEntry {
brandColor: string | null;
logoPath: string | null;
attachmentMaxBytes: number | null;
activityLogRetentionDays: number | null;
requireBoardApprovalForNewAgents: boolean;
feedbackDataSharingEnabled: boolean;
feedbackDataSharingConsentAt: string | null;
+1
View File
@@ -12,6 +12,7 @@ export interface Company {
budgetMonthlyCents: number;
spentMonthlyCents: number;
attachmentMaxBytes: number;
activityLogRetentionDays: number | null;
defaultResponsibleUserId: string | null;
requireBoardApprovalForNewAgents: boolean;
feedbackDataSharingEnabled: boolean;
@@ -12,12 +12,14 @@ const attachmentMaxBytesSchema = z
.int()
.min(1)
.max(MAX_COMPANY_ATTACHMENT_MAX_BYTES);
const activityLogRetentionDaysSchema = z.number().int().min(1).max(36_500).nullable();
export const createCompanySchema = z.object({
name: z.string().min(1),
description: z.string().optional().nullable(),
budgetMonthlyCents: z.number().int().nonnegative().optional().default(0),
attachmentMaxBytes: attachmentMaxBytesSchema.optional(),
activityLogRetentionDays: activityLogRetentionDaysSchema.optional(),
defaultResponsibleUserId: z.string().min(1).nullable().optional(),
});
@@ -36,6 +38,7 @@ export const updateCompanySchema = createCompanySchema
brandColor: brandColorSchema,
logoAssetId: logoAssetIdSchema,
attachmentMaxBytes: attachmentMaxBytesSchema.optional(),
activityLogRetentionDays: activityLogRetentionDaysSchema.optional(),
});
export type UpdateCompany = z.infer<typeof updateCompanySchema>;
@@ -0,0 +1,181 @@
import { randomUUID } from "node:crypto";
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
import { asc, eq } from "drizzle-orm";
import { activityLog, companies, createDb } from "@paperclipai/db";
import {
capActivityDetails,
logActivity,
MAX_ACTIVITY_DETAILS_BYTES,
} from "../services/activity-log.js";
import { pruneActivityLog } from "../services/activity-log-retention.js";
import {
getEmbeddedPostgresTestSupport,
startEmbeddedPostgresTestDatabase,
} from "./helpers/embedded-postgres.js";
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
if (!embeddedPostgresSupport.supported) {
console.warn(
`Skipping embedded Postgres activity retention tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`,
);
}
describe("capActivityDetails", () => {
it("leaves payloads under the cap unchanged", () => {
const details = { outcome: "ok" };
expect(capActivityDetails(details)).toBe(details);
});
it("replaces oversized payloads with a bounded truncation marker", () => {
const details = { message: `start-${"😀\"\\\n".repeat(30_000)}-end` };
const capped = capActivityDetails(details);
expect(capped).toMatchObject({
_paperclipTruncated: true,
_paperclipMaxBytes: MAX_ACTIVITY_DETAILS_BYTES,
});
expect(capped?._paperclipOriginalBytes).toBeGreaterThan(MAX_ACTIVITY_DETAILS_BYTES);
expect(capped?._paperclipPreview).toEqual(expect.stringContaining("start-"));
expect(Buffer.byteLength(JSON.stringify(capped), "utf8")).toBeLessThanOrEqual(
MAX_ACTIVITY_DETAILS_BYTES,
);
});
});
describeEmbeddedPostgres("activity log retention", () => {
let db!: ReturnType<typeof createDb>;
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
beforeAll(async () => {
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-activity-retention-");
db = createDb(tempDb.connectionString);
}, 20_000);
afterEach(async () => {
await db.delete(activityLog);
await db.delete(companies);
});
afterAll(async () => {
await tempDb?.cleanup();
});
it("prunes per-company policies without crossing company boundaries", async () => {
const retainedCompanyId = randomUUID();
const foreverCompanyId = randomUUID();
const shortRetentionCompanyId = randomUUID();
const now = new Date("2026-07-17T12:00:00.000Z");
await db.insert(companies).values([
{
id: retainedCompanyId,
name: "Thirty days",
issuePrefix: "THD",
activityLogRetentionDays: 30,
},
{
id: foreverCompanyId,
name: "Forever",
issuePrefix: "FOR",
activityLogRetentionDays: null,
},
{
id: shortRetentionCompanyId,
name: "Seven days",
issuePrefix: "SEV",
activityLogRetentionDays: 7,
},
]);
await db.insert(activityLog).values([
{
companyId: retainedCompanyId,
actorId: "system",
action: "retained.expired",
entityType: "company",
entityId: retainedCompanyId,
createdAt: new Date("2026-06-16T11:59:59.000Z"),
},
{
companyId: retainedCompanyId,
actorId: "system",
action: "retained.cutoff",
entityType: "company",
entityId: retainedCompanyId,
createdAt: new Date("2026-06-17T12:00:00.000Z"),
},
{
companyId: foreverCompanyId,
actorId: "system",
action: "forever.old",
entityType: "company",
entityId: foreverCompanyId,
createdAt: new Date("2020-01-01T00:00:00.000Z"),
},
{
companyId: shortRetentionCompanyId,
actorId: "system",
action: "short.expired",
entityType: "company",
entityId: shortRetentionCompanyId,
createdAt: new Date("2026-07-10T11:59:59.000Z"),
},
{
companyId: shortRetentionCompanyId,
actorId: "system",
action: "short.current",
entityType: "company",
entityId: shortRetentionCompanyId,
createdAt: new Date("2026-07-10T12:00:00.000Z"),
},
]);
await expect(pruneActivityLog(db, now)).resolves.toBe(2);
const remaining = await db
.select({ companyId: activityLog.companyId, action: activityLog.action })
.from(activityLog)
.orderBy(asc(activityLog.action));
expect(remaining).toEqual([
{ companyId: foreverCompanyId, action: "forever.old" },
{ companyId: retainedCompanyId, action: "retained.cutoff" },
{ companyId: shortRetentionCompanyId, action: "short.current" },
]);
});
it("caps details on the centralized activity write path", async () => {
const companyId = randomUUID();
await db.insert(companies).values({
id: companyId,
name: "Capped details",
issuePrefix: "CAP",
});
await logActivity(db, {
companyId,
actorType: "system",
actorId: "system",
action: "test.oversized",
entityType: "company",
entityId: companyId,
details: { output: "x".repeat(MAX_ACTIVITY_DETAILS_BYTES * 2) },
});
const row = await db
.select({ details: activityLog.details })
.from(activityLog)
.where(eq(activityLog.companyId, companyId))
.then((rows) => rows[0]);
expect(row?.details).toMatchObject({
_paperclipTruncated: true,
_paperclipMaxBytes: MAX_ACTIVITY_DETAILS_BYTES,
});
expect(Buffer.byteLength(JSON.stringify(row?.details), "utf8")).toBeLessThanOrEqual(
MAX_ACTIVITY_DETAILS_BYTES,
);
});
});
+3
View File
@@ -51,6 +51,7 @@ import {
toolAccessService,
} from "./services/index.js";
import { resolveWorktreeRunExecutionActivationState } from "./services/instance-settings.js";
import { startActivityLogRetention } from "./services/activity-log-retention.js";
import {
parseAdapterRegistryEnv,
reconcileAdapterAvailability,
@@ -826,6 +827,7 @@ export async function startServer(): Promise<StartedServer> {
let prepareHotRestartShutdown: ((signal: "SIGINT" | "SIGTERM") => Promise<{ skipDrain: boolean }>) | null = null;
let heartbeatSchedulerStopped = false;
let heartbeatSchedulerInterval: ReturnType<typeof setInterval> | null = null;
const stopActivityLogRetention = startActivityLogRetention(db as any);
const heartbeatSchedulerInFlight = new Set<Promise<void>>();
const trackHeartbeatSchedulerWork = (work: Promise<unknown>) => {
let tracked: Promise<void>;
@@ -1200,6 +1202,7 @@ export async function startServer(): Promise<StartedServer> {
{
const shutdown = async (signal: "SIGINT" | "SIGTERM") => {
heartbeatSchedulerStopped = true;
stopActivityLogRetention();
if (heartbeatSchedulerInterval) {
clearInterval(heartbeatSchedulerInterval);
heartbeatSchedulerInterval = null;
+5 -4
View File
@@ -18,6 +18,7 @@ import type { BetterAuthSessionResult } from "../auth/better-auth.js";
import { logger } from "./logger.js";
import { boardAuthService } from "../services/board-auth.js";
import { ensureHumanRoleDefaultGrants } from "../services/principal-access-compatibility.js";
import { capActivityDetails } from "../services/activity-log.js";
import { forbidden, unprocessable } from "../errors.js";
function hashToken(token: string) {
@@ -91,12 +92,12 @@ async function auditAgentJwtRunHeaderMismatch(
entityId: input.claimRunId,
...(isUuidLike(input.agentId) ? { agentId: input.agentId } : {}),
...(isUuidLike(input.claimRunId) ? { runId: input.claimRunId } : {}),
details: {
details: capActivityDetails({
claimRunId: input.claimRunId,
headerRunId: input.headerRunId,
method: input.method,
url: input.url,
},
}),
});
} catch (err) {
logger.warn(
@@ -119,10 +120,10 @@ async function auditAgentKeyMissingResponsibleUser(
entityType: "agent_api_key",
entityId: input.keyId,
...(isUuidLike(input.agentId) ? { agentId: input.agentId } : {}),
details: {
details: capActivityDetails({
method: input.method,
url: input.url,
},
}),
});
} catch (err) {
logger.warn(
@@ -0,0 +1,109 @@
import { and, asc, eq, inArray, isNotNull, lt } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import { activityLog, companies } from "@paperclipai/db";
import { logger } from "../middleware/logger.js";
const ACTIVITY_LOG_RETENTION_INTERVAL_MS = 60 * 60 * 1_000;
const DELETE_BATCH_SIZE = 1_000;
const MAX_BATCHES_PER_COMPANY = 100;
const DAY_MS = 24 * 60 * 60 * 1_000;
export async function pruneActivityLog(db: Db, now = new Date()): Promise<number> {
const retentionPolicies = await db
.select({
companyId: companies.id,
retentionDays: companies.activityLogRetentionDays,
})
.from(companies)
.where(isNotNull(companies.activityLogRetentionDays));
let totalDeleted = 0;
for (const policy of retentionPolicies) {
if (
policy.retentionDays === null
|| !Number.isInteger(policy.retentionDays)
|| policy.retentionDays < 1
|| policy.retentionDays > 36_500
) {
logger.warn(
{ companyId: policy.companyId, retentionDays: policy.retentionDays },
"Skipping invalid activity log retention policy",
);
continue;
}
const cutoff = new Date(now.getTime() - policy.retentionDays * DAY_MS);
let batches = 0;
let companyDeleted = 0;
while (batches < MAX_BATCHES_PER_COMPANY) {
const expiredIds = await db
.select({ id: activityLog.id })
.from(activityLog)
.where(and(
eq(activityLog.companyId, policy.companyId),
lt(activityLog.createdAt, cutoff),
))
.orderBy(asc(activityLog.createdAt), asc(activityLog.id))
.limit(DELETE_BATCH_SIZE)
.then((rows) => rows.map((row) => row.id));
if (expiredIds.length === 0) break;
const deleted = await db
.delete(activityLog)
.where(and(
eq(activityLog.companyId, policy.companyId),
inArray(activityLog.id, expiredIds),
))
.returning({ id: activityLog.id });
companyDeleted += deleted.length;
totalDeleted += deleted.length;
batches += 1;
if (expiredIds.length < DELETE_BATCH_SIZE) break;
}
if (batches >= MAX_BATCHES_PER_COMPANY) {
logger.warn(
{ companyId: policy.companyId, retentionDays: policy.retentionDays, companyDeleted },
"Activity log retention hit the per-company batch limit",
);
} else if (companyDeleted > 0) {
logger.info(
{ companyId: policy.companyId, retentionDays: policy.retentionDays, companyDeleted },
"Pruned expired activity log rows",
);
}
}
return totalDeleted;
}
export function startActivityLogRetention(
db: Db,
intervalMs: number = ACTIVITY_LOG_RETENTION_INTERVAL_MS,
): () => void {
let stopped = false;
let sweepInFlight: Promise<void> | null = null;
const runSweep = () => {
if (stopped || sweepInFlight) return;
sweepInFlight = pruneActivityLog(db)
.then(() => undefined)
.catch((err) => {
logger.warn({ err }, "Activity log retention sweep failed");
})
.finally(() => {
sweepInFlight = null;
});
};
runSweep();
const timer = setInterval(runSweep, intervalMs);
timer.unref?.();
return () => {
stopped = true;
clearInterval(timer);
};
}
+37 -3
View File
@@ -12,6 +12,7 @@ import type { PluginEventBus } from "./plugin-event-bus.js";
import { instanceSettingsService } from "./instance-settings.js";
const PLUGIN_EVENT_SET: ReadonlySet<string> = new Set(PLUGIN_EVENT_TYPES);
export const MAX_ACTIVITY_DETAILS_BYTES = 64 * 1024;
const ACTIVITY_ACTION_TO_PLUGIN_EVENT: Readonly<Record<string, PluginEventType>> = {
issue_comment_added: "issue.comment.created",
issue_comment_created: "issue.comment.created",
@@ -69,6 +70,38 @@ function readNonEmptyString(value: unknown) {
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
}
export function capActivityDetails(
details: Record<string, unknown> | null,
maxBytes: number = MAX_ACTIVITY_DETAILS_BYTES,
): Record<string, unknown> | null {
if (!details) return null;
const serialized = JSON.stringify(details);
const originalBytes = Buffer.byteLength(serialized, "utf8");
if (originalBytes <= maxBytes) return details;
const buildMarker = (preview: string) => ({
_paperclipTruncated: true,
_paperclipOriginalBytes: originalBytes,
_paperclipMaxBytes: maxBytes,
_paperclipPreview: preview,
});
let low = 0;
let high = serialized.length;
while (low < high) {
const midpoint = Math.ceil((low + high) / 2);
const candidate = buildMarker(serialized.slice(0, midpoint));
if (Buffer.byteLength(JSON.stringify(candidate), "utf8") <= maxBytes) {
low = midpoint;
} else {
high = midpoint - 1;
}
}
return buildMarker(serialized.slice(0, low));
}
export async function resolveResponsibleUserIdForActivity(db: Db, input: LogActivityInput) {
if (input.actorType === "user") return readNonEmptyString(input.actorId);
@@ -132,6 +165,7 @@ export async function logActivity(db: Db, input: LogActivityInput) {
const redactedDetails = sanitizedDetails
? redactCurrentUserValue(sanitizedDetails, currentUserRedactionOptions)
: null;
const cappedDetails = capActivityDetails(redactedDetails);
const responsibleUserId = await resolveResponsibleUserIdForActivity(db, input);
await db.insert(activityLog).values({
companyId: input.companyId,
@@ -143,7 +177,7 @@ export async function logActivity(db: Db, input: LogActivityInput) {
agentId: input.agentId ?? null,
runId: input.runId ?? null,
responsibleUserId,
details: redactedDetails,
details: cappedDetails,
});
publishLiveEvent({
@@ -158,7 +192,7 @@ export async function logActivity(db: Db, input: LogActivityInput) {
agentId: input.agentId ?? null,
runId: input.runId ?? null,
responsibleUserId,
details: redactedDetails,
details: cappedDetails,
},
});
@@ -174,7 +208,7 @@ export async function logActivity(db: Db, input: LogActivityInput) {
entityType: input.entityType,
companyId: input.companyId,
payload: {
...redactedDetails,
...cappedDetails,
agentId: input.agentId ?? null,
runId: input.runId ?? null,
responsibleUserId,
+5 -1
View File
@@ -18,6 +18,7 @@ import { ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY } from "@paperclipai/shared";
import { logger } from "../middleware/logger.js";
import { visibleIssueCondition } from "./issue-visibility.js";
import { classifyRunLiveness } from "./run-liveness.js";
import { capActivityDetails } from "./activity-log.js";
export interface ActivityFilters {
companyId: string;
@@ -582,7 +583,10 @@ export function activityService(db: Db) {
create: (data: typeof activityLog.$inferInsert) =>
db
.insert(activityLog)
.values(data)
.values({
...data,
details: capActivityDetails(data.details ?? null),
})
.returning()
.then((rows) => rows[0]),
};
+1
View File
@@ -137,6 +137,7 @@ export function companyService(db: Db) {
budgetMonthlyCents: companies.budgetMonthlyCents,
spentMonthlyCents: companies.spentMonthlyCents,
attachmentMaxBytes: companies.attachmentMaxBytes,
activityLogRetentionDays: companies.activityLogRetentionDays,
defaultResponsibleUserId: companies.defaultResponsibleUserId,
requireBoardApprovalForNewAgents: companies.requireBoardApprovalForNewAgents,
feedbackDataSharingEnabled: companies.feedbackDataSharingEnabled,
@@ -2669,6 +2669,10 @@ function buildManifestFromPackageFiles(
typeof paperclipCompany.attachmentMaxBytes === "number" && Number.isFinite(paperclipCompany.attachmentMaxBytes)
? Math.max(1, Math.floor(paperclipCompany.attachmentMaxBytes))
: null,
activityLogRetentionDays:
typeof paperclipCompany.activityLogRetentionDays === "number" && Number.isFinite(paperclipCompany.activityLogRetentionDays)
? Math.max(1, Math.min(36_500, Math.floor(paperclipCompany.activityLogRetentionDays)))
: null,
requireBoardApprovalForNewAgents:
typeof paperclipCompany.requireBoardApprovalForNewAgents === "boolean"
? paperclipCompany.requireBoardApprovalForNewAgents
@@ -3896,6 +3900,7 @@ export function companyPortabilityService(db: Db, storage?: StorageService) {
brandColor: company.brandColor ?? null,
logoPath: companyLogoPath,
attachmentMaxBytes: company.attachmentMaxBytes,
activityLogRetentionDays: company.activityLogRetentionDays,
requireBoardApprovalForNewAgents: company.requireBoardApprovalForNewAgents ? true : undefined,
feedbackDataSharingEnabled: company.feedbackDataSharingEnabled ? true : undefined,
feedbackDataSharingConsentAt: company.feedbackDataSharingConsentAt?.toISOString() ?? null,
@@ -4433,6 +4438,7 @@ export function companyPortabilityService(db: Db, storage?: StorageService) {
name: string;
requireBoardApprovalForNewAgents?: boolean | null;
attachmentMaxBytes?: number | null;
activityLogRetentionDays?: number | null;
} | null = null;
let companyAction: "created" | "updated" | "unchanged" = "unchanged";
@@ -4458,6 +4464,9 @@ export function companyPortabilityService(db: Db, storage?: StorageService) {
attachmentMaxBytes: include.company
? (sourceManifest.company?.attachmentMaxBytes ?? undefined)
: undefined,
activityLogRetentionDays: include.company
? (sourceManifest.company?.activityLogRetentionDays ?? undefined)
: undefined,
requireBoardApprovalForNewAgents: include.company
? (sourceManifest.company?.requireBoardApprovalForNewAgents ?? false)
: false,
@@ -4497,6 +4506,7 @@ export function companyPortabilityService(db: Db, storage?: StorageService) {
description: sourceManifest.company.description,
brandColor: sourceManifest.company.brandColor,
attachmentMaxBytes: sourceManifest.company.attachmentMaxBytes ?? undefined,
activityLogRetentionDays: sourceManifest.company.activityLogRetentionDays,
requireBoardApprovalForNewAgents: sourceManifest.company.requireBoardApprovalForNewAgents,
feedbackDataSharingEnabled: sourceManifest.company.feedbackDataSharingEnabled,
feedbackDataSharingConsentAt: sourceManifest.company.feedbackDataSharingConsentAt