Create background job runner and memory refresh jobs

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-04-19 20:42:01 -05:00
1 parent 22b4d8a176
commit 29f7355291
30 files changed
+18677 -107

No files matched your search

+68
View File
@@ -421,6 +421,23 @@ export const MEMORY_HOOK_KINDS = [
] as const;
export type MemoryHookKind = (typeof MEMORY_HOOK_KINDS)[number];
export const MEMORY_HOOK_EXTRACTION_MODES = [
"raw_capture",
"provider_managed",
"paperclip_managed",
] as const;
export type MemoryHookExtractionMode = (typeof MEMORY_HOOK_EXTRACTION_MODES)[number];
export const MEMORY_HOOK_RUN_MODES = ["sync", "async"] as const;
export type MemoryHookRunMode = (typeof MEMORY_HOOK_RUN_MODES)[number];
export const MEMORY_EXTRACTION_HARNESSES = [
"server_worker",
"plugin_worker",
"agent",
] as const;
export type MemoryExtractionHarness = (typeof MEMORY_EXTRACTION_HARNESSES)[number];
export const MEMORY_SOURCE_KINDS = [
"issue_comment",
"issue_document",
@@ -456,6 +473,57 @@ export type MemoryRetentionState = (typeof MEMORY_RETENTION_STATES)[number];
export const MEMORY_REVIEW_STATES = ["pending", "accepted", "rejected"] as const;
export type MemoryReviewState = (typeof MEMORY_REVIEW_STATES)[number];
export const BACKGROUND_JOB_BACKEND_KINDS = [
"server_worker",
"plugin_worker",
"agent_run",
] as const;
export type BackgroundJobBackendKind = (typeof BACKGROUND_JOB_BACKEND_KINDS)[number];
export const BACKGROUND_JOB_STATUSES = [
"active",
"paused",
"archived",
] as const;
export type BackgroundJobStatus = (typeof BACKGROUND_JOB_STATUSES)[number];
export const BACKGROUND_JOB_RUN_STATUSES = [
"queued",
"running",
"succeeded",
"failed",
"cancelled",
] as const;
export type BackgroundJobRunStatus = (typeof BACKGROUND_JOB_RUN_STATUSES)[number];
export const BACKGROUND_JOB_RUN_TRIGGERS = [
"manual",
"schedule",
"hook",
"retry",
] as const;
export type BackgroundJobRunTrigger = (typeof BACKGROUND_JOB_RUN_TRIGGERS)[number];
export const BACKGROUND_JOB_EVENT_TYPES = [
"queued",
"started",
"progress",
"log",
"checkpoint",
"completed",
"failed",
"cancelled",
] as const;
export type BackgroundJobEventType = (typeof BACKGROUND_JOB_EVENT_TYPES)[number];
export const BACKGROUND_JOB_EVENT_LEVELS = [
"debug",
"info",
"warn",
"error",
] as const;
export type BackgroundJobEventLevel = (typeof BACKGROUND_JOB_EVENT_LEVELS)[number];
// ---------------------------------------------------------------------------
// Plugin System — see doc/plugins/PLUGIN_SPEC.md for the full specification
// ---------------------------------------------------------------------------
+44
View File
@@ -66,6 +66,9 @@ export {
MEMORY_OPERATION_STATUSES,
MEMORY_TRIGGER_KINDS,
MEMORY_HOOK_KINDS,
MEMORY_HOOK_EXTRACTION_MODES,
MEMORY_HOOK_RUN_MODES,
MEMORY_EXTRACTION_HARNESSES,
MEMORY_SOURCE_KINDS,
MEMORY_EXTRACTION_JOB_STATUSES,
MEMORY_SCOPE_TYPES,
@@ -73,6 +76,12 @@ export {
MEMORY_SENSITIVITY_LABELS,
MEMORY_RETENTION_STATES,
MEMORY_REVIEW_STATES,
BACKGROUND_JOB_BACKEND_KINDS,
BACKGROUND_JOB_STATUSES,
BACKGROUND_JOB_RUN_STATUSES,
BACKGROUND_JOB_RUN_TRIGGERS,
BACKGROUND_JOB_EVENT_TYPES,
BACKGROUND_JOB_EVENT_LEVELS,
PLUGIN_API_VERSION,
PLUGIN_STATUSES,
PLUGIN_CATEGORIES,
@@ -152,6 +161,9 @@ export {
type MemoryOperationStatus,
type MemoryTriggerKind,
type MemoryHookKind,
type MemoryHookExtractionMode,
type MemoryHookRunMode,
type MemoryExtractionHarness,
type MemorySourceKind,
type MemoryExtractionJobStatus,
type MemoryScopeType,
@@ -159,6 +171,12 @@ export {
type MemorySensitivityLabel,
type MemoryRetentionState,
type MemoryReviewState,
type BackgroundJobBackendKind,
type BackgroundJobStatus,
type BackgroundJobRunStatus,
type BackgroundJobRunTrigger,
type BackgroundJobEventType,
type BackgroundJobEventLevel,
type PluginStatus,
type PluginCategory,
type PluginCapability,
@@ -306,6 +324,8 @@ export type {
MemoryProviderConfigPathSuggestion,
MemoryProviderConfigMetadata,
MemoryProviderDescriptor,
MemoryHookPolicy,
MemoryHookPolicyMap,
MemoryUsage,
MemoryGovernedScope,
MemoryPrincipalRef,
@@ -319,6 +339,8 @@ export type {
MemoryRecord,
MemoryOperation,
MemoryExtractionJob,
MemoryRefreshJobSourceCounts,
MemoryRefreshJobResult,
MemoryQueryResult,
MemoryCaptureResult,
MemoryForgetResult,
@@ -333,6 +355,10 @@ export type {
MemoryProviderCaptureOutput,
MemoryProviderForgetInput,
MemoryProviderForgetOutput,
BackgroundJob,
BackgroundJobRun,
BackgroundJobEvent,
BackgroundJobCostEvent,
BudgetPolicy,
BudgetPolicySummary,
BudgetIncident,
@@ -615,6 +641,8 @@ export {
memoryProviderHealthCheckSchema,
memoryProviderConfigPathSuggestionSchema,
memoryProviderConfigMetadataSchema,
memoryHookPolicySchema,
memoryHookPoliciesSchema,
createMemoryBindingSchema,
updateMemoryBindingSchema,
setCompanyMemoryBindingSchema,
@@ -627,9 +655,17 @@ export {
memoryCorrectSchema,
memoryReviewSchema,
memoryRetentionSweepSchema,
memoryRefreshJobSchema,
memoryListRecordsQuerySchema,
memoryListOperationsQuerySchema,
memoryListExtractionJobsQuerySchema,
createBackgroundJobSchema,
listBackgroundJobsQuerySchema,
createBackgroundJobRunSchema,
updateBackgroundJobRunProgressSchema,
createBackgroundJobEventSchema,
completeBackgroundJobRunSchema,
listBackgroundJobRunsQuerySchema,
memoryBindingTargetTypeSchema,
createAssetImageMetadataSchema,
createCompanyInviteSchema,
@@ -662,9 +698,17 @@ export {
type MemoryCorrect,
type MemoryReview,
type MemoryRetentionSweep,
type MemoryRefreshJob,
type MemoryListRecordsQuery,
type MemoryListOperationsQuery,
type MemoryListExtractionJobsQuery,
type CreateBackgroundJob,
type ListBackgroundJobsQuery,
type CreateBackgroundJobRun,
type UpdateBackgroundJobRunProgress,
type CreateBackgroundJobEvent,
type CompleteBackgroundJobRun,
type ListBackgroundJobRunsQuery,
type CreateAssetImageMetadata,
type CreateCompanyInvite,
type CreateOpenClawInvitePrompt,
@@ -0,0 +1,86 @@
import type {
BackgroundJobBackendKind,
BackgroundJobEventLevel,
BackgroundJobEventType,
BackgroundJobRunStatus,
BackgroundJobRunTrigger,
BackgroundJobStatus,
} from "../constants.js";
export interface BackgroundJob {
id: string;
companyId: string;
key: string;
jobType: string;
displayName: string;
description: string | null;
backendKind: BackgroundJobBackendKind;
status: BackgroundJobStatus;
config: Record<string, unknown>;
createdByAgentId: string | null;
createdByUserId: string | null;
sourceIssueId: string | null;
sourceProjectId: string | null;
createdAt: Date;
updatedAt: Date;
}
export interface BackgroundJobRun {
id: string;
companyId: string;
jobId: string | null;
jobKey: string;
jobType: string;
trigger: BackgroundJobRunTrigger;
status: BackgroundJobRunStatus;
requestedByActorType: "agent" | "user" | "system";
requestedByActorId: string;
requestedByAgentId: string | null;
requestedByUserId: string | null;
sourceIssueId: string | null;
sourceProjectId: string | null;
sourceAgentId: string | null;
heartbeatRunId: string | null;
totalItems: number | null;
processedItems: number;
succeededItems: number;
failedItems: number;
skippedItems: number;
progressPercent: number | null;
currentItem: string | null;
cancellationRequestedAt: Date | null;
cancelledAt: Date | null;
error: string | null;
result: Record<string, unknown> | null;
startedAt: Date | null;
finishedAt: Date | null;
durationMs: number | null;
createdAt: Date;
updatedAt: Date;
}
export interface BackgroundJobEvent {
id: string;
companyId: string;
runId: string;
eventType: BackgroundJobEventType;
level: BackgroundJobEventLevel;
message: string | null;
progressPercent: number | null;
totalItems: number | null;
processedItems: number | null;
succeededItems: number | null;
failedItems: number | null;
skippedItems: number | null;
currentItem: string | null;
details: Record<string, unknown> | null;
createdAt: Date;
}
export interface BackgroundJobCostEvent {
id: string;
companyId: string;
runId: string;
costEventId: string;
createdAt: Date;
}
+10
View File
@@ -129,6 +129,8 @@ export type {
MemoryProviderConfigPathSuggestion,
MemoryProviderConfigMetadata,
MemoryProviderDescriptor,
MemoryHookPolicy,
MemoryHookPolicyMap,
MemoryUsage,
MemoryGovernedScope,
MemoryPrincipalRef,
@@ -142,6 +144,8 @@ export type {
MemoryRecord,
MemoryOperation,
MemoryExtractionJob,
MemoryRefreshJobSourceCounts,
MemoryRefreshJobResult,
MemoryQueryResult,
MemoryCaptureResult,
MemoryForgetResult,
@@ -158,6 +162,12 @@ export type {
MemoryProviderForgetInput,
MemoryProviderForgetOutput,
} from "./memory.js";
export type {
BackgroundJob,
BackgroundJobRun,
BackgroundJobEvent,
BackgroundJobCostEvent,
} from "./background-job.js";
export type {
BudgetPolicy,
BudgetPolicySummary,
+35
View File
@@ -1,7 +1,10 @@
import type {
MemoryBindingTargetType,
MemoryExtractionHarness,
MemoryExtractionJobStatus,
MemoryHookExtractionMode,
MemoryHookKind,
MemoryHookRunMode,
MemoryOperationStatus,
MemoryOperationType,
MemoryPrincipalType,
@@ -13,6 +16,7 @@ import type {
MemorySourceKind,
MemoryTriggerKind,
} from "../constants.js";
import type { BackgroundJob, BackgroundJobRun } from "./background-job.js";
export interface MemoryProviderCapabilities {
browse: boolean;
@@ -85,6 +89,21 @@ export interface MemoryProviderDescriptor {
configMetadata: MemoryProviderConfigMetadata | null;
}
export interface MemoryHookPolicy {
enabled: boolean;
extractionMode: MemoryHookExtractionMode;
runMode: MemoryHookRunMode;
harness: MemoryExtractionHarness;
sensitivityLabel: MemorySensitivityLabel;
reviewState: MemoryReviewState;
retentionPolicy?: Record<string, unknown> | null;
modelProvider?: string | null;
model?: string | null;
config?: Record<string, unknown> | null;
}
export type MemoryHookPolicyMap = Partial<Record<MemoryHookKind, Partial<MemoryHookPolicy>>>;
export interface MemoryUsage {
provider: string;
model: string | null;
@@ -262,6 +281,21 @@ export interface MemoryExtractionJob {
updatedAt: Date;
}
export interface MemoryRefreshJobSourceCounts {
issue: number;
issue_comment: number;
issue_document: number;
run: number;
}
export interface MemoryRefreshJobResult {
job: BackgroundJob;
run: BackgroundJobRun;
dryRun: boolean;
sourceCounts: MemoryRefreshJobSourceCounts;
recordCount: number;
}
export interface MemoryQueryResult {
operation: MemoryOperation;
records: MemoryRecord[];
@@ -310,6 +344,7 @@ export interface MemoryProviderCaptureInput {
content: string;
summary?: string | null;
metadata?: Record<string, unknown>;
reviewState?: MemoryReviewState;
}
export interface MemoryProviderCaptureOutput {
@@ -0,0 +1,104 @@
import { z } from "zod";
import {
BACKGROUND_JOB_BACKEND_KINDS,
BACKGROUND_JOB_EVENT_LEVELS,
BACKGROUND_JOB_EVENT_TYPES,
BACKGROUND_JOB_RUN_STATUSES,
BACKGROUND_JOB_RUN_TRIGGERS,
BACKGROUND_JOB_STATUSES,
} from "../constants.js";
export const createBackgroundJobSchema = z
.object({
key: z.string().trim().min(1).max(128).regex(/^[a-z0-9][a-z0-9_.:-]*$/),
jobType: z.string().trim().min(1).max(128),
displayName: z.string().trim().min(1).max(200),
description: z.string().trim().max(2000).nullable().optional(),
backendKind: z.enum(BACKGROUND_JOB_BACKEND_KINDS).optional().default("server_worker"),
status: z.enum(BACKGROUND_JOB_STATUSES).optional().default("active"),
config: z.record(z.unknown()).optional().default({}),
sourceIssueId: z.string().uuid().nullable().optional(),
sourceProjectId: z.string().uuid().nullable().optional(),
})
.strict();
export const listBackgroundJobsQuerySchema = z
.object({
jobType: z.string().trim().min(1).max(128).optional(),
status: z.enum(BACKGROUND_JOB_STATUSES).optional(),
limit: z.coerce.number().int().positive().max(200).optional().default(50),
})
.strict();
export const createBackgroundJobRunSchema = z
.object({
jobId: z.string().uuid().nullable().optional(),
jobKey: z.string().trim().min(1).max(128).optional(),
jobType: z.string().trim().min(1).max(128).optional(),
trigger: z.enum(BACKGROUND_JOB_RUN_TRIGGERS).optional().default("manual"),
sourceIssueId: z.string().uuid().nullable().optional(),
sourceProjectId: z.string().uuid().nullable().optional(),
sourceAgentId: z.string().uuid().nullable().optional(),
heartbeatRunId: z.string().uuid().nullable().optional(),
totalItems: z.number().int().nonnegative().nullable().optional(),
config: z.record(z.unknown()).optional(),
})
.strict()
.refine((value) => value.jobId || (value.jobKey && value.jobType), {
message: "Provide jobId or both jobKey and jobType",
});
export const updateBackgroundJobRunProgressSchema = z
.object({
totalItems: z.number().int().nonnegative().nullable().optional(),
processedItems: z.number().int().nonnegative().optional(),
succeededItems: z.number().int().nonnegative().optional(),
failedItems: z.number().int().nonnegative().optional(),
skippedItems: z.number().int().nonnegative().optional(),
progressPercent: z.number().min(0).max(100).nullable().optional(),
currentItem: z.string().trim().max(500).nullable().optional(),
})
.strict();
export const createBackgroundJobEventSchema = z
.object({
eventType: z.enum(BACKGROUND_JOB_EVENT_TYPES),
level: z.enum(BACKGROUND_JOB_EVENT_LEVELS).optional().default("info"),
message: z.string().trim().max(4000).nullable().optional(),
progressPercent: z.number().min(0).max(100).nullable().optional(),
totalItems: z.number().int().nonnegative().nullable().optional(),
processedItems: z.number().int().nonnegative().nullable().optional(),
succeededItems: z.number().int().nonnegative().nullable().optional(),
failedItems: z.number().int().nonnegative().nullable().optional(),
skippedItems: z.number().int().nonnegative().nullable().optional(),
currentItem: z.string().trim().max(500).nullable().optional(),
details: z.record(z.unknown()).nullable().optional(),
})
.strict();
export const completeBackgroundJobRunSchema = z
.object({
status: z.enum(["succeeded", "failed", "cancelled"]),
error: z.string().trim().max(4000).nullable().optional(),
result: z.record(z.unknown()).nullable().optional(),
})
.strict();
export const listBackgroundJobRunsQuerySchema = z
.object({
jobId: z.string().uuid().optional(),
jobType: z.string().trim().min(1).max(128).optional(),
status: z.enum(BACKGROUND_JOB_RUN_STATUSES).optional(),
sourceIssueId: z.string().uuid().optional(),
sourceProjectId: z.string().uuid().optional(),
limit: z.coerce.number().int().positive().max(200).optional().default(50),
})
.strict();
export type CreateBackgroundJob = z.infer<typeof createBackgroundJobSchema>;
export type ListBackgroundJobsQuery = z.infer<typeof listBackgroundJobsQuerySchema>;
export type CreateBackgroundJobRun = z.infer<typeof createBackgroundJobRunSchema>;
export type UpdateBackgroundJobRunProgress = z.infer<typeof updateBackgroundJobRunProgressSchema>;
export type CreateBackgroundJobEvent = z.infer<typeof createBackgroundJobEventSchema>;
export type CompleteBackgroundJobRun = z.infer<typeof completeBackgroundJobRunSchema>;
export type ListBackgroundJobRunsQuery = z.infer<typeof listBackgroundJobRunsQuerySchema>;
+23
View File
@@ -246,6 +246,8 @@ export {
memoryProviderHealthCheckSchema,
memoryProviderConfigPathSuggestionSchema,
memoryProviderConfigMetadataSchema,
memoryHookPolicySchema,
memoryHookPoliciesSchema,
createMemoryBindingSchema,
updateMemoryBindingSchema,
setCompanyMemoryBindingSchema,
@@ -261,12 +263,15 @@ export {
memoryListRecordsQuerySchema,
memoryListOperationsQuerySchema,
memoryListExtractionJobsQuerySchema,
memoryRefreshJobSchema,
memoryBindingTargetTypeSchema,
type MemoryGovernedScopeInput,
type MemoryPrincipalRefInput,
type MemoryCitationInput,
type MemoryScopeInput,
type MemorySourceRefInput,
type MemoryHookPolicyInput,
type MemoryHookPoliciesInput,
type CreateMemoryBinding,
type UpdateMemoryBinding,
type SetCompanyMemoryBinding,
@@ -282,8 +287,26 @@ export {
type MemoryListRecordsQuery,
type MemoryListOperationsQuery,
type MemoryListExtractionJobsQuery,
type MemoryRefreshJob,
} from "./memory.js";
export {
createBackgroundJobSchema,
listBackgroundJobsQuerySchema,
createBackgroundJobRunSchema,
updateBackgroundJobRunProgressSchema,
createBackgroundJobEventSchema,
completeBackgroundJobRunSchema,
listBackgroundJobRunsQuerySchema,
type CreateBackgroundJob,
type ListBackgroundJobsQuery,
type CreateBackgroundJobRun,
type UpdateBackgroundJobRunProgress,
type CreateBackgroundJobEvent,
type CompleteBackgroundJobRun,
type ListBackgroundJobRunsQuery,
} from "./background-job.js";
export {
createFinanceEventSchema,
type CreateFinanceEvent,
+47
View File
@@ -1,7 +1,10 @@
import { z } from "zod";
import {
MEMORY_BINDING_TARGET_TYPES,
MEMORY_EXTRACTION_HARNESSES,
MEMORY_HOOK_EXTRACTION_MODES,
MEMORY_HOOK_KINDS,
MEMORY_HOOK_RUN_MODES,
MEMORY_OPERATION_STATUSES,
MEMORY_OPERATION_TYPES,
MEMORY_PRINCIPAL_TYPES,
@@ -127,6 +130,26 @@ export const memoryProviderConfigMetadataSchema = z
})
.strict();
export const memoryHookPolicySchema = z
.object({
enabled: z.boolean().optional(),
extractionMode: z.enum(MEMORY_HOOK_EXTRACTION_MODES).optional(),
runMode: z.enum(MEMORY_HOOK_RUN_MODES).optional(),
harness: z.enum(MEMORY_EXTRACTION_HARNESSES).optional(),
sensitivityLabel: z.enum(MEMORY_SENSITIVITY_LABELS).optional(),
reviewState: z.enum(MEMORY_REVIEW_STATES).optional(),
retentionPolicy: z.record(z.unknown()).nullable().optional(),
modelProvider: z.string().trim().min(1).max(128).nullable().optional(),
model: z.string().trim().min(1).max(200).nullable().optional(),
config: z.record(z.unknown()).nullable().optional(),
})
.strict();
export const memoryHookPoliciesSchema = z
.object(Object.fromEntries(MEMORY_HOOK_KINDS.map((kind) => [kind, memoryHookPolicySchema.optional()])))
.partial()
.strict();
export const createMemoryBindingSchema = z
.object({
key: z.string().trim().min(1).max(64).regex(/^[a-z0-9][a-z0-9_-]*$/, "Binding key must be lowercase letters, numbers, _ or -"),
@@ -300,6 +323,27 @@ export const memoryListExtractionJobsQuerySchema = z
})
.strict();
export const memoryRefreshJobSchema = z
.object({
bindingKey: z.string().trim().min(1).max(64).optional(),
scope: memoryScopeSchema.optional().default({}),
sourceKinds: z
.array(z.enum(["issue", "issue_comment", "issue_document", "run"]))
.min(1)
.max(4)
.optional()
.default(["issue", "issue_comment", "issue_document"]),
issueIds: z.array(z.string().uuid()).max(500).optional(),
projectId: z.string().uuid().nullable().optional(),
agentId: z.string().uuid().nullable().optional(),
runIds: z.array(z.string().uuid()).max(500).optional(),
since: z.coerce.date().nullable().optional(),
until: z.coerce.date().nullable().optional(),
dryRun: z.boolean().optional().default(false),
limit: z.number().int().positive().max(5000).optional().default(500),
})
.strict();
export const memoryBindingTargetTypeSchema = z.enum(MEMORY_BINDING_TARGET_TYPES);
export type MemoryScopeInput = z.infer<typeof memoryScopeSchema>;
@@ -307,6 +351,8 @@ export type MemoryGovernedScopeInput = z.infer<typeof memoryGovernedScopeSchema>
export type MemoryPrincipalRefInput = z.infer<typeof memoryPrincipalRefSchema>;
export type MemoryCitationInput = z.infer<typeof memoryCitationSchema>;
export type MemorySourceRefInput = z.infer<typeof memorySourceRefSchema>;
export type MemoryHookPolicyInput = z.infer<typeof memoryHookPolicySchema>;
export type MemoryHookPoliciesInput = z.infer<typeof memoryHookPoliciesSchema>;
export type CreateMemoryBinding = z.infer<typeof createMemoryBindingSchema>;
export type UpdateMemoryBinding = z.infer<typeof updateMemoryBindingSchema>;
export type SetCompanyMemoryBinding = z.infer<typeof setCompanyMemoryBindingSchema>;
@@ -322,3 +368,4 @@ export type MemoryRetentionSweep = z.infer<typeof memoryRetentionSweepSchema>;
export type MemoryListRecordsQuery = z.infer<typeof memoryListRecordsQuerySchema>;
export type MemoryListOperationsQuery = z.infer<typeof memoryListOperationsQuerySchema>;
export type MemoryListExtractionJobsQuery = z.infer<typeof memoryListExtractionJobsQuerySchema>;
export type MemoryRefreshJob = z.infer<typeof memoryRefreshJobSchema>;