Jannik/PaperClipAI
1
0 Fork 0
mirror of https://github.com/paperclipai/paperclip.git synced 2026-10-24 10:13:11 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat(runner): add partitioned native history storage

Browse Source 
Part 2 of the opt-in indexed history draft series. Validated at the integrated tip; activation remains disabled by default.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-09-29 16:20:49 -05:00
1 parent 8ecb181d9c
commit a4720aa56f
66 files changed
+15636 -493
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

packages/paperclip-runner/protocol/manifest.json
+10 -5
View File
@@ -20,12 +20,12 @@
{
"path": "schemas/command-v2.schema.json",
"id": "https://paperclip.dev/schemas/prp/v2/command.schema.json",
"sha256": "06bb3b3b0b420119007494c64e64a736fd5d58cb68d04ced843ada00c54668be"
"sha256": "7de0dd11ff77ae2bb39b048875dd44349946a0be47039382b12d81e95c9fbfed"
},
{
"path": "schemas/command.schema.json",
"id": "https://paperclip.dev/schemas/prp/v1/command.schema.json",
"sha256": "fc813e6b4d5b7e6890b496bd69848ada25dba58adbf7cf04163ca9e9bf2bcb79"
"sha256": "a35f8eddf3f0d36f01acac5023b848fc3feff1bfd45d2bb02c395d778e496bc9"
},
{
"path": "schemas/conformance-fixture.schema.json",
@@ -40,12 +40,12 @@
{
"path": "schemas/event-v2.schema.json",
"id": "https://paperclip.dev/schemas/prp/v2/event.schema.json",
"sha256": "d7528ee2a46ad20de831fd4ee4a0a51e8325a9855fed5a9b8c428dcebc39cc11"
"sha256": "5e60de9761dfc5568a331fd5de0190929c0be3a025c64486d296b6041a4cec64"
},
{
"path": "schemas/event.schema.json",
"id": "https://paperclip.dev/schemas/prp/v1/event.schema.json",
"sha256": "d48916630976ff7e850b6e2d7d94131761fda322f5d8a05212651ec46b0db19d"
"sha256": "40ecafed388a65d116e988251dc809768a3f8e39c3db122f2b0072f33871c61d"
},
{
"path": "schemas/fixture.schema.json",
@@ -62,6 +62,11 @@
"id": "https://paperclip.dev/schemas/prp/v1/native-execution.schema.json",
"sha256": "7894a809005354c5abdf81d5333a094750c8267473cd426a2c1bcccda41e0214"
},
{
"path": "schemas/output-body-reference.schema.json",
"id": "https://paperclip.dev/schemas/prp/v1/output-body-reference.schema.json",
"sha256": "40b5fdec157b3dfe8e49e280cbd7204e6f5f66e333bdaf536876ba7c81e95c19"
},
{
"path": "schemas/provider-descriptor.schema.json",
"id": "https://paperclip.dev/schemas/prp/v1/provider-descriptor.schema.json",
@@ -70,7 +75,7 @@
{
"path": "schemas/provider-event.schema.json",
"id": "https://paperclip.dev/schemas/prp/v1/provider-event.schema.json",
"sha256": "0b138a41bd3bd0b1a115f5536a2d7349537e79e36ede3b51d003f9f21ce73b50"
"sha256": "70f45e12b9fc893c4cd8e82d74865a1f1a4fff7e3563d59b0dc14cf11b4c6ae9"
},
{
"path": "schemas/question-adapter-fixture.schema.json",
packages/paperclip-runner/protocol/schemas/command-v2.schema.json
+1
View File
@@ -8,6 +8,7 @@
"schema": { "const": "paperclip.prp.command.v2" },
"commandId": { "type": "string", "minLength": 1, "maxLength": 160 },
"controllerSeq": { "type": "integer", "minimum": 1 },
"controllerEpoch": { "type": "string", "pattern": "^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$", "description": "Requires negotiated transport.command_epochs.v1; absent is the retained numeric namespace." },
"type": {
"enum": [
"run.prepare", "run.attach", "session.open", "turn.start", "turn.steer", "turn.interrupt", "turn.stop",
packages/paperclip-runner/protocol/schemas/command.schema.json
+1
View File
@@ -8,6 +8,7 @@
"schema": { "const": "paperclip.prp.command.v1" },
"commandId": { "type": "string", "minLength": 1, "maxLength": 160 },
"controllerSeq": { "type": "integer", "minimum": 1 },
"controllerEpoch": { "type": "string", "pattern": "^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$", "description": "Requires negotiated transport.command_epochs.v1; absent is the retained numeric namespace." },
"type": {
"enum": [
"run.prepare",
packages/paperclip-runner/protocol/schemas/event-v2.schema.json
+365 -42
View File
@@ -4,60 +4,300 @@
"title": "PRP v2 native event",
"type": "object",
"required": [
"schema", "sourceEventId", "sourceSeq", "sourceInstanceId", "sourceKind", "runId",
"eventType", "schemaVersion", "priority", "emittedAt", "payload"
"schema",
"sourceEventId",
"sourceSeq",
"sourceInstanceId",
"sourceKind",
"runId",
"eventType",
"schemaVersion",
"priority",
"emittedAt",
"payload"
],
"properties": {
"schema": { "const": "paperclip.prp.event.v2" },
"sourceEventId": { "type": "string", "minLength": 1, "maxLength": 160 },
"sourceSeq": { "type": "integer", "minimum": 1, "maximum": 9007199254740991 },
"sourceInstanceId": { "type": "string", "minLength": 1, "maxLength": 160 },
"sourceKind": { "enum": ["runner", "control_plane"] },
"runId": { "type": "string", "minLength": 1, "maxLength": 160 },
"normalizedSessionId": { "type": "string", "minLength": 1, "maxLength": 160 },
"turnId": { "type": "string", "minLength": 1, "maxLength": 160 },
"itemId": { "type": "string", "minLength": 1, "maxLength": 160 },
"eventType": {
"schema": {
"const": "paperclip.prp.event.v2"
},
"sourceEventId": {
"type": "string",
"minLength": 1,
"maxLength": 160
},
"sourceEpoch": {
"type": "string",
"pattern": "^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$"
},
"sourceSeq": {
"type": "integer",
"minimum": 1,
"maximum": 9007199254740991
},
"sourceInstanceId": {
"type": "string",
"minLength": 1,
"maxLength": 160
},
"sourceKind": {
"enum": [
"runner.connected", "runner.reconnected", "runner.reconciled", "runner.disconnected", "runner.draining", "runner.suspending", "runner.suspended", "runner.stopped", "runner.diagnostic",
"runtime.phase.changed", "sandbox.metric", "workspace.ready", "workspace.change.updated", "workspace.diff.recorded", "workspace.file.referenced",
"harness.starting", "harness.ready", "harness.exited", "harness.diagnostic", "plan.updated",
"tool.execution.started", "tool.execution.progressed", "tool.execution.completed", "research.started", "research.progressed", "research.completed",
"delegation.started", "delegation.updated", "delegation.completed", "model.route.changed", "model.verification.updated", "context.compacted",
"artifact.viewed", "artifact.generated", "review.mode.changed", "hook.started", "hook.completed", "memory.citation.referenced",
"safety.review.started", "safety.review.completed", "terminal.input.sent", "wait.started", "wait.completed", "provider.notice.recorded",
"session.starting", "session.started", "session.resuming", "session.resumed", "session.reconciled", "session.updated", "session.closed", "session.failed",
"session.capabilities.updated", "session.goal.snapshot", "session.goal.updated", "session.goal.cleared",
"turn.submitted", "turn.accepted", "turn.started", "turn.completed", "turn.failed", "turn.interrupted", "turn.cancelled",
"item.started", "item.delta", "item.completed", "item.failed", "usage.reported", "semantic_tool.input", "semantic_tool.result",
"mcp_app.discovered", "mcp_app.resource.resolved", "mcp_app.initializing", "mcp_app.ready", "mcp_app.tool_input", "mcp_app.tool_result", "mcp_app.action.requested", "mcp_app.action.resolved", "mcp_app.host_context.changed", "mcp_app.failed", "mcp_app.teardown",
"runtime_request.created", "runtime_request.resolved", "runtime_request.expired", "runtime_request.cancelled",
"interaction.request.proposed", "interaction.request.materialized", "interaction.request.rejected", "interaction.response.progressed", "interaction.response.resolved", "interaction.response.delivered",
"run.attached", "run.detached", "run.result.proposed", "run.result.accepted", "run.result.rejected",
"attention.request.proposed", "attention.request.routed", "attention.request.resolved", "attention.request.expired", "attention.request.superseded",
"work.assessment.recorded", "issue.status.decision.recorded", "issue.status.decision.applied", "issue.status.decision.rejected", "issue.status.decision.superseded", "run.terminal"
"runner",
"control_plane"
]
},
"schemaVersion": { "const": 2 },
"priority": { "enum": [0, 1, 2] },
"emittedAt": { "type": "string", "format": "date-time" },
"observedAt": { "type": "string", "format": "date-time" },
"payload": { "type": "object", "additionalProperties": true },
"debug": { "type": "object", "additionalProperties": true }
"runId": {
"type": "string",
"minLength": 1,
"maxLength": 160
},
"normalizedSessionId": {
"type": "string",
"minLength": 1,
"maxLength": 160
},
"turnId": {
"type": "string",
"minLength": 1,
"maxLength": 160
},
"itemId": {
"type": "string",
"minLength": 1,
"maxLength": 160
},
"eventType": {
"enum": [
"runner.connected",
"runner.reconnected",
"runner.reconciled",
"runner.disconnected",
"runner.draining",
"runner.suspending",
"runner.suspended",
"runner.stopped",
"runner.diagnostic",
"runtime.phase.changed",
"sandbox.metric",
"workspace.ready",
"workspace.change.updated",
"workspace.diff.recorded",
"workspace.file.referenced",
"harness.starting",
"harness.ready",
"harness.exited",
"harness.diagnostic",
"plan.updated",
"tool.execution.started",
"tool.execution.progressed",
"tool.execution.completed",
"research.started",
"research.progressed",
"research.completed",
"delegation.started",
"delegation.updated",
"delegation.completed",
"model.route.changed",
"model.verification.updated",
"context.compacted",
"artifact.viewed",
"artifact.generated",
"review.mode.changed",
"hook.started",
"hook.completed",
"memory.citation.referenced",
"safety.review.started",
"safety.review.completed",
"terminal.input.sent",
"wait.started",
"wait.completed",
"provider.notice.recorded",
"session.starting",
"session.started",
"session.resuming",
"session.resumed",
"session.reconciled",
"session.updated",
"session.closed",
"session.failed",
"session.capabilities.updated",
"session.goal.snapshot",
"session.goal.updated",
"session.goal.cleared",
"turn.submitted",
"turn.accepted",
"turn.started",
"turn.completed",
"turn.failed",
"turn.interrupted",
"turn.cancelled",
"output.body.chunk",
"item.started",
"item.delta",
"item.completed",
"item.failed",
"usage.reported",
"semantic_tool.input",
"semantic_tool.result",
"mcp_app.discovered",
"mcp_app.resource.resolved",
"mcp_app.initializing",
"mcp_app.ready",
"mcp_app.tool_input",
"mcp_app.tool_result",
"mcp_app.action.requested",
"mcp_app.action.resolved",
"mcp_app.host_context.changed",
"mcp_app.failed",
"mcp_app.teardown",
"runtime_request.created",
"runtime_request.resolved",
"runtime_request.expired",
"runtime_request.cancelled",
"interaction.request.proposed",
"interaction.request.materialized",
"interaction.request.rejected",
"interaction.response.progressed",
"interaction.response.resolved",
"interaction.response.delivered",
"run.attached",
"run.detached",
"run.result.proposed",
"run.result.accepted",
"run.result.rejected",
"attention.request.proposed",
"attention.request.routed",
"attention.request.resolved",
"attention.request.expired",
"attention.request.superseded",
"work.assessment.recorded",
"issue.status.decision.recorded",
"issue.status.decision.applied",
"issue.status.decision.rejected",
"issue.status.decision.superseded",
"run.terminal"
]
},
"schemaVersion": {
"const": 2
},
"priority": {
"enum": [
0,
1,
2
]
},
"emittedAt": {
"type": "string",
"format": "date-time"
},
"observedAt": {
"type": "string",
"format": "date-time"
},
"payload": {
"type": "object",
"additionalProperties": true
},
"debug": {
"type": "object",
"additionalProperties": true
},
"sourceEpochTransition": {
"type": "object",
"additionalProperties": false,
"required": [
"schema",
"runId",
"transitionId",
"fromEpoch",
"nextEpoch",
"finalOrdinal"
],
"properties": {
"schema": {
"const": "paperclip.prp.event-epoch.v1"
},
"runId": {
"type": "string",
"minLength": 1,
"maxLength": 240
},
"transitionId": {
"type": "string",
"pattern": "^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$"
},
"fromEpoch": {
"anyOf": [
{
"type": "string",
"pattern": "^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$"
},
{
"type": "null"
}
]
},
"nextEpoch": {
"type": "string",
"pattern": "^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$"
},
"finalOrdinal": {
"type": "integer",
"minimum": 1,
"maximum": 9007199254740991
}
}
}
},
"allOf": [
{
"if": { "properties": { "eventType": { "enum": ["session.goal.snapshot", "session.goal.updated"] } } },
"if": {
"properties": {
"eventType": {
"enum": [
"tool.execution.started",
"tool.execution.progressed",
"tool.execution.completed"
]
}
}
},
"then": {
"properties": {
"payload": {
"$ref": "https://paperclip.dev/schemas/prp/v1/provider-event.schema.json#/$defs/toolExecution"
}
}
}
},
{
"if": {
"properties": {
"eventType": {
"enum": [
"session.goal.snapshot",
"session.goal.updated"
]
}
}
},
"then": {
"properties": {
"payload": {
"type": "object",
"required": ["goal"],
"required": [
"goal"
],
"properties": {
"goal": {
"oneOf": [
{ "$ref": "https://paperclip.dev/schemas/prp/v2/session-goal.schema.json#/$defs/snapshot" },
{ "type": "null" }
{
"$ref": "https://paperclip.dev/schemas/prp/v2/session-goal.schema.json#/$defs/snapshot"
},
{
"type": "null"
}
]
}
},
@@ -67,19 +307,102 @@
}
},
{
"if": { "properties": { "eventType": { "const": "session.capabilities.updated" } } },
"if": {
"properties": {
"eventType": {
"const": "session.capabilities.updated"
}
}
},
"then": {
"properties": {
"payload": {
"type": "object",
"required": ["sessionGoals"],
"required": [
"sessionGoals"
],
"properties": {
"sessionGoals": { "$ref": "https://paperclip.dev/schemas/prp/v2/session-goal.schema.json#/$defs/capability" }
"sessionGoals": {
"$ref": "https://paperclip.dev/schemas/prp/v2/session-goal.schema.json#/$defs/capability"
}
},
"additionalProperties": true
}
}
}
},
{
"if": {
"properties": {
"eventType": {
"const": "output.body.chunk"
}
}
},
"then": {
"properties": {
"payload": {
"type": "object",
"required": [
"schema",
"body",
"offset",
"sha256",
"text"
],
"properties": {
"schema": {
"const": "paperclip.output.body.chunk.v1"
},
"body": {
"type": "object",
"required": [
"schema",
"bodyId",
"sha256",
"byteLength",
"mediaType"
],
"properties": {
"schema": {
"const": "paperclip.output.body.v1"
},
"bodyId": {
"type": "string",
"pattern": "^[a-f0-9]{64}$"
},
"sha256": {
"type": "string",
"pattern": "^[a-f0-9]{64}$"
},
"byteLength": {
"type": "string",
"pattern": "^[1-9][0-9]{0,6}$"
},
"mediaType": {
"const": "text/plain; charset=utf-8"
}
},
"additionalProperties": false
},
"offset": {
"type": "string",
"pattern": "^(0|[1-9][0-9]{0,6})$"
},
"sha256": {
"type": "string",
"pattern": "^[a-f0-9]{64}$"
},
"text": {
"type": "string",
"minLength": 1,
"maxLength": 32766
}
},
"additionalProperties": false
}
}
}
}
],
"additionalProperties": true
packages/paperclip-runner/protocol/schemas/event.schema.json
+706 -74
View File
@@ -17,105 +17,606 @@
"payload"
],
"properties": {
"schema": { "const": "paperclip.prp.event.v1" },
"sourceEventId": { "type": "string", "minLength": 1, "maxLength": 160 },
"sourceSeq": { "type": "integer", "minimum": 1, "maximum": 9007199254740991 },
"sourceInstanceId": { "type": "string", "minLength": 1, "maxLength": 160 },
"sourceKind": { "enum": ["runner", "control_plane"] },
"runId": { "type": "string", "minLength": 1, "maxLength": 160 },
"normalizedSessionId": { "type": "string", "minLength": 1, "maxLength": 160 },
"turnId": { "type": "string", "minLength": 1, "maxLength": 240 },
"itemId": { "type": "string", "minLength": 1, "maxLength": 240 },
"schema": {
"const": "paperclip.prp.event.v1"
},
"sourceEventId": {
"type": "string",
"minLength": 1,
"maxLength": 160
},
"sourceEpoch": {
"type": "string",
"pattern": "^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$"
},
"sourceSeq": {
"type": "integer",
"minimum": 1,
"maximum": 9007199254740991
},
"sourceInstanceId": {
"type": "string",
"minLength": 1,
"maxLength": 160
},
"sourceKind": {
"enum": [
"runner",
"control_plane"
]
},
"runId": {
"type": "string",
"minLength": 1,
"maxLength": 160
},
"normalizedSessionId": {
"type": "string",
"minLength": 1,
"maxLength": 160
},
"turnId": {
"type": "string",
"minLength": 1,
"maxLength": 240
},
"itemId": {
"type": "string",
"minLength": 1,
"maxLength": 240
},
"eventType": {
"enum": [
"runner.connected", "runner.reconnected", "runner.reconciled", "runner.disconnected", "runner.draining", "runner.backpressure", "runner.suspending", "runner.suspended", "runner.stopped", "runner.diagnostic",
"runtime.phase.changed", "sandbox.metric", "workspace.ready", "workspace.change.updated", "workspace.diff.recorded", "workspace.file.referenced",
"harness.starting", "harness.ready", "harness.exited", "harness.diagnostic",
"runner.connected",
"runner.reconnected",
"runner.reconciled",
"runner.disconnected",
"runner.draining",
"runner.backpressure",
"runner.suspending",
"runner.suspended",
"runner.stopped",
"runner.diagnostic",
"runtime.phase.changed",
"sandbox.metric",
"workspace.ready",
"workspace.change.updated",
"workspace.diff.recorded",
"workspace.file.referenced",
"harness.starting",
"harness.ready",
"harness.exited",
"harness.diagnostic",
"plan.updated",
"tool.execution.started", "tool.execution.progressed", "tool.execution.completed",
"research.started", "research.progressed", "research.completed",
"delegation.started", "delegation.updated", "delegation.completed",
"model.route.changed", "model.verification.updated", "context.compacted",
"artifact.viewed", "artifact.generated", "review.mode.changed",
"hook.started", "hook.completed", "memory.citation.referenced",
"safety.review.started", "safety.review.completed", "terminal.input.sent",
"wait.started", "wait.completed", "provider.notice.recorded",
"session.starting", "session.started", "session.resuming", "session.resumed", "session.reconciled", "session.updated", "session.closed", "session.failed",
"turn.submitted", "turn.accepted", "turn.started", "turn.completed", "turn.failed", "turn.interrupted", "turn.cancelled",
"item.started", "item.delta", "item.completed", "item.failed", "usage.reported",
"semantic_tool.input", "semantic_tool.result", "semantic_tool.reconciled",
"mcp_app.discovered", "mcp_app.resource.resolved", "mcp_app.initializing", "mcp_app.ready", "mcp_app.tool_input", "mcp_app.tool_result", "mcp_app.action.requested", "mcp_app.action.resolved", "mcp_app.host_context.changed", "mcp_app.failed", "mcp_app.teardown",
"runtime_request.created", "runtime_request.resolved", "runtime_request.expired", "runtime_request.cancelled",
"interaction.request.proposed", "interaction.request.materialized", "interaction.request.rejected", "interaction.response.progressed", "interaction.response.resolved", "interaction.response.delivered",
"run.attached", "run.detached", "run.result.proposed", "run.result.accepted", "run.result.rejected",
"attention.request.proposed", "attention.request.routed", "attention.request.resolved", "attention.request.expired", "attention.request.superseded",
"work.assessment.recorded", "issue.status.decision.recorded", "issue.status.decision.applied", "issue.status.decision.rejected", "issue.status.decision.superseded",
"tool.execution.started",
"tool.execution.progressed",
"tool.execution.completed",
"research.started",
"research.progressed",
"research.completed",
"delegation.started",
"delegation.updated",
"delegation.completed",
"model.route.changed",
"model.verification.updated",
"context.compacted",
"artifact.viewed",
"artifact.generated",
"review.mode.changed",
"hook.started",
"hook.completed",
"memory.citation.referenced",
"safety.review.started",
"safety.review.completed",
"terminal.input.sent",
"wait.started",
"wait.completed",
"provider.notice.recorded",
"session.starting",
"session.started",
"session.resuming",
"session.resumed",
"session.reconciled",
"session.updated",
"session.closed",
"session.failed",
"turn.submitted",
"turn.accepted",
"turn.started",
"turn.completed",
"turn.failed",
"turn.interrupted",
"turn.cancelled",
"output.body.chunk",
"item.started",
"item.delta",
"item.completed",
"item.failed",
"usage.reported",
"semantic_tool.input",
"semantic_tool.result",
"semantic_tool.reconciled",
"mcp_app.discovered",
"mcp_app.resource.resolved",
"mcp_app.initializing",
"mcp_app.ready",
"mcp_app.tool_input",
"mcp_app.tool_result",
"mcp_app.action.requested",
"mcp_app.action.resolved",
"mcp_app.host_context.changed",
"mcp_app.failed",
"mcp_app.teardown",
"runtime_request.created",
"runtime_request.resolved",
"runtime_request.expired",
"runtime_request.cancelled",
"interaction.request.proposed",
"interaction.request.materialized",
"interaction.request.rejected",
"interaction.response.progressed",
"interaction.response.resolved",
"interaction.response.delivered",
"run.attached",
"run.detached",
"run.result.proposed",
"run.result.accepted",
"run.result.rejected",
"attention.request.proposed",
"attention.request.routed",
"attention.request.resolved",
"attention.request.expired",
"attention.request.superseded",
"work.assessment.recorded",
"issue.status.decision.recorded",
"issue.status.decision.applied",
"issue.status.decision.rejected",
"issue.status.decision.superseded",
"run.terminal"
]
},
"schemaVersion": { "const": 1 },
"priority": { "enum": [0, 1, 2] },
"emittedAt": { "type": "string", "format": "date-time" },
"observedAt": { "type": "string", "format": "date-time" },
"schemaVersion": {
"const": 1
},
"priority": {
"enum": [
0,
1,
2
]
},
"emittedAt": {
"type": "string",
"format": "date-time"
},
"observedAt": {
"type": "string",
"format": "date-time"
},
"payload": {
"type": "object",
"properties": {
"channel": { "enum": ["progress", "final", "summary", "detail", "unknown"] },
"providerPhase": { "type": "string" },
"providerMethod": { "type": "string" }
"channel": {
"enum": [
"progress",
"final",
"summary",
"detail",
"unknown"
]
},
"providerPhase": {
"type": "string"
},
"providerMethod": {
"type": "string"
}
},
"additionalProperties": true
},
"debug": { "type": "object", "additionalProperties": true }
"debug": {
"type": "object",
"additionalProperties": true
},
"sourceEpochTransition": {
"type": "object",
"additionalProperties": false,
"required": [
"schema",
"runId",
"transitionId",
"fromEpoch",
"nextEpoch",
"finalOrdinal"
],
"properties": {
"schema": {
"const": "paperclip.prp.event-epoch.v1"
},
"runId": {
"type": "string",
"minLength": 1,
"maxLength": 240
},
"transitionId": {
"type": "string",
"pattern": "^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$"
},
"fromEpoch": {
"anyOf": [
{
"type": "string",
"pattern": "^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$"
},
{
"type": "null"
}
]
},
"nextEpoch": {
"type": "string",
"pattern": "^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$"
},
"finalOrdinal": {
"type": "integer",
"minimum": 1,
"maximum": 9007199254740991
}
}
}
},
"allOf": [
{ "if": { "properties": { "eventType": { "const": "plan.updated" } } }, "then": { "properties": { "payload": { "$ref": "https://paperclip.dev/schemas/prp/v1/provider-event.schema.json#/$defs/plan" } } } },
{ "if": { "properties": { "eventType": { "enum": ["tool.execution.started", "tool.execution.progressed", "tool.execution.completed"] } } }, "then": { "properties": { "payload": { "$ref": "https://paperclip.dev/schemas/prp/v1/provider-event.schema.json#/$defs/toolExecution" } } } },
{ "if": { "properties": { "eventType": { "enum": ["research.started", "research.progressed", "research.completed"] } } }, "then": { "properties": { "payload": { "$ref": "https://paperclip.dev/schemas/prp/v1/provider-event.schema.json#/$defs/research" } } } },
{ "if": { "properties": { "eventType": { "enum": ["delegation.started", "delegation.updated", "delegation.completed"] } } }, "then": { "properties": { "payload": { "$ref": "https://paperclip.dev/schemas/prp/v1/provider-event.schema.json#/$defs/delegation" } } } },
{ "if": { "properties": { "eventType": { "const": "model.route.changed" } } }, "then": { "properties": { "payload": { "$ref": "https://paperclip.dev/schemas/prp/v1/provider-event.schema.json#/$defs/modelRoute" } } } },
{ "if": { "properties": { "eventType": { "const": "model.verification.updated" } } }, "then": { "properties": { "payload": { "$ref": "https://paperclip.dev/schemas/prp/v1/provider-event.schema.json#/$defs/modelVerification" } } } },
{ "if": { "properties": { "eventType": { "const": "context.compacted" } } }, "then": { "properties": { "payload": { "$ref": "https://paperclip.dev/schemas/prp/v1/provider-event.schema.json#/$defs/contextCompacted" } } } },
{ "if": { "properties": { "eventType": { "const": "artifact.viewed" } } }, "then": { "properties": { "payload": { "$ref": "https://paperclip.dev/schemas/prp/v1/provider-event.schema.json#/$defs/artifactViewed" } } } },
{ "if": { "properties": { "eventType": { "const": "artifact.generated" } } }, "then": { "properties": { "payload": { "$ref": "https://paperclip.dev/schemas/prp/v1/provider-event.schema.json#/$defs/artifactGenerated" } } } },
{ "if": { "properties": { "eventType": { "const": "review.mode.changed" } } }, "then": { "properties": { "payload": { "$ref": "https://paperclip.dev/schemas/prp/v1/provider-event.schema.json#/$defs/reviewMode" } } } },
{ "if": { "properties": { "eventType": { "enum": ["hook.started", "hook.completed"] } } }, "then": { "properties": { "payload": { "$ref": "https://paperclip.dev/schemas/prp/v1/provider-event.schema.json#/$defs/hook" } } } },
{ "if": { "properties": { "eventType": { "const": "memory.citation.referenced" } } }, "then": { "properties": { "payload": { "$ref": "https://paperclip.dev/schemas/prp/v1/provider-event.schema.json#/$defs/memoryCitation" } } } },
{ "if": { "properties": { "eventType": { "enum": ["safety.review.started", "safety.review.completed"] } } }, "then": { "properties": { "payload": { "$ref": "https://paperclip.dev/schemas/prp/v1/provider-event.schema.json#/$defs/safetyReview" } } } },
{ "if": { "properties": { "eventType": { "const": "terminal.input.sent" } } }, "then": { "properties": { "payload": { "$ref": "https://paperclip.dev/schemas/prp/v1/provider-event.schema.json#/$defs/terminalInput" } } } },
{ "if": { "properties": { "eventType": { "enum": ["wait.started", "wait.completed"] } } }, "then": { "properties": { "payload": { "$ref": "https://paperclip.dev/schemas/prp/v1/provider-event.schema.json#/$defs/wait" } } } },
{ "if": { "properties": { "eventType": { "const": "provider.notice.recorded" } } }, "then": { "properties": { "payload": { "$ref": "https://paperclip.dev/schemas/prp/v1/provider-event.schema.json#/$defs/providerNotice" } } } },
{
"if": { "properties": { "eventType": { "const": "workspace.file.referenced" } } },
"if": {
"properties": {
"eventType": {
"const": "plan.updated"
}
}
},
"then": {
"properties": {
"payload": { "$ref": "https://paperclip.dev/schemas/prp/v1/workspace-file-reference.schema.json" }
"payload": {
"$ref": "https://paperclip.dev/schemas/prp/v1/provider-event.schema.json#/$defs/plan"
}
}
}
},
{
"if": { "properties": { "eventType": { "enum": ["workspace.change.updated", "workspace.diff.recorded"] } } },
"if": {
"properties": {
"eventType": {
"enum": [
"tool.execution.started",
"tool.execution.progressed",
"tool.execution.completed"
]
}
}
},
"then": {
"properties": {
"payload": { "$ref": "https://paperclip.dev/schemas/prp/v1/workspace-diff.schema.json" }
"payload": {
"$ref": "https://paperclip.dev/schemas/prp/v1/provider-event.schema.json#/$defs/toolExecution"
}
}
}
},
{
"if": { "properties": { "eventType": { "const": "workspace.diff.recorded" } } },
"if": {
"properties": {
"eventType": {
"enum": [
"research.started",
"research.progressed",
"research.completed"
]
}
}
},
"then": {
"properties": {
"payload": {
"$ref": "https://paperclip.dev/schemas/prp/v1/provider-event.schema.json#/$defs/research"
}
}
}
},
{
"if": {
"properties": {
"eventType": {
"enum": [
"delegation.started",
"delegation.updated",
"delegation.completed"
]
}
}
},
"then": {
"properties": {
"payload": {
"$ref": "https://paperclip.dev/schemas/prp/v1/provider-event.schema.json#/$defs/delegation"
}
}
}
},
{
"if": {
"properties": {
"eventType": {
"const": "model.route.changed"
}
}
},
"then": {
"properties": {
"payload": {
"$ref": "https://paperclip.dev/schemas/prp/v1/provider-event.schema.json#/$defs/modelRoute"
}
}
}
},
{
"if": {
"properties": {
"eventType": {
"const": "model.verification.updated"
}
}
},
"then": {
"properties": {
"payload": {
"$ref": "https://paperclip.dev/schemas/prp/v1/provider-event.schema.json#/$defs/modelVerification"
}
}
}
},
{
"if": {
"properties": {
"eventType": {
"const": "context.compacted"
}
}
},
"then": {
"properties": {
"payload": {
"$ref": "https://paperclip.dev/schemas/prp/v1/provider-event.schema.json#/$defs/contextCompacted"
}
}
}
},
{
"if": {
"properties": {
"eventType": {
"const": "artifact.viewed"
}
}
},
"then": {
"properties": {
"payload": {
"$ref": "https://paperclip.dev/schemas/prp/v1/provider-event.schema.json#/$defs/artifactViewed"
}
}
}
},
{
"if": {
"properties": {
"eventType": {
"const": "artifact.generated"
}
}
},
"then": {
"properties": {
"payload": {
"$ref": "https://paperclip.dev/schemas/prp/v1/provider-event.schema.json#/$defs/artifactGenerated"
}
}
}
},
{
"if": {
"properties": {
"eventType": {
"const": "review.mode.changed"
}
}
},
"then": {
"properties": {
"payload": {
"$ref": "https://paperclip.dev/schemas/prp/v1/provider-event.schema.json#/$defs/reviewMode"
}
}
}
},
{
"if": {
"properties": {
"eventType": {
"enum": [
"hook.started",
"hook.completed"
]
}
}
},
"then": {
"properties": {
"payload": {
"$ref": "https://paperclip.dev/schemas/prp/v1/provider-event.schema.json#/$defs/hook"
}
}
}
},
{
"if": {
"properties": {
"eventType": {
"const": "memory.citation.referenced"
}
}
},
"then": {
"properties": {
"payload": {
"$ref": "https://paperclip.dev/schemas/prp/v1/provider-event.schema.json#/$defs/memoryCitation"
}
}
}
},
{
"if": {
"properties": {
"eventType": {
"enum": [
"safety.review.started",
"safety.review.completed"
]
}
}
},
"then": {
"properties": {
"payload": {
"$ref": "https://paperclip.dev/schemas/prp/v1/provider-event.schema.json#/$defs/safetyReview"
}
}
}
},
{
"if": {
"properties": {
"eventType": {
"const": "terminal.input.sent"
}
}
},
"then": {
"properties": {
"payload": {
"$ref": "https://paperclip.dev/schemas/prp/v1/provider-event.schema.json#/$defs/terminalInput"
}
}
}
},
{
"if": {
"properties": {
"eventType": {
"enum": [
"wait.started",
"wait.completed"
]
}
}
},
"then": {
"properties": {
"payload": {
"$ref": "https://paperclip.dev/schemas/prp/v1/provider-event.schema.json#/$defs/wait"
}
}
}
},
{
"if": {
"properties": {
"eventType": {
"const": "provider.notice.recorded"
}
}
},
"then": {
"properties": {
"payload": {
"$ref": "https://paperclip.dev/schemas/prp/v1/provider-event.schema.json#/$defs/providerNotice"
}
}
}
},
{
"if": {
"properties": {
"eventType": {
"const": "workspace.file.referenced"
}
}
},
"then": {
"properties": {
"payload": {
"$ref": "https://paperclip.dev/schemas/prp/v1/workspace-file-reference.schema.json"
}
}
}
},
{
"if": {
"properties": {
"eventType": {
"enum": [
"workspace.change.updated",
"workspace.diff.recorded"
]
}
}
},
"then": {
"properties": {
"payload": {
"$ref": "https://paperclip.dev/schemas/prp/v1/workspace-diff.schema.json"
}
}
}
},
{
"if": {
"properties": {
"eventType": {
"const": "workspace.diff.recorded"
}
}
},
"then": {
"properties": {
"payload": {
"allOf": [
{ "$ref": "https://paperclip.dev/schemas/prp/v1/workspace-diff.schema.json" },
{ "type": "object", "properties": { "complete": { "const": true } } }
{
"$ref": "https://paperclip.dev/schemas/prp/v1/workspace-diff.schema.json"
},
{
"type": "object",
"properties": {
"complete": {
"const": true
}
}
}
]
}
}
}
},
{
"if": { "properties": { "eventType": { "enum": ["semantic_tool.input", "mcp_app.tool_input"] } } },
"if": {
"properties": {
"eventType": {
"enum": [
"semantic_tool.input",
"mcp_app.tool_input"
]
}
}
},
"then": {
"properties": {
"payload": {
@@ -123,8 +624,17 @@
"properties": {
"semantic_tool": {
"allOf": [
{ "$ref": "https://paperclip.dev/schemas/prp/v1/semantic-tool.schema.json" },
{ "type": "object", "properties": { "phase": { "const": "input" } } }
{
"$ref": "https://paperclip.dev/schemas/prp/v1/semantic-tool.schema.json"
},
{
"type": "object",
"properties": {
"phase": {
"const": "input"
}
}
}
]
}
},
@@ -134,7 +644,16 @@
}
},
{
"if": { "properties": { "eventType": { "enum": ["semantic_tool.result", "mcp_app.tool_result"] } } },
"if": {
"properties": {
"eventType": {
"enum": [
"semantic_tool.result",
"mcp_app.tool_result"
]
}
}
},
"then": {
"properties": {
"payload": {
@@ -142,8 +661,17 @@
"properties": {
"semantic_tool": {
"allOf": [
{ "$ref": "https://paperclip.dev/schemas/prp/v1/semantic-tool.schema.json" },
{ "type": "object", "properties": { "phase": { "const": "result" } } }
{
"$ref": "https://paperclip.dev/schemas/prp/v1/semantic-tool.schema.json"
},
{
"type": "object",
"properties": {
"phase": {
"const": "result"
}
}
}
]
}
},
@@ -153,7 +681,13 @@
}
},
{
"if": { "properties": { "eventType": { "const": "semantic_tool.reconciled" } } },
"if": {
"properties": {
"eventType": {
"const": "semantic_tool.reconciled"
}
}
},
"then": {
"properties": {
"payload": {
@@ -161,8 +695,17 @@
"properties": {
"semantic_tool": {
"allOf": [
{ "$ref": "https://paperclip.dev/schemas/prp/v1/semantic-tool.schema.json" },
{ "type": "object", "properties": { "phase": { "const": "reconciled" } } }
{
"$ref": "https://paperclip.dev/schemas/prp/v1/semantic-tool.schema.json"
},
{
"type": "object",
"properties": {
"phase": {
"const": "reconciled"
}
}
}
]
}
},
@@ -172,18 +715,107 @@
}
},
{
"if": { "properties": { "eventType": { "const": "run.terminal" } } },
"if": {
"properties": {
"eventType": {
"const": "run.terminal"
}
}
},
"then": {
"properties": {
"payload": { "$ref": "https://paperclip.dev/schemas/prp/v1/terminal.schema.json" }
"payload": {
"$ref": "https://paperclip.dev/schemas/prp/v1/terminal.schema.json"
}
}
}
},
{
"if": { "properties": { "eventType": { "const": "run.result.proposed" } } },
"if": {
"properties": {
"eventType": {
"const": "run.result.proposed"
}
}
},
"then": {
"properties": {
"payload": { "$ref": "https://paperclip.dev/schemas/prp/v1/result.schema.json" }
"payload": {
"$ref": "https://paperclip.dev/schemas/prp/v1/result.schema.json"
}
}
}
},
{
"if": {
"properties": {
"eventType": {
"const": "output.body.chunk"
}
}
},
"then": {
"properties": {
"payload": {
"type": "object",
"required": [
"schema",
"body",
"offset",
"sha256",
"text"
],
"properties": {
"schema": {
"const": "paperclip.output.body.chunk.v1"
},
"body": {
"type": "object",
"required": [
"schema",
"bodyId",
"sha256",
"byteLength",
"mediaType"
],
"properties": {
"schema": {
"const": "paperclip.output.body.v1"
},
"bodyId": {
"type": "string",
"pattern": "^[a-f0-9]{64}$"
},
"sha256": {
"type": "string",
"pattern": "^[a-f0-9]{64}$"
},
"byteLength": {
"type": "string",
"pattern": "^[1-9][0-9]{0,6}$"
},
"mediaType": {
"const": "text/plain; charset=utf-8"
}
},
"additionalProperties": false
},
"offset": {
"type": "string",
"pattern": "^(0|[1-9][0-9]{0,6})$"
},
"sha256": {
"type": "string",
"pattern": "^[a-f0-9]{64}$"
},
"text": {
"type": "string",
"minLength": 1,
"maxLength": 32766
}
},
"additionalProperties": false
}
}
}
}
packages/paperclip-runner/protocol/schemas/output-body-reference.schema.json
+15
View File
@@ -0,0 +1,15 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://paperclip.dev/schemas/prp/v1/output-body-reference.schema.json",
"title": "Immutable output body reference",
"type": "object",
"required": ["schema", "bodyId", "sha256", "byteLength", "mediaType"],
"properties": {
"schema": { "const": "paperclip.output.body.v1" },
"bodyId": { "type": "string", "pattern": "^[a-f0-9]{64}$" },
"sha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" },
"byteLength": { "type": "string", "pattern": "^[1-9][0-9]{0,6}$" },
"mediaType": { "const": "text/plain; charset=utf-8" }
},
"additionalProperties": false
}
packages/paperclip-runner/protocol/schemas/provider-event.schema.json
+2 -1
View File
@@ -62,7 +62,8 @@
"output": { "type": ["string", "null"], "maxLength": 65536 },
"outputBytes": { "type": "integer", "minimum": 0 },
"outputTruncated": { "type": "boolean" },
"outputDigest": { "type": ["string", "null"], "pattern": "^sha256:[a-f0-9]{64}$" }
"outputDigest": { "type": ["string", "null"], "pattern": "^sha256:[a-f0-9]{64}$" },
"outputBody": { "$ref": "https://paperclip.dev/schemas/prp/v1/output-body-reference.schema.json" }
},
"additionalProperties": false
},
packages/paperclip-runner/runner/Cargo.lock
Generated
+88 -3
View File
@@ -940,6 +940,18 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "fallible-iterator"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649"
[[package]]
name = "fallible-streaming-iterator"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
[[package]]
name = "fancy-regex"
version = "0.19.0"
@@ -1162,6 +1174,15 @@ dependencies = [
"tracing",
]
[[package]]
name = "hashbrown"
version = "0.16.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
dependencies = [
"foldhash",
]
[[package]]
name = "hashbrown"
version = "0.17.1"
@@ -1173,6 +1194,15 @@ dependencies = [
"foldhash",
]
[[package]]
name = "hashlink"
version = "0.12.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a596f1b20ed2cc5ecac41a164aaebc7258057060f06c0cf7a2ba3991ee7990fb"
dependencies = [
"hashbrown 0.17.1",
]
[[package]]
name = "heck"
version = "0.5.0"
@@ -1445,7 +1475,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07aa2048142242915a31d35844fb311e0e53fcca590c3a0a40dcf1b841fa09eb"
dependencies = [
"equivalent",
"hashbrown",
"hashbrown 0.17.1",
]
[[package]]
@@ -1548,6 +1578,17 @@ version = "0.2.189"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
[[package]]
name = "libsqlite3-sys"
version = "0.38.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1d20bef17f513b9b3004532233187769cd072d790971f4e4da0e346eb6401e8"
dependencies = [
"cc",
"pkg-config",
"vcpkg",
]
[[package]]
name = "linux-raw-sys"
version = "0.12.1"
@@ -1581,7 +1622,7 @@ version = "0.18.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d317b4b9eb398e6acce275758ec6125535505e7a146fb1a9b8bda2451b0ff4c"
dependencies = [
"hashbrown",
"hashbrown 0.17.1",
]
[[package]]
@@ -1761,6 +1802,7 @@ dependencies = [
"num-bigint",
"num-traits",
"reqwest",
"rusqlite",
"rustix",
"rustls",
"rustls-native-certs",
@@ -2074,7 +2116,7 @@ dependencies = [
"ahash",
"fluent-uri",
"getrandom 0.3.4",
"hashbrown",
"hashbrown 0.17.1",
"itoa",
"micromap",
"parking_lot",
@@ -2181,6 +2223,31 @@ dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "rsqlite-vfs"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c"
dependencies = [
"hashbrown 0.16.1",
"thiserror",
]
[[package]]
name = "rusqlite"
version = "0.40.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23f2a97da3e3873c73cb2a2e71b35c40ff95e0b1eefa8d72d8499a6928c3b5b3"
dependencies = [
"bitflags",
"fallible-iterator",
"fallible-streaming-iterator",
"hashlink",
"libsqlite3-sys",
"smallvec",
"sqlite-wasm-rs",
]
[[package]]
name = "rustc-hash"
version = "2.1.3"
@@ -2490,6 +2557,18 @@ dependencies = [
"der",
]
[[package]]
name = "sqlite-wasm-rs"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75"
dependencies = [
"cc",
"js-sys",
"rsqlite-vfs",
"wasm-bindgen",
]
[[package]]
name = "stable_deref_trait"
version = "1.2.1"
@@ -2889,6 +2968,12 @@ dependencies = [
"vsimd",
]
[[package]]
name = "vcpkg"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
[[package]]
name = "version_check"
version = "0.9.5"
packages/paperclip-runner/runner/Cargo.toml
+1
View File
@@ -30,6 +30,7 @@ rustls-native-certs = "0.8"
rustls-pemfile = "2.2"
reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls"] }
rustix = { version = "1.1", features = ["fs"] }
rusqlite = { version = "=0.40.2", features = ["bundled"] }
sha1 = "0.10"
tungstenite = { version = "0.28", default-features = false, features = ["handshake", "rustls-tls-native-roots"] }
toml_edit = "0.22"
packages/paperclip-runner/runner/crates/runner-core/Cargo.toml
+1
View File
@@ -27,6 +27,7 @@ rustls-native-certs.workspace = true
rustls-pemfile.workspace = true
reqwest.workspace = true
rustix.workspace = true
rusqlite.workspace = true
sha1.workspace = true
tungstenite.workspace = true
toml_edit.workspace = true
packages/paperclip-runner/runner/crates/runner-core/src/acpx_event_payload.rs
+2 -1
View File
@@ -1,3 +1,4 @@
use serde::{Deserialize, Serialize};
use std::collections::BTreeSet;
use serde_json::Value;
@@ -30,7 +31,7 @@ pub enum AcpxRuntimeEventKind {
Done,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize)]
pub enum AcpxTurnStatus {
Completed,
Failed,
packages/paperclip-runner/runner/crates/runner-core/src/acpx_event_scope.rs
+2 -1
View File
@@ -1,3 +1,4 @@
use serde::{Deserialize, Serialize};
use std::collections::BTreeSet;
use crate::acpx_sidecar_transport::AcpxSidecarEvent;
@@ -12,7 +13,7 @@ const MAX_SETTLED_TURN_IDS: usize = 4_096;
/// The sidecar transport validates framing and sequence identity. This scope
/// validates that a well-formed event still belongs to the run and turn that
/// runnerd is currently executing.
#[derive(Clone, Debug, PartialEq, Eq)]
#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)]
pub struct AcpxEventScope {
run_id: String,
active_turn_id: Option<String>,
packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs
+165 -2
View File
@@ -378,6 +378,10 @@ impl AcpxProviderDescriptor {
#[serde(rename_all = "camelCase", deny_unknown_fields)]
struct AcpxDurableState {
schema: String,
#[serde(default)]
indexed_receipts: bool,
#[serde(default)]
last_receipt_batch: Option<String>,
launch_profile_digest: String,
lifecycle: String,
descriptor: AcpxProviderDescriptor,
@@ -410,6 +414,8 @@ impl AcpxDurableState {
) -> Self {
Self {
schema: ACPX_PROVIDER_STATE_SCHEMA.to_owned(),
indexed_receipts: false,
last_receipt_batch: None,
launch_profile_digest,
lifecycle: "prepared".to_owned(),
descriptor,
@@ -455,6 +461,12 @@ impl AcpxDurableState {
| "suspended"
| "closed"
)
|| self.last_receipt_batch.as_ref().is_some_and(|batch| {
!self.indexed_receipts
|| batch.parse::<u64>().ok().is_none_or(|value| {
value == 0 || value > i64::MAX as u64 || value.to_string() != *batch
})
})
|| self.next_event_sequence == 0
|| self.pending_events.len() > MAX_PENDING_EVENTS
|| self.pending_events.iter().any(|event| {
@@ -520,6 +532,7 @@ pub struct AcpxCommandExecutor {
session: Option<AcpxProviderSession>,
restore_checked: bool,
restore_error: Option<DurableRunnerError>,
persistence_error: Option<DurableRunnerError>,
launch_profile: Option<AcpxLaunchProfile>,
}
@@ -538,6 +551,7 @@ impl AcpxCommandExecutor {
session: None,
restore_checked: false,
restore_error: None,
persistence_error: None,
launch_profile: config.acpx_launch_profile.clone(),
}
}
@@ -558,6 +572,7 @@ impl AcpxCommandExecutor {
}
fn restore(&mut self) -> Result<(), DurableRunnerError> {
self.assert_persistence_known()?;
if self.restore_checked {
return Ok(());
}
@@ -698,7 +713,15 @@ impl AcpxCommandExecutor {
expected,
self.launch_profile.as_ref(),
)?;
let mut session = AcpxProviderSession::start(&config).map_err(|error| {
let started = if state.indexed_receipts {
AcpxProviderSession::start_with_indexed_receipts(
&config,
&self.state_dir.join("acpx-provider-receipts.sqlite"),
)
} else {
AcpxProviderSession::start(&config)
};
let mut session = started.map_err(|error| {
DurableRunnerError::invalid(format!("failed to start ACPX provider: {error}"))
})?;
if session.identity().profile_digest != state.descriptor.command_digest {
@@ -710,7 +733,32 @@ impl AcpxCommandExecutor {
Ok(session)
}
fn save_state(&self) -> Result<(), DurableRunnerError> {
fn assert_persistence_known(&self) -> Result<(), DurableRunnerError> {
match &self.persistence_error {
Some(error) => Err(error.clone()),
None => Ok(()),
}
}
fn save_state(&mut self) -> Result<(), DurableRunnerError> {
self.assert_persistence_known()?;
let result = self.persist_state();
if self
.state
.as_ref()
.is_some_and(|state| state.indexed_receipts)
{
if let Err(error) = &result {
// The in-memory batch ID is not evidence of a durable outbox.
// A failed rename/fsync may be ambiguous. No later poll, drain
// or ACK may publish that candidate without recovery.
self.persistence_error = Some(error.clone());
}
}
result
}
fn persist_state(&self) -> Result<(), DurableRunnerError> {
let state = self
.state
.as_ref()
@@ -764,6 +812,14 @@ impl AcpxCommandExecutor {
let tool_set = authorized_tool_set(payload)?;
let launch_profile_digest = self.launch_profile_digest()?;
if let Some(state) = self.state.as_ref() {
if state.indexed_receipts
!= (payload.get("durability").and_then(Value::as_str)
== Some("durability.indexed_state.v1"))
{
return Err(DurableRunnerError::invalid(
"ACPX receipt format cannot change across a retained run",
));
}
if state.descriptor != descriptor || state.tool_set != tool_set {
return Err(DurableRunnerError::invalid(
"ACPX provider or authorized tool contract changed across the durable run",
@@ -780,6 +836,11 @@ impl AcpxCommandExecutor {
tool_set,
launch_profile_digest,
));
self.state
.as_mut()
.expect("prepared ACPX state exists")
.indexed_receipts = payload.get("durability").and_then(Value::as_str)
== Some("durability.indexed_state.v1");
self.save_state()?;
}
Ok(CommandExecution::result(json!({
@@ -1406,6 +1467,24 @@ impl AcpxCommandExecutor {
DurableRunnerError::invalid(format!("ACPX provider failed: {error}"))
})?;
let Some(events) = events else { break };
let batch = self
.session
.as_ref()
.and_then(AcpxProviderSession::pending_event_batch);
if batch.is_some()
&& self
.state
.as_ref()
.and_then(|state| state.last_receipt_batch.as_ref())
== batch.as_ref()
{
self.session
.as_mut()
.expect("polled session exists")
.acknowledge_event_batch()
.map_err(|error| DurableRunnerError::invalid(error.to_string()))?;
continue;
}
let mut provider_turn_settled = false;
for event in events {
let normalized = project_acpx_state_event(&self.context, &event)
@@ -1505,7 +1584,19 @@ impl AcpxCommandExecutor {
if provider_turn_settled {
self.context.provider_turn_id = None;
}
self.state
.as_mut()
.expect("polled ACPX state exists")
.last_receipt_batch = batch;
self.save_state()?;
// The bounded provider batch and receipts survive until the outer
// provider outbox is durable. A lost acknowledgement replays the
// same batch ID rather than allocating a second event prefix.
self.session
.as_mut()
.expect("polled session exists")
.acknowledge_event_batch()
.map_err(|error| DurableRunnerError::invalid(error.to_string()))?;
}
Ok(())
}
@@ -1610,6 +1701,7 @@ impl CommandExecutor for AcpxCommandExecutor {
}
fn retained_events(&mut self) -> Result<Vec<PolledEvent>, DurableRunnerError> {
self.assert_persistence_known()?;
// Explicit drain runs while control traffic suppresses provider polling.
// Expose the already-retained suffix so runnerd can commit and ACK it
// before suspension, without restoring or advancing the provider.
@@ -1623,6 +1715,7 @@ impl CommandExecutor for AcpxCommandExecutor {
}
fn acknowledge_events(&mut self, count: usize) -> Result<(), DurableRunnerError> {
self.assert_persistence_known()?;
if count == 0 {
return Ok(());
}
@@ -1640,6 +1733,19 @@ impl CommandExecutor for AcpxCommandExecutor {
}
fn shutdown(&mut self) -> Result<(), DurableRunnerError> {
if let Some(error) = self.persistence_error.clone() {
// Still reap our own live child, but never publish successful
// retirement after an uncertain durability boundary.
if let Some(session) = self.session.as_mut() {
session
.shutdown("ACPX persistence failed")
.map_err(|cleanup| {
DurableRunnerError::invalid(format!("{error}; cleanup failed: {cleanup}"))
})?;
}
self.session = None;
return Err(error);
}
// A replacement durable runner may reach terminal reconciliation
// before any provider command or event poll. Restore the persisted
// session first so cleanup cannot succeed merely because this process
@@ -1942,6 +2048,59 @@ mod tests {
fs::remove_dir_all(directory).unwrap();
}
#[test]
fn failed_indexed_outbox_publication_cannot_be_drained_or_acknowledged() {
let directory = temporary_directory("indexed-outbox-failure");
let profile = AcpxLaunchProfile {
authority_digest: format!("sha256:{}", "d".repeat(64)),
command: directory.join("sidecar"),
args: Vec::new(),
artifacts: Vec::new(),
};
let config = test_config(&directory, Some(profile.clone()));
let mut executor = AcpxCommandExecutor::with_runner_config(&directory, &config);
let operations = Vec::new();
let mut state = AcpxDurableState::new(
serde_json::from_value(descriptor("codex")).unwrap(),
AuthorizedToolSet {
schema: TOOL_SET_SCHEMA.to_owned(),
schema_version: 1,
catalog_digest: authorized_tool_catalog_digest(&operations).unwrap(),
operations,
},
profile.canonical_digest().unwrap(),
);
state.indexed_receipts = true;
executor.state = Some(state);
executor.restore_checked = true;
executor.save_state().unwrap();
let original = fs::read(executor.state_path()).unwrap();
let state = executor.state.as_mut().unwrap();
state.last_receipt_batch = Some("17".to_owned());
state
.push(NormalizedProviderEvent {
event_type: "harness.diagnostic".to_owned(),
priority: EventPriority::P0,
payload: json!({"code":"unpublished"}),
})
.unwrap();
// A real publication error after writing and syncing the candidate.
let previous = directory.join("previous.json");
fs::rename(executor.state_path(), &previous).unwrap();
fs::create_dir(executor.state_path()).unwrap();
let error = executor.save_state().unwrap_err().to_string();
assert!(error.contains("failed to atomically replace"), "{error}");
fs::remove_dir(executor.state_path()).unwrap();
fs::rename(previous, executor.state_path()).unwrap();
assert!(executor.poll_events().is_err());
assert!(executor.retained_events().is_err());
assert!(executor.acknowledge_events(1).is_err());
assert!(executor.save_state().is_err());
assert_eq!(executor.state.as_ref().unwrap().pending_events.len(), 1);
assert_eq!(fs::read(executor.state_path()).unwrap(), original);
fs::remove_dir_all(directory).unwrap();
}
#[test]
fn admits_only_exact_qualified_claude_and_codex_descriptors() {
for agent in ["claude", "codex"] {
@@ -2218,6 +2377,7 @@ mod tests {
assert!(!marker.exists());
let non_attach_error = attached
.execute(&Command {
controller_epoch: None,
schema: "paperclip.prp.command.v1".to_owned(),
command_id: "command-before-attach".to_owned(),
controller_seq: 1,
@@ -2349,6 +2509,7 @@ mod tests {
let mut drifted = AcpxCommandExecutor::with_runner_config(&directory, &drifted_config);
let drift_error = drifted
.execute(&Command {
controller_epoch: None,
schema: "paperclip.prp.command.v1".to_owned(),
command_id: "command-drift".to_owned(),
controller_seq: 1,
@@ -2364,6 +2525,7 @@ mod tests {
.contains("launch profile digest does not match runner startup"));
let retry_error = drifted
.execute(&Command {
controller_epoch: None,
schema: "paperclip.prp.command.v1".to_owned(),
command_id: "command-drift-retry".to_owned(),
controller_seq: 2,
@@ -2382,6 +2544,7 @@ mod tests {
let mut recovered = AcpxCommandExecutor::with_runner_config(&directory, &config);
let snapshot = recovered
.execute(&Command {
controller_epoch: None,
schema: "paperclip.prp.command.v1".to_owned(),
command_id: "command-1".to_owned(),
controller_seq: 1,
packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_session.rs
+150 -7
View File
@@ -8,6 +8,7 @@ use crate::acpx_provider_state::{
is_reserved_terminal_operation, AcpxProviderState, AcpxProviderStateEvent, PRP_BLOCK_TOOL_NAME,
PRP_COMPLETION_TOOL_NAME,
};
use crate::acpx_receipt_ledger::AcpxReceiptLedger;
use crate::acpx_sidecar_transport::{AcpxSidecarTransport, AcpxSidecarTransportConfig};
use crate::generated_acpx_sidecar_contract::{
GeneratedAcpxSidecarCommand, GENERATED_ACPX_SIDECAR_PROTOCOL_VERSION,
@@ -219,10 +220,29 @@ pub struct AcpxProviderSession {
working_directory: PathBuf,
closed: bool,
transport_terminated: bool,
receipt_ledger: Option<AcpxReceiptLedger>,
}
impl AcpxProviderSession {
pub fn start(config: &AcpxProviderSessionConfig) -> Result<Self, LocalRunnerError> {
Self::start_inner(config, None)
}
/// Indexed sessions retain exact receipts on disk and require their caller
/// to acknowledge each event batch after its own durable handoff.
pub fn start_with_indexed_receipts(
config: &AcpxProviderSessionConfig,
path: &Path,
) -> Result<Self, LocalRunnerError> {
config.validate()?;
let ledger = AcpxReceiptLedger::open(path, config)?;
Self::start_inner(config, Some(ledger))
}
fn start_inner(
config: &AcpxProviderSessionConfig,
mut receipt_ledger: Option<AcpxReceiptLedger>,
) -> Result<Self, LocalRunnerError> {
config.validate()?;
let mut tool_bridge = ProviderToolBridge::default();
tool_bridge
@@ -230,7 +250,7 @@ impl AcpxProviderSession {
.map_err(|error| {
LocalRunnerError::invalid(format!("ACPX authorized tools are invalid: {error}"))
})?;
let reserved_tool_bridge = reserved_terminal_tool_bridge()?;
let mut reserved_tool_bridge = reserved_terminal_tool_bridge()?;
let mut transport =
AcpxSidecarTransport::start_for_agent(&config.transport, &config.agent)?;
let bootstrap = bootstrap(&mut transport, config);
@@ -241,6 +261,19 @@ impl AcpxProviderSession {
return Err(with_cleanup_error(error, cleanup));
}
};
if let Some(ledger) = &mut receipt_ledger {
if let Err(error) = ledger.checkpoint(
&identity,
&state,
&mut tool_bridge,
&mut reserved_tool_bridge,
vec![],
None,
) {
let cleanup = transport.shutdown();
return Err(with_cleanup_error(error, cleanup));
}
}
Ok(Self {
transport,
config: config.clone(),
@@ -252,6 +285,7 @@ impl AcpxProviderSession {
working_directory: config.working_directory.clone(),
closed: false,
transport_terminated: false,
receipt_ledger,
})
}
@@ -267,6 +301,20 @@ impl AcpxProviderSession {
&self.state
}
pub fn pending_event_batch(&self) -> Option<String> {
self.receipt_ledger
.as_ref()
.and_then(AcpxReceiptLedger::pending_batch)
}
/// The executor must save its projected event outbox before calling this.
pub fn acknowledge_event_batch(&mut self) -> Result<(), LocalRunnerError> {
if let Some(ledger) = &mut self.receipt_ledger {
ledger.acknowledge()?;
}
Ok(())
}
pub fn catalog_revision(&self) -> u64 {
self.catalog_revision
}
@@ -308,6 +356,14 @@ impl AcpxProviderSession {
"ACPX provider session already has an active turn",
));
}
if let Some(ledger) = &self.receipt_ledger {
if ledger.pending_batch().is_some() {
return Err(LocalRunnerError::invalid(
"ACPX previous event batch has not been durably acknowledged",
));
}
ledger.validate_turn(turn_id)?;
}
let rotate_turn_identity_ledger = self.state.settled_turn_identity_capacity_reached();
let identity_validation = if rotate_turn_identity_ledger {
self.state
@@ -365,6 +421,28 @@ impl AcpxProviderSession {
"ACPX reserved tool receipt rotation failed: {error}"
))));
}
let mut next_state = self.state.clone();
next_state.begin_turn(turn_id)?;
if let Some(ledger) = &mut self.receipt_ledger {
next_tool_bridge = ProviderToolBridge::default();
next_tool_bridge
.prepare(self.config.tool_set.clone())
.map_err(|error| LocalRunnerError::invalid(error.to_string()))?;
next_reserved_tool_bridge = reserved_terminal_tool_bridge()?;
ledger.bind(
turn_id,
&mut next_tool_bridge,
&mut next_reserved_tool_bridge,
)?;
ledger.checkpoint(
&self.identity,
&next_state,
&mut next_tool_bridge,
&mut next_reserved_tool_bridge,
vec![],
None,
)?;
}
let response = match self.transport.request(
GeneratedAcpxSidecarCommand::TurnStart,
json!({"turnId":turn_id,"message":message}),
@@ -377,9 +455,7 @@ impl AcpxProviderSession {
"ACPX sidecar did not confirm the requested turn",
)));
}
if let Err(error) = self.state.begin_turn(turn_id) {
return Err(self.fail_closed(error));
}
self.state = next_state;
self.tool_bridge = next_tool_bridge;
self.reserved_tool_bridge = next_reserved_tool_bridge;
Ok(response)
@@ -417,6 +493,13 @@ impl AcpxProviderSession {
timeout: Duration,
) -> Result<Option<Vec<AcpxProviderStateEvent>>, LocalRunnerError> {
self.ensure_open()?;
if let Some(events) = self
.receipt_ledger
.as_ref()
.and_then(AcpxReceiptLedger::events)
{
return Ok(Some(events.to_vec()));
}
let event = match self.transport.poll_event(timeout) {
Ok(event) => event,
Err(error) => return Err(self.fail_closed(error)),
@@ -450,14 +533,21 @@ impl AcpxProviderSession {
// the server for completion feedback before resolving
// the provider call. The result is still reconciled
// by the reserved receipt ledger below.
if next_bridge.has_call_receipt(call_id) {
if next_bridge.has_call_receipt(call_id).map_err(|error| {
self.fail_closed(LocalRunnerError::invalid(error.to_string()))
})? {
return Err(self.fail_closed(LocalRunnerError::invalid(
"ACPX reused a dynamic call id for a reserved terminal invocation",
)));
}
&mut next_reserved_bridge
} else {
if next_reserved_bridge.has_call_receipt(call_id) {
if next_reserved_bridge
.has_call_receipt(call_id)
.map_err(|error| {
self.fail_closed(LocalRunnerError::invalid(error.to_string()))
})?
{
return Err(self.fail_closed(LocalRunnerError::invalid(
"ACPX reused a reserved call id for a dynamic tool invocation",
)));
@@ -490,7 +580,12 @@ impl AcpxProviderSession {
))));
}
} else {
let replayed = next_bridge.has_completed_call(&result.call_id);
let replayed =
next_bridge
.has_completed_call(&result.call_id)
.map_err(|error| {
self.fail_closed(LocalRunnerError::invalid(error.to_string()))
})?;
if let Err(error) =
next_bridge.apply_result(crate::provider_bridge::ToolResult {
call_id: result.call_id.clone(),
@@ -571,6 +666,18 @@ impl AcpxProviderSession {
reconciled_events.push(event);
}
}
if let Some(ledger) = &mut self.receipt_ledger {
if let Err(error) = ledger.checkpoint(
&self.identity,
&next_state,
&mut next_bridge,
&mut next_reserved_bridge,
reconciled_events.clone(),
None,
) {
return Err(self.fail_closed(error));
}
}
self.state = next_state;
self.tool_bridge = next_bridge;
self.reserved_tool_bridge = next_reserved_bridge;
@@ -614,6 +721,18 @@ impl AcpxProviderSession {
"error":Value::Null,
})
};
if let Some(ledger) = &mut self.receipt_ledger {
if let Err(error) = ledger.checkpoint(
&self.identity,
&next_state,
&mut next_bridge,
&mut next_reserved_bridge,
vec![],
Some(result.clone()),
) {
return Err(self.fail_closed(error));
}
}
let response = match self
.transport
.request(GeneratedAcpxSidecarCommand::ToolResolve, resolution)
@@ -622,6 +741,18 @@ impl AcpxProviderSession {
Err(error) => return Err(self.fail_closed(error)),
};
self.verify_resolution(&response, "tool")?;
if let Some(ledger) = &mut self.receipt_ledger {
if let Err(error) = ledger.checkpoint(
&self.identity,
&next_state,
&mut next_bridge,
&mut next_reserved_bridge,
vec![],
None,
) {
return Err(self.fail_closed(error));
}
}
self.state = next_state;
self.tool_bridge = next_bridge;
self.reserved_tool_bridge = next_reserved_bridge;
@@ -661,6 +792,18 @@ impl AcpxProviderSession {
Err(error) => return Err(self.fail_closed(error)),
};
self.verify_resolution(&response, "input")?;
if let Some(ledger) = &mut self.receipt_ledger {
if let Err(error) = ledger.checkpoint(
&self.identity,
&next_state,
&mut self.tool_bridge,
&mut self.reserved_tool_bridge,
vec![],
None,
) {
return Err(self.fail_closed(error));
}
}
self.state = next_state;
Ok(())
}
packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_state.rs
+6 -5
View File
@@ -1,3 +1,4 @@
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
use serde_json::Value;
@@ -24,7 +25,7 @@ pub(crate) fn is_reserved_terminal_operation(operation_id: &str) -> bool {
matches!(operation_id, PRP_COMPLETION_TOOL_NAME | PRP_BLOCK_TOOL_NAME)
}
#[derive(Clone, Debug, PartialEq)]
#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)]
pub struct AcpxPendingTool {
pub operation_id: String,
pub input: Value,
@@ -32,7 +33,7 @@ pub struct AcpxPendingTool {
input_bytes: usize,
}
#[derive(Clone, Debug, PartialEq)]
#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)]
pub struct AcpxSemanticResult {
pub call_id: String,
pub operation_id: String,
@@ -41,7 +42,7 @@ pub struct AcpxSemanticResult {
pub(crate) result_digest: String,
}
#[derive(Clone, Debug, PartialEq)]
#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)]
pub enum AcpxProviderStateEvent {
Activity(NormalizedProviderEvent),
ToolCall {
@@ -79,7 +80,7 @@ pub enum AcpxProviderStateEvent {
},
}
#[derive(Clone, Debug, PartialEq)]
#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)]
struct PendingInput {
runtime_request_id: String,
value_bytes: usize,
@@ -92,7 +93,7 @@ struct PendingInput {
/// turn authority before payload decoding. Transport commands remain outside
/// this reducer so callers can commit a pending resolution only after the
/// corresponding sidecar request succeeds.
#[derive(Clone, Debug, PartialEq)]
#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)]
pub struct AcpxProviderState {
scope: AcpxEventScope,
provider_requests: u64,
packages/paperclip-runner/runner/crates/runner-core/src/acpx_receipt_ledger.rs
+199
View File
@@ -0,0 +1,199 @@
//! Bounded ACPX current state, durable delivery intent and exact tool receipts.
//! The session publishes candidates only after the receipt transaction commits.
use crate::indexed_revision::Revision;
use std::path::Path;
use serde::{Deserialize, Serialize};
use crate::acpx_provider_session::{AcpxProviderSessionConfig, AcpxProviderSessionIdentity};
use crate::acpx_provider_state::{AcpxProviderState, AcpxProviderStateEvent};
use crate::indexed_store::{ExactReceipt, IndexedStore};
use crate::local_runner::LocalRunnerError;
use crate::provider_bridge::{ProviderToolBridge, ToolResult};
const SCHEMA: &str = "paperclip.runner.acpx-receipts.v1";
const STATE_KEY: &str = "acpx-session";
#[derive(Clone, Deserialize, Serialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
struct Checkpoint {
schema: String,
run_id: String,
identity: AcpxProviderSessionIdentity,
state: AcpxProviderState,
dynamic: ProviderToolBridge,
reserved: ProviderToolBridge,
events: Vec<AcpxProviderStateEvent>,
delivery: Option<ToolResult>,
}
pub(crate) struct AcpxReceiptLedger {
store: IndexedStore,
generation: Revision,
run_id: String,
previous: Option<Checkpoint>,
}
fn invalid(error: impl std::fmt::Display) -> LocalRunnerError {
LocalRunnerError::invalid(format!("ACPX receipt storage: {error}"))
}
impl AcpxReceiptLedger {
/// Called before spawning. An interrupted active lifetime must pass the
/// executor's recovery admission; opening receipts cannot grant it a new
/// provider process or silently discard unfinished delivery.
pub(crate) fn open(
path: &Path,
config: &AcpxProviderSessionConfig,
) -> Result<Self, LocalRunnerError> {
let binding = format!("acpx/{}", config.normalized_session_id);
let store = IndexedStore::open(path, &binding, config.expected_identity.is_none())
.map_err(invalid)?;
let current = store.read_state(STATE_KEY).map_err(invalid)?;
if current.is_none() && config.expected_identity.is_some() {
return Err(invalid("retained current receipt authority is missing"));
}
let previous: Option<Checkpoint> = current
.as_ref()
.map(|value| serde_json::from_slice(&value.bytes).map_err(invalid))
.transpose()?;
if let Some(previous) = &previous {
if previous.schema != SCHEMA
|| previous.identity.normalized_session_id != config.normalized_session_id
|| previous.state.run_id() != previous.run_id
|| previous.state.active_turn_id().is_some()
|| previous.state.has_pending_requests()
|| previous.dynamic.pending_calls().next().is_some()
|| previous.reserved.pending_calls().next().is_some()
|| !previous.events.is_empty()
|| previous.delivery.is_some()
{
return Err(invalid(
"prior session requires reconciliation before restart",
));
}
if config.expected_identity.as_ref() != Some(&previous.identity) {
return Err(invalid("retained session identity was not admitted"));
}
}
Ok(Self {
store,
generation: current.map_or(Revision::Absent, |value| value.generation),
run_id: config.run_id.clone(),
previous,
})
}
pub(crate) fn validate_turn(&self, turn_id: &str) -> Result<(), LocalRunnerError> {
if self
.store
.receipt(&format!("acpx-turns/{}", self.run_id), turn_id)
.map_err(invalid)?
.is_some()
{
return Err(invalid("reused a settled turn identity"));
}
Ok(())
}
pub(crate) fn bind(
&self,
turn_id: &str,
dynamic: &mut ProviderToolBridge,
reserved: &mut ProviderToolBridge,
) -> Result<(), LocalRunnerError> {
let namespace = format!("acpx/{}/{turn_id}", self.run_id);
dynamic
.bind_indexed_receipts(self.store.clone(), format!("{namespace}/dynamic"))
.map_err(invalid)?;
reserved
.bind_indexed_receipts(self.store.clone(), format!("{namespace}/reserved"))
.map_err(invalid)
}
pub(crate) fn events(&self) -> Option<&[AcpxProviderStateEvent]> {
self.previous
.as_ref()
.filter(|value| !value.events.is_empty())
.map(|value| value.events.as_slice())
}
pub(crate) fn pending_batch(&self) -> Option<String> {
self.events().map(|_| self.generation.to_string())
}
pub(crate) fn acknowledge(&mut self) -> Result<(), LocalRunnerError> {
let Some(mut next) = self.previous.clone() else {
return Ok(());
};
if next.events.is_empty() {
return Ok(());
}
next.events.clear();
self.commit(next, vec![])
}
pub(crate) fn checkpoint(
&mut self,
identity: &AcpxProviderSessionIdentity,
state: &AcpxProviderState,
dynamic: &mut ProviderToolBridge,
reserved: &mut ProviderToolBridge,
events: Vec<AcpxProviderStateEvent>,
delivery: Option<ToolResult>,
) -> Result<(), LocalRunnerError> {
if self.events().is_some() {
return Err(invalid(
"previous event batch has not been durably acknowledged",
));
}
let mut receipts = dynamic.checkpoint_indexed_receipts().map_err(invalid)?;
receipts.extend(reserved.checkpoint_indexed_receipts().map_err(invalid)?);
if let Some(previous) = &self.previous {
if let Some(turn) = previous
.state
.active_turn_id()
.filter(|_| state.active_turn_id().is_none())
{
receipts.push(ExactReceipt {
namespace: format!("acpx-turns/{}", previous.run_id),
key: turn.to_owned(),
bytes: b"settled".to_vec(),
});
}
}
let next = Checkpoint {
schema: SCHEMA.to_owned(),
run_id: self.run_id.clone(),
identity: identity.clone(),
state: state.clone(),
dynamic: dynamic.clone(),
reserved: reserved.clone(),
events,
delivery,
};
self.commit(next, receipts)
}
fn commit(
&mut self,
next: Checkpoint,
receipts: Vec<ExactReceipt>,
) -> Result<(), LocalRunnerError> {
let bytes = serde_json::to_vec(&next).map_err(invalid)?;
self.generation = match self
.store
.commit(STATE_KEY, self.generation, bytes, receipts)
{
Ok(generation) => generation,
Err(error) => {
// A response can be lost after durable commit. This writer may
// neither retry with a stale candidate nor release a result.
self.generation = Revision::Fenced;
return Err(invalid(error));
}
};
self.previous = Some(next);
Ok(())
}
}
packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-acpx-sidecar.rs
+82
View File
@@ -20,6 +20,12 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
.find(|pair| pair[0] == "--mode")
.map(|pair| pair[1].as_str())
.unwrap_or("happy");
let history_calls: usize = args
.windows(2)
.find(|pair| pair[0] == "--calls")
.map(|pair| pair[1].parse())
.transpose()?
.unwrap_or(64);
let profile_digest = args
.windows(2)
.find(|pair| pair[0] == "--profile-digest")
@@ -39,6 +45,82 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
.get("command")
.and_then(Value::as_str)
.ok_or("request command is missing")?;
if mode == "indexed-many-tools" {
if command == "tool.resolve" {
if let Some(marker) = args
.windows(2)
.find(|pair| pair[0] == "--resolution-marker")
{
std::fs::write(&marker[1], b"tool.resolve received")?;
}
}
write_json(
&mut stdout,
&bootstrap_success(id, command, &request, mode, profile_digest),
)?;
let params = request.get("params").unwrap_or(&Value::Null);
let turn = params
.get("turnId")
.and_then(Value::as_str)
.unwrap_or("turn-1");
let next_call = if command == "turn.start" {
Some(0)
} else if command == "tool.resolve" {
let call = params["callId"].as_str().ok_or("missing call identity")?;
write_turn_event(
&mut stdout,
next_sequence,
"runtime.event",
"run-1",
turn,
json!({
"type":"semantic_result", "callId":call, "operationId":"issues.read", "ok":true,
"result": params["result"],
}),
)?;
next_sequence += 1;
let index: usize = call
.strip_prefix("history-call-")
.ok_or("bad call identity")?
.parse()?;
Some(index + 1)
} else {
None
};
if let Some(index) = next_call {
if index < history_calls {
write_turn_event(
&mut stdout,
next_sequence,
"runtime.tool_called",
"run-1",
turn,
json!({"callId":format!("history-call-{index}"),"operationId":"issues.read","input":{"index":index}}),
)?;
} else {
// Ancient exact replay after crossing the former 4096-call ceiling.
write_turn_event(
&mut stdout,
next_sequence,
"runtime.event",
"run-1",
turn,
json!({"type":"semantic_result","callId":"history-call-0","operationId":"issues.read","ok":true,"result":{"index":0}}),
)?;
next_sequence += 1;
write_turn_event(
&mut stdout,
next_sequence,
"runtime.turn_terminal",
"run-1",
turn,
json!({"status":"completed"}),
)?;
}
next_sequence += 1;
}
continue;
}
if mode == "goals" && command.starts_with("session.goal.") {
let params = request.get("params").cloned().unwrap_or_else(|| json!({}));
match command {
packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs
+87 -2
View File
@@ -26,6 +26,25 @@ fn argument(args: &[String], name: &str) -> Option<String> {
.cloned()
}
fn opened_thread(state: &FakeState, args: &[String]) -> Value {
let mut thread = json!({
"id": state.thread_id,
"sessionId": "codex-account-session",
"historyMode": if args.iter().any(|arg| arg == "--legacy-history-mode")
|| argument(args, "--state-file").is_some_and(|path| {
Path::new(&path).with_file_name("legacy-history-mode").exists()
}) {
"legacy"
} else {
"paginated"
},
});
if args.iter().any(|arg| arg == "--missing-history-mode") {
thread.as_object_mut().unwrap().remove("historyMode");
}
thread
}
fn send(value: Value) -> io::Result<()> {
let mut stdout = io::stdout().lock();
serde_json::to_writer(&mut stdout, &value)?;
@@ -563,6 +582,25 @@ fn send_structured_activity(state: &FakeState) -> io::Result<()> {
}))
}
// Explicit credential-free history qualification. Keep each provider frame
// bounded and emit ordinary completed message items on one active turn.
fn send_history_growth(state: &FakeState, mib: u64) -> io::Result<()> {
for index in 0..mib * 4 {
let prefix = format!("history-{index}: ");
let text = format!("{prefix}{}", "h".repeat(256 * 1024 - prefix.len()));
let item = if index % 2 == 0 {
json!({"id":format!("history-{index}"),"type":"agentMessage","status":"completed","text":text})
} else {
json!({"id":format!("history-{index}"),"type":"commandExecution","status":"completed","command":"qualification-output","aggregatedOutput":text,"exitCode":0})
};
send(json!({"method":"item/completed", "params":{
"threadId":state.thread_id,"turnId":state.active_turn_id,
"item":item
}}))?;
}
Ok(())
}
fn send_runtime_request_flood(
state: &FakeState,
interrupt_count: u64,
@@ -617,6 +655,15 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
.iter()
.any(|value| value == "--opencode-proxy-runtime-question");
let emit_runtime_elicitation = args.iter().any(|value| value == "--runtime-elicitation");
let history_growth_mib = argument(&args, "--history-growth-mib")
.map(|value| value.parse::<u64>())
.transpose()?
.unwrap_or(0);
let soak_output_on_steer = args.iter().any(|value| value == "--soak-output-on-steer");
let mut soak_output_sequence = 0_u64;
if history_growth_mib > 1024 * 1024 {
return Err("qualification size exceeds 1 TiB".into());
}
let emit_structured_activity = args.iter().any(|value| value == "--structured-activity");
let emit_split_event_burst = args.iter().any(|value| value == "--split-event-burst");
let split_event_suffix_count = argument(&args, "--split-event-suffix-count")
@@ -1024,6 +1071,11 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
}
"initialized" => {}
"thread/start" => {
if args.iter().any(|arg| arg == "--require-paginated-history")
&& message.pointer("/params/historyMode") != Some(&json!("paginated"))
{
return Err("thread/start must explicitly select paginated history".into());
}
if require_external_sandbox
&& (message.pointer("/params/sandbox") != Some(&json!("danger-full-access"))
|| message.pointer("/params/permissions").is_some())
@@ -1066,7 +1118,7 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
}
send(json!({
"id": id,
"result": {"thread": {"id": state.thread_id, "sessionId": "codex-account-session"}}
"result": {"thread": opened_thread(&state, &args)}
}))?;
if agent_created_goal {
send(json!({
@@ -1116,7 +1168,7 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
}
send(json!({
"id": id,
"result": {"thread": {"id": state.thread_id, "sessionId": "codex-account-session"}}
"result": {"thread": opened_thread(&state, &args)}
}))?;
if args.iter().any(|arg| arg == "--resume-usage-snapshot")
&& state.active_turn_id.is_none()
@@ -1152,6 +1204,25 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
}
}
"thread/turns/list" => {
if args
.iter()
.any(|arg| arg == "--unbounded-completed-history")
{
if args.iter().any(|arg| arg == "--settle-during-history") {
state.active_turn_id = None;
save_state(&state_path, &state)?;
}
let page = message
.pointer("/params/cursor")
.and_then(Value::as_str)
.and_then(|cursor| cursor.parse::<u64>().ok())
.unwrap_or(0);
send(json!({"id": id, "result": {
"data": [{"id": format!("old-turn-{page}"), "status": "completed", "items": [], "itemsView": "notLoaded"}],
"nextCursor": (page + 1).to_string(),
}}))?;
continue;
}
if args
.iter()
.any(|arg| arg == "--require-lightweight-history")
@@ -1665,6 +1736,11 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
send_opencode_proxy_runtime_question(&state)?;
} else if emit_runtime_elicitation {
send_runtime_elicitation(&state)?;
} else if history_growth_mib > 0 {
if turn_start_count == 1 {
send_history_growth(&state, history_growth_mib)?;
}
finish_turn(&state_path, &mut state, "completed")?;
} else if emit_structured_activity {
send_structured_activity(&state)?;
finish_turn(&state_path, &mut state, "completed")?;
@@ -1788,6 +1864,15 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
}
"turn/steer" => {
send(json!({"id": id, "result": {"accepted": true}}))?;
if soak_output_on_steer && state.active_turn_id.is_some() {
soak_output_sequence += 1;
let text = format!("soak-output-{soak_output_sequence}: {}", "s".repeat(8192));
send(json!({"method":"item/completed", "params": {
"threadId": state.thread_id, "turnId": state.active_turn_id,
"item": {"id": format!("soak-output-{soak_output_sequence}"),
"type":"agentMessage", "text":text}
}}))?;
}
if linger_after_turn_start {
send(json!({
"method": "item/completed",
packages/paperclip-runner/runner/crates/runner-core/src/bin/paperclip-runnerd.rs
+293 -14
View File
@@ -8,8 +8,9 @@ use std::time::Duration;
use std::os::unix::fs::{OpenOptionsExt, PermissionsExt};
use paperclip_runner_core::durable::{
capture_bootstrap_ticket, redact_diagnostic_text, run_durable_runner, AcpxLaunchProfile,
DurableRunnerConfig, OpenCodeLaunchProfile, QualifiedLaunchArtifact,
capture_bootstrap_ticket, redact_diagnostic_text, run_durable_runner,
run_indexed_durable_runner, AcpxLaunchProfile, DurableRunnerConfig, OpenCodeLaunchProfile,
QualifiedLaunchArtifact,
};
use paperclip_runner_core::local_runner::{run_local_runner, LocalRunnerError, RunnerConfig};
use paperclip_runner_core::native_provider_backend::NativeProviderCommandExecutor;
@@ -261,14 +262,7 @@ fn usize_value(args: &[String], name: &str, default: usize) -> Result<usize, Loc
})
}
fn run_durable(args: &[String]) -> Result<(), LocalRunnerError> {
let ticket = capture_bootstrap_ticket()
.map_err(|error| LocalRunnerError::invalid(error.to_string()))?
.ok_or_else(|| {
LocalRunnerError::invalid(
"PAPERCLIP_RUNNER_BOOTSTRAP_TICKET is required for durable mode",
)
})?;
fn durable_config(args: &[String]) -> Result<DurableRunnerConfig, LocalRunnerError> {
let duration = |name: &str, default: u64| {
optional_u64(args, name).map(|value| Duration::from_millis(value.unwrap_or(default)))
};
@@ -312,7 +306,7 @@ fn run_durable(args: &[String]) -> Result<(), LocalRunnerError> {
"--ca-bundle-path is accepted only with wss://",
));
}
let config = DurableRunnerConfig {
Ok(DurableRunnerConfig {
connect_url,
ca_bundle_path,
state_dir: state_dir.clone(),
@@ -332,10 +326,26 @@ fn run_durable(args: &[String]) -> Result<(), LocalRunnerError> {
reconnect_delay: duration("--reconnect-delay-ms", 250)?,
reconnect_grace: optional_u64(args, "--reconnect-grace-ms")?.map(Duration::from_millis),
max_runtime: duration("--max-runtime-ms", 0)?,
};
})
}
fn run_durable(args: &[String]) -> Result<(), LocalRunnerError> {
let ticket = capture_bootstrap_ticket()
.map_err(|error| LocalRunnerError::invalid(error.to_string()))?
.ok_or_else(|| {
LocalRunnerError::invalid(
"PAPERCLIP_RUNNER_BOOTSTRAP_TICKET is required for durable mode",
)
})?;
let config = durable_config(args)?;
let state_dir = config.state_dir.clone();
let executor = NativeProviderCommandExecutor::with_runner_config(state_dir, &config);
run_durable_runner(config, ticket, executor)
.map_err(|error| LocalRunnerError::invalid(error.to_string()))
(if args.iter().any(|argument| argument == "--indexed-state") {
run_indexed_durable_runner(config, ticket, executor)
} else {
run_durable_runner(config, ticket, executor)
})
.map_err(|error| LocalRunnerError::invalid(error.to_string()))
}
fn run(args: &[String]) -> Result<(), LocalRunnerError> {
@@ -343,6 +353,9 @@ fn run(args: &[String]) -> Result<(), LocalRunnerError> {
println!("{}", build_metadata());
return Ok(());
}
if args.first().is_some_and(|argument| argument == "storage") {
return run_storage(args);
}
if args.iter().any(|argument| argument == "--connect-url")
|| args.iter().any(|argument| argument == "--listen-address")
|| args.iter().any(|argument| argument == "--listen-port")
@@ -368,6 +381,208 @@ fn run(args: &[String]) -> Result<(), LocalRunnerError> {
})
}
fn run_storage(args: &[String]) -> Result<(), LocalRunnerError> {
use base64::Engine;
use paperclip_runner_core::indexed_store::IndexedStore;
let operation = args.get(1).map(String::as_str).unwrap_or("");
if operation == "snapshot-session" {
use paperclip_runner_core::indexed_session_snapshot::{snapshot_session, ExpectedState};
let states = [
ExpectedState {
generation: value(args, "--runner-generation")?,
sha256: value(args, "--runner-sha256")?,
},
ExpectedState {
generation: value(args, "--provider-generation")?,
sha256: value(args, "--provider-sha256")?,
},
];
let result = snapshot_session(
&PathBuf::from(value(args, "--directory")?),
&PathBuf::from(value(args, "--destination")?),
states,
)
.map_err(|e| LocalRunnerError::invalid(e.to_string()))?;
println!("{result}");
return Ok(());
}
if operation == "inspect-state" {
return paperclip_runner_core::indexed_inspection::stream(
&PathBuf::from(value(args, "--path")?),
std::io::stdout().lock(),
)
.map_err(|error| LocalRunnerError::invalid(error.to_string()));
}
if operation == "read-receipt" {
let bytes = paperclip_runner_core::indexed_store::read_only_receipt(
&PathBuf::from(value(args, "--path")?),
&value(args, "--binding")?,
value(args, "--generation")?
.parse::<paperclip_runner_core::indexed_revision::Revision>()
.map_err(|_| LocalRunnerError::invalid("invalid proof generation"))?,
&value(args, "--namespace")?,
&value(args, "--key")?,
)
.map_err(|e| LocalRunnerError::invalid(e.to_string()))?;
println!(
"{}",
json!(bytes.map(|b| base64::engine::general_purpose::STANDARD.encode(b)))
);
return Ok(());
}
if operation == "rpc" {
let store = IndexedStore::open(
&PathBuf::from(value(args, "--path")?),
&value(args, "--binding")?,
optional_value(args, "--create")?.as_deref() == Some("true"),
)
.map_err(|e| LocalRunnerError::invalid(e.to_string()))?;
return paperclip_runner_core::indexed_rpc::serve(
store,
std::io::stdin().lock(),
std::io::stdout().lock(),
)
.map_err(|e| LocalRunnerError::invalid(e.to_string()));
}
if operation == "archive-controller" {
paperclip_runner_core::indexed_archive::controller(
&PathBuf::from(value(args, "--directory")?),
&PathBuf::from(value(args, "--destination")?),
&value(args, "--digest")?,
)
.map_err(|e| LocalRunnerError::invalid(e.to_string()))?;
println!("{}", json!({"archive":"complete"}));
return Ok(());
}
if operation == "seal" {
paperclip_runner_core::indexed_archive::seal(
&PathBuf::from(value(args, "--directory")?),
&value(args, "--generation")?,
)
.map_err(|e| LocalRunnerError::invalid(e.to_string()))?;
println!("{}", json!({"seal":"complete"}));
return Ok(());
}
if operation == "archive-prepare" || operation == "archive-finish" {
let runner = PathBuf::from(value(args, "--directory")?);
let archive = PathBuf::from(value(args, "--destination")?);
let map = |e: paperclip_runner_core::durable::DurableRunnerError| {
LocalRunnerError::invalid(e.to_string())
};
if operation == "archive-prepare" {
paperclip_runner_core::indexed_archive::prepare(
&runner,
&archive,
&value(args, "--generation")?,
&value(args, "--digest")?,
)
.map_err(map)?;
println!("{}", json!({"archive":"prepared"}));
} else {
let finished =
paperclip_runner_core::indexed_archive::finish(&runner, &archive).map_err(map)?;
println!(
"{}",
json!({"archive":if finished {"complete"} else {"absent"}})
);
}
return Ok(());
}
if operation == "stage-legacy" {
let config = durable_config(args)?;
let destination = PathBuf::from(value(args, "--destination")?);
let fence_id = value(args, "--fence-id")?;
let map = |e: paperclip_runner_core::durable::DurableRunnerError| {
LocalRunnerError::invalid(e.to_string())
};
let runner = paperclip_runner_core::durable::DurableStateStore::stage_legacy(
&config,
&destination,
&fence_id,
)
.map_err(map)?;
let provider =
paperclip_runner_core::provider_backend::CodexCommandExecutor::with_runner_config(
&config.state_dir,
&config,
)
.stage_legacy(&destination, &fence_id)
.map_err(map)?;
println!(
"{}",
json!({"schema":"paperclip.runner.prepared-legacy-session.v1","runner":runner,"provider":provider})
);
return Ok(());
}
if !["partitions", "maintain", "relocate", "backup"].contains(&operation) {
return Err(LocalRunnerError::invalid(
"storage requires partitions, maintain, relocate, or backup",
));
}
let path = PathBuf::from(value(args, "--path")?);
let binding = value(args, "--binding")?;
let store = IndexedStore::open(&path, &binding, false)
.map_err(|e| LocalRunnerError::invalid(e.to_string()))?;
let decode = |value: String| -> Result<String, LocalRunnerError> {
if value.len() > 2800 {
return Err(LocalRunnerError::invalid(
"partition cursor exceeds capacity",
));
}
let bytes = base64::engine::general_purpose::STANDARD
.decode(value)
.map_err(|_| LocalRunnerError::invalid("invalid base64 partition cursor"))?;
String::from_utf8(bytes)
.map_err(|_| LocalRunnerError::invalid("invalid partition cursor UTF-8"))
};
let mapped = |error: paperclip_runner_core::durable::DurableRunnerError| {
LocalRunnerError::invalid(error.to_string())
};
match operation {
"partitions" => {
let after = optional_value(args, "--after")?.map(decode).transpose()?;
let entries = store
.partitions(after.as_deref(), usize_value(args, "--limit", 32)?)
.map_err(mapped)?;
let rows:Vec<_>=entries.iter().map(|entry|json!({"start":base64::engine::general_purpose::STANDARD.encode(&entry.lower_key),"file":entry.file,"records":entry.records.to_string(),"bytes":entry.bytes.to_string()})).collect();
println!(
"{}",
json!({"schema":"paperclip.receipt-partitions.v1","partitions":rows,"nextAfter":entries.last().map(|entry|base64::engine::general_purpose::STANDARD.encode(&entry.lower_key))})
);
}
"maintain" => {
let steps = usize_value(args, "--steps", 1)?;
if !(1..=1000).contains(&steps) {
return Err(LocalRunnerError::invalid(
"maintenance steps must be 1 through 1000",
));
}
for _ in 0..steps {
store.maintain().map_err(mapped)?;
std::thread::yield_now();
}
println!("{}", json!({"maintenanceSteps":steps}));
}
"relocate" => {
let lower = decode(value(args, "--partition-start")?)?;
store
.relocate_partition(&lower, &PathBuf::from(value(args, "--destination")?))
.map_err(mapped)?;
println!("{}", json!({"relocation":"started"}));
}
"backup" => {
let destination = PathBuf::from(value(args, "--destination")?);
store.begin_backup(&destination).map_err(mapped)?;
while !store.backup_step().map_err(mapped)? {
std::thread::sleep(Duration::from_millis(10));
}
println!("{}", json!({"backup":"complete","destination":destination}));
}
_ => unreachable!(),
}
Ok(())
}
fn run_main(args: Vec<String>) -> ExitCode {
let diagnostics_directory = diagnostic_directory(&args);
install_diagnostic_panic_hook(diagnostics_directory.clone());
@@ -394,6 +609,70 @@ fn run_main(args: Vec<String>) -> ExitCode {
mod tests {
use super::*;
#[test]
fn storage_commands_back_up_and_inspect_an_existing_private_store() {
use paperclip_runner_core::indexed_store::{ExactReceipt, IndexedStore};
let directory =
std::env::temp_dir().join(format!("runner-storage-cli-{}", uuid::Uuid::new_v4()));
let path = directory.join("state.sqlite");
let store = IndexedStore::open(&path, "operator-test", true).unwrap();
store
.commit(
"current",
0,
b"checkpoint".to_vec(),
vec![ExactReceipt {
namespace: "effects".into(),
key: "ancient".into(),
bytes: b"receipt".to_vec(),
}],
)
.unwrap();
drop(store);
let args = |operation: &str, extra: Vec<String>| {
let mut arguments = vec![
"storage".into(),
operation.into(),
"--path".into(),
path.to_str().unwrap().into(),
"--binding".into(),
"operator-test".into(),
];
arguments.extend(extra);
arguments
};
run(&args("partitions", vec![])).unwrap();
run(&args("maintain", vec!["--steps".into(), "2".into()])).unwrap();
let destination = directory.join("backup");
run(&args(
"backup",
vec!["--destination".into(), destination.to_str().unwrap().into()],
))
.unwrap();
let restored = IndexedStore::open(
&destination.join("authority.sqlite"),
"operator-test",
false,
)
.unwrap();
assert_eq!(
restored.read_state("current").unwrap().unwrap().bytes,
b"checkpoint"
);
assert_eq!(
restored.receipt("effects", "ancient").unwrap(),
Some(b"receipt".to_vec())
);
drop(restored);
assert!(run(&args("maintain", vec!["--steps".into(), "1001".into()])).is_err());
assert!(run(&args(
"backup",
vec!["--destination".into(), destination.to_str().unwrap().into()]
))
.is_err());
fs::remove_dir_all(directory).unwrap();
}
#[test]
fn build_metadata_advertises_the_remote_transport_contract() {
let metadata = build_metadata();
packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs
+353 -144
View File
@@ -1,3 +1,6 @@
use crate::indexed_revision::Revision;
use crate::indexed_store::{ExactReceipt, IndexedStore};
use crate::process_generation::ProcessGeneration;
use std::collections::{BTreeMap, BTreeSet, VecDeque};
use std::fs::{self, OpenOptions};
use std::io::Write;
@@ -514,14 +517,14 @@ pub enum CodexProviderEvent {
completed_turn_authoritative: bool,
completed_turn_observed_by_process: bool,
completion_reconciles_exit: bool,
process_generation: u64,
completed_turn_process_generation: Option<u64>,
process_generation: ProcessGeneration,
completed_turn_process_generation: Option<ProcessGeneration>,
},
}
#[derive(Clone, Debug, PartialEq, Eq)]
struct CompletedTurnAuthority {
process_generation: u64,
process_generation: ProcessGeneration,
provider_turn_id: String,
}
@@ -529,29 +532,72 @@ struct CompletedTurnAuthority {
struct SettledProviderTurnIds {
ids: BTreeSet<String>,
filter: DurableReplayFilter,
index: Option<(IndexedStore, String, Revision)>,
}
impl SettledProviderTurnIds {
fn insert(&mut self, provider_turn_id: String) -> bool {
if self.contains(&provider_turn_id) {
return true;
fn bind(&mut self, store: IndexedStore, namespace: String) -> Result<(), LocalRunnerError> {
if !self.filter.is_empty() {
return Err(LocalRunnerError::invalid(
"legacy turn filter requires fenced migration",
));
}
if self.ids.len() >= MAX_SETTLED_PROVIDER_TURN_IDS {
return false;
let generation = store
.read_state(&format!("identities/{namespace}"))
.map_err(|error| LocalRunnerError::invalid(error.to_string()))?
.map_or(Revision::Absent, |state| state.generation);
self.index = Some((store, namespace, generation));
for id in std::mem::take(&mut self.ids) {
self.insert(id)?;
}
self.ids.insert(provider_turn_id)
Ok(())
}
fn contains(&self, provider_turn_id: &str) -> bool {
self.ids.contains(provider_turn_id)
fn insert(&mut self, provider_turn_id: String) -> Result<bool, LocalRunnerError> {
if self.contains(&provider_turn_id)? {
return Ok(true);
}
if let Some((store, namespace, generation)) = self.index.as_mut() {
*generation = store
.commit(
&format!("identities/{namespace}"),
*generation,
b"{}".to_vec(),
vec![ExactReceipt {
namespace: namespace.clone(),
key: provider_turn_id,
bytes: b"settled".to_vec(),
}],
)
.map_err(|error| LocalRunnerError::invalid(error.to_string()))?;
return Ok(true);
}
if self.ids.len() >= MAX_SETTLED_PROVIDER_TURN_IDS {
return Ok(false);
}
Ok(self.ids.insert(provider_turn_id))
}
fn contains(&self, provider_turn_id: &str) -> Result<bool, LocalRunnerError> {
if self.ids.contains(provider_turn_id) {
return Ok(true);
}
if let Some((store, namespace, _)) = &self.index {
return store
.receipt(namespace, provider_turn_id)
.map(|record| record.is_some())
.map_err(|error| LocalRunnerError::invalid(error.to_string()));
}
Ok(false)
}
fn at_capacity(&self) -> bool {
self.ids.len() >= MAX_SETTLED_PROVIDER_TURN_IDS || !self.filter.is_empty()
self.index.is_none()
&& (self.ids.len() >= MAX_SETTLED_PROVIDER_TURN_IDS || !self.filter.is_empty())
}
fn restore(&mut self, provider_turn_id: String) -> Result<(), LocalRunnerError> {
if !self.insert(provider_turn_id) {
if !self.insert(provider_turn_id)? {
return Err(LocalRunnerError::invalid(
"Codex restored provider turn identity epoch exceeded its exact capacity",
));
@@ -569,7 +615,7 @@ impl SettledProviderTurnIds {
.map_err(|error| LocalRunnerError::invalid(error.to_string()))?;
self.filter = replay_filter;
for provider_turn_id in provider_turn_ids {
if !self.insert(provider_turn_id) {
if !self.insert(provider_turn_id)? {
return Err(LocalRunnerError::invalid(
"Codex restored provider turn identity epoch exceeded its exact capacity",
));
@@ -634,7 +680,6 @@ pub struct CodexProvider {
stderr_tail: BoundedLogBuffer,
config: CodexProviderConfig,
authorized_tools: Vec<AuthorizedTool>,
next_request_id: u64,
thread_id: String,
provider_session_id: Option<String>,
active_provider_turn_id: Option<String>,
@@ -645,20 +690,20 @@ pub struct CodexProvider {
pending_tool_requests: BTreeMap<String, PendingToolRequest>,
completed_tool_call_ids: BTreeSet<String>,
durable_tool_call_replays: bool,
require_paginated_history: bool,
pending_tool_request_bytes: usize,
pending_runtime_requests: BTreeMap<String, PendingRuntimeRequest>,
pending_runtime_request_bytes: usize,
runtime_request_scope: [u8; 16],
next_runtime_request_sequence: u64,
expected_shutdown: bool,
process_generation: u64,
process_generation: ProcessGeneration,
completed_turn_authority: Option<CompletedTurnAuthority>,
active_turn_result_authoritative: bool,
completion_reconciliation_pending: bool,
goal_allows_autonomous_turns: bool,
ambiguous_turn_start_pending: bool,
settled_provider_turn_ids: SettledProviderTurnIds,
descendant_thread_ids: BTreeSet<String>,
descendant_thread_ids: SettledProviderTurnIds,
notification_identity_diagnostics: usize,
rejected_accepted_turn: Option<RejectedAcceptedTurn>,
quarantined: bool,
@@ -671,7 +716,7 @@ pub struct CodexProvider {
}
struct ProviderExitDrain {
process_generation: u64,
process_generation: ProcessGeneration,
deadline: std::time::Instant,
timed_out: bool,
warning_emitted: bool,
@@ -804,7 +849,7 @@ impl CodexProvider {
config,
std::iter::empty(),
resume_thread_id,
1,
1.into(),
None,
None,
)
@@ -819,7 +864,7 @@ impl CodexProvider {
config,
authorized_tools,
resume_thread_id,
1,
1.into(),
None,
None,
)
@@ -829,7 +874,7 @@ impl CodexProvider {
config: &CodexProviderConfig,
authorized_tools: impl IntoIterator<Item = AuthorizedTool>,
resume_thread_id: Option<&str>,
process_generation: u64,
process_generation: ProcessGeneration,
opencode_launch_profile: Option<&OpenCodeLaunchProfile>,
completion_contract: Option<(&str, &[String])>,
) -> Result<Self, LocalRunnerError> {
@@ -840,6 +885,7 @@ impl CodexProvider {
process_generation,
opencode_launch_profile,
completion_contract,
false,
&mut |_| Ok(()),
)
}
@@ -848,9 +894,10 @@ impl CodexProvider {
config: &CodexProviderConfig,
authorized_tools: impl IntoIterator<Item = AuthorizedTool>,
resume_thread_id: Option<&str>,
process_generation: u64,
process_generation: ProcessGeneration,
opencode_launch_profile: Option<&OpenCodeLaunchProfile>,
completion_contract: Option<(&str, &[String])>,
require_paginated_history: bool,
observe: &mut dyn FnMut(ProviderStartupObservation) -> Result<(), LocalRunnerError>,
) -> Result<Self, LocalRunnerError> {
config.validate()?;
@@ -964,7 +1011,6 @@ impl CodexProvider {
),
config: config.clone(),
authorized_tools,
next_request_id: 1,
thread_id: String::new(),
provider_session_id: None,
active_provider_turn_id: None,
@@ -975,11 +1021,11 @@ impl CodexProvider {
pending_tool_requests: BTreeMap::new(),
completed_tool_call_ids: BTreeSet::new(),
durable_tool_call_replays: false,
require_paginated_history: require_paginated_history && config.provider == "codex",
pending_tool_request_bytes: 0,
pending_runtime_requests: BTreeMap::new(),
pending_runtime_request_bytes: 0,
runtime_request_scope,
next_runtime_request_sequence: 1,
expected_shutdown: false,
process_generation,
completed_turn_authority: None,
@@ -988,7 +1034,7 @@ impl CodexProvider {
goal_allows_autonomous_turns: false,
ambiguous_turn_start_pending: false,
settled_provider_turn_ids: SettledProviderTurnIds::default(),
descendant_thread_ids: BTreeSet::new(),
descendant_thread_ids: SettledProviderTurnIds::default(),
notification_identity_diagnostics: 0,
rejected_accepted_turn: None,
quarantined: false,
@@ -1063,10 +1109,27 @@ impl CodexProvider {
"thread/resume"
} else {
params_object.insert("experimentalRawEvents".to_owned(), json!(false));
if provider.require_paginated_history {
params_object.insert("historyMode".to_owned(), json!("paginated"));
}
"thread/start"
};
stage = ProviderStartupStage::ThreadOpen;
let opened = provider.request(method, params)?;
// Indexed Paperclip storage must not silently create or resume a
// legacy provider history. This verifies only the history API;
// paginated history alone does not prove bounded model-context
// reconstruction inside the provider on a cold resume.
if provider.require_paginated_history
&& opened
.pointer("/thread/historyMode")
.and_then(Value::as_str)
!= Some("paginated")
{
return Err(LocalRunnerError::invalid(
"codex_history_mode_unsupported: indexed sessions require paginated provider history",
));
}
provider.thread_id = opened
.pointer("/thread/id")
.or_else(|| opened.get("threadId"))
@@ -1117,7 +1180,7 @@ impl CodexProvider {
self.process.id()
}
pub(crate) fn process_generation(&self) -> u64 {
pub(crate) fn process_generation(&self) -> ProcessGeneration {
self.process_generation
}
@@ -1363,13 +1426,13 @@ impl CodexProvider {
pub(crate) fn restore_completed_turn_authority(
&mut self,
authoritative: bool,
process_generation: Option<u64>,
process_generation: Option<ProcessGeneration>,
provider_turn_id: Option<&str>,
) -> Result<(), LocalRunnerError> {
self.completed_turn_authority = authoritative.then(|| CompletedTurnAuthority {
// Legacy state did not record the generation. Generation zero is
// deliberately older than every supervised process generation.
process_generation: process_generation.unwrap_or(0),
process_generation: process_generation.unwrap_or_default(),
provider_turn_id: provider_turn_id
.unwrap_or("durable-completed-turn")
.to_owned(),
@@ -1391,11 +1454,54 @@ impl CodexProvider {
Ok(())
}
pub(crate) fn restore_descendant_thread_identities(&mut self, identities: &BTreeSet<String>) {
// Exact provider-confirmed lineage lives as long as the root session.
// Evicting it would turn later child progress into a root integrity fault.
pub(crate) fn restore_descendant_thread_identities(
&mut self,
identities: &BTreeSet<String>,
) -> Result<(), LocalRunnerError> {
for id in identities {
self.descendant_thread_ids.restore(id.clone())?;
}
Ok(())
}
pub(crate) fn bind_indexed_turn_receipts(
&mut self,
store: IndexedStore,
binding: String,
) -> Result<(), LocalRunnerError> {
self.settled_provider_turn_ids
.bind(store.clone(), format!("turns/{binding}"))?;
self.descendant_thread_ids
.extend(identities.iter().cloned());
.bind(store, format!("descendants/{binding}"))
}
// Resolve only identities carried by this frame. Restoring lineage never
// enumerates all children, and a failed storage read is not an unknown child.
fn notification_descendants(
&self,
params: &Value,
) -> Result<BTreeSet<String>, LocalRunnerError> {
let mut known = BTreeSet::new();
for path in [
"/threadId",
"/thread/id",
"/turn/threadId",
"/thread/parentThreadId",
"/thread/source/subAgent/thread_spawn/parent_thread_id",
"/thread/source/subAgent/threadSpawn/parentThreadId",
"/thread/source/subagent/thread_spawn/parent_thread_id",
] {
if let Some(id) = params
.pointer(path)
.and_then(Value::as_str)
.filter(|id| !id.is_empty() && id.len() <= 240)
{
if self.descendant_thread_ids.contains(id)? {
known.insert(id.to_owned());
}
}
}
Ok(known)
}
pub(crate) fn restore_settled_turn_identities(
@@ -1407,7 +1513,7 @@ impl CodexProvider {
.restore_all(provider_turn_ids, replay_filter)
}
pub(crate) fn completed_turn_authority(&self) -> Option<(u64, &str)> {
pub(crate) fn completed_turn_authority(&self) -> Option<(ProcessGeneration, &str)> {
self.completed_turn_authority.as_ref().map(|authority| {
(
authority.process_generation,
@@ -1436,11 +1542,16 @@ impl CodexProvider {
"durable provider rollover requires its startup ownership observer",
));
}
self.restart_idle_identity_epoch_observed(&mut |_| Ok(()))
let next = self
.process_generation
.next()
.ok_or_else(|| LocalRunnerError::invalid("fresh provider lifetime unavailable"))?;
self.restart_idle_identity_epoch_observed(next, &mut |_| Ok(()))
}
pub(crate) fn restart_idle_identity_epoch_observed(
&mut self,
next_generation: ProcessGeneration,
observe: &mut dyn FnMut(ProviderStartupObservation) -> Result<(), LocalRunnerError>,
) -> Result<(), LocalRunnerError> {
if self.active_provider_turn_id.is_some() || self.ambiguous_turn_start_pending {
@@ -1449,9 +1560,11 @@ impl CodexProvider {
));
}
let next_generation = self.process_generation.checked_add(1).ok_or_else(|| {
LocalRunnerError::invalid("Codex process generation exhausted during epoch rollover")
})?;
if !next_generation.is_successor_of(self.process_generation) {
return Err(LocalRunnerError::invalid(
"provider rollover differs from its prepared lifetime",
));
}
let config = self.config.clone();
let authorized_tools = self.authorized_tools.clone();
let thread_id = self.thread_id.clone();
@@ -1477,6 +1590,7 @@ impl CodexProvider {
contract.criterion_ids.as_slice(),
)
}),
self.require_paginated_history,
observe,
)?;
replacement.durable_tool_call_replays = durable_tool_call_replays;
@@ -1737,7 +1851,7 @@ impl CodexProvider {
return Err(error);
}
};
if self.settled_provider_turn_ids.contains(&provider_turn_id) {
if self.settled_provider_turn_ids.contains(&provider_turn_id)? {
return Err(self.reject_accepted_reused_turn_identity(provider_turn_id));
}
// Only a validated provider turn identity proves that replacement
@@ -1771,7 +1885,7 @@ impl CodexProvider {
))
})?;
if self.settled_provider_turn_ids.contains(&provider_turn_id) {
if self.settled_provider_turn_ids.contains(&provider_turn_id)? {
return Err(self.reject_accepted_reused_turn_identity(provider_turn_id));
}
@@ -1893,7 +2007,10 @@ impl CodexProvider {
let mut cursor = Value::Null;
let mut cursors = BTreeSet::new();
let mut turns = BTreeMap::new();
for _ in 0..10_000 {
// An active turn belongs to the newest metadata window. If it settles
// between the status read and paging, reread status rather than walking
// arbitrarily old completed turns looking for work that no longer exists.
for _ in 0..4 {
let page = self
.request(
"thread/turns/list",
@@ -1910,6 +2027,11 @@ impl CodexProvider {
let data = page.get("data").and_then(Value::as_array).ok_or_else(|| {
LocalRunnerError::invalid("codex_history_incomplete: turn page omitted data")
})?;
if data.len() > 100 {
return Err(LocalRunnerError::invalid(
"codex_history_incomplete: oversized turn metadata page",
));
}
for turn in data {
let id = bounded_provider_turn_id(turn.get("id").and_then(Value::as_str))?;
if !matches!(
@@ -1928,9 +2050,7 @@ impl CodexProvider {
let next = page.get("nextCursor").cloned().unwrap_or(Value::Null);
if next.is_null() || found_active {
if !found_active {
return Err(LocalRunnerError::invalid(
"codex_history_incomplete: active thread has no active turn",
));
break;
}
snapshot["thread"]["turns"] = Value::Array(turns.into_values().collect());
return Ok(snapshot);
@@ -1948,8 +2068,25 @@ impl CodexProvider {
}
cursor = next;
}
let mut current = self.request(
"thread/read",
json!({"threadId": self.thread_id, "includeTurns": false}),
)?;
if current.pointer("/thread/id").and_then(Value::as_str) != Some(self.thread_id.as_str()) {
return Err(LocalRunnerError::invalid(
"Codex thread/read returned a different thread",
));
}
if current
.pointer("/thread/status/type")
.and_then(Value::as_str)
== Some("idle")
{
current["thread"]["turns"] = json!([]);
return Ok(current);
}
Err(LocalRunnerError::invalid(
"codex_history_incomplete: turn page limit exceeded",
"codex_history_incomplete: active turn absent from current history window",
))
}
@@ -2306,7 +2443,9 @@ impl CodexProvider {
if params
.get("threadId")
.and_then(Value::as_str)
.is_some_and(|id| self.descendant_thread_ids.contains(id))
.map(|id| self.descendant_thread_ids.contains(id))
.transpose()?
.unwrap_or(false)
{
return self.reject_descendant_request(rpc_id, method);
}
@@ -2320,7 +2459,7 @@ impl CodexProvider {
self.active_provider_turn_id.as_deref(),
&self.settled_provider_turn_ids,
&params,
) {
)? {
return self.reject_post_terminal_request(rpc_id, method);
}
let active_turn_id = self.active_provider_turn_id.as_deref().ok_or_else(|| {
@@ -2406,7 +2545,8 @@ impl CodexProvider {
"Codex emitted too many pending tool calls",
));
}
if !completed_replay
if !self.durable_tool_call_replays
&& !completed_replay
&& self.completed_tool_call_ids.len() >= MAX_COMPLETED_TOOL_CALL_IDS
{
return Err(LocalRunnerError::invalid(
@@ -2437,7 +2577,9 @@ impl CodexProvider {
if params
.get("threadId")
.and_then(Value::as_str)
.is_some_and(|id| self.descendant_thread_ids.contains(id))
.map(|id| self.descendant_thread_ids.contains(id))
.transpose()?
.unwrap_or(false)
{
return self.reject_descendant_request(rpc_id, method);
}
@@ -2451,7 +2593,7 @@ impl CodexProvider {
self.active_provider_turn_id.as_deref(),
&self.settled_provider_turn_ids,
&params,
) {
)? {
return self.reject_post_terminal_request(rpc_id, method);
}
let active_turn_id = self.active_provider_turn_id.clone().ok_or_else(|| {
@@ -2521,18 +2663,14 @@ impl CodexProvider {
}),
}));
}
let request_sequence = self.next_runtime_request_sequence;
self.next_runtime_request_sequence = self
.next_runtime_request_sequence
.checked_add(1)
.ok_or_else(|| {
LocalRunnerError::invalid("Codex runtime request sequence overflowed")
})?;
// This is an identity, not event order. Pending duplicate RPCs
// returned above keep their original durable request identity.
let request_nonce = new_runtime_request_scope()?;
let request_id = scoped_runtime_request_id(
&self.runtime_request_scope,
&active_turn_id,
&provider_request_id,
request_sequence,
&request_nonce,
);
self.pending_runtime_requests
.insert(request_id.clone(), pending);
@@ -2558,7 +2696,7 @@ impl CodexProvider {
let identity = match classify_notification_thread(
method,
&self.thread_id,
&self.descendant_thread_ids,
&self.notification_descendants(&params)?,
&params,
) {
Ok(identity) => identity,
@@ -2569,28 +2707,31 @@ impl CodexProvider {
let candidate = notification_thread_id(&params)
.filter(|id| !id.is_empty() && id.len() <= 240 && *id != self.thread_id)
.map(str::to_owned);
let verified = candidate.as_ref().is_some_and(|candidate| {
self.request(
let verified = if let Some(candidate) = candidate.as_ref() {
if let Ok(metadata) = self.request(
"thread/read",
json!({"threadId": candidate, "includeTurns": false}),
)
.ok()
.is_some_and(|metadata| {
) {
let known = self.notification_descendants(&metadata)?;
metadata.pointer("/thread/id").and_then(Value::as_str)
== Some(candidate.as_str())
&& matches!(
classify_notification_thread(
"thread/started",
&self.thread_id,
&self.descendant_thread_ids,
&known,
&metadata
),
Ok(NotificationThread::Descendant)
)
})
});
} else {
false
}
} else {
false
};
if verified {
let mut known = self.descendant_thread_ids.clone();
let mut known = self.notification_descendants(&params)?;
known.insert(candidate.expect("verified candidate"));
match classify_notification_thread(method, &self.thread_id, &known, &params)
{
@@ -2615,22 +2756,14 @@ impl CodexProvider {
if identity == NotificationThread::Descendant {
let id =
notification_thread_id(&params).expect("classified descendant has an identity");
let newly_known = match remember_descendant_thread(
&mut self.descendant_thread_ids,
id,
) {
Ok(newly_known) => newly_known,
Err(code) => {
return Ok(Some(CodexProviderEvent::ResourceLimit {
diagnostic: json!({
"code": code, "recoverable": false, "classification": "resource_capacity",
"message": "Codex reached the child-thread inventory limit. Reconcile child work before continuing in a fresh provider session.",
"method": bounded_method(method), "limit": MAX_DESCENDANT_THREAD_IDS,
"expectedThreadId": self.thread_id, "receivedThreadId": id,
}),
}))
}
};
let newly_known = !self.descendant_thread_ids.contains(id)?;
if !self.descendant_thread_ids.insert(id.to_owned())? {
return Ok(Some(CodexProviderEvent::ResourceLimit {
diagnostic: json!({"code": "provider_descendant_capacity_exhausted", "recoverable": false,
"classification": "resource_capacity", "limit": MAX_DESCENDANT_THREAD_IDS,
"message": "Codex reached the legacy child-thread inventory limit."}),
}));
}
// Retain each discovered child's effect inventory independently of
// the informational diagnostic budget, then bound repeated progress.
if !newly_known && self.notification_identity_diagnostics >= 32 {
@@ -2670,7 +2803,9 @@ impl CodexProvider {
if notification_turn_id.is_some()
&& notification_turn_id != self.active_provider_turn_id.as_deref()
&& notification_turn_id
.is_some_and(|turn_id| self.settled_provider_turn_ids.contains(turn_id))
.map(|turn_id| self.settled_provider_turn_ids.contains(turn_id))
.transpose()?
.unwrap_or(false)
{
self.notification_identity_diagnostics += 1;
if self.notification_identity_diagnostics > 32 {
@@ -2716,12 +2851,10 @@ impl CodexProvider {
"turn_binding_mismatch",
)));
}
if let Err(code) = remember_spawned_descendants(
&mut self.descendant_thread_ids,
&self.thread_id,
method,
&params,
) {
let mut spawned = BTreeSet::new();
if let Err(code) =
remember_spawned_descendants(&mut spawned, &self.thread_id, method, &params)
{
if code == "provider_descendant_capacity_exhausted" {
return Ok(Some(CodexProviderEvent::ResourceLimit {
diagnostic: json!({"code": code, "recoverable": false,
@@ -2732,6 +2865,9 @@ impl CodexProvider {
}
return Ok(Some(self.identity_failure(method, &params, code)));
}
for id in spawned {
self.descendant_thread_ids.restore(id)?;
}
if let Some(terminal_event_type) = terminal_event_type {
if self.active_provider_turn_id.is_none() {
return Err(LocalRunnerError::invalid(
@@ -2754,7 +2890,7 @@ impl CodexProvider {
};
if !self
.settled_provider_turn_ids
.insert(provider_turn_id.clone())
.insert(provider_turn_id.clone())?
{
return Err(LocalRunnerError::invalid(
"Codex provider turn identity epoch reached its exact capacity",
@@ -2815,7 +2951,11 @@ impl CodexProvider {
self.pending_tool_request_bytes = self
.pending_tool_request_bytes
.saturating_sub(completed.retained_bytes);
self.completed_tool_call_ids.insert(result.call_id.clone());
// Durable backends own exact replay authority. Retaining a second
// per-turn set here would reintroduce a lifetime call allowance.
if !self.durable_tool_call_replays {
self.completed_tool_call_ids.insert(result.call_id.clone());
}
}
Ok(())
}
@@ -2919,10 +3059,7 @@ impl CodexProvider {
method: &str,
params: Value,
) -> Result<Value, ProviderRequestError> {
let request_id = self.next_request_id;
self.next_request_id = self.next_request_id.checked_add(1).ok_or_else(|| {
ProviderRequestError::Rejected(LocalRunnerError::invalid("Codex request id exhausted"))
})?;
let request_id = format!("paperclip-rpc-{}", uuid::Uuid::new_v4());
self.send_frame(&json!({"id": request_id, "method": method, "params": params}))
.map_err(ProviderRequestError::Ambiguous)?;
loop {
@@ -2964,7 +3101,7 @@ impl CodexProvider {
}
ProviderRequestError::Ambiguous(error)
})?;
if message.get("id").and_then(Value::as_u64) == Some(request_id)
if message.get("id").and_then(Value::as_str) == Some(request_id.as_str())
&& message.get("method").is_none()
{
if let (Some(trace), Some(frame_id)) = (self.trace.as_mut(), trace_frame_id) {
@@ -3507,11 +3644,11 @@ fn request_targets_non_active_turn(
active_turn_id: Option<&str>,
settled_turn_ids: &SettledProviderTurnIds,
params: &Value,
) -> bool {
let requested_turn_id = runtime_request_turn_id(params);
requested_turn_id.is_some_and(|requested| {
settled_turn_ids.contains(requested) || active_turn_id != Some(requested)
})
) -> Result<bool, LocalRunnerError> {
let Some(requested) = runtime_request_turn_id(params) else {
return Ok(false);
};
Ok(active_turn_id != Some(requested) || settled_turn_ids.contains(requested)?)
}
fn runtime_request_turn_id(params: &Value) -> Option<&str> {
@@ -3835,7 +3972,7 @@ fn scoped_runtime_request_id(
scope: &[u8; 16],
turn_id: &str,
provider_request_id: &str,
request_sequence: u64,
request_nonce: &[u8; 16],
) -> String {
let mut digest = Sha256::new();
digest.update(scope);
@@ -3844,7 +3981,7 @@ fn scoped_runtime_request_id(
digest.update([0]);
digest.update(provider_request_id.as_bytes());
digest.update([0]);
digest.update(request_sequence.to_be_bytes());
digest.update(request_nonce);
format!("runtime-request-{:x}", digest.finalize())
}
@@ -3990,16 +4127,18 @@ mod tests {
let script = r#"
turn=0
while IFS= read -r line; do
request_id="${line#*\"id\":\"}"
request_id="${request_id%%\"*}"
case "$line" in
*'"method":"initialize"'*) printf '%s\n' '{"id":1,"result":{}}' ;;
*'"method":"thread/start"'*) printf '%s\n' '{"id":2,"result":{"thread":{"id":"reader-tail-thread"}}}' ;;
*'"method":"initialize"'*) printf '{"id":"%s","result":{}}\n' "$request_id" ;;
*'"method":"thread/start"'*) printf '{"id":"%s","result":{"thread":{"id":"reader-tail-thread"}}}\n' "$request_id" ;;
*'"method":"turn/start"'*)
turn=$((turn + 1))
if [ "$turn" = 1 ]; then
printf '%s\n' '{"id":3,"result":{"turn":{"id":"reader-tail-1"}}}'
printf '{"id":"%s","result":{"turn":{"id":"reader-tail-1"}}}\n' "$request_id"
printf '%s\n' '{"method":"turn/completed","params":{"turn":{"id":"reader-tail-1","status":"completed"}}}'
else
printf '%s\n' '{"id":4,"error":{}}'
printf '{"id":"%s","error":{}}\n' "$request_id"
printf '%s\n' '{"method":"turn/started","params":{"turn":{"id":"reader-tail-2"}}}'
printf '%s\n' '{"method":"turn/completed","params":{"turn":{"id":"reader-tail-2","status":"completed"}}}'
if __FLOOD_STDOUT__; then
@@ -4048,7 +4187,7 @@ done
fn warm_attachment_accepts_normalized_usage_for_the_completed_turn() {
let mut provider = completion_tail_provider();
provider
.restore_completed_turn_authority(true, Some(1), Some("reader-tail-1"))
.restore_completed_turn_authority(true, Some(1.into()), Some("reader-tail-1"))
.unwrap();
provider.active_provider_turn_id = None;
provider
@@ -4250,7 +4389,7 @@ done
assert_eq!(process_generation, 1);
assert_eq!(
completed_turn_process_generation,
(boundary != "turn/completed").then_some(1)
(boundary != "turn/completed").then_some(1.into())
);
break;
}
@@ -4319,7 +4458,7 @@ done
matches!(provider.poll().unwrap(), Some(CodexProviderEvent::Notification { method, params }) if method == "configWarning" && params["code"] == "provider_stdout_drain_timeout")
);
// A stale generation's timeout is not inherited by a new owned epoch.
provider.exit_drain.as_mut().unwrap().process_generation = 0;
provider.exit_drain.as_mut().unwrap().process_generation = 0.into();
assert!(
matches!(provider.poll().unwrap(), Some(CodexProviderEvent::Notification { method, .. }) if method == "turn/started")
);
@@ -4422,9 +4561,10 @@ done
&config,
[],
None,
1,
1.into(),
None,
None,
false,
&mut |observation| match observation {
ProviderStartupObservation::Spawned {
process_id,
@@ -4641,20 +4781,20 @@ done
.is_err());
let scope = [7u8; 16];
assert_eq!(
scoped_runtime_request_id(&scope, "turn-1", "41", 1),
scoped_runtime_request_id(&scope, "turn-1", "41", 1),
scoped_runtime_request_id(&scope, "turn-1", "41", &[1; 16]),
scoped_runtime_request_id(&scope, "turn-1", "41", &[1; 16]),
);
assert_ne!(
scoped_runtime_request_id(&scope, "turn-1", "41", 1),
scoped_runtime_request_id(&scope, "turn-2", "41", 1),
scoped_runtime_request_id(&scope, "turn-1", "41", &[1; 16]),
scoped_runtime_request_id(&scope, "turn-2", "41", &[1; 16]),
);
assert_ne!(
scoped_runtime_request_id(&scope, "turn-1", "41", 1),
scoped_runtime_request_id(&[8u8; 16], "turn-1", "41", 1),
scoped_runtime_request_id(&scope, "turn-1", "41", &[1; 16]),
scoped_runtime_request_id(&[8u8; 16], "turn-1", "41", &[1; 16]),
);
assert_ne!(
scoped_runtime_request_id(&scope, "turn-1", "41", 1),
scoped_runtime_request_id(&scope, "turn-1", "41", 2),
scoped_runtime_request_id(&scope, "turn-1", "41", &[1; 16]),
scoped_runtime_request_id(&scope, "turn-1", "41", &[2; 16]),
);
assert!(codex_question_response(
&pending,
@@ -4780,6 +4920,68 @@ done
.contains("option ids must be unique"));
}
#[test]
fn indexed_turn_and_descendant_identities_outlive_the_memory_epoch() {
let directory =
std::env::temp_dir().join(format!("paperclip-identities-{}", uuid::Uuid::new_v4()));
let path = directory.join("provider.sqlite");
{
let store = IndexedStore::open(&path, "session", true).unwrap();
let mut turns = SettledProviderTurnIds::default();
let mut children = SettledProviderTurnIds::default();
turns.bind(store.clone(), "turns/session".into()).unwrap();
children.bind(store, "descendants/session".into()).unwrap();
for index in 0..5_000 {
assert!(turns.insert(format!("turn-{index}")).unwrap());
assert!(children.insert(format!("child-{index}")).unwrap());
}
assert!(turns.ids.is_empty() && children.ids.is_empty());
assert!(!turns.at_capacity() && !children.at_capacity());
}
let store = IndexedStore::open(&path, "session", false).unwrap();
let mut turns = SettledProviderTurnIds::default();
let mut children = SettledProviderTurnIds::default();
turns.bind(store.clone(), "turns/session".into()).unwrap();
children
.bind(store.clone(), "descendants/session".into())
.unwrap();
assert!(turns.contains("turn-0").unwrap());
assert!(children.contains("child-0").unwrap());
assert!(!children.contains("turn-0").unwrap());
// Damaged storage must never turn an ancient identity into a new one.
let db = rusqlite::Connection::open(&path).unwrap();
let target = "descendants/session\0child-0";
let mut cursor: Option<String> = None;
let mut partition = None;
loop {
let page = crate::indexed_partitions::partitions(&db, cursor.as_deref(), 128).unwrap();
if page.is_empty() {
break;
}
for route in &page {
if route.lower_key.as_str() <= target {
partition = Some(route.file.clone());
} else {
break;
}
}
let passed_target = page
.last()
.is_some_and(|route| route.lower_key.as_str() > target);
cursor = page.last().map(|route| route.lower_key.clone());
if passed_target {
break;
}
}
let partition = partition.expect("routing page covers the descendant receipt key");
let shard = rusqlite::Connection::open(directory.join(partition)).unwrap();
shard.execute("UPDATE receipt_rows SET digest=zeroblob(32) WHERE namespace='descendants/session' AND receipt_key='child-0'", []).unwrap();
drop(shard);
assert!(children.contains("child-0").is_err());
drop((db, children, turns, store));
std::fs::remove_dir_all(directory).unwrap();
}
#[test]
fn finds_only_active_turns_during_resume() {
let snapshot = json!({"thread": {"turns": [
@@ -4814,67 +5016,74 @@ done
#[test]
fn rejects_requests_bound_to_any_non_active_turn_nonfatally() {
let mut turn_one_settled = SettledProviderTurnIds::default();
turn_one_settled.insert("turn-1".to_owned());
turn_one_settled.insert("turn-1".to_owned()).unwrap();
let mut turn_two_settled = SettledProviderTurnIds::default();
turn_two_settled.insert("turn-2".to_owned());
turn_two_settled.insert("turn-2".to_owned()).unwrap();
let no_settled_turns = SettledProviderTurnIds::default();
assert!(request_targets_non_active_turn(
Some("turn-2"),
&turn_one_settled,
&json!({"turnId": "turn-1"}),
));
)
.unwrap());
assert!(request_targets_non_active_turn(
Some("turn-2"),
&turn_one_settled,
&json!({"turnId": "turn-0"}),
));
)
.unwrap());
assert!(request_targets_non_active_turn(
None,
&turn_two_settled,
&json!({"turnId": "turn-1"}),
));
)
.unwrap());
assert!(request_targets_non_active_turn(
Some("turn-1"),
&turn_one_settled,
&json!({"turnId": "turn-1"}),
));
)
.unwrap());
assert!(!request_targets_non_active_turn(
Some("turn-2"),
&turn_one_settled,
&json!({"turnId": "turn-2"}),
));
assert!(!request_targets_non_active_turn(
None,
&no_settled_turns,
&json!({}),
));
)
.unwrap());
assert!(!request_targets_non_active_turn(None, &no_settled_turns, &json!({}),).unwrap());
assert!(request_targets_non_active_turn(
Some("turn-2"),
&turn_one_settled,
&json!({"request": {"turnId": "turn-1"}}),
));
)
.unwrap());
assert!(!request_targets_non_active_turn(
Some("turn-2"),
&turn_one_settled,
&json!({"request": {"turnId": "turn-2"}}),
));
)
.unwrap());
}
#[test]
fn settled_provider_turn_history_never_evicts_exact_identities() {
let mut settled = SettledProviderTurnIds::default();
for index in 0..MAX_SETTLED_PROVIDER_TURN_IDS {
assert!(settled.insert(format!("turn-{index}")));
assert!(settled.insert(format!("turn-{index}")).unwrap());
}
assert_eq!(settled.ids.len(), MAX_SETTLED_PROVIDER_TURN_IDS);
assert!(settled.contains("turn-0"));
assert!(settled.contains("turn-1"));
assert!(settled.contains("turn-0").unwrap());
assert!(settled.contains("turn-1").unwrap());
assert!(settled.at_capacity());
assert!(!settled.insert(format!("turn-{MAX_SETTLED_PROVIDER_TURN_IDS}")));
assert!(!settled.contains(&format!("turn-{MAX_SETTLED_PROVIDER_TURN_IDS}")));
assert!(!settled
.insert(format!("turn-{MAX_SETTLED_PROVIDER_TURN_IDS}"))
.unwrap());
assert!(!settled
.contains(&format!("turn-{MAX_SETTLED_PROVIDER_TURN_IDS}"))
.unwrap());
assert_eq!(settled.ids.len(), MAX_SETTLED_PROVIDER_TURN_IDS);
assert!(settled.contains("turn-0"));
assert!(settled.contains("turn-0").unwrap());
assert!(settled.filter.is_empty());
}
@@ -4884,7 +5093,7 @@ done
settled.restore("turn-restored".to_owned()).unwrap();
assert!(settled.contains("turn-restored"));
assert!(settled.contains("turn-restored").unwrap());
}
#[test]
@@ -4898,8 +5107,8 @@ done
)
.unwrap();
assert!(settled.contains("turn-older"));
assert!(settled.contains("turn-latest"));
assert!(settled.contains("turn-older").unwrap());
assert!(settled.contains("turn-latest").unwrap());
}
#[test]
packages/paperclip-runner/runner/crates/runner-core/src/durable/command_epochs.rs
+46
View File
@@ -0,0 +1,46 @@
use serde::{Deserialize, Serialize};
use super::DurableRunnerError;
pub(crate) const CAPABILITY: &str = "transport.command_epochs.v1";
pub(crate) fn valid_epoch(value: &str) -> bool {
uuid::Uuid::parse_str(value).is_ok_and(|id| {
id.get_version_num() == 4
&& id.get_variant() == uuid::Variant::RFC4122
&& id.to_string() == value
})
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub(crate) struct CommandEpochTransition {
pub schema: String,
pub run_id: String,
pub transition_id: String,
pub from_epoch: Option<String>,
pub next_epoch: String,
pub final_ordinal: u64,
}
impl CommandEpochTransition {
pub fn validate(&self, run_id: &str) -> Result<(), DurableRunnerError> {
if self.schema != "paperclip.prp.command-epoch.v1"
|| self.run_id != run_id
|| !valid_epoch(&self.transition_id)
|| !valid_epoch(&self.next_epoch)
|| self
.from_epoch
.as_deref()
.is_some_and(|id| !valid_epoch(id))
|| self.from_epoch.as_ref() == Some(&self.next_epoch)
|| self.final_ordinal == 0
|| self.final_ordinal > 9_007_199_254_740_991
{
return Err(DurableRunnerError::invalid(
"invalid command epoch transition",
));
}
Ok(())
}
}
packages/paperclip-runner/runner/crates/runner-core/src/durable/event_epochs.rs
+49
View File
@@ -0,0 +1,49 @@
use serde::{Deserialize, Serialize};
use super::DurableRunnerError;
pub(crate) const CAPABILITY: &str = "transport.event_epochs.v1";
pub(crate) fn valid_epoch(value: &str) -> bool {
uuid::Uuid::parse_str(value).is_ok_and(|id| {
id.get_version_num() == 4
&& id.get_variant() == uuid::Variant::RFC4122
&& id.to_string() == value
})
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub(crate) struct EventEpochTransition {
pub schema: String,
pub run_id: String,
pub transition_id: String,
pub from_epoch: Option<String>,
pub next_epoch: String,
pub final_ordinal: u64,
}
impl EventEpochTransition {
pub fn validate(&self, run_id: &str) -> Result<(), DurableRunnerError> {
if self.schema != "paperclip.prp.event-epoch.v1"
|| self.run_id != run_id
|| !valid_epoch(&self.transition_id)
|| !valid_epoch(&self.next_epoch)
|| self
.from_epoch
.as_deref()
.is_some_and(|id| !valid_epoch(id))
|| self.from_epoch.as_ref() == Some(&self.next_epoch)
|| self.final_ordinal == 0
|| self.final_ordinal > 9_007_199_254_740_991
{
return Err(DurableRunnerError::invalid(
"invalid event epoch transition",
));
}
Ok(())
}
}
pub(crate) const DEFAULT_LIMIT: u64 = 1_048_576;
pub(crate) const MAX_ORDINAL: u64 = 9_007_199_254_740_991;
packages/paperclip-runner/runner/crates/runner-core/src/durable/mod.rs
+55 -5
View File
@@ -1,5 +1,8 @@
mod command_epochs;
mod event_epochs;
mod runner;
mod state;
mod storage_control;
mod transport;
use std::error::Error;
@@ -12,11 +15,11 @@ use sha2::{Digest, Sha256};
use crate::stable_identity::{is_stable_id, DURABLE_STABLE_ID_CHARS, SHORT_STABLE_ID_CHARS};
pub use runner::{
run_durable_runner, CommandExecution, CommandExecutor, PolledEvent,
run_durable_runner, run_indexed_durable_runner, CommandExecution, CommandExecutor, PolledEvent,
TerminalDeliveryReconciliation,
};
pub(crate) use state::{
create_private_temporary_file, open_private_regular_file, redact_text,
create_private_temporary_file, open_private_regular_file, redact_output_text, redact_text,
sanitize_semantic_tool_input, sanitize_value, verify_private_directory,
};
pub use state::{
@@ -38,17 +41,64 @@ pub fn redact_diagnostic_text(input: &str) -> String {
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct DurableRunnerError(String);
pub struct DurableRunnerError {
message: String,
storage_capacity: bool,
storage_wait_cancelled: bool,
transport_unavailable: bool,
}
impl DurableRunnerError {
pub fn invalid(message: impl Into<String>) -> Self {
Self(message.into())
Self {
message: message.into(),
storage_capacity: false,
storage_wait_cancelled: false,
transport_unavailable: false,
}
}
pub(crate) fn storage_capacity(message: impl Into<String>) -> Self {
Self {
message: message.into(),
storage_capacity: true,
storage_wait_cancelled: false,
transport_unavailable: false,
}
}
pub(crate) fn storage_wait_cancelled(message: impl Into<String>) -> Self {
Self {
message: message.into(),
storage_capacity: false,
storage_wait_cancelled: true,
transport_unavailable: false,
}
}
pub(crate) fn transport_unavailable(message: impl Into<String>) -> Self {
Self {
message: message.into(),
storage_capacity: false,
storage_wait_cancelled: false,
transport_unavailable: true,
}
}
pub(crate) fn is_transport_unavailable(&self) -> bool {
self.transport_unavailable
}
pub(crate) fn is_storage_capacity(&self) -> bool {
self.storage_capacity
}
pub(crate) fn is_storage_wait_cancelled(&self) -> bool {
self.storage_wait_cancelled
}
}
impl Display for DurableRunnerError {
fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result {
formatter.write_str(&self.0)
formatter.write_str(&self.message)
}
}
packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs
+474 -50
View File
@@ -228,6 +228,7 @@ fn apply_authority_rotation(
let reconnect_count = state.reconnect_count.saturating_add(1);
let mut diagnostics = state.diagnostics.clone();
let mut rotated = DurableState::new(&next);
state.retain_indexed_authority(&mut rotated);
if let Some(mut transition) = state.warm_transition.clone() {
transition.phase = "activating".to_owned();
rotated.warm_transition = Some(transition);
@@ -362,12 +363,33 @@ impl CumulativeAckPersistence {
}
pub fn run_durable_runner<E: CommandExecutor>(
config: DurableRunnerConfig,
bootstrap_ticket: BootstrapTicket,
executor: E,
) -> Result<(), DurableRunnerError> {
run_durable_runner_with_storage(config, bootstrap_ticket, executor, false)
}
pub fn run_indexed_durable_runner<E: CommandExecutor>(
config: DurableRunnerConfig,
bootstrap_ticket: BootstrapTicket,
executor: E,
) -> Result<(), DurableRunnerError> {
run_durable_runner_with_storage(config, bootstrap_ticket, executor, true)
}
fn run_durable_runner_with_storage<E: CommandExecutor>(
mut config: DurableRunnerConfig,
bootstrap_ticket: BootstrapTicket,
mut executor: E,
indexed: bool,
) -> Result<(), DurableRunnerError> {
config.validate()?;
let store = DurableStateStore::new(&config.state_dir)?;
let store = if indexed {
DurableStateStore::new_indexed(&config.state_dir)?
} else {
DurableStateStore::new(&config.state_dir)?
};
let (mut state, recovered) = store.load_or_create(&config)?;
if state.lifecycle == "revoked"
|| (state.lifecycle == "stopped"
@@ -376,7 +398,11 @@ pub fn run_durable_runner<E: CommandExecutor>(
{
return Ok(());
}
if recovered && state.warm_transition.is_none() {
if recovered
&& state.warm_transition.is_none()
&& state.event_epoch_transition.is_none()
&& state.next_source_seq < super::event_epochs::MAX_ORDINAL - 32_768
{
state.reconnect_count = state.reconnect_count.saturating_add(1);
state.record_diagnostic("runner restored its durable identity after process recovery");
state.enqueue_event(
@@ -392,6 +418,11 @@ pub fn run_durable_runner<E: CommandExecutor>(
// retry after the trust decision.
let mut endpoint = RunnerTransportEndpoint::new(&config.connect_url, &config.run_id)?;
let started = Instant::now();
let runtime_deadline = if config.max_runtime.is_zero() {
None
} else {
started.checked_add(config.max_runtime)
};
let mut bootstrap_ticket = Some(bootstrap_ticket);
let mut lease: Option<LeaseCredential> = None;
let mut authenticated_once = false;
@@ -620,7 +651,39 @@ pub fn run_durable_runner<E: CommandExecutor>(
}
state.lifecycle = "ready".to_owned();
state.recoverable_failure = None;
store.save(&state)?;
let (saved, lane) = super::storage_control::run(
(transport, connection, lease, next_lease_renewal),
pressure_control_identity(&state, &config),
welcome.lease_renewal_version == Some(1),
runtime_deadline,
|| store.save(&state),
);
(transport, connection, lease, next_lease_renewal) = lane;
if let Err(error) = saved {
if error.is_storage_wait_cancelled() {
let _ = shutdown_preserving_cleanup(&state, &mut executor);
}
return Err(error);
}
let (rotated, lane) = rotate_event_epoch_if_needed(
(transport, connection, lease, next_lease_renewal),
&mut state,
&store,
&config,
welcome.event_epoch_limit,
runtime_deadline,
);
(transport, connection, lease, next_lease_renewal) = lane;
if let Err(error) = rotated {
if error.is_storage_wait_cancelled() {
let _ = shutdown_preserving_cleanup(&state, &mut executor);
return Err(error);
}
state.record_diagnostic(format!("event epoch reconnect scheduled: {error}"));
store.save(&state)?;
disconnected_since = Some(Instant::now());
continue;
}
let mut sent_source_seq = state.acked_source_seq;
let mut ack_persistence = CumulativeAckPersistence::default();
@@ -634,17 +697,19 @@ pub fn run_durable_runner<E: CommandExecutor>(
welcome.warm_transition_version,
connection.protocol_version,
)?;
let (result, lifecycle) =
process_command(&mut state, &store, &config, &mut executor, &command)?;
let (executed, lane) = execute_command_with_capacity_control(
(transport, connection, lease, next_lease_renewal),
&mut state,
&store,
&config,
&mut executor,
&command,
welcome.lease_renewal_version == Some(1),
runtime_deadline,
);
(transport, connection, lease, next_lease_renewal) = lane;
let (result, lifecycle) = executed?;
let next_authority = next_authority.filter(|_| completed_attachment(&result));
if let Some(durable_lifecycle) = lifecycle.durable_state() {
persist_lifecycle_before_command_delivery(
&mut state,
&store,
durable_lifecycle,
&result,
)?;
}
lifecycle_after_reply = lifecycle_after_reply.merge(lifecycle);
let delivery = (|| {
if let Some(next) = &next_authority {
@@ -753,6 +818,32 @@ pub fn run_durable_runner<E: CommandExecutor>(
if !config.max_runtime.is_zero() && started.elapsed() >= config.max_runtime {
break;
}
let (rotated, lane) = rotate_event_epoch_if_needed(
(transport, connection, lease, next_lease_renewal),
&mut state,
&store,
&config,
welcome.event_epoch_limit,
runtime_deadline,
);
(transport, connection, lease, next_lease_renewal) = lane;
match rotated {
Ok(true) => {
sent_source_seq = state.acked_source_seq;
ack_persistence = CumulativeAckPersistence::default();
}
Ok(false) => {}
Err(error) => {
if error.is_storage_wait_cancelled() {
let _ = shutdown_preserving_cleanup(&state, &mut executor);
return Err(error);
}
state.record_diagnostic(format!("event epoch reconnect scheduled: {error}"));
store.save(&state)?;
disconnected_since = Some(Instant::now());
break;
}
}
if let Err(error) = send_outbox(
&mut transport,
&state,
@@ -813,14 +904,29 @@ pub fn run_durable_runner<E: CommandExecutor>(
// Consuming the last cumulative ACK must not let a new output
// suffix overtake the stop/suspend already queued behind it.
let control_message = transport.receive_json();
poll_executor_events_when_control_idle(
&mut state,
&store,
&config,
&mut executor,
sent_source_seq,
&control_message,
)?;
let (polled, lane) = super::storage_control::run(
(transport, connection, lease, next_lease_renewal),
pressure_control_identity(&state, &config),
welcome.lease_renewal_version == Some(1),
runtime_deadline,
|| {
poll_executor_events_when_control_idle(
&mut state,
&store,
&config,
&mut executor,
sent_source_seq,
&control_message,
)
},
);
(transport, connection, lease, next_lease_renewal) = lane;
if let Err(error) = polled {
if error.is_storage_wait_cancelled() {
let _ = shutdown_preserving_cleanup(&state, &mut executor);
}
return Err(error);
}
let message = match control_message {
Ok(Some(message)) => message,
Ok(None) => continue,
@@ -851,16 +957,78 @@ pub fn run_durable_runner<E: CommandExecutor>(
lease_renewal_deadline(current_unix_ms()?, connection.expires_at_unix_ms);
}
Some("ack") => {
if !state.ack_is_current(message.get("payload").unwrap_or(&Value::Null))? {
continue;
}
let acked = message
.pointer("/payload/ackedSourceSeq")
.and_then(Value::as_u64)
.ok_or_else(|| DurableRunnerError::invalid("ACK cursor is required"))?;
ack_persistence.apply(
&mut state,
&store,
acked,
connection.protocol_version,
)?;
let version = connection.protocol_version;
let mut pressure_identity = pressure_control_identity(&state, &config);
pressure_identity.acked_source_seq = acked;
let (acked, lane) = super::storage_control::run(
(transport, connection, lease, next_lease_renewal),
pressure_identity,
welcome.lease_renewal_version == Some(1),
runtime_deadline,
|| ack_persistence.apply(&mut state, &store, acked, version),
);
(transport, connection, lease, next_lease_renewal) = lane;
if let Err(error) = acked {
if error.is_storage_wait_cancelled() {
let _ = shutdown_preserving_cleanup(&state, &mut executor);
}
return Err(error);
}
}
Some("event_epoch_committed") => {
let transition: super::event_epochs::EventEpochTransition =
serde_json::from_value(
message.get("payload").cloned().unwrap_or(Value::Null),
)
.map_err(|e| DurableRunnerError::invalid(e.to_string()))?;
if welcome.event_epoch_limit.is_none()
|| state.last_event_epoch_transition.as_ref() != Some(&transition)
{
return Err(DurableRunnerError::invalid(
"unsolicited event epoch commit receipt",
));
}
}
Some("command_epoch_rotate") => {
if !welcome.command_epochs {
return Err(DurableRunnerError::invalid(
"command epoch rotation was not negotiated",
));
}
let transition: super::command_epochs::CommandEpochTransition =
serde_json::from_value(
message.get("payload").cloned().unwrap_or(Value::Null),
)
.map_err(|e| DurableRunnerError::invalid(e.to_string()))?;
let (rotated, lane) = super::storage_control::run(
(transport, connection, lease, next_lease_renewal),
pressure_control_identity(&state, &config),
welcome.lease_renewal_version == Some(1),
runtime_deadline,
|| store.rotate_command_epoch(&mut state, &transition),
);
(transport, connection, lease, next_lease_renewal) = lane;
rotated?;
if let Err(error) = transport.send_json(&control_envelope(
&state,
&connection,
"command_epoch_committed",
serde_json::to_value(&transition)
.map_err(|e| DurableRunnerError::invalid(e.to_string()))?,
)) {
disconnected_since.get_or_insert_with(Instant::now);
state.record_diagnostic(error.to_string());
state.reconnect_count = state.reconnect_count.saturating_add(1);
store.save(&state)?;
break;
}
}
Some("command") => {
let command: Command =
@@ -876,17 +1044,19 @@ pub fn run_durable_runner<E: CommandExecutor>(
welcome.warm_transition_version,
connection.protocol_version,
)?;
let (result, lifecycle) =
process_command(&mut state, &store, &config, &mut executor, &command)?;
let (executed, lane) = execute_command_with_capacity_control(
(transport, connection, lease, next_lease_renewal),
&mut state,
&store,
&config,
&mut executor,
&command,
welcome.lease_renewal_version == Some(1),
runtime_deadline,
);
(transport, connection, lease, next_lease_renewal) = lane;
let (result, lifecycle) = executed?;
let next_authority = next_authority.filter(|_| completed_attachment(&result));
if let Some(durable_lifecycle) = lifecycle.durable_state() {
persist_lifecycle_before_command_delivery(
&mut state,
&store,
durable_lifecycle,
&result,
)?;
}
let delivery = (|| {
if let Some(next) = &next_authority {
return deliver_warm_attachment(
@@ -1036,11 +1206,212 @@ pub fn run_durable_runner<E: CommandExecutor>(
}
}
fn lease_renewal_deadline(now: u64, expires_at: u64) -> u64 {
/// No provider callback runs while an event namespace is closing. Pending RPCs
/// remain live; new commands wait in the bounded transport backlog. Reading the
/// socket directly prevents a deferred command from hiding the commit receipt.
fn rotate_event_epoch_if_needed(
mut lane: super::storage_control::Lane,
state: &mut DurableState,
store: &DurableStateStore,
config: &DurableRunnerConfig,
limit: Option<u64>,
deadline: Option<Instant>,
) -> (
Result<bool, DurableRunnerError>,
super::storage_control::Lane,
) {
if state.event_epoch_transition.is_none()
&& limit.is_none_or(|limit| state.highest_source_seq() < limit)
{
return (Ok(false), lane);
}
if limit.is_none() {
return (
Err(DurableRunnerError::invalid(
"pending event epoch capability was downgraded",
)),
lane,
);
}
let (prepared, next_lane) = super::storage_control::run(
lane,
pressure_control_identity(state, config),
true,
deadline,
|| store.prepare_event_epoch(state),
);
lane = next_lane;
let transition = match prepared {
Ok(t) => t,
Err(e) => return (Err(e), lane),
};
let mut sent = state.acked_source_seq;
let mut announced = false;
let mut stop_deadline: Option<Instant> = None;
loop {
let (transport, connection, lease, renewal) = &mut lane;
let polled = (|| {
let now = current_unix_ms()?;
if deadline.is_some_and(|d| Instant::now() >= d)
|| stop_deadline.is_some_and(|d| Instant::now() >= d)
|| now >= connection.expires_at_unix_ms
{
return Err(DurableRunnerError::storage_wait_cancelled(
"event epoch wait exceeded authenticated lifetime",
));
}
send_outbox(transport, state, &mut sent, connection.protocol_version)?;
if now >= *renewal {
let credential = lease.as_mut().ok_or_else(|| {
DurableRunnerError::invalid("event epoch requires live lease")
})?;
credential.renewal_requested = true;
transport.send_json(&control_envelope(
state,
connection,
"lease_renew",
json!({
"connectionLeaseExpiresAtUnixMs": connection.expires_at_unix_ms,
"connectionLeaseRevocationEpoch": connection.revocation_epoch,
}),
))?;
*renewal = now.saturating_add(
5000.min(connection.expires_at_unix_ms.saturating_sub(now) / 2)
.max(1),
);
}
if !announced && state.acked_source_seq == transition.final_ordinal {
transport.send_json(&control_envelope(
state,
connection,
"event_epoch_rotate",
serde_json::to_value(&transition)
.map_err(|e| DurableRunnerError::invalid(e.to_string()))?,
))?;
announced = true;
}
let Some(message) = transport.receive_control_during_epoch()? else {
return Ok(None);
};
validate_control_identity(&message, state, Some(connection))?;
match message.get("kind").and_then(Value::as_str) {
Some("ack") => {
if state.ack_is_current(message.get("payload").unwrap_or(&Value::Null))? {
let ack = message.pointer("/payload/ackedSourceSeq").and_then(Value::as_u64).ok_or_else(|| DurableRunnerError::invalid("ACK required"))?;
state.apply_ack(ack, connection.protocol_version)?;
}
}
Some("event_epoch_committed") => {
let received: super::event_epochs::EventEpochTransition = serde_json::from_value(message.get("payload").cloned().unwrap_or(Value::Null)).map_err(|e| DurableRunnerError::invalid(e.to_string()))?;
if !announced || received != transition { return Err(DurableRunnerError::invalid("event epoch acknowledgement changed its exact intent")); }
return Ok(Some(received));
}
Some("lease_renewed") => {
apply_lease_renewal(&message, connection, lease.as_mut().ok_or_else(|| DurableRunnerError::invalid("live lease required"))?)?;
*renewal = lease_renewal_deadline(current_unix_ms()?, connection.expires_at_unix_ms);
}
Some("ping") => transport.send_json(&control_envelope(state, connection, "pong", json!({"lifecycle":state.lifecycle,"sourceEpoch":state.source_epoch,"ackedSourceSeq":state.acked_source_seq})))?,
Some("revoke") => {
if message.pointer("/payload/revocationEpoch").and_then(Value::as_u64).is_none_or(|epoch| epoch <= connection.revocation_epoch) {
return Err(DurableRunnerError::invalid("invalid revocation during event epoch transition"));
}
return Err(DurableRunnerError::storage_wait_cancelled("event epoch authority was revoked"));
},
Some("command") => {
let command: Command = serde_json::from_value(message.get("payload").cloned().unwrap_or(Value::Null)).map_err(|e| DurableRunnerError::invalid(e.to_string()))?;
command.validate()?;
if super::storage_control::fresh_stop(&command, state.controller_epoch.as_deref(), state.last_controller_command_seq) {
stop_deadline.get_or_insert_with(|| Instant::now() + Duration::from_secs(10));
}
transport.defer_control(message)?;
}
Some("command_epoch_rotate") => transport.defer_control(message)?,
_ => return Err(DurableRunnerError::invalid("unexpected control during event epoch transition")),
}
Ok(None)
})();
match polled {
Err(error) => return (Err(error), lane),
Ok(Some(receipt)) => {
let (committed, next_lane) = super::storage_control::run(
lane,
pressure_control_identity(state, config),
true,
deadline,
|| store.commit_event_epoch(state, &receipt),
);
return (committed.map(|()| true), next_lane);
}
Ok(None) => {}
}
}
}
pub(super) fn lease_renewal_deadline(now: u64, expires_at: u64) -> u64 {
now.saturating_add(expires_at.saturating_sub(now) / 2)
}
fn apply_lease_renewal(
fn pressure_control_identity(state: &DurableState, config: &DurableRunnerConfig) -> DurableState {
let mut identity = DurableState::new(config);
identity
.runner_instance_id
.clone_from(&state.runner_instance_id);
identity
.environment_lease_id
.clone_from(&state.environment_lease_id);
identity.run_id.clone_from(&state.run_id);
identity
.normalized_session_id
.clone_from(&state.normalized_session_id);
identity.turn_id.clone_from(&state.turn_id);
identity.item_id.clone_from(&state.item_id);
state.copy_event_ack_authority(&mut identity);
identity.acked_source_seq = state.acked_source_seq;
identity.next_source_seq = state.next_source_seq;
identity.last_controller_command_seq = state.last_controller_command_seq;
identity
.controller_epoch
.clone_from(&state.controller_epoch);
identity.lifecycle.clone_from(&state.lifecycle);
identity
}
fn execute_command_with_capacity_control<E: CommandExecutor>(
lane: super::storage_control::Lane,
state: &mut DurableState,
store: &DurableStateStore,
config: &DurableRunnerConfig,
executor: &mut E,
command: &Command,
renew: bool,
deadline: Option<Instant>,
) -> (
Result<(StoredCommandResult, CommandLifecycle), DurableRunnerError>,
super::storage_control::Lane,
) {
let mut identity = pressure_control_identity(state, config);
if command.controller_epoch == identity.controller_epoch {
identity.last_controller_command_seq = identity
.last_controller_command_seq
.max(command.controller_seq);
}
let (result, lane) = super::storage_control::run(lane, identity, renew, deadline, || {
let (result, lifecycle) = process_command(state, store, config, executor, command)?;
if let Some(durable_lifecycle) = lifecycle.durable_state() {
persist_lifecycle_before_command_delivery(state, store, durable_lifecycle, &result)?;
}
Ok((result, lifecycle))
});
if result
.as_ref()
.is_err_and(|error| error.is_storage_wait_cancelled())
{
let _ = shutdown_preserving_cleanup(state, executor);
}
(result, lane)
}
pub(super) fn apply_lease_renewal(
message: &Value,
connection: &mut ConnectionMetadata,
credential: &mut LeaseCredential,
@@ -1204,6 +1575,9 @@ fn wait_for_terminal_result_ack(
.pointer("/payload/ackedSourceSeq")
.and_then(Value::as_u64)
.ok_or_else(|| DurableRunnerError::invalid("ACK cursor is required"))?;
if !state.ack_is_current(message.get("payload").unwrap_or(&Value::Null))? {
continue;
}
ack_persistence.apply(state, store, acked, connection.protocol_version)?;
}
Some("ping") => transport.send_json(&control_envelope(
@@ -1268,6 +1642,9 @@ fn wait_for_old_authority_outbox_ack(
.pointer("/payload/ackedSourceSeq")
.and_then(Value::as_u64)
.ok_or_else(|| DurableRunnerError::invalid("ACK cursor is required"))?;
if !state.ack_is_current(message.get("payload").unwrap_or(&Value::Null))? {
continue;
}
state.apply_ack(acked, connection.protocol_version)?;
store.save(state)?;
}
@@ -1382,6 +1759,7 @@ fn deliver_warm_attachment(
connection.lease_id.clone(),
connection.expires_at_unix_ms,
connection.revocation_epoch,
state.source_epoch.clone(),
)?;
if let Some(pending) = &state.warm_transition {
if pending.phase != "prepared"
@@ -1458,7 +1836,8 @@ fn deliver_warm_attachment(
if message
.pointer("/payload/ackedSourceSeq")
.and_then(Value::as_u64)
== Some(state.acked_source_seq) => {}
== Some(state.acked_source_seq)
&& state.ack_is_current(message.get("payload").unwrap_or(&Value::Null))? => {}
_ => {
return Err(DurableRunnerError::invalid(
"warm attachment result fence received unrelated control",
@@ -1725,15 +2104,17 @@ fn process_command<E: CommandExecutor>(
}
match state.begin_command(command)? {
CommandDisposition::Replay(result) => {
let lifecycle =
if result.status == "pending" || state.pending_provider_cleanup.is_some() {
// Its terminal delivery was already reconciled. Replaying
// the old receipt must not replace the newer command cursor
// with another terminal-delivery fence at an older sequence.
CommandLifecycle::Continue
} else {
CommandLifecycle::for_terminal(command)
};
let lifecycle = if result.status == "pending"
|| state.pending_provider_cleanup.is_some()
|| command.controller_epoch != state.controller_epoch
{
// Its terminal delivery was already reconciled. Replaying
// the old receipt must not replace the newer command cursor
// with another terminal-delivery fence at an older sequence.
CommandLifecycle::Continue
} else {
CommandLifecycle::for_terminal(command)
};
return Ok((result, lifecycle));
}
CommandDisposition::Reject(result) => {
@@ -1747,6 +2128,7 @@ fn process_command<E: CommandExecutor>(
store.save(state)?;
let mut execution = match executor.execute(command) {
Ok(execution) => execution,
Err(error) if error.is_storage_wait_cancelled() => return Err(error),
Err(error) => {
// Failure facts may follow a full retained provider backlog. Only
// the non-restoring FIFO is legal here; regular poll can launch.
@@ -1909,7 +2291,7 @@ fn command_result_envelope(
})
}
fn control_envelope(
pub(super) fn control_envelope(
state: &DurableState,
connection: &ConnectionMetadata,
kind: &str,
@@ -2214,6 +2596,7 @@ mod tests {
fn command(command_type: &str) -> Command {
Command {
controller_epoch: None,
schema: "paperclip.prp.command.v1".to_owned(),
command_id: "command_1".to_owned(),
controller_seq: 1,
@@ -4426,6 +4809,47 @@ mod tests {
fs::remove_dir_all(directory).unwrap();
}
#[test]
fn closed_command_epoch_terminal_replay_cannot_stop_current_work() {
for kind in ["runner.suspend", "runner.shutdown"] {
let directory =
std::env::temp_dir().join(format!("historical-terminal-{}", uuid::Uuid::new_v4()));
let config = config(directory.clone());
let store = DurableStateStore::new_indexed(&directory).unwrap();
let (mut state, _) = store.load_or_create(&config).unwrap();
let old = command(kind);
state.begin_command(&old).unwrap();
store.save(&state).unwrap();
let original = state
.complete_command(&old, json!({"exact": "settled terminal"}))
.unwrap();
// The prior process completed terminal delivery and cleanup. New
// work is live under this same run, then its command lane rotates.
state.lifecycle = "ready".into();
store.save(&state).unwrap();
let transition = super::super::command_epochs::CommandEpochTransition {
schema: "paperclip.prp.command-epoch.v1".into(),
run_id: state.run_id.clone(),
transition_id: uuid::Uuid::new_v4().to_string(),
from_epoch: None,
next_epoch: uuid::Uuid::new_v4().to_string(),
final_ordinal: 1,
};
store.rotate_command_epoch(&mut state, &transition).unwrap();
let before = state.clone();
let mut executor = CountingExecutor { calls: 0 };
let (result, disposition) =
process_command(&mut state, &store, &config, &mut executor, &old).unwrap();
assert_eq!(result, original);
assert_eq!(disposition, CommandLifecycle::Continue);
assert_eq!(state, before);
assert_eq!(executor.calls, 0);
drop(state);
drop(store);
fs::remove_dir_all(directory).unwrap();
}
}
#[test]
fn completed_shutdown_replay_still_stops_after_delivery() {
let directory = std::env::temp_dir().join(format!(
packages/paperclip-runner/runner/crates/runner-core/src/durable/state.rs
+1197 -9
View File
File diff suppressed because it is too large. Load diff
packages/paperclip-runner/runner/crates/runner-core/src/durable/storage_control.rs
+194
View File
@@ -0,0 +1,194 @@
//! Capacity recovery retains the synchronous command/effect stack, while the
//! authenticated control lane remains live. Deferred messages are bounded and
//! replayed in order only after the original storage operation is durable.
use super::runner::{apply_lease_renewal, control_envelope, lease_renewal_deadline};
use super::transport::{
current_unix_ms, validate_control_identity, AuthenticatedTransport, ConnectionMetadata,
LeaseCredential,
};
use super::{Command, DurableRunnerError, DurableState};
use serde_json::json;
use std::{
cell::RefCell,
rc::Rc,
time::{Duration, Instant},
};
pub(super) type Lane = (
AuthenticatedTransport,
ConnectionMetadata,
Option<LeaseCredential>,
u64,
);
pub(super) fn run<T>(
lane: Lane,
identity: DurableState,
renew: bool,
deadline: Option<Instant>,
operation: impl FnOnce() -> Result<T, DurableRunnerError>,
) -> (Result<T, DurableRunnerError>, Lane) {
let context = Rc::new(RefCell::new(Some(lane)));
let callback_context = context.clone();
let mut ack_cursor = identity.acked_source_seq;
let mut disconnected = false;
let result = crate::storage_capacity::with_wait(
move || {
let mut context = callback_context.borrow_mut();
let (transport, connection, lease, next_renewal) =
context.as_mut().expect("live pressure control lane");
let now = current_unix_ms()?;
if deadline.is_some_and(|deadline| Instant::now() >= deadline)
|| now >= connection.expires_at_unix_ms
{
return Err(DurableRunnerError::invalid(
"execution or authenticated lease expired during storage pressure",
));
}
let started = Instant::now();
let polled = if disconnected {
Ok(())
} else {
(|| {
if renew && now >= *next_renewal {
let credential = lease.as_mut().ok_or_else(|| {
DurableRunnerError::invalid("storage wait requires live lease")
})?;
credential.renewal_requested = true;
transport.send_json(&control_envelope(
&identity,
connection,
"lease_renew",
json!({
"connectionLeaseExpiresAtUnixMs": connection.expires_at_unix_ms,
"connectionLeaseRevocationEpoch": connection.revocation_epoch,
}),
))?;
*next_renewal = now.saturating_add(
5_000.min(
connection
.expires_at_unix_ms
.saturating_sub(now)
.saturating_div(2)
.max(1),
),
);
}
if let Some(message) = transport.receive_control_during_storage_wait()? {
validate_control_identity(&message, &identity, Some(connection))?;
match message.get("kind").and_then(serde_json::Value::as_str) {
Some("lease_renewed") => {
apply_lease_renewal(&message, connection, lease.as_mut().ok_or_else(|| DurableRunnerError::invalid("lease renewal requires live credential"))?)?;
*next_renewal = lease_renewal_deadline(current_unix_ms()?, connection.expires_at_unix_ms);
}
Some("ping") => transport.send_json(&control_envelope(&identity, connection, "pong", json!({
"lifecycle": identity.lifecycle, "ackedSourceSeq": identity.acked_source_seq,
"storagePressure": true,
})))?,
Some("revoke") => {
if message.pointer("/payload/revocationEpoch").and_then(serde_json::Value::as_u64).is_none_or(|epoch| epoch <= connection.revocation_epoch) {
return Err(DurableRunnerError::invalid("invalid revocation during storage pressure"));
}
return Err(DurableRunnerError::invalid("authenticated capability revoked during storage pressure"));
}
Some("command") => {
let command: Command = serde_json::from_value(message.get("payload").cloned().unwrap_or_default()).map_err(|error| DurableRunnerError::invalid(error.to_string()))?;
command.validate()?;
if fresh_stop(&command, identity.controller_epoch.as_deref(), identity.last_controller_command_seq) {
// This is an authenticated request to stop, not evidence
// that the command committed. Unwind to physical owned
// cleanup; never send a successful terminal receipt.
return Err(DurableRunnerError::invalid("authenticated stop requested during storage pressure"));
}
transport.defer_control(message)?;
}
Some("command_epoch_rotate") => {
// Rotation is durable work. Keep it behind the current
// storage operation; the normal loop validates capability
// and the exact current namespace before committing it.
let transition: super::command_epochs::CommandEpochTransition = serde_json::from_value(message.get("payload").cloned().unwrap_or_default()).map_err(|e| DurableRunnerError::invalid(e.to_string()))?;
transition.validate(&identity.run_id)?;
transport.defer_control(message)?;
}
Some("event_epoch_committed") => {
let transition: super::event_epochs::EventEpochTransition = serde_json::from_value(message.get("payload").cloned().unwrap_or_default()).map_err(|e| DurableRunnerError::invalid(e.to_string()))?;
transition.validate(&identity.run_id)?;
transport.defer_control(message)?;
}
Some("ack") => {
if !identity.ack_is_current(message.get("payload").unwrap_or(&serde_json::Value::Null))? { return Ok(()); }
let acked = message.pointer("/payload/ackedSourceSeq").and_then(serde_json::Value::as_u64).ok_or_else(|| DurableRunnerError::invalid("ACK cursor is required"))?;
if acked < ack_cursor || acked > identity.highest_source_seq() {
return Err(DurableRunnerError::invalid("cumulative ACK is outside the durable pressure snapshot"));
}
ack_cursor = acked;
transport.defer_control(message)?;
}
_ => return Err(DurableRunnerError::invalid("unexpected control during storage pressure")),
}
}
Ok(())
})()
};
match polled {
// Keep the exact write/effect stack while its existing lease is
// valid. After capacity returns, the ordinary runner loop
// reconnects and replays its durable result. A lost socket does
// not renew authority, suppress revocation, or admit new work.
Err(error) if error.is_transport_unavailable() => disconnected = true,
Err(error) => return Err(error),
Ok(()) => {}
}
// Busy ping/ACK traffic cannot turn failed writes into a spin loop.
std::thread::sleep(Duration::from_millis(25).saturating_sub(started.elapsed()));
Ok(())
},
operation,
);
let lane = context
.borrow_mut()
.take()
.expect("pressure control lane restored");
(result, lane)
}
pub(super) fn fresh_stop(command: &Command, epoch: Option<&str>, ordinal: u64) -> bool {
command.controller_epoch.as_deref() == epoch
&& command.controller_seq > ordinal
&& matches!(
command.command_type.as_str(),
"turn.stop"
| "turn.interrupt"
| "run.cancel"
| "session.close"
| "session.destroy"
| "runner.shutdown"
| "runner.suspend"
)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn storage_pressure_stop_cannot_cross_a_command_epoch() {
let epoch = uuid::Uuid::new_v4().to_string();
for kind in [
"turn.stop",
"turn.interrupt",
"run.cancel",
"session.close",
"session.destroy",
"runner.shutdown",
"runner.suspend",
] {
let mut command: Command = serde_json::from_value(json!({"schema":"paperclip.prp.command.v1", "commandId":"old-stop", "controllerSeq":9007199254740990_u64, "type":kind, "issuedAt":"2026-09-29T00:00:00Z", "payload":{}})).unwrap();
assert!(!fresh_stop(&command, Some(&epoch), 1));
command.controller_epoch = Some(epoch.clone());
command.controller_seq = 2;
assert!(fresh_stop(&command, Some(&epoch), 1));
assert!(!fresh_stop(&command, Some(&epoch), 2));
command.controller_epoch = Some(uuid::Uuid::new_v4().to_string());
assert!(!fresh_stop(&command, Some(&epoch), 1));
}
}
}
packages/paperclip-runner/runner/crates/runner-core/src/durable/storage_pressure_tests.rs
+448
View File
@@ -0,0 +1,448 @@
// Included in transport's private test module to exercise the actual mutually
// authenticated, encrypted lane and normal durable runner command loop.
#[test]
fn storage_wait_distinguishes_socket_loss_from_invalid_protocol() {
assert!(map_websocket_error(tungstenite::Error::ConnectionClosed).is_transport_unavailable());
assert!(
map_websocket_error(tungstenite::Error::Io(std::io::Error::from(
std::io::ErrorKind::BrokenPipe
)))
.is_transport_unavailable()
);
assert!(map_websocket_error(tungstenite::Error::Protocol(
tungstenite::error::ProtocolError::ResetWithoutClosingHandshake
))
.is_transport_unavailable());
assert!(!map_websocket_error(tungstenite::Error::Protocol(
tungstenite::error::ProtocolError::SendAfterClosing
))
.is_transport_unavailable());
assert!(!DurableRunnerError::invalid("authentication failed").is_transport_unavailable());
}
#[test]
fn storage_pressure_coalesces_ack_debt_without_reordering_commands() {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let client = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
let (_server, _) = listener.accept().unwrap();
let mut transport = AuthenticatedTransport {
socket: RunnerSocket::Listen(WebSocket::from_raw_socket(
client,
tungstenite::protocol::Role::Client,
None,
)),
secure_channel: SecureChannel::client(&[1; 32], b"challenge", b"server", b"client")
.unwrap(),
max_frame_bytes: 16 * 1024 * 1024,
deferred: Default::default(),
deferred_bytes: 0,
};
for seq in 0..10_000 {
transport
.defer_control(json!({"kind":"ack","payload":{"ackedSourceSeq":seq}}))
.unwrap();
}
let command = json!({"kind":"command","payload":{"commandId":"after-acks"}});
transport.defer_control(command.clone()).unwrap();
for seq in 10_000..20_000 {
transport
.defer_control(json!({"kind":"ack","payload":{"ackedSourceSeq":seq}}))
.unwrap();
}
assert_eq!(transport.deferred.len(), 3);
assert!(transport
.defer_control(json!({"kind":"ack","payload":{"ackedSourceSeq":7}}))
.is_err());
assert_eq!(
transport.receive_json().unwrap().unwrap()["payload"]["ackedSourceSeq"],
9_999
);
assert_eq!(transport.receive_json().unwrap().unwrap(), command);
assert_eq!(
transport.receive_json().unwrap().unwrap()["payload"]["ackedSourceSeq"],
19_999
);
assert_eq!(transport.deferred_bytes, 0);
for _ in 0..128 {
transport.defer_control(command.clone()).unwrap();
}
assert!(transport.defer_control(command).is_err());
assert_eq!(transport.deferred.len(), 128);
}
#[test]
fn storage_pressure_recovers_without_reexecuting_effects_or_losing_the_lease() {
storage_pressure_transport_case(None, false, None);
}
#[test]
fn storage_pressure_revocation_stops_owned_execution_while_sqlite_remains_full() {
storage_pressure_transport_case(Some("revoke"), false, None);
}
#[test]
fn storage_pressure_stop_is_responsive_without_a_false_terminal_receipt() {
storage_pressure_transport_case(Some("stop"), false, None);
}
#[test]
fn storage_pressure_recovers_a_command_delivered_in_the_welcome() {
storage_pressure_transport_case(None, true, None);
}
#[test]
fn storage_pressure_recovers_after_a_socket_disconnect_without_repeating_the_effect() {
storage_pressure_transport_case(None, false, Some(false));
}
#[test]
fn storage_pressure_disconnect_cannot_extend_the_authenticated_lease() {
storage_pressure_transport_case(None, false, Some(true));
}
fn storage_pressure_transport_case(
cancel: Option<&'static str>,
pending_welcome: bool,
disconnected_expiry: Option<bool>,
) {
use crate::indexed_store::{ExactReceipt, IndexedStore};
fn accept_before(listener: &TcpListener, deadline: std::time::Instant) -> TcpStream {
listener.set_nonblocking(true).unwrap();
loop {
match listener.accept() {
Ok((stream, _)) => {
stream.set_nonblocking(false).unwrap();
return stream;
}
Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
assert!(
std::time::Instant::now() < deadline,
"runner did not reconnect before the test deadline"
);
thread::sleep(Duration::from_millis(10));
}
Err(error) => panic!("controller accept failed: {error}"),
}
}
}
fn receive_before(
socket: &mut WebSocket<TcpStream>,
secure: &mut SecureChannel,
config: &DurableRunnerConfig,
deadline: std::time::Instant,
) -> Value {
// Runtime storage waits use the test's execution deadline, not the
// transport helper's short authentication deadline. Parallel fsync
// qualifications can legitimately exceed the latter after admission.
socket
.get_mut()
.set_read_timeout(Some(Duration::from_secs(1)))
.unwrap();
loop {
if let Some(frame) =
receive_plain_optional_until(socket, config.max_frame_bytes, Some(deadline))
.unwrap()
{
return secure.decrypt(&frame, true).unwrap();
}
}
}
struct Executor {
store: IndexedStore,
effects: Arc<AtomicUsize>,
shutdowns: Arc<AtomicUsize>,
}
impl super::super::CommandExecutor for Executor {
fn execute(
&mut self,
command: &Command,
) -> Result<super::super::CommandExecution, DurableRunnerError> {
if command.command_type == "session.open" {
self.effects.fetch_add(1, Ordering::SeqCst);
self.store.commit(
"provider",
1,
b"completed".to_vec(),
vec![ExactReceipt {
namespace: "effects".into(),
key: "one-effect".into(),
bytes: vec![b'x'; 2 * 1024 * 1024],
}],
)?;
}
Ok(super::super::CommandExecution::result(
json!({"status":"completed"}),
))
}
fn shutdown(&mut self) -> Result<(), DurableRunnerError> {
self.shutdowns.fetch_add(1, Ordering::SeqCst);
Ok(())
}
}
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let mut cfg = config(listener.local_addr().unwrap().port());
cfg.max_runtime = Duration::from_secs(90);
let directory =
std::env::temp_dir().join(format!("paperclip-capacity-{}", uuid::Uuid::new_v4()));
cfg.state_dir = directory.join("runner");
let store = IndexedStore::open(
&directory.join("provider/state.sqlite"),
"pressure-test",
true,
)
.unwrap();
store
.commit("provider", 0, b"before".to_vec(), vec![])
.unwrap();
store.test_capacity(true);
let server_store = store.clone();
let server_cfg = cfg.clone();
let state = test_state(&cfg);
let server = thread::spawn(move || {
let deadline = std::time::Instant::now() + server_cfg.max_runtime;
let stream = accept_before(&listener, deadline);
stream
.set_read_timeout(Some(Duration::from_secs(15)))
.unwrap();
let mut socket = accept(stream).unwrap();
let mut connection_id = "connection_1";
// Keep the intentional expiry case short. Recovery cases leave time
// for a real FULL-synchronous commit under the full suite's I/O load.
let expires = current_unix_ms().unwrap()
+ if disconnected_expiry == Some(true) {
4_000
} else {
30_000
};
let mut secure = server_authenticate(
&mut socket,
&server_cfg,
&state,
ServerCredential {
token: "bootstrap-secret",
kind: "bootstrap",
lease_id: None,
expires_at_unix_ms: expires,
revocation_epoch: 0,
},
);
let open = json!({
"schema":"paperclip.prp.command.v1", "commandId":"command_open", "controllerSeq":1,
"type":"session.open", "issuedAt":"2026-09-28T00:00:00Z", "payload":{},
});
let mut greeting = welcome(
&state,
connection_id,
Some("lease-secret"),
expires,
0,
if pending_welcome {
vec![open.clone()]
} else {
vec![]
},
);
greeting["payload"]["connectionLeaseRenewalVersion"] = json!(1);
send_secure(&mut socket, &mut secure, &server_cfg, &greeting);
if !pending_welcome {
send_secure(
&mut socket,
&mut secure,
&server_cfg,
&control(&state, connection_id, "command", open.clone()),
);
}
send_secure(
&mut socket,
&mut secure,
&server_cfg,
&control(&state, connection_id, "ack", json!({"ackedSourceSeq":0})),
);
send_secure(
&mut socket,
&mut secure,
&server_cfg,
&control(&state, connection_id, "ping", json!({})),
);
let mut pong = false;
loop {
let message = receive_before(&mut socket, &mut secure, &server_cfg, deadline);
match message["kind"].as_str().unwrap() {
"pong" => {
assert_eq!(message["payload"]["storagePressure"], true);
pong = true;
if let Some(expire) = disconnected_expiry {
drop(socket);
if expire {
return;
}
thread::sleep(Duration::from_millis(200));
server_store.test_capacity(false);
let stream = accept_before(&listener, deadline);
stream
.set_read_timeout(Some(Duration::from_secs(15)))
.unwrap();
socket = accept(stream).unwrap();
secure = server_authenticate(
&mut socket,
&server_cfg,
&state,
ServerCredential {
token: "lease-secret",
kind: "lease",
lease_id: Some("lease_1"),
expires_at_unix_ms: expires,
revocation_epoch: 1,
},
);
connection_id = "connection_2";
send_secure(
&mut socket,
&mut secure,
&server_cfg,
&welcome(&state, connection_id, None, expires, 0, vec![open.clone()]),
);
continue;
}
if let Some(cancel) = cancel {
let message = if cancel == "revoke" {
control(
&state,
connection_id,
"revoke",
json!({"revocationEpoch":2}),
)
} else {
control(
&state,
connection_id,
"command",
json!({
"schema":"paperclip.prp.command.v1", "commandId":"command_stop", "controllerSeq":2,
"type":"turn.stop", "issuedAt":"2026-09-28T00:00:01Z", "payload":{},
}),
)
};
send_secure(&mut socket, &mut secure, &server_cfg, &message);
return;
}
}
"lease_renew" => {
assert!(pong, "the ping must remain responsive before renewal");
send_secure(
&mut socket,
&mut secure,
&server_cfg,
&control(
&state,
connection_id,
"lease_renewed",
json!({
"previousExpiresAtUnixMs":expires, "connectionLeaseExpiresAtUnixMs":current_unix_ms().unwrap()+60_000,
"connectionLeaseRevocationEpoch":1,
}),
),
);
server_store.test_capacity(false);
}
"command_result" => {
assert!(pong);
assert_eq!(message["payload"]["commandId"], "command_open");
assert_eq!(message["payload"]["status"], "completed");
break;
}
other => panic!("unexpected pressure response: {other}"),
}
}
send_secure(
&mut socket,
&mut secure,
&server_cfg,
&control(
&state,
connection_id,
"command",
json!({
"schema":"paperclip.prp.command.v1", "commandId":"command_shutdown", "controllerSeq":2,
"type":"runner.shutdown", "issuedAt":"2026-09-28T00:00:01Z", "payload":{},
}),
),
);
let message = receive_before(&mut socket, &mut secure, &server_cfg, deadline);
assert_eq!(message["payload"]["commandId"], "command_shutdown");
send_secure(
&mut socket,
&mut secure,
&server_cfg,
&control(
&state,
connection_id,
"command_result_ack",
json!({
"commandId":"command_shutdown", "commandType":"runner.shutdown", "controllerSeq":2, "status":"completed",
}),
),
);
});
let effects = Arc::new(AtomicUsize::new(0));
let shutdowns = Arc::new(AtomicUsize::new(0));
let result = super::super::run_indexed_durable_runner(
cfg.clone(),
BootstrapTicket::new("bootstrap-secret".into()).unwrap(),
Executor {
store: store.clone(),
effects: effects.clone(),
shutdowns: shutdowns.clone(),
},
);
if let Err(error) = &result {
eprintln!("pressure case cancel={cancel:?} welcome={pending_welcome} disconnected_expiry={disconnected_expiry:?}: {error}");
}
assert!(
server.join().is_ok(),
"pressure controller failed; runner result: {result:?}"
);
assert_eq!(effects.load(Ordering::SeqCst), 1);
assert_eq!(shutdowns.load(Ordering::SeqCst), 1);
let current = store.read_state("provider").unwrap().unwrap();
if cancel.is_some() || disconnected_expiry == Some(true) {
let error = result.unwrap_err();
assert!(error.is_storage_wait_cancelled());
assert!(
error.to_string().contains(if cancel == Some("revoke") {
"capability revoked"
} else if cancel == Some("stop") {
"stop requested"
} else {
"lease expired"
}),
"{error}"
);
assert_eq!(current.generation, 1);
assert_eq!(current.bytes, b"before");
assert!(store.receipt("effects", "one-effect").unwrap().is_none());
// The effect happened before capacity failed. A stopped runner cannot
// label it completed or execute it again when it later reopens.
let runner = super::super::DurableStateStore::new_indexed(&cfg.state_dir).unwrap();
let (mut recovered, existed) = runner.load_or_create(&cfg).unwrap();
assert!(existed);
let open: Command = serde_json::from_value(json!({
"schema":"paperclip.prp.command.v1", "commandId":"command_open", "controllerSeq":1,
"type":"session.open", "issuedAt":"2026-09-28T00:00:00Z", "payload":{},
}))
.unwrap();
assert!(
matches!(recovered.begin_command(&open).unwrap(), super::super::CommandDisposition::Replay(result) if result.status == "indeterminate")
);
} else {
result.unwrap();
assert_eq!(current.generation, 2);
assert_eq!(current.bytes, b"completed");
assert_eq!(
store.receipt("effects", "one-effect").unwrap().unwrap(),
vec![b'x'; 2 * 1024 * 1024]
);
}
drop(store);
std::fs::remove_dir_all(directory).unwrap();
}
packages/paperclip-runner/runner/crates/runner-core/src/durable/transport.rs
+297 -2
View File
@@ -25,6 +25,9 @@ use super::{
};
const SECURE_FRAME_SCHEMA: &str = "paperclip.runner.secure-frame.v1";
// Reconnect with fresh challenge-derived keys before a connection-local nonce
// counter can become a lifetime limit or exceed the exact JSON representation.
const MAX_SECURE_CHANNEL_FRAMES: u64 = 1_048_576;
const AUTH_TIMEOUT: Duration = Duration::from_secs(2);
const WELCOME_TIMEOUT: Duration = Duration::from_secs(2);
const RUNTIME_READ_TIMEOUT: Duration = Duration::from_millis(250);
@@ -735,6 +738,9 @@ pub(crate) struct Welcome {
pub(crate) acked_source_seq: Option<u64>,
pub(crate) pending_commands: Vec<Command>,
pub(crate) warm_transition_version: Option<u64>,
pub(crate) command_epochs: bool,
pub(crate) event_epoch_limit: Option<u64>,
pub(crate) source_epoch: Option<String>,
pub(crate) lease_renewal_version: Option<u64>,
pub(crate) warm_transition: Option<Value>,
pub(crate) warm_transition_phase: Option<String>,
@@ -826,6 +832,9 @@ impl SecureChannel {
server_direction: bool,
) -> Result<Value, DurableRunnerError> {
let counter = self.send_counter;
if counter >= MAX_SECURE_CHANNEL_FRAMES {
return Err(DurableRunnerError::invalid("secure channel rekey required"));
}
let direction = if server_direction { b"P3S1" } else { b"P3C1" };
let label = if server_direction {
"core_to_client"
@@ -860,6 +869,9 @@ impl SecureChannel {
frame: &Value,
server_direction: bool,
) -> Result<Value, DurableRunnerError> {
if self.receive_counter >= MAX_SECURE_CHANNEL_FRAMES {
return Err(DurableRunnerError::invalid("secure channel rekey required"));
}
if frame.get("schema").and_then(Value::as_str) != Some(SECURE_FRAME_SCHEMA) {
return Err(DurableRunnerError::invalid(
"unauthenticated plaintext control frame was rejected",
@@ -911,6 +923,8 @@ pub(crate) struct AuthenticatedTransport {
socket: RunnerSocket,
secure_channel: SecureChannel,
max_frame_bytes: usize,
deferred: std::collections::VecDeque<(Value, usize)>,
deferred_bytes: usize,
}
#[derive(Debug)]
@@ -998,6 +1012,8 @@ impl AuthenticatedTransport {
"protocolMin": PROTOCOL_MIN_VERSION,
"protocolMax": PROTOCOL_VERSION,
"warmTransitionVersion": 1,
"durability": "durability.indexed_state.v1",
"outputBodies": crate::output_body::CAPABILITY,
"runnerInstanceId": state.runner_instance_id,
"environmentLeaseId": state.environment_lease_id,
"runId": state.run_id,
@@ -1013,6 +1029,14 @@ impl AuthenticatedTransport {
},
},
});
if state.indexed_durability() {
hello["payload"]["commandEpochs"] = json!(super::command_epochs::CAPABILITY);
hello["payload"]["eventEpochs"] = json!(super::event_epochs::CAPABILITY);
hello["payload"]["eventResume"] = state.event_resume();
}
if let Some(epoch) = &state.controller_epoch {
hello["payload"]["resume"]["controllerEpoch"] = json!(epoch);
}
if let Some(transition) = &state.warm_transition {
hello["payload"]["warmTransitionId"] = json!(transition.receipt.transition_id);
}
@@ -1088,6 +1112,8 @@ impl AuthenticatedTransport {
socket,
secure_channel,
max_frame_bytes: config.max_frame_bytes,
deferred: Default::default(),
deferred_bytes: 0,
};
// Ticket validation and lease persistence happen before the server
// emits welcome. Keep that work bounded without assuming a loaded
@@ -1113,6 +1139,20 @@ impl AuthenticatedTransport {
expected_lease,
challenge.selected_version,
)?;
if welcome.command_epochs
!= (challenge.command_epochs.as_deref() == Some(super::command_epochs::CAPABILITY))
{
return Err(DurableRunnerError::invalid(
"welcome changed the command epoch capability",
));
}
if welcome.event_epoch_limit != challenge.event_epoch_limit
|| welcome.source_epoch != state.source_epoch
{
return Err(DurableRunnerError::invalid(
"welcome changed the event epoch contract",
));
}
// Authentication can wait longer for control-plane validation, but
// the steady-state runner loop must return to provider polling
// promptly when no control message is available.
@@ -1142,9 +1182,99 @@ impl AuthenticatedTransport {
}
pub(crate) fn receive_json(&mut self) -> Result<Option<Value>, DurableRunnerError> {
if let Some((value, bytes)) = self.deferred.pop_front() {
self.deferred_bytes -= bytes;
return Ok(Some(value));
}
self.receive_control_during_storage_wait()
}
pub(crate) fn receive_control_during_storage_wait(
&mut self,
) -> Result<Option<Value>, DurableRunnerError> {
self.receive_json_until(None)
}
pub(crate) fn receive_control_during_epoch(
&mut self,
) -> Result<Option<Value>, DurableRunnerError> {
if let Some(index) = self.deferred.iter().position(|(value, _)| {
!matches!(
value.get("kind").and_then(Value::as_str),
Some("command" | "command_epoch_rotate")
)
}) {
let (value, bytes) = self
.deferred
.remove(index)
.expect("located deferred epoch control");
self.deferred_bytes -= bytes;
return Ok(Some(value));
}
self.receive_json_until(None)
}
pub(crate) fn defer_control(&mut self, value: Value) -> Result<(), DurableRunnerError> {
let bytes = serde_json::to_vec(&value)
.map_err(|error| DurableRunnerError::invalid(error.to_string()))?
.len();
// Authenticated cumulative ACKs with no intervening command can be
// collapsed. A large sent prefix must not fill the control backlog
// simply because its receipts arrive while local storage is full.
if value.get("kind").and_then(Value::as_str) == Some("ack") {
let next = value
.pointer("/payload/ackedSourceSeq")
.and_then(Value::as_u64);
let previous = self.deferred.iter().rev().find_map(|(previous, _)| {
(previous.get("kind").and_then(Value::as_str) == Some("ack")
&& previous.pointer("/payload/sourceEpoch")
== value.pointer("/payload/sourceEpoch"))
.then(|| {
previous
.pointer("/payload/ackedSourceSeq")
.and_then(Value::as_u64)
})
.flatten()
});
if next.is_none() || previous.zip(next).is_some_and(|(old, new)| new < old) {
return Err(DurableRunnerError::invalid(
"cumulative ACK regressed in storage control backlog",
));
}
if let Some((previous, previous_bytes)) = self.deferred.back_mut() {
if previous.get("kind").and_then(Value::as_str) == Some("ack")
&& previous.pointer("/payload/sourceEpoch")
== value.pointer("/payload/sourceEpoch")
&& value
.pointer("/payload/ackedSourceSeq")
.and_then(Value::as_u64)
.zip(
previous
.pointer("/payload/ackedSourceSeq")
.and_then(Value::as_u64),
)
.is_some_and(|(new, old)| new >= old)
&& self.deferred_bytes - *previous_bytes + bytes <= 32 * 1024 * 1024
{
self.deferred_bytes = self.deferred_bytes - *previous_bytes + bytes;
*previous = value;
*previous_bytes = bytes;
return Ok(());
}
}
}
if self.deferred.len() >= 128
|| self.deferred_bytes.saturating_add(bytes) > 32 * 1024 * 1024
{
return Err(DurableRunnerError::invalid(
"storage pressure control backlog is full",
));
}
self.deferred.push_back((value, bytes));
self.deferred_bytes += bytes;
Ok(())
}
fn receive_json_until(
&mut self,
deadline: Option<Instant>,
@@ -1182,6 +1312,20 @@ struct AuthChallenge {
#[serde(default)]
warm_transition_version: Option<u64>,
#[serde(default)]
durability: Option<String>,
#[serde(default)]
output_bodies: Option<String>,
#[serde(default)]
command_epochs: Option<String>,
#[serde(default)]
event_epochs: Option<String>,
#[serde(default)]
event_resume: Option<Value>,
#[serde(default)]
event_epoch_limit: Option<u64>,
#[serde(default)]
command_resume: Option<Value>,
#[serde(default)]
warm_transition_id: Option<String>,
}
@@ -1249,6 +1393,55 @@ fn validate_challenge(
"authentication challenge is expired or selected an unsupported protocol",
));
}
if state.indexed_durability()
&& (challenge.durability.as_deref() != Some("durability.indexed_state.v1")
|| challenge.output_bodies.as_deref() != Some(crate::output_body::CAPABILITY))
{
return Err(DurableRunnerError::invalid(
"indexed durability capability was not authenticated",
));
}
if challenge
.command_epochs
.as_deref()
.is_some_and(|v| v != super::command_epochs::CAPABILITY)
|| (state.controller_epoch.is_some()
&& challenge.command_epochs.as_deref() != Some(super::command_epochs::CAPABILITY))
{
return Err(DurableRunnerError::invalid(
"command epoch capability was not authenticated",
));
}
if challenge.command_epochs.is_some()
&& challenge.command_resume
!= Some(json!({
"controllerEpoch": state.controller_epoch, "lastControllerCommandSeq": state.last_controller_command_seq,
}))
{
return Err(DurableRunnerError::invalid(
"authentication challenge changed the command resume cursor",
));
}
if challenge
.event_epochs
.as_deref()
.is_some_and(|v| v != super::event_epochs::CAPABILITY)
|| ((state.source_epoch.is_some() || state.event_epoch_transition.is_some())
&& challenge.event_epochs.is_none())
|| (challenge.event_epochs.is_some()
&& (challenge.selected_version < 2
|| !state.indexed_durability()
|| challenge.event_resume != Some(state.event_resume())
|| challenge.event_epoch_limit.is_none_or(|limit| {
!(4..=super::event_epochs::DEFAULT_LIMIT).contains(&limit)
})))
|| (challenge.event_epochs.is_none()
&& (challenge.event_resume.is_some() || challenge.event_epoch_limit.is_some()))
{
return Err(DurableRunnerError::invalid(
"event epoch contract was not authenticated exactly",
));
}
if state.warm_transition.as_ref().is_some_and(|transition| {
challenge.warm_transition_version != Some(1)
|| challenge.warm_transition_id.as_deref()
@@ -1296,6 +1489,27 @@ fn challenge_signing_bytes(challenge: &AuthChallenge) -> Vec<u8> {
"credentialExpiresAtUnixMs": challenge.credential_expires_at_unix_ms,
"revocationEpoch": challenge.revocation_epoch,
});
if let Some(capability) = &challenge.event_epochs {
body["eventEpochs"] = json!(capability);
}
if let Some(resume) = &challenge.event_resume {
body["eventResume"] = resume.clone();
}
if let Some(limit) = challenge.event_epoch_limit {
body["eventEpochLimit"] = json!(limit);
}
if let Some(resume) = &challenge.command_resume {
body["commandResume"] = resume.clone();
}
if let Some(capability) = &challenge.command_epochs {
body["commandEpochs"] = json!(capability);
}
if let Some(durability) = &challenge.durability {
body["durability"] = json!(durability);
}
if let Some(capability) = &challenge.output_bodies {
body["outputBodies"] = json!(capability);
}
if let Some(version) = challenge.warm_transition_version {
body["warmTransitionVersion"] = json!(version);
}
@@ -1429,8 +1643,21 @@ fn validate_welcome(
},
lease,
acked_source_seq: payload.get("ackedSourceSeq").and_then(Value::as_u64),
source_epoch: payload
.get("sourceEpoch")
.and_then(Value::as_str)
.map(str::to_owned),
event_epoch_limit: if payload.get("eventEpochs").and_then(Value::as_str)
== Some(super::event_epochs::CAPABILITY)
{
payload.get("eventEpochLimit").and_then(Value::as_u64)
} else {
None
},
pending_commands,
warm_transition_version: payload.get("warmTransitionVersion").and_then(Value::as_u64),
command_epochs: payload.get("commandEpochs").and_then(Value::as_str)
== Some(super::command_epochs::CAPABILITY),
lease_renewal_version: payload
.get("connectionLeaseRenewalVersion")
.and_then(Value::as_u64),
@@ -1594,7 +1821,7 @@ fn receive_plain_optional_until(
.map_err(map_websocket_error)?,
Ok(Message::Pong(_)) => {}
Ok(Message::Close(_)) => {
return Err(DurableRunnerError::invalid(
return Err(DurableRunnerError::transport_unavailable(
"WebSocket peer closed the connection",
))
}
@@ -1617,7 +1844,28 @@ fn receive_plain_optional_until(
}
fn map_websocket_error(error: tungstenite::Error) -> DurableRunnerError {
DurableRunnerError::invalid(format!("WebSocket transport failed: {error}"))
let disconnected = match &error {
tungstenite::Error::ConnectionClosed
| tungstenite::Error::AlreadyClosed
| tungstenite::Error::Protocol(
tungstenite::error::ProtocolError::ResetWithoutClosingHandshake,
) => true,
tungstenite::Error::Io(error) => matches!(
error.kind(),
io::ErrorKind::ConnectionReset
| io::ErrorKind::ConnectionAborted
| io::ErrorKind::BrokenPipe
| io::ErrorKind::UnexpectedEof
| io::ErrorKind::NotConnected
),
_ => false,
};
let message = format!("WebSocket transport failed: {error}");
if disconnected {
DurableRunnerError::transport_unavailable(message)
} else {
DurableRunnerError::invalid(message)
}
}
fn random_nonce() -> Result<String, DurableRunnerError> {
@@ -1710,6 +1958,7 @@ fn hex_nibble(byte: u8) -> Result<u8, DurableRunnerError> {
#[cfg(test)]
mod tests {
include!("storage_pressure_tests.rs");
use std::net::{IpAddr, Ipv4Addr, TcpListener};
use std::path::PathBuf;
use std::sync::atomic::{AtomicUsize, Ordering};
@@ -1788,6 +2037,13 @@ mod tests {
revocation_epoch: server_credential.revocation_epoch,
server_proof: String::new(),
warm_transition_version: None,
command_epochs: None,
event_epochs: None,
event_resume: None,
event_epoch_limit: None,
command_resume: None,
durability: Some("durability.indexed_state.v1".into()),
output_bodies: Some(crate::output_body::CAPABILITY.into()),
warm_transition_id: None,
};
let signing = challenge_signing_bytes(&challenge);
@@ -1821,6 +2077,8 @@ mod tests {
"credentialLeaseId": &challenge.credential_lease_id,
"revocationEpoch": challenge.revocation_epoch,
"serverProof": &challenge.server_proof,
"durability": &challenge.durability,
"outputBodies": &challenge.output_bodies,
},
}),
config.max_frame_bytes,
@@ -2265,6 +2523,36 @@ mod tests {
assert!(server.decrypt(&json!({"kind": "command"}), true).is_err());
}
#[test]
fn secure_channel_rekeys_before_counter_exhaustion_and_rejects_old_ciphertext() {
let key = [7_u8; 32];
let mut client = SecureChannel::client(&key, b"challenge", b"server", b"client").unwrap();
let mut server = SecureChannel::server(&key, b"challenge", b"server", b"client").unwrap();
let old_frame = client.encrypt(br#"{"old":true}"#, false).unwrap();
client.send_counter = MAX_SECURE_CHANNEL_FRAMES - 1;
server.receive_counter = MAX_SECURE_CHANNEL_FRAMES - 1;
let last = client.encrypt(br#"{"last":true}"#, false).unwrap();
assert_eq!(server.decrypt(&last, true).unwrap(), json!({"last": true}));
assert!(client
.encrypt(br#"{}"#, false)
.unwrap_err()
.to_string()
.contains("rekey required"));
assert!(server
.decrypt(&last, true)
.unwrap_err()
.to_string()
.contains("rekey required"));
let mut client =
SecureChannel::client(&key, b"new-challenge", b"new-server", b"client").unwrap();
let mut server =
SecureChannel::server(&key, b"new-challenge", b"new-server", b"client").unwrap();
assert!(server.decrypt(&old_frame, true).is_err());
let new = client.encrypt(br#"{"new":true}"#, false).unwrap();
assert_eq!(new["counter"], 0);
assert_eq!(server.decrypt(&new, true).unwrap(), json!({"new": true}));
}
#[test]
fn control_identity_mismatch_fails_closed() {
let config = config(3000);
@@ -2318,6 +2606,13 @@ mod tests {
revocation_epoch: 0,
server_proof: String::new(),
warm_transition_version: None,
command_epochs: None,
event_epochs: None,
event_resume: None,
event_epoch_limit: None,
command_resume: None,
durability: None,
output_bodies: None,
warm_transition_id: None,
};
let signing = challenge_signing_bytes(&challenge);
packages/paperclip-runner/runner/crates/runner-core/src/indexed_archive.rs
+624
View File
@@ -0,0 +1,624 @@
//! Bounded cold-epoch transfer. The durable intent fences new connections and
//! a lifetime lock excludes every live runner/maintenance/backup connection.
use crate::durable::{open_private_regular_file, verify_private_directory, DurableRunnerError};
use crate::indexed_lifetime::{archive_fence, StoreLifetime};
use crate::indexed_revision::Revision;
use crate::indexed_store::storage_error;
use rusqlite::{Connection, OpenFlags, OptionalExtension};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::{
fs,
io::{Read, Write},
path::{Path, PathBuf},
};
type Result<T> = std::result::Result<T, DurableRunnerError>;
const LOCATOR: &str = "runner-state.json";
const FILES: [&str; 6] = [
"runner-state.sqlite",
"runner-state.sqlite-wal",
"runner-state.sqlite-shm",
"runner-state.sqlite.receipts",
"runner-state.sqlite.routing",
"runner-state.json",
];
#[derive(Deserialize, Serialize, PartialEq, Debug)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
struct Transfer {
schema: String,
generation: String,
state_sha256: String,
files: Vec<String>,
}
fn invalid(message: &str) -> DurableRunnerError {
DurableRunnerError::invalid(message)
}
fn present(path: &Path) -> Result<bool> {
match fs::symlink_metadata(path) {
Ok(_) => Ok(true),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(false),
Err(e) => Err(storage_error(e)),
}
}
fn sync(path: &Path) -> Result<()> {
fs::File::open(path)
.and_then(|f| f.sync_all())
.map_err(storage_error)
}
fn read_json<T: serde::de::DeserializeOwned>(path: &Path) -> Result<T> {
let file = open_private_regular_file(path).map_err(storage_error)?;
let mut bytes = Vec::new();
file.take(4097)
.read_to_end(&mut bytes)
.map_err(storage_error)?;
if bytes.len() > 4096 {
return Err(invalid("native_indexed_archive_oversized"));
}
serde_json::from_slice(&bytes).map_err(storage_error)
}
fn publish(path: &Path, value: &impl Serialize) -> Result<()> {
let temporary = path.with_extension(format!("{}.tmp", uuid::Uuid::new_v4()));
let mut options = fs::OpenOptions::new();
options.write(true).create_new(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
options.mode(0o600);
}
let mut file = options.open(&temporary).map_err(storage_error)?;
let result = (|| {
file.write_all(&serde_json::to_vec(value).map_err(storage_error)?)
.map_err(storage_error)?;
file.sync_all().map_err(storage_error)?;
// A conflicting owner cannot replace either a file or a dangling link.
fs::hard_link(&temporary, path).map_err(storage_error)?;
sync(path.parent().unwrap())
})();
let _ = fs::remove_file(temporary);
result
}
fn verify_file(path: &Path) -> Result<()> {
if path.file_name().is_some_and(|name| {
name == "runner-state.sqlite.receipts" || name == "runner-state.sqlite.routing"
}) {
verify_private_directory(path)
} else {
open_private_regular_file(path)
.map(|_| ())
.map_err(storage_error)
}
}
fn database(directory: &Path) -> Result<Connection> {
for file in &FILES[..3] {
let path = directory.join(file);
if *file == FILES[0] || present(&path)? {
verify_file(&path)?;
}
}
let db = Connection::open_with_flags(
directory.join(FILES[0]),
OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NOFOLLOW,
)
.map_err(storage_error)?;
db.execute_batch("PRAGMA query_only=ON; PRAGMA trusted_schema=OFF; PRAGMA cache_size=-4096; PRAGMA busy_timeout=5000;").map_err(storage_error)?;
Ok(db)
}
fn verify_snapshot(
directory: &Path,
generation: &str,
digest: &str,
allow_backup: bool,
) -> Result<()> {
let locator: serde_json::Value = read_json(&directory.join("runner-state.json"))?;
if locator["schema"] != "paperclip.runner.durable.state.indexed.v1" {
return Err(invalid("native_indexed_archive_locator_invalid"));
}
let db = database(directory)?;
let binding: (i64, String) = db
.query_row(
"SELECT schema_version,binding FROM store_binding WHERE singleton=1",
[],
|r| Ok((r.get(0)?, r.get(1)?)),
)
.map_err(storage_error)?;
if ![1, 2].contains(&binding.0) || locator["binding"].as_str() != Some(&binding.1) {
return Err(invalid("native_indexed_archive_binding_changed"));
}
let has_routing_tree: bool = db.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name='receipt_routing_heads')", [], |r| r.get(0)).map_err(storage_error)?;
if has_routing_tree {
verify_private_directory(&directory.join("runner-state.sqlite.routing"))?;
}
crate::indexed_partitions::validate_backup(&db)?;
if !allow_backup
&& db
.query_row(
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='receipt_backup'",
[],
|_| Ok(()),
)
.optional()
.map_err(storage_error)?
.is_some()
&& db
.query_row("SELECT 1 FROM receipt_backup LIMIT 1", [], |_| Ok(()))
.optional()
.map_err(storage_error)?
.is_some()
{
return Err(invalid(
"storage_pressure: finish the indexed backup before archiving its source",
));
}
let (stored_generation, bytes, stored_digest): (Revision, Vec<u8>, Vec<u8>) = db.query_row("SELECT generation,CASE WHEN length(bytes)<=33554432 THEN bytes END,CASE WHEN length(digest)=32 THEN digest END FROM current_state WHERE key='runner'", [], |r| Ok((r.get(0)?,r.get(1)?,r.get(2)?))).map_err(storage_error)?;
let actual = Sha256::digest(&bytes);
let actual_hex: String = actual.iter().map(|byte| format!("{byte:02x}")).collect();
if stored_generation.to_string() != generation
|| stored_digest != actual.as_slice()
|| actual_hex != digest
{
return Err(invalid("native_indexed_archive_changed"));
}
let state: serde_json::Value = serde_json::from_slice(&bytes).map_err(storage_error)?;
if state["lifecycle"] != "suspended" {
return Err(invalid("native_indexed_archive_unsettled"));
}
Ok(())
}
fn transfer(archive: &Path) -> Result<Option<Transfer>> {
let path = archive.join("runner-transfer.json");
if !present(&path)? {
return Ok(None);
}
let value: Transfer = read_json(&path)?;
if value.schema != "paperclip.runner.indexed-archive.v1"
|| value
.generation
.parse::<Revision>()
.ok()
.filter(|n| *n != Revision::Absent)
.map(|n| n.to_string())
.as_ref()
!= Some(&value.generation)
|| value.state_sha256.len() != 64
|| !value
.state_sha256
.bytes()
.all(|c| c.is_ascii_digit() || (b'a'..=b'f').contains(&c))
|| !value.files.iter().any(|f| f == FILES[0])
|| !value.files.iter().any(|f| f == LOCATOR)
|| value.files.len() > FILES.len()
|| value
.files
.iter()
.enumerate()
.any(|(i, f)| !FILES.contains(&f.as_str()) || value.files[..i].contains(f))
{
return Err(invalid("native_indexed_archive_invalid"));
}
Ok(Some(value))
}
fn fence(runner: &Path, archive: &Path) -> Result<PathBuf> {
let path = archive_fence(&runner.join(FILES[0]));
let value = serde_json::json!({"schema":"paperclip.runner.archive-fence.v1","archive":archive});
if present(&path)? {
if read_json::<serde_json::Value>(&path)? != value {
return Err(invalid("native_indexed_archive_conflict"));
}
} else {
publish(&path, &value)?;
}
Ok(path)
}
/// Caller has already proven process-tree death and controller ownership. This
/// independently excludes storage operators and checks the exact current row.
pub fn prepare(runner: &Path, archive: &Path, generation: &str, digest: &str) -> Result<()> {
verify_private_directory(runner)?;
verify_private_directory(archive)?;
let runner = runner.canonicalize().map_err(storage_error)?;
let archive = archive.canonicalize().map_err(storage_error)?;
let _lifetime = StoreLifetime::exclusive(&runner.join(FILES[0]))?;
let _operation = crate::indexed_partitions::PartitionLock::acquire(&runner.join(FILES[0]))?;
verify_snapshot(&runner, generation, digest, false)?;
let mut files = Vec::new();
for name in FILES {
let path = runner.join(name);
if present(&path)? {
verify_file(&path)?;
files.push(name.to_owned());
}
}
let intent = Transfer {
schema: "paperclip.runner.indexed-archive.v1".into(),
generation: generation.into(),
state_sha256: digest.into(),
files,
};
if let Some(existing) = transfer(&archive)? {
if existing != intent {
return Err(invalid("native_indexed_archive_changed"));
}
} else {
publish(&archive.join("runner-transfer.json"), &intent)?;
}
fence(&runner, &archive)?;
Ok(())
}
fn rename_new(source: &Path, target: &Path) -> Result<()> {
#[cfg(any(target_os = "linux", target_os = "android", target_vendor = "apple"))]
{
rustix::fs::renameat_with(
rustix::fs::CWD,
source,
rustix::fs::CWD,
target,
rustix::fs::RenameFlags::NOREPLACE,
)
.map_err(storage_error)
}
#[cfg(not(any(target_os = "linux", target_os = "android", target_vendor = "apple")))]
{
let _ = (source, target);
Err(invalid("indexed archive is unqualified on this platform"))
}
}
pub fn finish(runner: &Path, archive: &Path) -> Result<bool> {
verify_private_directory(runner)?;
verify_private_directory(archive)?;
let runner = runner.canonicalize().map_err(storage_error)?;
let archive = archive.canonicalize().map_err(storage_error)?;
let Some(intent) = transfer(&archive)? else {
return Ok(false);
};
if !present(&archive.join("control-plane"))? {
return Err(invalid("native_indexed_archive_controller_missing"));
}
verify_private_directory(&archive.join("control-plane"))?;
let _source_lifetime = StoreLifetime::exclusive(&runner.join(FILES[0]))?;
let _target_lifetime = StoreLifetime::exclusive(&archive.join(FILES[0]))?;
let _operation = crate::indexed_partitions::PartitionLock::acquire(&runner.join(FILES[0]))?;
let published = present(&archive.join(LOCATOR))?;
if published {
for name in &intent.files {
if present(&runner.join(name))? {
return Err(invalid("native_indexed_archive_conflict"));
}
}
}
let marker = fence(&runner, &archive)?;
if !published {
for name in FILES
.into_iter()
.filter(|name| intent.files.iter().any(|f| f == name))
{
let source = runner.join(name);
let target = archive.join(name);
if present(&source)? {
verify_file(&source)?;
rename_new(&source, &target)?;
sync(&runner)?;
sync(&archive)?;
} else {
verify_file(&target)?;
}
}
}
verify_snapshot(&archive, &intent.generation, &intent.state_sha256, false)?;
fs::remove_file(marker).map_err(storage_error)?;
sync(&runner)?;
Ok(true)
}
/// Transfer a standalone controller while excluding its storage subprocess and
/// online backup. The directory rename is atomic; a retry observes the archived
/// locator. PostgreSQL locators have no local database and are moved by the host.
pub fn controller(source: &Path, target: &Path, expected_digest: &str) -> Result<()> {
verify_private_directory(source)?;
verify_private_directory(
source
.parent()
.ok_or_else(|| invalid("controller parent missing"))?,
)?;
verify_private_directory(
target
.parent()
.ok_or_else(|| invalid("archive parent missing"))?,
)?;
let source = source.canonicalize().map_err(storage_error)?;
let target = target
.parent()
.unwrap()
.canonicalize()
.map_err(storage_error)?
.join(
target
.file_name()
.ok_or_else(|| invalid("archive filename missing"))?,
);
let locator: serde_json::Value = read_json(&source.join("control-plane-state.json"))?;
let file = locator["location"]["file"]
.as_str()
.ok_or_else(|| invalid("controller locator missing"))?;
if locator["schema"] != "paperclip.runner.authority-locator.v1"
|| locator["location"]["kind"] != "sqlite"
|| !file.ends_with(".sqlite")
|| file.is_empty()
|| !file
.bytes()
.all(|c| c.is_ascii_alphanumeric() || b"_.-".contains(&c))
{
return Err(invalid("controller archive locator invalid"));
}
let path = source.join(file);
let _lifetime = StoreLifetime::exclusive(&path)?;
let _operation = crate::indexed_partitions::PartitionLock::acquire(&path)?;
for suffix in ["", "-wal", "-shm"] {
let sidecar = PathBuf::from(format!("{}{suffix}", path.display()));
if suffix.is_empty() || present(&sidecar)? {
crate::indexed_store::verify_sqlite_file(&sidecar).map_err(storage_error)?;
}
}
let db = Connection::open_with_flags(
&path,
OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NOFOLLOW,
)
.map_err(storage_error)?;
db.execute_batch("PRAGMA query_only=ON; PRAGMA trusted_schema=OFF; PRAGMA cache_size=-4096;")
.map_err(storage_error)?;
let native = db
.query_row(
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='store_binding'",
[],
|_| Ok(()),
)
.optional()
.map_err(storage_error)?
.is_some();
let (binding, bytes, digest): (String, Vec<u8>, Vec<u8>) = if native {
crate::indexed_partitions::validate_backup(&db)?;
if db
.query_row("SELECT 1 FROM receipt_backup LIMIT 1", [], |_| Ok(()))
.optional()
.map_err(storage_error)?
.is_some()
{
return Err(invalid(
"storage_pressure: finish the controller backup before archiving",
));
}
db.query_row("SELECT binding,CASE WHEN length(bytes)<=16777216 THEN bytes END,digest FROM store_binding,current_state WHERE singleton=1 AND key='authority'", [], |r| Ok((r.get(0)?,r.get(1)?,r.get(2)?))).map_err(storage_error)?
} else {
let (binding,body,digest):(String,String,String)=db.query_row("SELECT binding,CASE WHEN length(body)<=16777216 THEN body END,digest FROM authority_binding,authority_state WHERE authority_binding.singleton=1 AND authority_state.singleton=1", [], |r| Ok((r.get(0)?,r.get(1)?,r.get(2)?))).map_err(storage_error)?;
let actual = format!("{:x}", Sha256::digest(body.as_bytes()));
if digest != actual {
return Err(invalid("controller archive digest mismatch"));
}
(
binding,
body.as_bytes().to_vec(),
Sha256::digest(body.as_bytes()).to_vec(),
)
};
if locator["location"]["binding"].as_str() != Some(&binding)
|| Sha256::digest(&bytes).as_slice() != digest
|| format!("{:x}", Sha256::digest(&bytes)) != expected_digest
{
return Err(invalid("controller archive authority changed"));
}
drop(db);
rename_new(&source, &target)?;
sync(source.parent().unwrap())?;
sync(target.parent().unwrap())
}
/// The server proves process-tree death; this fence also excludes operator
/// connections and file transfer while sealing the exact empty checkpoint.
pub fn seal(runner: &Path, expected_generation: &str) -> Result<()> {
verify_private_directory(runner)?;
let runner = runner.canonicalize().map_err(storage_error)?;
crate::legacy_indexed_import::require_active_directory(&runner)?;
let path = runner.join(FILES[0]);
let _lifetime = StoreLifetime::exclusive(&path)?;
if present(&archive_fence(&path))? {
return Err(invalid(
"storage_pressure: indexed archive must finish before sealing",
));
}
let _operation = crate::indexed_partitions::PartitionLock::acquire(&path)?;
let locator: serde_json::Value = read_json(&runner.join(LOCATOR))?;
if locator["schema"] != "paperclip.runner.durable.state.indexed.v1" {
return Err(invalid("indexed_state_locator_invalid"));
}
for name in &FILES[..3] {
let file = runner.join(name);
if *name == FILES[0] || present(&file)? {
verify_file(&file)?;
}
}
let mut db = Connection::open_with_flags(
&path,
OpenFlags::SQLITE_OPEN_READ_WRITE | OpenFlags::SQLITE_OPEN_NOFOLLOW,
)
.map_err(storage_error)?;
db.execute_batch("PRAGMA trusted_schema=OFF; PRAGMA synchronous=FULL; PRAGMA cache_size=-4096; PRAGMA busy_timeout=5000;").map_err(storage_error)?;
let tx = db
.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)
.map_err(storage_error)?;
let binding: (i64, String) = tx
.query_row(
"SELECT schema_version,binding FROM store_binding WHERE singleton=1",
[],
|r| Ok((r.get(0)?, r.get(1)?)),
)
.map_err(storage_error)?;
if ![1, 2].contains(&binding.0) || locator["binding"].as_str() != Some(&binding.1) {
return Err(invalid("indexed_state_binding_mismatch"));
}
crate::indexed_partitions::validate_backup(&tx)
.map_err(|_| invalid("indexed_state_backup_incomplete"))?;
let (generation,bytes,digest):(Revision,Vec<u8>,Vec<u8>)=tx.query_row("SELECT generation,CASE WHEN length(bytes)<=33554432 THEN bytes END,CASE WHEN length(digest)=32 THEN digest END FROM current_state WHERE key='runner'",[],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?))).map_err(storage_error)?;
if Sha256::digest(&bytes).as_slice() != digest {
return Err(invalid("indexed_state_digest_mismatch"));
}
let mut state: serde_json::Value = serde_json::from_slice(&bytes).map_err(storage_error)?;
let next = state["nextSourceSeq"].as_u64();
let acked = state["ackedSourceSeq"].as_u64();
if generation.to_string() != expected_generation
|| generation == Revision::Absent
|| ![Some("ready"), Some("suspended")].contains(&state["lifecycle"].as_str())
|| next.is_none()
|| next.unwrap() > 9_007_199_254_740_991
|| acked.and_then(|n| n.checked_add(1)) != next
|| !state["pendingTerminalDelivery"].is_null()
|| !state["pendingProviderCleanup"].is_null()
{
return Err(invalid("indexed_state_seal_stale_or_unsettled"));
}
if binding.0 == 2 {
let pending:Option<(Revision,Vec<u8>,Vec<u8>)>=tx.query_row("SELECT generation,CASE WHEN length(body)<=33554432 THEN body END,digest FROM receipt_prepare WHERE state_key='runner'",[],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?))).optional().map_err(storage_error)?;
if pending.is_some_and(|row| row != (generation, bytes.clone(), digest.clone())) {
return Err(invalid("indexed_state_prepared_commit_mismatch"));
}
}
state["lifecycle"] = serde_json::json!("suspended");
let body = serde_json::to_vec(&state).map_err(storage_error)?;
if body.len() > 33_554_432 {
return Err(invalid("indexed_state_snapshot_invalid"));
}
let digest = Sha256::digest(&body).to_vec();
let next_revision = generation.next()?;
tx.execute(
"UPDATE current_state SET bytes=?1,digest=?2,generation=?3 WHERE key='runner'",
rusqlite::params![body, digest, next_revision],
)
.map_err(storage_error)?;
if binding.0 == 2 {
tx.execute(
"UPDATE receipt_prepare SET body=?1,digest=?2,generation=?3 WHERE state_key='runner'",
rusqlite::params![body, digest, next_revision],
)
.map_err(storage_error)?;
}
tx.commit().map_err(storage_error)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::indexed_store::IndexedStore;
#[test]
fn pending_session_migration_fences_sealing_before_storage_is_opened() {
let root =
std::env::temp_dir().join(format!("indexed-seal-migration-{}", uuid::Uuid::new_v4()));
let runner = root.join("runner");
fs::create_dir_all(&runner).unwrap();
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
for path in [&root, &runner] {
fs::set_permissions(path, fs::Permissions::from_mode(0o700)).unwrap();
}
}
publish(
&root.join("indexed-migration.json"),
&serde_json::json!({
"schema": "paperclip.runner.legacy-activation.v1", "phase": "activating"
}),
)
.unwrap();
assert!(seal(&runner, "1")
.unwrap_err()
.to_string()
.contains("migration is pending"));
assert_eq!(fs::read_dir(&runner).unwrap().count(), 0);
fs::remove_dir_all(root).unwrap();
}
#[test]
fn active_connections_and_interrupted_backups_block_archive_without_moving_history() {
for opaque in [false, true] {
let root =
std::env::temp_dir().join(format!("indexed-archive-{}", uuid::Uuid::new_v4()));
fs::create_dir(&root).unwrap();
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
fs::set_permissions(&root, fs::Permissions::from_mode(0o700)).unwrap();
}
let archive = root.join("archive");
fs::create_dir(&archive).unwrap();
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
fs::set_permissions(&archive, fs::Permissions::from_mode(0o700)).unwrap();
}
let mut store = IndexedStore::open(&root.join(FILES[0]), "runner/test", true).unwrap();
let bytes = br#"{"lifecycle":"suspended"}"#;
let mut generation = store
.commit("runner", 0, bytes.to_vec(), vec![])
.unwrap()
.to_string();
if opaque {
drop(store);
let db = Connection::open(root.join(FILES[0])).unwrap();
db.execute(
"UPDATE current_state SET generation=?1 WHERE key='runner'",
[i64::MAX],
)
.unwrap();
drop(db);
store = IndexedStore::open(&root.join(FILES[0]), "runner/test", false).unwrap();
generation = store
.commit("runner", i64::MAX as u64, bytes.to_vec(), vec![])
.unwrap()
.to_string();
assert!(generation.starts_with("r:"));
}
publish(&root.join(LOCATOR),&serde_json::json!({"schema":"paperclip.runner.durable.state.indexed.v1","binding":"runner/test"})).unwrap();
let digest: String = Sha256::digest(bytes)
.iter()
.map(|b| format!("{b:02x}"))
.collect();
assert!(prepare(&root, &archive, &generation, &digest)
.unwrap_err()
.to_string()
.contains("open connection"));
store.begin_backup(&root.join("backup")).unwrap();
drop(store);
assert!(prepare(&root, &archive, &generation, &digest)
.unwrap_err()
.to_string()
.contains("finish the indexed backup"));
let store = IndexedStore::open(&root.join(FILES[0]), "runner/test", false).unwrap();
while !store.backup_step().unwrap() {}
drop(store);
prepare(&root, &archive, &generation, &digest).unwrap();
assert!(
IndexedStore::open(&root.join(FILES[0]), "runner/test", false)
.unwrap_err()
.to_string()
.contains("archive must finish")
);
assert!(!archive.join(FILES[0]).exists());
fs::create_dir(archive.join("control-plane")).unwrap();
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
fs::set_permissions(
archive.join("control-plane"),
fs::Permissions::from_mode(0o700),
)
.unwrap();
}
// A crash between root and companion/locator movement rolls forward.
rename_new(&root.join(FILES[0]), &archive.join(FILES[0])).unwrap();
assert!(finish(&root, &archive).unwrap());
assert!(finish(&root, &archive).unwrap());
let archived =
IndexedStore::open(&archive.join(FILES[0]), "runner/test", false).unwrap();
assert_eq!(archived.read_state("runner").unwrap().unwrap().bytes, bytes);
drop(archived);
fs::remove_dir_all(root).unwrap();
}
}
}
packages/paperclip-runner/runner/crates/runner-core/src/indexed_backup.rs
+627
View File
@@ -0,0 +1,627 @@
//! Online, resumable snapshots. A short root snapshot pins immutable source
//! partitions; bounded copies select only the insertion ordinals present at
//! that snapshot. Appends, splits and relocation continue between copy steps.
//! The destination is published only after every range verifies exactly.
use super::*;
struct Backup {
id: String,
destination: PathBuf,
staging: PathBuf,
phase: String,
filename: String,
}
fn valid_filename(filename: &str) -> bool {
filename.len() <= 120
&& filename.ends_with(".sqlite")
&& !filename.starts_with('.')
&& filename
.bytes()
.all(|b| b.is_ascii_alphanumeric() || b"-_.".contains(&b))
}
fn job(db: &Connection) -> Result<Option<Backup>> {
db.query_row(
"SELECT id,destination,staging,phase,COALESCE((SELECT filename FROM receipt_backup_filename WHERE singleton=1),'authority.sqlite') FROM receipt_backup WHERE singleton=1",
[],
|r| {
Ok(Backup {
id: r.get(0)?,
destination: PathBuf::from(r.get::<_, String>(1)?),
staging: PathBuf::from(r.get::<_, String>(2)?),
phase: r.get(3)?,
filename: r.get(4)?,
})
},
)
.optional()
.map_err(error).and_then(|job| {
if job.as_ref().is_some_and(|job| !valid_filename(&job.filename)) {
return Err(invalid("invalid backup database filename"));
}
Ok(job)
})
}
fn private_directory(path: &Path) -> Result<()> {
let mut builder = fs::DirBuilder::new();
#[cfg(unix)]
{
use std::os::unix::fs::DirBuilderExt;
builder.mode(0o700);
}
builder.create(path).map_err(error)?;
sync_directory(path.parent().unwrap())
}
fn sync_directory(path: &Path) -> Result<()> {
fs::File::open(path)
.and_then(|f| f.sync_all())
.map_err(error)
}
fn publish_directory(source: &Path, destination: &Path) -> Result<()> {
// The earlier existence check is only diagnostic. Publication itself must
// refuse a directory/symlink created by another owner during the copy.
#[cfg(any(target_os = "linux", target_os = "android", target_vendor = "apple"))]
{
rustix::fs::renameat_with(
rustix::fs::CWD,
source,
rustix::fs::CWD,
destination,
rustix::fs::RenameFlags::NOREPLACE,
)
.map_err(error)
}
#[cfg(not(any(target_os = "linux", target_os = "android", target_vendor = "apple")))]
{
let _ = (source, destination);
Err(invalid(
"atomic backup publication is unqualified on this platform",
))
}
}
fn create_file(path: &Path) -> Result<()> {
let mut options = fs::OpenOptions::new();
options.create_new(true).write(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
options.mode(0o600);
}
options.open(path).and_then(|f| f.sync_all()).map_err(error)
}
fn root(path: &Path) -> Result<Connection> {
crate::durable::verify_private_directory(path.parent().unwrap())?;
for suffix in ["", "-wal", "-shm"] {
match crate::indexed_store::verify_sqlite_file(Path::new(&format!(
"{}{suffix}",
path.display()
))) {
Ok(_) => {}
Err(e) if !suffix.is_empty() && e.kind() == std::io::ErrorKind::NotFound => {}
Err(e) => return Err(error(e)),
}
}
let db = Connection::open_with_flags(
path,
rusqlite::OpenFlags::SQLITE_OPEN_READ_WRITE | rusqlite::OpenFlags::SQLITE_OPEN_NOFOLLOW,
)
.map_err(error)?;
db.execute_batch("PRAGMA journal_mode=WAL; PRAGMA synchronous=FULL; PRAGMA trusted_schema=OFF; PRAGMA cache_size=-4096;").map_err(error)?;
Ok(db)
}
/// Called before any activated store can serve a snapshot.
pub(crate) fn validate_backup(db: &Connection) -> Result<()> {
let marker: Option<i64> = db
.query_row(
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='backup_manifest'",
[],
|r| r.get(0),
)
.optional()
.map_err(error)?;
if marker.is_some()
&& db
.query_row(
"SELECT complete FROM backup_manifest WHERE singleton=1",
[],
|r| r.get::<_, i64>(0),
)
.map_err(error)?
!= 1
{
return Err(invalid(
"incomplete backup cannot serve execution authority",
));
}
Ok(())
}
pub(crate) fn begin_backup(db: &mut Connection, destination: &Path, filename: &str) -> Result<()> {
if !valid_filename(filename) {
return Err(invalid("invalid backup database filename"));
}
let parent = destination
.parent()
.ok_or_else(|| invalid("backup has no parent"))?;
crate::durable::verify_private_directory(parent)?;
let destination = parent.canonicalize().map_err(error)?.join(
destination
.file_name()
.ok_or_else(|| invalid("backup has no filename"))?,
);
if let Some(active) = job(db)? {
if active.destination != destination || active.filename != filename {
return Err(invalid("storage_pressure: another backup is active"));
}
return Ok(());
}
if destination.try_exists().map_err(error)? {
return Err(invalid("backup destination already exists"));
}
let id = uuid::Uuid::new_v4().to_string();
let staging = parent
.canonicalize()
.map_err(error)?
.join(format!(".receipt-backup-{id}"));
private_directory(&staging)?;
let tx = db
.transaction_with_behavior(TransactionBehavior::Immediate)
.map_err(error)?;
tx.execute(
"INSERT INTO receipt_backup VALUES(1,?1,?2,?3,'preparing')",
params![id, destination.to_str(), staging.to_str()],
)
.map_err(error)?;
tx.execute(
"INSERT OR REPLACE INTO receipt_backup_filename VALUES(1,?1)",
[filename],
)
.map_err(error)?;
tx.commit().map_err(error)?;
Ok(())
}
fn prepare(db: &mut Connection, job: &Backup) -> Result<()> {
crate::durable::verify_private_directory(&job.staging)?;
let path = job.staging.join(&job.filename);
// Preparing has not published a snapshot. Restart only this owned temporary
// root; no receipt files have been copied yet and no destination is visible.
for suffix in ["-wal", "-shm", ""] {
let file = PathBuf::from(format!("{}{suffix}", path.display()));
match crate::indexed_store::verify_sqlite_file(&file) {
Ok(_) => fs::remove_file(file).map_err(error)?,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
Err(e) => return Err(error(e)),
}
}
create_file(&path)?;
// Only bounded current/redo state and two routing pointers live here. The
// immutable tree root is the snapshot inventory; no O(H) pin/backlog copy
// happens while the operation fence is held.
db.execute("VACUUM INTO ?1", [path.to_str()])
.map_err(error)?;
let target = root(&path)?;
// A backup of a restored store may already contain a completed marker.
target.execute_batch("BEGIN IMMEDIATE;
DROP TABLE IF EXISTS backup_manifest;
DROP TABLE IF EXISTS backup_copy;
CREATE TABLE backup_manifest(singleton INTEGER PRIMARY KEY CHECK(singleton=1), id TEXT NOT NULL, complete INTEGER NOT NULL CHECK(complete IN(0,1)));
CREATE TABLE backup_copy(singleton INTEGER PRIMARY KEY CHECK(singleton=1), after_lower TEXT, lower_key TEXT, source TEXT, target TEXT, records INTEGER, bytes INTEGER, fingerprint BLOB, cursor INTEGER NOT NULL DEFAULT 0, verifying INTEGER NOT NULL DEFAULT 0, verified_after TEXT);
INSERT INTO backup_copy(singleton) VALUES(1);
DELETE FROM receipt_migration;
DELETE FROM receipt_backup; DELETE FROM receipt_backup_filename;
COMMIT;").map_err(error)?;
routes::rebase_snapshot(&target, db)?;
target
.execute("INSERT INTO backup_manifest VALUES(1,?1,0)", [&job.id])
.map_err(error)?;
target
.execute_batch("PRAGMA wal_checkpoint(TRUNCATE)")
.map_err(error)?;
drop(target);
crate::durable::open_private_regular_file(&path)
.and_then(|f| f.sync_all())
.map_err(error)?;
sync_directory(&job.staging)?;
db.execute(
"UPDATE receipt_backup SET phase='copying' WHERE singleton=1",
[],
)
.map_err(error)?;
Ok(())
}
fn release(db: &mut Connection, job: &Backup) -> Result<bool> {
let target = root(&job.destination.join(&job.filename))?;
let marker: (String, i64) = target
.query_row(
"SELECT id,complete FROM backup_manifest WHERE singleton=1",
[],
|r| Ok((r.get(0)?, r.get(1)?)),
)
.map_err(error)?;
if marker != (job.id.clone(), 1) {
return Err(invalid("published backup identity differs"));
}
sync_directory(job.destination.parent().unwrap())?;
db.execute_batch(
"BEGIN IMMEDIATE; DELETE FROM receipt_backup; DELETE FROM receipt_backup_filename; COMMIT;",
)
.map_err(error)?;
Ok(true)
}
/// Returns true only after an independently openable snapshot is published.
/// The caller yields its operation fence after each <=128-record/4-MiB page.
pub(crate) fn backup_step(db: &mut Connection) -> Result<bool> {
let Some(job) = job(db)? else {
return Ok(true);
};
if job.destination.try_exists().map_err(error)? {
return release(db, &job);
}
if job.phase == "preparing" {
prepare(db, &job)?;
return Ok(false);
}
if job.phase != "copying" {
return Err(invalid("invalid backup phase"));
}
let path = job.staging.join(&job.filename);
let mut target_root = root(&path)?;
let marker: (String, i64) = target_root
.query_row(
"SELECT id,complete FROM backup_manifest WHERE singleton=1",
[],
|r| Ok((r.get(0)?, r.get(1)?)),
)
.map_err(error)?;
if marker.0 != job.id {
return Err(invalid("backup identity differs"));
}
if marker.1 == 1 {
target_root
.execute_batch("PRAGMA wal_checkpoint(TRUNCATE)")
.map_err(error)?;
drop(target_root);
crate::durable::open_private_regular_file(&path)
.and_then(|f| f.sync_all())
.map_err(error)?;
sync_directory(&job.staging)?;
publish_directory(&job.staging, &job.destination)?;
return release(db, &job);
}
let (verifying, verified_after): (bool, Option<String>) = target_root
.query_row(
"SELECT verifying,verified_after FROM backup_copy WHERE singleton=1",
[],
|r| Ok((r.get(0)?, r.get(1)?)),
)
.map_err(error)?;
if !verifying {
type CopyRow = (String, String, Option<String>, i64, i64, Vec<u8>, i64);
let active: bool = target_root
.query_row(
"SELECT lower_key IS NOT NULL FROM backup_copy WHERE singleton=1",
[],
|r| r.get(0),
)
.map_err(error)?;
if !active {
let after: Option<String> = target_root
.query_row(
"SELECT after_lower FROM backup_copy WHERE singleton=1",
[],
|r| r.get(0),
)
.map_err(error)?;
if let Some(row) = routes::next(&target_root, "routes", after.as_deref())? {
target_root.execute("UPDATE backup_copy SET lower_key=?1,source=?2,target=NULL,records=?3,bytes=?4,fingerprint=?5,cursor=0 WHERE singleton=1",
params![row.lower, row.file, row.records, row.bytes, row.fingerprint]).map_err(error)?;
}
}
let next: Option<CopyRow> = target_root.query_row("SELECT lower_key,source,target,records,bytes,CASE WHEN length(fingerprint)=32 THEN fingerprint END,cursor FROM backup_copy WHERE lower_key IS NOT NULL AND singleton=1", [], |r| Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?,r.get(5)?,r.get(6)?))).optional().map_err(error)?;
if let Some((lower, source, target, count, size, sha, cursor)) = next {
if count < 0 || cursor < 0 || cursor > count {
return Err(invalid("invalid backup cursor"));
}
let target = match target {
Some(file) => file,
None => {
let file = local_name(&path, &new_shard(&directory(&path)?)?)?;
target_root
.execute(
"UPDATE backup_copy SET target=?2 WHERE lower_key=?1 AND singleton=1",
params![lower, file],
)
.map_err(error)?;
file
}
};
// Resolve source using the live root, including relocated private volumes.
let source_db = open_shard(&resolve_file(db, &source)?, false)?;
let mut query=source_db.prepare("SELECT ordinal,namespace,receipt_key,CASE WHEN length(body)<=33554432 THEN body END,CASE WHEN length(digest)=32 THEN digest END FROM receipt_rows WHERE ordinal>?1 AND ordinal<=?2 ORDER BY ordinal LIMIT 128").map_err(error)?;
let mut rows = query.query(params![cursor, count]).map_err(error)?;
let mut batch = Vec::new();
let mut bytes = 0;
let mut after = cursor;
while let Some(row) = rows.next().map_err(error)? {
let ordinal: i64 = row.get(0).map_err(error)?;
let body: Vec<u8> = row.get(3).map_err(error)?;
let digest: Vec<u8> = row.get(4).map_err(error)?;
verify(&body, &digest)?;
if !batch.is_empty() && bytes + body.len() > PAGE_BYTES {
break;
}
if ordinal != after + 1 {
return Err(invalid("backup source ordinal gap"));
}
after = ordinal;
bytes += body.len();
batch.push(ExactReceipt {
namespace: row.get(1).map_err(error)?,
key: row.get(2).map_err(error)?,
bytes: body,
});
}
drop(rows);
drop(query);
drop(source_db);
if after == cursor && cursor < count {
return Err(invalid("backup source is incomplete"));
}
insert_batch(
&mut open_shard(&resolve_file(&target_root, &target)?, false)?,
&batch,
)?;
if after < count {
target_root
.execute(
"UPDATE backup_copy SET cursor=?2 WHERE lower_key=?1 AND singleton=1",
params![lower, after],
)
.map_err(error)?;
return Ok(false);
}
if totals(&target_root, &target)? != (count, size, sha.clone()) {
return Err(invalid("backup partition verification failed"));
}
let tx = target_root
.transaction_with_behavior(TransactionBehavior::Immediate)
.map_err(error)?;
routes::put(
&tx,
"routes",
routes::Route {
lower: lower.clone(),
file: target,
records: count,
bytes: size,
fingerprint: sha,
},
)?;
tx.execute(
"UPDATE backup_copy SET after_lower=?1,lower_key=NULL,source=NULL,target=NULL,records=NULL,bytes=NULL,fingerprint=NULL,cursor=0 WHERE singleton=1",
[&lower],
)
.map_err(error)?;
tx.commit().map_err(error)?;
return Ok(false);
}
target_root
.execute("UPDATE backup_copy SET verifying=1 WHERE singleton=1", [])
.map_err(error)?;
return Ok(false);
}
if let Some(row) = routes::next_local(&target_root, verified_after.as_deref())? {
target_root
.execute(
"UPDATE backup_copy SET verified_after=?1 WHERE singleton=1",
[row.lower],
)
.map_err(error)?;
return Ok(false);
}
target_root
.execute_batch(
"BEGIN IMMEDIATE;
UPDATE backup_manifest SET complete=1 WHERE singleton=1;
DROP TABLE backup_copy;
COMMIT; PRAGMA wal_checkpoint(TRUNCATE);",
)
.map_err(error)?;
drop(target_root);
crate::durable::open_private_regular_file(&path)
.and_then(|f| f.sync_all())
.map_err(error)?;
sync_directory(&job.staging)?;
publish_directory(&job.staging, &job.destination)?;
release(db, &job)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::indexed_store::IndexedStore;
#[test]
fn publication_does_not_replace_a_racing_empty_directory_or_symlink() {
let home = std::env::temp_dir().join(format!(
"paperclip-backup-publication-{}",
uuid::Uuid::new_v4()
));
fs::create_dir(&home).unwrap();
let source = home.join("prepared");
let destination = home.join("backup");
fs::create_dir(&source).unwrap();
fs::write(source.join("evidence"), b"verified snapshot").unwrap();
fs::create_dir(&destination).unwrap();
assert!(publish_directory(&source, &destination).is_err());
assert!(source.join("evidence").exists());
assert!(fs::read_dir(&destination).unwrap().next().is_none());
fs::remove_dir(&destination).unwrap();
#[cfg(unix)]
{
std::os::unix::fs::symlink(home.join("missing"), &destination).unwrap();
assert!(publish_directory(&source, &destination).is_err());
assert!(fs::symlink_metadata(&destination).unwrap().is_symlink());
fs::remove_file(&destination).unwrap();
}
publish_directory(&source, &destination).unwrap();
assert_eq!(
fs::read(destination.join("evidence")).unwrap(),
b"verified snapshot"
);
fs::remove_dir_all(home).unwrap();
}
fn receipt(i: u64) -> ExactReceipt {
ExactReceipt {
namespace: "effects".into(),
key: format!("effect-{i:020}"),
bytes: vec![(i % 251) as u8; 1024],
}
}
#[test]
fn snapshot_restores_exact_prefix_while_live_writes_and_splits_continue_across_restarts() {
let home = std::env::temp_dir().join(format!("paperclip-backup-{}", uuid::Uuid::new_v4()));
let path = home.join("source.sqlite");
let mut store = IndexedStore::open(&path, "backup-test", true).unwrap();
store
.commit(
"authority",
0,
b"snapshot-state".to_vec(),
(0..500).map(receipt).collect(),
)
.unwrap();
let db = Connection::open(&path).unwrap();
db.execute(
"UPDATE receipt_storage_config SET partition_bytes=32768",
[],
)
.unwrap();
for _ in 0..80 {
store.maintain().unwrap();
}
let destination = home.join("backup");
store.begin_backup(&destination).unwrap();
assert!(!store.backup_step().unwrap());
assert!(!destination.exists());
let staging: String = db
.query_row("SELECT staging FROM receipt_backup", [], |r| r.get(0))
.unwrap();
assert!(IndexedStore::open(
&Path::new(&staging).join("authority.sqlite"),
"backup-test",
false
)
.is_err());
let mut complete = false;
let mut generation = 1;
let mut verification_steps = 0;
for i in 0..200 {
store
.commit(
"authority",
generation,
b"live-state".to_vec(),
vec![receipt(500 + i)],
)
.unwrap();
generation += 1;
for _ in 0..4 {
store.maintain().unwrap();
}
if i % 2 == 0 {
drop(store);
store = IndexedStore::open(&path, "backup-test", false).unwrap();
store.begin_backup(&destination).unwrap();
}
let snapshot = root(&Path::new(&staging).join("authority.sqlite")).unwrap();
if snapshot
.query_row("SELECT verifying FROM backup_copy", [], |r| {
r.get::<_, bool>(0)
})
.unwrap()
{
verification_steps += 1;
}
drop(snapshot);
if store.backup_step().unwrap() {
complete = true;
break;
}
}
assert!(complete);
assert!(
verification_steps > 2,
"verification must yield across routing pages and restarts"
);
assert_eq!(
db.query_row("SELECT count(*) FROM receipt_backup", [], |r| r
.get::<_, i64>(0))
.unwrap(),
0
);
let restored =
IndexedStore::open(&destination.join("authority.sqlite"), "backup-test", false)
.unwrap();
let state = restored.read_state("authority").unwrap().unwrap();
assert_eq!(state.generation, 1);
assert_eq!(state.bytes, b"snapshot-state");
for i in [0, 127, 499] {
assert_eq!(
restored.receipt("effects", &receipt(i).key).unwrap(),
Some(receipt(i).bytes)
);
}
assert_eq!(
restored.receipt("effects", &receipt(500).key).unwrap(),
None
);
assert_eq!(
store.read_state("authority").unwrap().unwrap().generation,
generation
);
assert!(store
.receipt("effects", &receipt(500).key)
.unwrap()
.is_some());
drop(restored);
drop(store);
drop(db);
fs::remove_dir_all(home).unwrap();
}
#[test]
fn interrupted_preparation_can_restart_and_incomplete_source_cannot_publish() {
let home =
std::env::temp_dir().join(format!("paperclip-backup-fault-{}", uuid::Uuid::new_v4()));
let path = home.join("source.sqlite");
let store = IndexedStore::open(&path, "backup-test", true).unwrap();
store
.commit("authority", 0, b"state".to_vec(), vec![receipt(0)])
.unwrap();
let destination = home.join("backup");
store.begin_backup(&destination).unwrap();
drop(store);
let db = Connection::open(&path).unwrap();
let active = job(&db).unwrap().unwrap();
create_file(&active.staging.join("authority.sqlite")).unwrap();
let store = IndexedStore::open(&path, "backup-test", false).unwrap();
store.begin_backup(&destination).unwrap();
assert!(!store.backup_step().unwrap());
let (_, file) = route(&db, "effects\0effect").unwrap();
let shard = open_shard(&resolve_file(&db, &file).unwrap(), false).unwrap();
shard.execute("DELETE FROM receipt_rows", []).unwrap();
drop(shard);
assert!(store.backup_step().is_err());
assert!(!destination.exists());
assert_eq!(
db.query_row("SELECT count(*) FROM receipt_backup", [], |r| r
.get::<_, i64>(0))
.unwrap(),
1
);
drop(store);
drop(db);
fs::remove_dir_all(home).unwrap();
}
}
packages/paperclip-runner/runner/crates/runner-core/src/indexed_commit.rs
+198
View File
@@ -0,0 +1,198 @@
//! One retry identity per current row, never a lifetime list. It shares the
//! root transaction with current state and receipt redo. A caller retries the
//! same storage request, not the command that produced it.
use crate::indexed_revision::Revision;
use crate::{
durable::DurableRunnerError,
indexed_store::{storage_error, ExactReceipt},
indexed_work::WorkChange,
};
use rusqlite::{params, Connection, OptionalExtension};
type Result<T> = std::result::Result<T, DurableRunnerError>;
pub(crate) fn initialize(db: &Connection) -> Result<()> {
db.execute_batch("CREATE TABLE IF NOT EXISTS current_commit (state_key TEXT PRIMARY KEY, operation_id TEXT NOT NULL, generation INTEGER NOT NULL, digest BLOB NOT NULL, expected_generation TEXT) WITHOUT ROWID;").map_err(storage_error)?;
let has_predecessor: bool = db.query_row("SELECT EXISTS(SELECT 1 FROM pragma_table_info('current_commit') WHERE name='expected_generation')", [], |r| r.get(0)).map_err(storage_error)?;
if !has_predecessor {
db.execute_batch("ALTER TABLE current_commit ADD COLUMN expected_generation TEXT;")
.map_err(storage_error)?;
}
Ok(())
}
fn fingerprint(
key: &str,
generation: Revision,
predecessor: Option<Revision>,
bytes: &[u8],
receipts: &[ExactReceipt],
work: &[WorkChange],
) -> Vec<u8> {
use sha2::{Digest, Sha256};
let mut hash = Sha256::new();
let mut field = |bytes: &[u8]| {
hash.update((bytes.len() as u64).to_be_bytes());
hash.update(bytes);
};
field(if predecessor.is_some() {
b"paperclip.indexed-commit.v2"
} else {
b"paperclip.indexed-commit.v1"
});
field(key.as_bytes());
field(&generation.fingerprint());
if let Some(previous) = predecessor {
field(&previous.fingerprint());
}
field(bytes);
field(&(receipts.len() as u64).to_be_bytes());
for receipt in receipts {
field(receipt.namespace.as_bytes());
field(receipt.key.as_bytes());
field(&receipt.bytes);
}
field(&(work.len() as u64).to_be_bytes());
for change in work {
field(change.collection.as_bytes());
field(change.key.as_bytes());
for value in [&change.expected_digest, &change.bytes] {
field(&[u8::from(value.is_some())]);
if let Some(bytes) = value {
field(bytes);
}
}
}
hash.finalize().to_vec()
}
pub(crate) fn record(
db: &Connection,
key: &str,
operation: &str,
expected: Revision,
generation: Revision,
bytes: &[u8],
receipts: &[ExactReceipt],
work: &[WorkChange],
) -> Result<()> {
db.execute("INSERT INTO current_commit(state_key,operation_id,generation,digest,expected_generation) VALUES(?1,?2,?3,?4,?5) ON CONFLICT(state_key) DO UPDATE SET operation_id=excluded.operation_id,generation=excluded.generation,digest=excluded.digest,expected_generation=excluded.expected_generation", params![key,operation,generation,fingerprint(key,generation,Some(expected),bytes,receipts,work),expected.to_string()]).map_err(storage_error)?;
Ok(())
}
pub(crate) fn replay(
db: &Connection,
key: &str,
operation: &str,
expected: Revision,
bytes: &[u8],
receipts: &[ExactReceipt],
work: &[WorkChange],
) -> Result<Option<Revision>> {
let stamp: Option<(String, Revision, Vec<u8>, Option<String>)> = db.query_row("SELECT operation_id,generation,CASE WHEN length(digest)=32 THEN digest END,expected_generation FROM current_commit WHERE state_key=?1", [key], |r| Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?))).optional().map_err(storage_error)?;
let Some((id, generation, digest, predecessor)) = stamp else {
return Ok(None);
};
if id != operation {
return Ok(None);
}
let predecessor = predecessor
.map(|value| value.parse::<Revision>())
.transpose()?;
let follows = match predecessor {
Some(value) => value == expected && generation != expected,
None => {
matches!((expected.legacy(), generation.legacy()), (Some(old), Some(new)) if old.checked_add(1) == Some(new))
}
};
if !follows || digest != fingerprint(key, generation, predecessor, bytes, receipts, work) {
return Err(DurableRunnerError::invalid(
"indexed commit retry identity changed",
));
}
let current = crate::indexed_store::read_state(db, key)?;
if !current.is_some_and(|state| state.generation == generation && state.bytes == bytes) {
return Err(DurableRunnerError::invalid(
"indexed commit retry was superseded",
));
}
Ok(Some(generation))
}
#[cfg(test)]
mod tests {
use super::*;
use sha2::{Digest, Sha256};
#[test]
fn retained_retry_stamp_survives_schema_upgrade() {
let db = Connection::open_in_memory().unwrap();
db.execute_batch("CREATE TABLE current_commit(state_key TEXT PRIMARY KEY,operation_id TEXT,generation INTEGER,digest BLOB); CREATE TABLE current_state(key TEXT PRIMARY KEY,generation INTEGER,bytes BLOB,digest BLOB);").unwrap();
db.execute(
"INSERT INTO current_state VALUES('state',2,?1,?2)",
params![b"accepted", Sha256::digest(b"accepted").as_slice()],
)
.unwrap();
db.execute(
"INSERT INTO current_commit VALUES('state','old-operation',2,?1)",
[fingerprint("state", 2.into(), None, b"accepted", &[], &[])],
)
.unwrap();
initialize(&db).unwrap();
initialize(&db).unwrap();
assert_eq!(
replay(
&db,
"state",
"old-operation",
1.into(),
b"accepted",
&[],
&[]
)
.unwrap(),
Some(2.into())
);
assert!(replay(
&db,
"state",
"old-operation",
0.into(),
b"accepted",
&[],
&[]
)
.is_err());
}
#[test]
fn opaque_retry_stamp_binds_the_predecessor_and_rejects_its_corruption() {
let db = Connection::open_in_memory().unwrap();
initialize(&db).unwrap();
db.execute_batch("CREATE TABLE current_state(key TEXT PRIMARY KEY,generation INTEGER,bytes BLOB,digest BLOB);").unwrap();
let previous = Revision::from(i64::MAX as u64);
let next = previous.next().unwrap();
db.execute(
"INSERT INTO current_state VALUES('state',?1,?2,?3)",
params![next, b"accepted", Sha256::digest(b"accepted").as_slice()],
)
.unwrap();
record(
&db,
"state",
"operation",
previous,
next,
b"accepted",
&[],
&[],
)
.unwrap();
assert_eq!(
replay(&db, "state", "operation", previous, b"accepted", &[], &[]).unwrap(),
Some(next)
);
db.execute("UPDATE current_commit SET expected_generation='1'", [])
.unwrap();
assert!(replay(&db, "state", "operation", 1.into(), b"accepted", &[], &[]).is_err());
}
}
packages/paperclip-runner/runner/crates/runner-core/src/indexed_inspection.rs
+295
View File
@@ -0,0 +1,295 @@
//! Read-only current-state inspection for remote recovery. No receipt/history
//! enumeration, owner recovery, SQLite checkpoint, or writable connection.
use crate::indexed_revision::Revision;
use crate::{
durable::{open_private_regular_file, verify_private_directory, DurableRunnerError},
indexed_store::{read_state, storage_error},
};
use base64::{engine::general_purpose::STANDARD, Engine};
use rusqlite::{Connection, OpenFlags, OptionalExtension};
use serde_json::{json, Value};
use sha2::{Digest, Sha256};
use std::{
io::{Read, Write},
path::{Path, PathBuf},
};
type Result<T> = std::result::Result<T, DurableRunnerError>;
fn invalid(message: &str) -> DurableRunnerError {
DurableRunnerError::invalid(message)
}
pub const INSPECTION_CHUNK_BYTES: usize = 192 * 1024;
pub fn inspect(path: &Path) -> Result<Value> {
let parent = path
.parent()
.ok_or_else(|| invalid("indexed inspection has no parent"))?;
verify_private_directory(parent)?;
let key = match path.file_name().and_then(|name| name.to_str()) {
Some("runner-state.json") => "runner",
Some("codex-provider-state.json") => "codex-provider",
_ => return Err(invalid("unsupported indexed inspection locator")),
};
let mut bytes = Vec::new();
open_private_regular_file(path)
.map_err(storage_error)?
.take(4097)
.read_to_end(&mut bytes)
.map_err(storage_error)?;
if bytes.len() > 4096 {
return Err(invalid("indexed inspection locator exceeds capacity"));
}
let locator: Value = serde_json::from_slice(&bytes).map_err(storage_error)?;
let schema = if key == "runner" {
"paperclip.runner.durable.state.indexed.v1"
} else {
"paperclip.runner.codex-provider-state.indexed.v1"
};
let binding = locator["binding"]
.as_str()
.filter(|s| !s.is_empty() && s.len() <= 1024 && !s.chars().any(char::is_control))
.ok_or_else(|| invalid("invalid indexed inspection binding"))?;
if locator["schema"] != schema {
return Err(invalid("invalid indexed inspection schema"));
}
let database = parent
.canonicalize()
.map_err(storage_error)?
.join(path.with_extension("sqlite").file_name().unwrap());
let _lifetime = crate::indexed_lifetime::StoreLifetime::existing_shared(&database)?;
for suffix in ["", "-wal", "-shm"] {
match crate::indexed_store::verify_sqlite_file(&PathBuf::from(format!(
"{}{suffix}",
database.display()
))) {
Ok(_) => {}
Err(e) if !suffix.is_empty() && e.kind() == std::io::ErrorKind::NotFound => {}
Err(e) => return Err(storage_error(e)),
}
}
let db = Connection::open_with_flags(
&database,
OpenFlags::SQLITE_OPEN_READ_ONLY
| OpenFlags::SQLITE_OPEN_NOFOLLOW
| OpenFlags::SQLITE_OPEN_NO_MUTEX,
)
.map_err(storage_error)?;
db.busy_timeout(std::time::Duration::from_secs(5))
.map_err(storage_error)?;
db.execute_batch(
"PRAGMA query_only=ON; PRAGMA trusted_schema=OFF; PRAGMA cache_size=-4096; BEGIN;",
)
.map_err(storage_error)?;
let stored: (i64, String) = db
.query_row(
"SELECT schema_version,binding FROM store_binding WHERE singleton=1",
[],
|r| Ok((r.get(0)?, r.get(1)?)),
)
.map_err(storage_error)?;
if ![1, 2].contains(&stored.0) || stored.1 != binding {
return Err(invalid("indexed inspection binding mismatch"));
}
crate::indexed_partitions::validate_backup(&db)?;
let preparation = match read_state(&db, "legacy-import")? {
None => Value::Null,
Some(imported) => {
if imported.bytes.len() > 16384 {
return Err(invalid("indexed inspection import exceeds capacity"));
}
let progress: Value = serde_json::from_slice(&imported.bytes).map_err(storage_error)?;
if progress["schema"] != "paperclip.runner.local-import.v1"
|| progress["binding"] != binding
|| !progress["prepared"].is_object()
{
return Err(invalid("indexed inspection import is incomplete"));
}
progress["prepared"].clone()
}
};
let snapshot = read_state(&db, key)?
.ok_or_else(|| invalid("indexed inspection current state is missing"))?;
if snapshot.generation == 0 {
return Err(invalid("indexed inspection generation is invalid"));
}
let digest = Sha256::digest(&snapshot.bytes);
if stored.0 == 2 {
let pending: Option<(Revision, Vec<u8>, Vec<u8>)> = db.query_row(
"SELECT generation,CASE WHEN length(body)<=33554432 THEN body END,CASE WHEN length(digest)=32 THEN digest END FROM receipt_prepare WHERE state_key=?1",
[key], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?))).optional().map_err(storage_error)?;
if pending.is_some_and(|(generation, body, hash)| {
generation != snapshot.generation || body != snapshot.bytes || hash != digest.as_slice()
}) {
return Err(invalid(
"indexed inspection has conflicting prepared receipts",
));
}
}
let mut state: Value = serde_json::from_slice(&snapshot.bytes).map_err(storage_error)?;
if !state.is_object() {
return Err(invalid("indexed inspection current state is invalid"));
}
if key == "runner" {
let next = state["nextSourceSeq"]
.as_u64()
.filter(|n| *n <= 9_007_199_254_740_991);
let acked = state["ackedSourceSeq"]
.as_u64()
.filter(|n| *n <= 9_007_199_254_740_991);
let (Some(next), Some(acked)) = (next, acked) else {
return Err(invalid("indexed inspection cursor is invalid"));
};
if next <= acked {
return Err(invalid("indexed inspection cursor is invalid"));
}
// A proof must preserve pending delivery as a negative fence, without
// restoring or reading the historical outbox bodies.
state["outbox"] = if next == acked + 1 {
json!([])
} else {
json!([{ "indexedPending": true }])
};
}
Ok(
json!({ "state": state, "generation": snapshot.generation.to_string(), "stateDigest": format!("{digest:x}"), "preparation": preparation }),
)
}
/// Emit one consistent snapshot in bounded wire frames. EOF, a missing footer,
/// or a changed digest cannot be interpreted as an empty/settled authority.
pub fn stream(path: &Path, mut output: impl Write) -> Result<()> {
let bytes = serde_json::to_vec(&inspect(path)?).map_err(storage_error)?;
let header = json!({ "schema": "paperclip.indexed-inspection.v1", "byteLength": bytes.len(), "sha256": format!("{:x}", Sha256::digest(&bytes)) });
serde_json::to_writer(&mut output, &header).map_err(storage_error)?;
output.write_all(b"\n").map_err(storage_error)?;
for (index, chunk) in bytes.chunks(INSPECTION_CHUNK_BYTES).enumerate() {
serde_json::to_writer(
&mut output,
&json!({ "index": index, "bytes": STANDARD.encode(chunk) }),
)
.map_err(storage_error)?;
output.write_all(b"\n").map_err(storage_error)?;
}
output
.write_all(b"{\"complete\":true}\n")
.and_then(|_| output.flush())
.map_err(storage_error)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::indexed_store::{ExactReceipt, IndexedStore};
struct Fixture {
root: PathBuf,
path: PathBuf,
store: IndexedStore,
}
impl Fixture {
fn new(next: u64) -> Self {
let root =
std::env::temp_dir().join(format!("paperclip-inspection-{}", uuid::Uuid::new_v4()));
let store =
IndexedStore::open(&root.join("runner-state.sqlite"), "inspection", true).unwrap();
let path = root.join("runner-state.json");
let mut file = std::fs::OpenOptions::new();
file.write(true).create_new(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
file.mode(0o600);
}
file.open(&path).unwrap().write_all(br#"{"schema":"paperclip.runner.durable.state.indexed.v1","binding":"inspection"}"#).unwrap();
store.commit("runner", 0, serde_json::to_vec(&json!({"nextSourceSeq":next,"ackedSourceSeq":1,"lifecycle":"ready","current":"exact"})).unwrap(), vec![ExactReceipt {
namespace: "old-output".into(), key: "ancient".into(), bytes: vec![b'x'; 2 * 1024 * 1024],
}]).unwrap();
Self { root, path, store }
}
}
impl Drop for Fixture {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.root);
}
}
#[test]
fn current_inspection_preserves_pending_delivery_and_does_not_restore_history() {
for (next, outbox) in [(2, json!([])), (3, json!([{"indexedPending":true}]))] {
let f = Fixture::new(next);
let before = f.store.read_state("runner").unwrap().unwrap();
let observed = inspect(&f.path).unwrap();
assert_eq!(observed["state"]["outbox"], outbox);
assert_eq!(observed["generation"], "1");
assert!(serde_json::to_vec(&observed).unwrap().len() < 1000);
assert_eq!(f.store.read_state("runner").unwrap().unwrap(), before);
assert_eq!(
f.store
.receipt("old-output", "ancient")
.unwrap()
.unwrap()
.len(),
2 * 1024 * 1024
);
}
}
#[test]
fn inspection_rejects_conflicting_redo_without_recovering_it() {
let f = Fixture::new(2);
let db = Connection::open(f.root.join("runner-state.sqlite")).unwrap();
// Copy the actual schema's redo columns, changing only the current
// generation. No side-effect/receipt repair is authorized by inspection.
db.execute("INSERT INTO receipt_prepare(singleton,state_key,generation,body,digest,records,fingerprint) SELECT 1,key,generation+1,bytes,digest,0,zeroblob(32) FROM current_state WHERE key='runner'", []).unwrap();
assert!(inspect(&f.path)
.unwrap_err()
.to_string()
.contains("conflicting prepared"));
assert_eq!(
db.query_row(
"SELECT generation FROM current_state WHERE key='runner'",
[],
|r| r.get::<_, i64>(0)
)
.unwrap(),
1
);
}
#[test]
fn missing_activated_state_is_not_an_empty_snapshot() {
let f = Fixture::new(2);
let db = Connection::open(f.root.join("runner-state.sqlite")).unwrap();
db.execute("DELETE FROM current_state WHERE key='runner'", [])
.unwrap();
assert!(inspect(&f.path)
.unwrap_err()
.to_string()
.contains("current state is missing"));
}
#[test]
fn streams_one_snapshot_in_bounded_ordered_digest_checked_frames() {
let f = Fixture::new(2);
f.store.commit("runner", 1, serde_json::to_vec(&json!({"nextSourceSeq":2,"ackedSourceSeq":1,"activeInput":"界".repeat(500_000)})).unwrap(), vec![]).unwrap();
let mut wire = Vec::new();
stream(&f.path, &mut wire).unwrap();
let rows: Vec<_> = wire
.split(|b| *b == b'\n')
.filter(|line| !line.is_empty())
.collect();
let header: Value = serde_json::from_slice(rows[0]).unwrap();
let mut snapshot = Vec::new();
for (index, row) in rows[1..rows.len() - 1].iter().enumerate() {
assert!(row.len() <= INSPECTION_CHUNK_BYTES * 4 / 3 + 128);
let chunk: Value = serde_json::from_slice(row).unwrap();
assert_eq!(chunk["index"], index);
snapshot.extend(STANDARD.decode(chunk["bytes"].as_str().unwrap()).unwrap());
}
assert_eq!(header["byteLength"], snapshot.len());
assert_eq!(header["sha256"], format!("{:x}", Sha256::digest(&snapshot)));
assert_eq!(
serde_json::from_slice::<Value>(rows.last().unwrap()).unwrap(),
json!({"complete":true})
);
assert_eq!(
serde_json::from_slice::<Value>(&snapshot).unwrap()["generation"],
"2"
);
}
}
packages/paperclip-runner/runner/crates/runner-core/src/indexed_lifetime.rs
+97
View File
@@ -0,0 +1,97 @@
//! A connection owns a shared lifetime fence until SQLite has closed. Moving a
//! store requires the exclusive fence; per-operation locks alone are too short.
use crate::durable::{open_private_regular_file, verify_private_directory, DurableRunnerError};
use crate::indexed_store::storage_error;
use std::{
fs,
path::{Path, PathBuf},
};
type Result<T> = std::result::Result<T, DurableRunnerError>;
pub(crate) struct StoreLifetime(fs::File);
pub(crate) fn archive_fence(path: &Path) -> PathBuf {
let mut name = path.as_os_str().to_os_string();
name.push(".archive");
PathBuf::from(name)
}
impl StoreLifetime {
pub(crate) fn existing_exclusive(path: &Path) -> Result<Self> {
let file =
open_private_regular_file(&PathBuf::from(format!("{}.lifetime", path.display())))
.map_err(storage_error)?;
file.try_lock().map_err(|_| {
DurableRunnerError::invalid(
"storage_pressure: indexed store still has an open connection",
)
})?;
Ok(Self(file))
}
pub(crate) fn existing_shared(path: &Path) -> Result<Self> {
let file =
open_private_regular_file(&PathBuf::from(format!("{}.lifetime", path.display())))
.map_err(storage_error)?;
file.try_lock_shared().map_err(|_| {
DurableRunnerError::invalid("storage_pressure: indexed store is being archived")
})?;
let lock = Self(file);
match fs::symlink_metadata(archive_fence(path)) {
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(lock),
_ => Err(DurableRunnerError::invalid(
"indexed archive must finish before reading receipts",
)),
}
}
fn file(path: &Path) -> Result<fs::File> {
verify_private_directory(
path.parent()
.ok_or_else(|| DurableRunnerError::invalid("store has no parent"))?,
)?;
let mut name = path.as_os_str().to_os_string();
name.push(".lifetime");
let path = PathBuf::from(name);
let mut options = fs::OpenOptions::new();
options.write(true).create_new(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
options.mode(0o600);
}
match options.open(&path) {
Ok(file) => file.sync_all().map_err(storage_error)?,
Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {}
Err(e) => return Err(storage_error(e)),
}
open_private_regular_file(&path).map_err(storage_error)
}
pub(crate) fn shared(path: &Path) -> Result<Self> {
let file = Self::file(path)?;
file.try_lock_shared().map_err(|_| {
DurableRunnerError::invalid("storage_pressure: indexed store is being archived")
})?;
let lock = Self(file);
match fs::symlink_metadata(archive_fence(path)) {
Ok(_) => {
return Err(DurableRunnerError::invalid(
"storage_pressure: indexed archive must finish before opening the store",
))
}
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
Err(e) => return Err(storage_error(e)),
}
Ok(lock)
}
pub(crate) fn exclusive(path: &Path) -> Result<Self> {
let file = Self::file(path)?;
file.try_lock().map_err(|_| {
DurableRunnerError::invalid(
"storage_pressure: indexed store still has an open connection",
)
})?;
Ok(Self(file))
}
}
impl Drop for StoreLifetime {
fn drop(&mut self) {
let _ = self.0.unlock();
}
}
packages/paperclip-runner/runner/crates/runner-core/src/indexed_partitions.rs
+1784
View File
File diff suppressed because it is too large. Load diff
packages/paperclip-runner/runner/crates/runner-core/src/indexed_revision.rs
+164
View File
@@ -0,0 +1,164 @@
//! Exact CAS identities. Retained integer revisions keep their representation;
//! at the old boundary they switch permanently to opaque commit identities.
//! Revision order is never used to infer event or receipt order.
use crate::durable::DurableRunnerError;
use rusqlite::types::{FromSql, FromSqlError, FromSqlResult, ToSqlOutput, Value, ValueRef};
use rusqlite::ToSql;
use std::{fmt, str::FromStr};
use uuid::Uuid;
#[derive(Clone, Copy, Debug, PartialEq, Eq, Default)]
pub enum Revision {
#[default]
Absent,
Legacy(u64),
Opaque(Uuid),
/// A writer with an indeterminate outcome cannot re-admit writes.
Fenced,
}
impl Revision {
pub fn next(self) -> Result<Self, DurableRunnerError> {
match self {
Self::Absent => Ok(Self::Legacy(1)),
Self::Legacy(n) if n > 0 && n < i64::MAX as u64 => Ok(Self::Legacy(n + 1)),
Self::Legacy(n) if n == i64::MAX as u64 => Ok(Self::Opaque(Uuid::new_v4())),
Self::Opaque(previous) => {
let next = Uuid::new_v4();
if next == previous {
return Err(DurableRunnerError::invalid(
"fresh indexed revision unavailable",
));
}
Ok(Self::Opaque(next))
}
_ => Err(DurableRunnerError::invalid("indexed writer is fenced")),
}
}
pub(crate) fn legacy(self) -> Option<u64> {
match self {
Self::Absent => Some(0),
Self::Legacy(n) => Some(n),
_ => None,
}
}
pub(crate) fn fingerprint(self) -> Vec<u8> {
// Preserve v1 retry fingerprints for retained integer commits.
match self.legacy() {
Some(n) => n.to_be_bytes().to_vec(),
None => self.to_string().into_bytes(),
}
}
}
impl From<u64> for Revision {
fn from(value: u64) -> Self {
match value {
0 => Self::Absent,
n if n <= i64::MAX as u64 => Self::Legacy(n),
_ => Self::Fenced,
}
}
}
impl PartialEq<u64> for Revision {
fn eq(&self, other: &u64) -> bool {
self.legacy() == Some(*other)
}
}
impl fmt::Display for Revision {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Absent => f.write_str("0"),
Self::Legacy(n) => write!(f, "{n}"),
Self::Opaque(id) => write!(f, "r:{id}"),
Self::Fenced => f.write_str("fenced"),
}
}
}
impl FromStr for Revision {
type Err = DurableRunnerError;
fn from_str(value: &str) -> Result<Self, Self::Err> {
if let Some(id) = value
.strip_prefix("r:")
.and_then(|s| Uuid::parse_str(s).ok())
{
if id.get_version_num() == 4
&& id.get_variant() == uuid::Variant::RFC4122
&& value == format!("r:{id}")
{
return Ok(Self::Opaque(id));
}
} else if let Ok(n) = value.parse::<u64>() {
if n <= i64::MAX as u64 && n.to_string() == value {
return Ok(n.into());
}
}
Err(DurableRunnerError::invalid("invalid indexed revision"))
}
}
impl FromSql for Revision {
fn column_result(value: ValueRef<'_>) -> FromSqlResult<Self> {
match value {
ValueRef::Integer(n) if n > 0 => Ok(Self::Legacy(n as u64)),
ValueRef::Text(bytes) => {
match std::str::from_utf8(bytes).ok().and_then(|s| s.parse().ok()) {
Some(revision @ Self::Opaque(_)) => Ok(revision),
_ => Err(FromSqlError::InvalidType),
}
}
_ => Err(FromSqlError::InvalidType),
}
}
}
impl ToSql for Revision {
fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
match self {
Self::Legacy(n) if *n > 0 && *n <= i64::MAX as u64 => {
Ok(ToSqlOutput::Owned(Value::Integer(*n as i64)))
}
Self::Opaque(_) => Ok(ToSqlOutput::Owned(Value::Text(self.to_string()))),
_ => Err(rusqlite::Error::InvalidParameterName(
"invalid persisted revision".into(),
)),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn transitions_at_exact_boundary_and_never_wraps() {
let last = Revision::from(i64::MAX as u64);
let first = last.next().unwrap();
let second = first.next().unwrap();
assert!(matches!(first, Revision::Opaque(_)));
assert_ne!(first, second);
for revision in [
Revision::Absent,
Revision::from(9_007_199_254_740_993),
last,
first,
second,
] {
assert_eq!(revision.to_string().parse::<Revision>().unwrap(), revision);
}
assert!(Revision::Fenced.next().is_err());
}
#[test]
fn rejects_ambiguous_or_invalid_encodings() {
for value in [
"01",
"-1",
"9223372036854775808",
"r:00000000-0000-0000-0000-000000000001",
"fenced",
] {
assert!(value.parse::<Revision>().is_err(), "{value}");
}
assert!(Revision::column_result(ValueRef::Integer(0)).is_err());
assert!(Revision::column_result(ValueRef::Text(b"1")).is_err());
}
}
packages/paperclip-runner/runner/crates/runner-core/src/indexed_routes.rs
+924
View File
@@ -0,0 +1,924 @@
//! Immutable, bounded B+tree pages. SQLite publishes only a root reference in
//! the same transaction as current authority. New pages are synced first, so a
//! rollback can leave unreachable pages but never a partially published tree.
//! Old pages stay immutable while an online snapshot holds their root.
use super::*;
use serde::{Deserialize, Serialize};
use std::io::{Read, Write};
const FANOUT: usize = 32;
// Includes worst-case JSON escaping of every admitted key/path in 32 rows.
const MAX_PAGE_BYTES: usize = 2 * 1024 * 1024;
#[path = "indexed_routes_gc.rs"]
mod gc;
pub(crate) use gc::RoutingCollector;
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
pub(super) struct Route {
pub lower: String,
pub file: String,
pub records: i64,
pub bytes: i64,
pub fingerprint: Vec<u8>,
}
impl Route {
pub(super) fn empty(lower: String, file: String) -> Self {
Self {
lower,
file,
records: 0,
bytes: 0,
fingerprint: vec![0; 32],
}
}
fn splittable_bytes(&self) -> i64 {
if self.records > 1 {
self.bytes
} else {
0
}
}
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
struct PageRef {
sha: String,
/// The current root fits in one bounded SQLite value. Updating the root
/// alone must not create/fsync a new filesystem object on every receipt.
#[serde(default, skip_serializing_if = "Option::is_none")]
inline: Option<String>,
// Only an incomplete backup may reference another store. A finished
// backup verifies every reachable page is local before publication.
#[serde(default, skip_serializing_if = "Option::is_none")]
origin: Option<String>,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
struct Edge {
lower: String,
largest: i64,
page: PageRef,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(tag = "kind")]
enum Node {
Leaf { rows: Vec<Route> },
Branch { level: u16, children: Vec<Edge> },
}
impl Node {
fn level(&self) -> u16 {
match self {
Self::Leaf { .. } => 0,
Self::Branch { level, .. } => *level,
}
}
fn edge(&self, page: PageRef) -> Edge {
match self {
Self::Leaf { rows } => Edge {
lower: rows[0].lower.clone(),
largest: rows.iter().map(Route::splittable_bytes).max().unwrap(),
page,
},
Self::Branch { children, .. } => Edge {
lower: children[0].lower.clone(),
largest: children.iter().map(|r| r.largest).max().unwrap(),
page,
},
}
}
fn validate(&self) -> Result<()> {
let keys: Vec<&str> = match self {
Self::Leaf { rows } => {
if rows.iter().any(|r| {
r.lower.len() > 4096
|| r.file.len() > 4096
|| r.records < 0
|| r.bytes < 0
|| r.fingerprint.len() != 32
}) {
return Err(invalid("invalid receipt routing row"));
}
rows.iter().map(|r| r.lower.as_str()).collect()
}
Self::Branch { level, children } => {
if *level == 0
|| children
.iter()
.any(|r| r.largest < 0 || !valid_ref(&r.page) || r.page.inline.is_some())
{
return Err(invalid("invalid receipt routing branch"));
}
children.iter().map(|r| r.lower.as_str()).collect()
}
};
if keys.is_empty()
|| keys.len() > FANOUT
|| keys.iter().any(|k| k.len() > 4096)
|| keys.windows(2).any(|p| p[0] >= p[1])
{
return Err(invalid("invalid receipt routing order/fanout"));
}
Ok(())
}
}
fn valid_ref(page: &PageRef) -> bool {
page.sha.len() == 64
&& page
.sha
.bytes()
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
&& page
.inline
.as_ref()
.is_none_or(|value| value.len() <= MAX_PAGE_BYTES)
&& page
.origin
.as_ref()
.is_none_or(|p| p.len() <= 4096 && Path::new(p).is_absolute())
}
fn root_directory(db: &Connection) -> Result<PathBuf> {
Ok(PathBuf::from(format!(
"{}.routing",
db.path()
.ok_or_else(|| invalid("routing root has no path"))?
)))
}
fn private_directory(path: &Path) -> Result<()> {
let mut builder = fs::DirBuilder::new();
#[cfg(unix)]
{
use std::os::unix::fs::DirBuilderExt;
builder.mode(0o700);
}
match builder.create(path) {
Ok(()) => fs::File::open(path.parent().unwrap())
.and_then(|f| f.sync_all())
.map_err(error)?,
Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {}
Err(e) => return Err(error(e)),
}
crate::durable::verify_private_directory(path)
}
fn page_path(db: &Connection, page: &PageRef, create: bool) -> Result<PathBuf> {
if !valid_ref(page) {
return Err(invalid("invalid receipt routing reference"));
}
let root = page
.origin
.as_ref()
.map(PathBuf::from)
.unwrap_or(root_directory(db)?);
let directory = root.join(&page.sha[..2]).join(&page.sha[2..4]);
for p in [&root, &root.join(&page.sha[..2]), &directory] {
if create {
private_directory(p)?;
} else {
crate::durable::verify_private_directory(p)?;
}
}
Ok(directory.join(format!("{}.json", &page.sha[4..])))
}
fn read_node(db: &Connection, page: &PageRef) -> Result<Node> {
#[cfg(test)]
READ_PAGES.with(|reads| reads.set(reads.get() + 1));
if !valid_ref(page) {
return Err(invalid("invalid receipt routing reference"));
}
let bytes = if let Some(value) = &page.inline {
value.as_bytes().to_vec()
} else {
let path = page_path(db, page, false)?;
let file = crate::durable::open_private_regular_file(&path).map_err(error)?;
let mut bytes = Vec::new();
file.take((MAX_PAGE_BYTES + 1) as u64)
.read_to_end(&mut bytes)
.map_err(error)?;
bytes
};
if bytes.len() > MAX_PAGE_BYTES || format!("{:x}", Sha256::digest(&bytes)) != page.sha {
return Err(invalid("receipt routing page digest/size mismatch"));
}
let mut node: Node = serde_json::from_slice(&bytes).map_err(error)?;
node.validate()?;
if let Node::Branch { children, .. } = &mut node {
for child in children {
if child.page.origin.is_none() {
child.page.origin = page.origin.clone();
}
}
}
Ok(node)
}
fn write_node(db: &Connection, node: &Node) -> Result<Edge> {
node.validate()?;
let bytes = serde_json::to_vec(node).map_err(error)?;
if bytes.len() > MAX_PAGE_BYTES {
return Err(invalid("receipt routing page exceeds bound"));
}
let page = PageRef {
sha: format!("{:x}", Sha256::digest(&bytes)),
inline: None,
origin: None,
};
let path = page_path(db, &page, true)?;
// A deterministic file name makes interrupted/repeated staging idempotent.
// Write through a fresh temporary inode and publish without replacing a
// pre-existing page. A conflicting existing file is an integrity failure.
let temporary = path.with_extension(format!("{}.tmp", uuid::Uuid::new_v4()));
let mut options = fs::OpenOptions::new();
options.create_new(true).write(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
options.mode(0o600);
}
let mut file = options.open(&temporary).map_err(error)?;
file.write_all(&bytes)
.and_then(|_| file.sync_all())
.map_err(error)?;
drop(file);
match fs::hard_link(&temporary, &path) {
Ok(()) => {}
Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
read_node(db, &page)?;
}
Err(e) => return Err(error(e)),
}
fs::remove_file(&temporary).map_err(error)?;
fs::File::open(path.parent().unwrap())
.and_then(|f| f.sync_all())
.map_err(error)?;
Ok(node.edge(page))
}
fn inline_node(db: &Connection, node: &Node) -> Result<Edge> {
node.validate()?;
let value = serde_json::to_string(node).map_err(error)?;
if value.len() > MAX_PAGE_BYTES {
return Err(invalid("receipt routing root exceeds bound"));
}
// Keep an owned directory even before the first root subdivision; archive
// and backup publication always carry the same complete store shape.
private_directory(&root_directory(db)?)?;
Ok(node.edge(PageRef {
sha: format!("{:x}", Sha256::digest(value.as_bytes())),
inline: Some(value),
origin: None,
}))
}
pub(super) fn initialize(db: &Connection) -> Result<()> {
db.execute_batch("CREATE TABLE IF NOT EXISTS receipt_routing_heads(name TEXT PRIMARY KEY, root TEXT NOT NULL) WITHOUT ROWID;").map_err(error)
}
fn head(db: &Connection, name: &str) -> Result<Option<Edge>> {
let value: Option<String> = db.query_row("SELECT CASE WHEN length(root)<=8388608 THEN root END FROM receipt_routing_heads WHERE name=?1", [name], |r| r.get(0)).optional().map_err(error)?;
value
.map(|v| serde_json::from_str::<Edge>(&v).map_err(error))
.transpose()
}
fn publish(db: &Connection, name: &str, edge: Option<Edge>) -> Result<()> {
if let Some(edge) = edge {
db.execute("INSERT INTO receipt_routing_heads VALUES(?1,?2) ON CONFLICT(name) DO UPDATE SET root=excluded.root", params![name, serde_json::to_string(&edge).map_err(error)?]).map_err(error)?;
} else {
db.execute("DELETE FROM receipt_routing_heads WHERE name=?1", [name])
.map_err(error)?;
}
Ok(())
}
fn checked_child(db: &Connection, edge: &Edge, level: Option<u16>) -> Result<Node> {
let node = read_node(db, &edge.page)?;
let actual = node.edge(edge.page.clone());
if actual.lower != edge.lower
|| actual.largest != edge.largest
|| level.is_some_and(|l| l != node.level())
{
return Err(invalid("receipt routing child differs from parent"));
}
Ok(node)
}
pub(super) fn floor(db: &Connection, name: &str, key: &str) -> Result<Option<Route>> {
let Some(mut edge) = head(db, name)? else {
return Ok(None);
};
let mut level = None;
loop {
match checked_child(db, &edge, level)? {
Node::Leaf { rows } => {
return Ok(rows.into_iter().rev().find(|r| r.lower.as_str() <= key))
}
Node::Branch { level: l, children } => {
let Some(next) = children.into_iter().rev().find(|r| r.lower.as_str() <= key)
else {
return Ok(None);
};
edge = next;
level = Some(l - 1);
}
}
}
}
pub(super) fn exact(db: &Connection, name: &str, key: &str) -> Result<Option<Route>> {
Ok(floor(db, name, key)?.filter(|r| r.lower == key))
}
pub(super) fn next(db: &Connection, name: &str, after: Option<&str>) -> Result<Option<Route>> {
next_page(db, name, after, false)
}
/// Verify only the next route's path. Background publication persists its
/// cursor between calls instead of holding the live source lock for a tree walk.
pub(super) fn next_local(db: &Connection, after: Option<&str>) -> Result<Option<Route>> {
next_page(db, "routes", after, true)
}
fn next_page(
db: &Connection,
name: &str,
after: Option<&str>,
local: bool,
) -> Result<Option<Route>> {
fn search(
db: &Connection,
edge: &Edge,
level: Option<u16>,
after: Option<&str>,
local: bool,
) -> Result<Option<Route>> {
if local && edge.page.origin.is_some() {
return Err(invalid("snapshot references external routing pages"));
}
match checked_child(db, edge, level)? {
Node::Leaf { rows } => Ok(rows
.into_iter()
.find(|r| after.is_none_or(|a| r.lower.as_str() > a))),
Node::Branch { level, children } => {
let start = after
.map(|a| {
children
.partition_point(|r| r.lower.as_str() <= a)
.saturating_sub(1)
})
.unwrap_or(0);
for child in &children[start..] {
if let Some(row) = search(db, child, Some(level - 1), after, local)? {
return Ok(Some(row));
}
}
Ok(None)
}
}
}
match head(db, name)? {
Some(edge) => search(db, &edge, None, after, local),
None if local => Err(invalid("snapshot has no routing head")),
None => Ok(None),
}
}
pub(super) fn oversized(db: &Connection, threshold: i64) -> Result<Option<Route>> {
let Some(mut edge) = head(db, "routes")? else {
return Ok(None);
};
let mut level = None;
loop {
let node = checked_child(db, &edge, level)?;
if edge.largest <= threshold {
return Ok(None);
}
match node {
Node::Leaf { rows } => {
return Ok(rows.into_iter().find(|r| r.splittable_bytes() > threshold))
}
Node::Branch { level: l, children } => {
edge = children
.into_iter()
.find(|r| r.largest > threshold)
.ok_or_else(|| invalid("missing oversized route"))?;
level = Some(l - 1);
}
}
}
}
fn save_nodes(db: &Connection, node: Node, inline: bool) -> Result<Vec<Edge>> {
match node {
Node::Leaf { mut rows } if rows.len() > FANOUT => {
let right = rows.split_off(rows.len() / 2);
Ok(vec![
write_node(db, &Node::Leaf { rows })?,
write_node(db, &Node::Leaf { rows: right })?,
])
}
Node::Branch {
level,
mut children,
} if children.len() > FANOUT => {
let right = children.split_off(children.len() / 2);
Ok(vec![
write_node(db, &Node::Branch { level, children })?,
write_node(
db,
&Node::Branch {
level,
children: right,
},
)?,
])
}
node => Ok(vec![if inline {
inline_node(db, &node)?
} else {
write_node(db, &node)?
}]),
}
}
pub(super) fn put(db: &Connection, name: &str, row: Route) -> Result<()> {
fn insert(
db: &Connection,
edge: &Edge,
expected: Option<u16>,
row: Route,
inline: bool,
) -> Result<(u16, Vec<Edge>)> {
let mut node = checked_child(db, edge, expected)?;
let level = node.level();
match &mut node {
Node::Leaf { rows } => {
let i = rows.partition_point(|r| r.lower < row.lower);
if rows.get(i).is_some_and(|r| r.lower == row.lower) {
rows[i] = row;
} else {
rows.insert(i, row);
}
}
Node::Branch { level, children } => {
let i = children
.partition_point(|r| r.lower <= row.lower)
.saturating_sub(1);
let (_, replacements) = insert(db, &children[i], Some(*level - 1), row, false)?;
children.splice(i..=i, replacements);
}
}
Ok((level, save_nodes(db, node, inline)?))
}
let root = match head(db, name)? {
Some(edge) => {
let (level, mut replacements) = insert(db, &edge, None, row, true)?;
if replacements.len() == 1 {
replacements.remove(0)
} else {
inline_node(
db,
&Node::Branch {
level: level
.checked_add(1)
.ok_or_else(|| invalid("invalid routing depth"))?,
children: replacements,
},
)?
}
}
None => inline_node(db, &Node::Leaf { rows: vec![row] })?,
};
publish(db, name, Some(root))
}
pub(super) fn remove(db: &Connection, name: &str, key: &str) -> Result<()> {
fn delete(
db: &Connection,
edge: Edge,
expected: Option<u16>,
key: &str,
inline: bool,
) -> Result<Option<Edge>> {
let mut node = checked_child(db, &edge, expected)?;
match &mut node {
Node::Leaf { rows } => {
rows.retain(|r| r.lower != key);
if rows.is_empty() {
return Ok(None);
}
}
Node::Branch { level, children } => {
let i = children
.partition_point(|r| r.lower.as_str() <= key)
.saturating_sub(1);
let replacement = delete(db, children[i].clone(), Some(*level - 1), key, false)?;
children.splice(i..=i, replacement);
if children.is_empty() {
return Ok(None);
}
}
}
Ok(Some(if inline {
inline_node(db, &node)?
} else {
write_node(db, &node)?
}))
}
let root = match head(db, name)? {
Some(edge) => delete(db, edge, None, key, true)?,
None => None,
};
publish(db, name, root)
}
pub(super) fn clear(db: &Connection, name: &str) -> Result<()> {
publish(db, name, None)
}
pub(super) fn rebase_snapshot(db: &Connection, source: &Connection) -> Result<()> {
let origin = root_directory(source)?
.canonicalize()
.map_err(error)?
.to_str()
.ok_or_else(|| invalid("invalid routing source path"))?
.to_owned();
let mut edge = head(db, "routes")?.ok_or_else(|| invalid("missing snapshot routing head"))?;
if edge.page.origin.is_some() {
return Err(invalid(
"snapshot source references an external routing store",
));
}
edge.page.origin = Some(origin);
publish(db, "routes", Some(edge))?;
clear(db, "retired")
}
pub(super) fn verify_local(db: &Connection) -> Result<()> {
fn walk(db: &Connection, edge: &Edge, level: Option<u16>) -> Result<()> {
if edge.page.origin.is_some() {
return Err(invalid("snapshot references external routing pages"));
}
if let Node::Branch { level, children } = checked_child(db, edge, level)? {
for child in children {
walk(db, &child, Some(level - 1))?;
}
}
Ok(())
}
walk(
db,
&head(db, "routes")?.ok_or_else(|| invalid("snapshot has no routing head"))?,
None,
)
}
#[cfg(test)]
thread_local! { static READ_PAGES: std::cell::Cell<usize> = const { std::cell::Cell::new(0) }; }
#[cfg(test)]
mod tests {
use super::*;
fn fixture() -> (PathBuf, Connection) {
let directory =
std::env::temp_dir().join(format!("paperclip-routing-{}", uuid::Uuid::new_v4()));
private_directory(&directory).unwrap();
let path = directory.join("root.sqlite");
let mut options = fs::OpenOptions::new();
options.create_new(true).write(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
options.mode(0o600);
}
options.open(&path).unwrap();
let db = Connection::open(path).unwrap();
initialize(&db).unwrap();
db.execute_batch("CREATE TABLE receipt_backup(singleton INTEGER PRIMARY KEY); CREATE TABLE receipt_prepare(singleton INTEGER PRIMARY KEY);").unwrap();
(directory, db)
}
fn row(id: usize) -> Route {
Route {
lower: format!("range-{id:08}"),
file: format!("root.sqlite.receipts/{id}.sqlite"),
records: 2,
bytes: (id + 1) as i64,
fingerprint: vec![id as u8; 32],
}
}
#[test]
fn routing_root_subdivides_and_reads_one_path_after_thousands_of_ranges() {
let (directory, mut db) = fixture();
let tx = db.transaction().unwrap();
for i in 0..1200 {
put(&tx, "routes", row(i)).unwrap();
}
tx.commit().unwrap();
let edge = head(&db, "routes").unwrap().unwrap();
let height = checked_child(&db, &edge, None).unwrap().level();
assert!(
height >= 2,
"test must split the routing root, not only leaves"
);
assert_eq!(
db.query_row("SELECT count(*) FROM receipt_routing_heads", [], |r| r
.get::<_, i64>(0))
.unwrap(),
1
);
assert!(
db.query_row("SELECT length(root) FROM receipt_routing_heads", [], |r| {
r.get::<_, i64>(0)
})
.unwrap()
< 32768
);
for id in [0, 599, 1199] {
READ_PAGES.with(|n| n.set(0));
assert_eq!(exact(&db, "routes", &row(id).lower).unwrap(), Some(row(id)));
assert_eq!(READ_PAGES.with(|n| n.get()), height as usize + 1);
}
assert_eq!(oversized(&db, 1199).unwrap(), Some(row(1199)));
let mut cursor = None;
for id in 0..1200 {
READ_PAGES.with(|n| n.set(0));
let found = next_local(&db, cursor.as_deref()).unwrap().unwrap();
assert_eq!(found, row(id));
assert!(READ_PAGES.with(|n| n.get()) <= 2 * (height as usize + 1));
cursor = Some(found.lower);
}
assert_eq!(next(&db, "routes", cursor.as_deref()).unwrap(), None);
drop(db);
let db = Connection::open(directory.join("root.sqlite")).unwrap();
assert_eq!(exact(&db, "routes", &row(0).lower).unwrap(), Some(row(0)));
fs::remove_dir_all(directory).unwrap();
}
#[test]
fn root_rollback_preserves_exact_routes_and_missing_pages_are_not_misses() {
let (directory, mut db) = fixture();
for i in 0..80 {
put(&db, "routes", row(i)).unwrap();
}
let before = head(&db, "routes").unwrap().unwrap();
{
let tx = db.transaction().unwrap();
for i in 80..160 {
put(&tx, "routes", row(i)).unwrap();
}
// Crash/rollback before the root transaction commits.
}
assert_eq!(head(&db, "routes").unwrap().unwrap().page, before.page);
assert_eq!(exact(&db, "routes", &row(0).lower).unwrap(), Some(row(0)));
assert_eq!(exact(&db, "routes", &row(159).lower).unwrap(), None);
let Node::Branch { children, .. } = read_node(&db, &before.page).unwrap() else {
panic!("root did not subdivide");
};
let path = page_path(&db, &children.last().unwrap().page, false).unwrap();
fs::write(&path, b"changed routing node").unwrap();
assert!(exact(&db, "routes", &row(159).lower).is_err());
fs::remove_file(&path).unwrap();
assert!(exact(&db, "routes", &row(159).lower).is_err());
fs::remove_dir_all(directory).unwrap();
}
#[test]
fn immutable_snapshot_root_survives_writes_and_becomes_independent_before_publication() {
let (source_directory, source) = fixture();
let (target_directory, target) = fixture();
for i in 0..80 {
put(&source, "routes", row(i)).unwrap();
}
publish(&target, "routes", head(&source, "routes").unwrap()).unwrap();
rebase_snapshot(&target, &source).unwrap();
assert!(verify_local(&target).is_err());
assert!(next_local(&target, None).is_err());
for i in 80..120 {
put(&source, "routes", row(i)).unwrap();
}
assert_eq!(exact(&target, "routes", &row(119).lower).unwrap(), None);
for i in 0..80 {
let mut expected = row(i);
expected.file = format!("root.sqlite.receipts/copied-{i}.sqlite");
assert_eq!(
exact(&target, "routes", &row(i).lower).unwrap(),
Some(row(i))
);
put(&target, "routes", expected).unwrap();
if i == 39 {
assert!(next_local(&target, Some(&row(78).lower)).is_err());
}
}
verify_local(&target).unwrap();
drop(source);
fs::remove_dir_all(source_directory).unwrap();
assert_eq!(
exact(&target, "routes", &row(0).lower)
.unwrap()
.unwrap()
.file,
"root.sqlite.receipts/copied-0.sqlite"
);
drop(target);
fs::remove_dir_all(target_directory).unwrap();
}
#[test]
fn retired_inventory_is_paged_and_can_release_every_entry() {
let (directory, db) = fixture();
for i in 0..80 {
put(&db, "retired", row(i)).unwrap();
}
for i in 0..80 {
assert_eq!(next(&db, "retired", None).unwrap(), Some(row(i)));
remove(&db, "retired", &row(i).lower).unwrap();
}
assert!(head(&db, "retired").unwrap().is_none());
fs::remove_dir_all(directory).unwrap();
}
fn collect_cycle(db: &Connection, collector: &mut RoutingCollector) -> usize {
let mut most_reads = 0;
for _ in 0..100_000 {
READ_PAGES.with(|n| n.set(0));
let complete = collector.step(db).unwrap();
most_reads = most_reads.max(READ_PAGES.with(|n| n.get()));
if complete {
return most_reads;
}
}
panic!("routing collection did not finish its bounded steps");
}
fn page_count(directory: &Path) -> usize {
fs::read_dir(directory)
.unwrap()
.map(|entry| {
let path = entry.unwrap().path();
if path.is_dir() {
page_count(&path)
} else {
1
}
})
.sum()
}
#[test]
fn collection_reclaims_rolled_back_and_replaced_pages_but_preserves_both_live_trees() {
let (directory, mut db) = fixture();
for i in 0..1200 {
put(&db, "routes", row(i)).unwrap();
}
for i in 0..80 {
put(&db, "retired", row(i + 2000)).unwrap();
}
{
let tx = db.transaction().unwrap();
for i in 1200..1280 {
put(&tx, "routes", row(i)).unwrap();
}
assert!(RoutingCollector::default().step(&tx).is_err());
}
let path = root_directory(&db).unwrap();
let before = page_count(&path);
let mut collector = RoutingCollector::default();
let most_reads = collect_cycle(&db, &mut collector);
assert!(most_reads <= 7, "one collection step traversed history");
let after = page_count(&path);
assert!(
after < before / 4,
"obsolete pages were not reclaimed: {before} -> {after}"
);
for i in 0..1200 {
assert_eq!(exact(&db, "routes", &row(i).lower).unwrap(), Some(row(i)));
}
for i in 0..80 {
assert_eq!(
exact(&db, "retired", &row(i + 2000).lower).unwrap(),
Some(row(i + 2000))
);
}
assert!(exact(&db, "routes", &row(1279).lower).unwrap().is_none());
// Restart and a second complete sweep retain precisely the same live pages.
drop(collector);
drop(db);
let db = Connection::open(directory.join("root.sqlite")).unwrap();
collect_cycle(&db, &mut RoutingCollector::default());
assert_eq!(page_count(&path), after);
fs::remove_dir_all(directory).unwrap();
}
#[test]
fn collection_checks_fresh_roots_and_holds_all_pages_until_backup_finishes() {
let (directory, db) = fixture();
let (target_directory, target) = fixture();
for i in 0..80 {
put(&db, "routes", row(i)).unwrap();
}
publish(&target, "routes", head(&db, "routes").unwrap()).unwrap();
rebase_snapshot(&target, &db).unwrap();
db.execute("INSERT INTO receipt_backup VALUES(1)", [])
.unwrap();
let mut collector = RoutingCollector::default();
for i in 0..80 {
let mut updated = row(i);
updated.bytes += 1000;
put(&db, "routes", updated).unwrap();
assert!(!collector.step(&db).unwrap());
}
for i in 0..80 {
assert_eq!(
exact(&target, "routes", &row(i).lower).unwrap(),
Some(row(i))
);
// Localize the backup before releasing its pin.
put(&target, "routes", row(i)).unwrap();
}
verify_local(&target).unwrap();
db.execute("DELETE FROM receipt_backup", []).unwrap();
// Begin enumeration, then republish historical content. Liveness comes
// from today's roots, never a stale mark captured when a scan started.
assert!(!collector.step(&db).unwrap());
for i in 0..80 {
put(&db, "routes", row(i)).unwrap();
}
collect_cycle(&db, &mut collector);
for i in 0..80 {
assert_eq!(exact(&db, "routes", &row(i).lower).unwrap(), Some(row(i)));
}
drop(db);
fs::remove_dir_all(directory).unwrap();
assert_eq!(
exact(&target, "routes", &row(0).lower).unwrap(),
Some(row(0))
);
fs::remove_dir_all(target_directory).unwrap();
}
#[test]
fn collection_holds_pending_prepares_and_rejects_corrupt_pages_without_deleting_them() {
let (directory, db) = fixture();
for i in 0..80 {
put(&db, "routes", row(i)).unwrap();
}
let obsolete = write_node(
&db,
&Node::Leaf {
rows: vec![row(9000)],
},
)
.unwrap();
let path = page_path(&db, &obsolete.page, false).unwrap();
db.execute("INSERT INTO receipt_prepare VALUES(1)", [])
.unwrap();
let mut collector = RoutingCollector::default();
for _ in 0..100 {
assert!(!collector.step(&db).unwrap());
}
assert!(path.exists());
db.execute("DELETE FROM receipt_prepare", []).unwrap();
fs::write(&path, b"damaged").unwrap();
let mut rejected = false;
for _ in 0..10_000 {
match collector.step(&db) {
Err(_) => {
rejected = true;
break;
}
Ok(true) => break,
Ok(false) => {}
}
}
assert!(rejected);
assert!(path.exists());
assert_eq!(exact(&db, "routes", &row(0).lower).unwrap(), Some(row(0)));
fs::remove_dir_all(directory).unwrap();
}
#[test]
fn collection_removes_only_owned_temporaries_and_refuses_unknown_roots() {
let (directory, db) = fixture();
put(&db, "routes", row(0)).unwrap();
let page = write_node(&db, &Node::Leaf { rows: vec![row(9)] }).unwrap();
let path = page_path(&db, &page.page, false).unwrap();
let temporary = path.with_extension(format!("{}.tmp", uuid::Uuid::new_v4()));
fs::hard_link(&path, &temporary).unwrap();
publish(&db, "future-root", head(&db, "routes").unwrap()).unwrap();
assert!(RoutingCollector::default().step(&db).is_err());
assert!(temporary.exists());
clear(&db, "future-root").unwrap();
collect_cycle(&db, &mut RoutingCollector::default());
assert!(!temporary.exists());
assert!(!path.exists());
assert_eq!(exact(&db, "routes", &row(0).lower).unwrap(), Some(row(0)));
fs::remove_dir_all(directory).unwrap();
}
#[cfg(unix)]
#[test]
fn collection_never_follows_a_page_symlink() {
let (directory, db) = fixture();
put(&db, "routes", row(0)).unwrap();
let page = write_node(&db, &Node::Leaf { rows: vec![row(9)] }).unwrap();
let path = page_path(&db, &page.page, false).unwrap();
let outside = directory.join("preserve-me");
fs::rename(&path, &outside).unwrap();
std::os::unix::fs::symlink(&outside, &path).unwrap();
let mut collector = RoutingCollector::default();
let mut rejected = false;
for _ in 0..100 {
if collector.step(&db).is_err() {
rejected = true;
break;
}
}
assert!(rejected);
assert!(fs::symlink_metadata(&path)
.unwrap()
.file_type()
.is_symlink());
assert!(outside.exists());
fs::remove_dir_all(directory).unwrap();
}
}
packages/paperclip-runner/runner/crates/runner-core/src/indexed_routes_gc.rs
+166
View File
@@ -0,0 +1,166 @@
//! Collect obsolete index pages without a history-sized mark set or inventory.
//! A page contains its immutable lower key and level, so two current-root point
//! lookups establish reachability. The caller holds the partition operation lock
//! through that check and unlink. An online backup pins every source page until
//! its independent copy is published. Receipt contents are never collected here.
use super::*;
#[derive(Default)]
pub(crate) struct RoutingCollector {
// At most three open directory iterators: root, two-digit prefix, leaf.
// Restarting loses only scan progress, never a deletion authorization.
directories: Vec<(PathBuf, fs::ReadDir)>,
}
fn hex(value: &str, length: usize) -> bool {
value.len() == length
&& value
.bytes()
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
}
/// A corrupt current path is an error, never evidence that a page is garbage.
fn reachable(db: &Connection, name: &str, candidate: &Edge, level: u16) -> Result<bool> {
let Some(mut edge) = head(db, name)? else {
if name == "routes" {
return Err(invalid("missing routing authority prevents collection"));
}
return Ok(false);
};
let mut expected = None;
loop {
if edge.page.origin.is_some() {
return Err(invalid("collection requires local routing authority"));
}
let node = checked_child(db, &edge, expected)?;
if node.level() <= level {
return Ok(node.level() == level
&& edge.page.inline.is_none()
&& edge.page.sha == candidate.page.sha);
}
let Node::Branch { level, children } = node else {
unreachable!("leaf level is zero");
};
let Some(next) = children
.into_iter()
.rev()
.find(|child| child.lower <= candidate.lower)
else {
return Ok(false);
};
edge = next;
expected = Some(level - 1);
}
}
impl RoutingCollector {
/// Inspect at most one directory entry and one bounded page. Returns true at
/// the end of a scan. Caller must recover pending writes first and retain
/// PartitionLock; no current-root transaction may span this call.
pub(crate) fn step(&mut self, db: &Connection) -> Result<bool> {
if !db.is_autocommit() {
return Err(invalid(
"routing collection cannot run inside a transaction",
));
}
let held: bool = db
.query_row(
"SELECT EXISTS(SELECT 1 FROM receipt_backup) OR EXISTS(SELECT 1 FROM receipt_prepare)",
[],
|r| r.get(0),
)
.map_err(error)?;
if held {
return Ok(false);
}
let unknown_head: bool = db
.query_row(
"SELECT EXISTS(SELECT 1 FROM receipt_routing_heads WHERE name NOT IN ('routes','retired'))",
[],
|r| r.get(0),
)
.map_err(error)?;
if unknown_head {
return Err(invalid("unknown routing root prevents collection"));
}
if self.directories.is_empty() {
let root = root_directory(db)?;
crate::durable::verify_private_directory(&root)?;
let entries = fs::read_dir(&root).map_err(error)?;
self.directories.push((root, entries));
return Ok(false);
}
let depth = self.directories.len();
let (_, entries) = self.directories.last_mut().unwrap();
let Some(entry) = entries.next() else {
self.directories.pop();
return Ok(self.directories.is_empty());
};
let entry = entry.map_err(error)?;
let name = entry.file_name();
let name = name
.to_str()
.ok_or_else(|| invalid("invalid routing directory entry"))?;
let path = entry.path();
// A different store handle may have collected this entry between steps.
let metadata = match fs::symlink_metadata(&path) {
Ok(metadata) => metadata,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(false),
Err(e) => return Err(error(e)),
};
if depth < 3 {
if !hex(name, 2) {
return Err(invalid("invalid routing directory prefix"));
}
crate::durable::verify_private_directory(&path)?;
self.directories
.push((path.clone(), fs::read_dir(path).map_err(error)?));
return Ok(false);
}
if !metadata.is_file() || metadata.file_type().is_symlink() {
return Err(invalid("routing collection encountered a nonregular page"));
}
let parent = path.parent().unwrap();
let prefix = parent
.parent()
.unwrap()
.file_name()
.unwrap()
.to_str()
.unwrap();
let suffix = parent.file_name().unwrap().to_str().unwrap();
if let Some(stem) = name.strip_suffix(".json").filter(|s| hex(s, 60)) {
let page = PageRef {
sha: format!("{prefix}{suffix}{stem}"),
inline: None,
origin: None,
};
let node = read_node(db, &page)?;
let candidate = node.edge(page);
if reachable(db, "routes", &candidate, node.level())?
|| reachable(db, "retired", &candidate, node.level())?
{
return Ok(false);
}
} else {
// Only our exact unpublished temporary filename can be reclaimed.
// Writers hold the same lock, so none can still be writing it.
let Some((stem, temporary)) = name.split_once('.') else {
return Err(invalid("invalid routing page filename"));
};
if !hex(stem, 60)
|| !temporary.strip_suffix(".tmp").is_some_and(|id| {
uuid::Uuid::parse_str(id).is_ok_and(|uuid| uuid.to_string() == id)
})
{
return Err(invalid("invalid routing temporary filename"));
}
drop(crate::durable::open_private_regular_file(&path).map_err(error)?);
}
fs::remove_file(&path).map_err(error)?;
fs::File::open(parent)
.and_then(|f| f.sync_all())
.map_err(error)?;
Ok(false)
}
}
packages/paperclip-runner/runner/crates/runner-core/src/indexed_rpc.rs
+137
View File
@@ -0,0 +1,137 @@
//! Private stdio port for standalone controllers. Bounded requests reuse the
//! same partitioned store as runner/provider state, including its redo protocol.
use crate::{
durable::DurableRunnerError,
indexed_store::{ExactReceipt, IndexedStore},
};
use base64::{engine::general_purpose::STANDARD, Engine};
use serde_json::{json, Value};
use std::io::{BufRead, Write};
type Result<T> = std::result::Result<T, DurableRunnerError>;
fn invalid(message: &str) -> DurableRunnerError {
DurableRunnerError::invalid(message)
}
fn field<'a>(value: &'a Value, key: &str) -> Result<&'a str> {
value[key]
.as_str()
.ok_or_else(|| invalid("invalid storage RPC field"))
}
fn number(value: &Value, key: &str) -> Result<u64> {
let text = field(value, key)?;
let number = text
.parse::<u64>()
.map_err(|_| invalid("invalid storage RPC integer"))?;
if text != number.to_string() {
return Err(invalid("invalid storage RPC integer"));
}
Ok(number)
}
fn bytes(value: &Value, key: &str) -> Result<Vec<u8>> {
STANDARD
.decode(field(value, key)?)
.map_err(|_| invalid("invalid storage RPC bytes"))
}
fn dispatch(store: &IndexedStore, value: &Value) -> Result<Value> {
let input = &value["input"];
Ok(match field(value,"operation")? {
"ready"=>Value::Null,
"load"=>store.read_state(field(input,"key")?)?.map(|state|json!({"generation":state.generation.to_string(),"bytes":STANDARD.encode(state.bytes)})).unwrap_or(Value::Null),
"get"=>store.receipt(field(input,"namespace")?,field(input,"key")?)?.map(|body|json!(STANDARD.encode(body))).unwrap_or(Value::Null),
"getWork"=>store.read_work(field(input,"collection")?,field(input,"key")?)?.map(work_value).unwrap_or(Value::Null),
"workPage"=>json!(store.work_page(field(input,"collection")?,field(input,"after")?,number(input,"limit")? as usize)?.into_iter().map(work_value).collect::<Vec<_>>()),
"page"=>{
let records=store.receipt_page(field(input,"namespace")?,field(input,"after")?,number(input,"limit")? as usize,number(input,"byteBudget")? as usize)?;
json!(records.into_iter().map(|r|json!({"key":r.key,"bytes":STANDARD.encode(r.bytes)})).collect::<Vec<_>>())
},
"commit"=>{
let records=input["receipts"].as_array().ok_or_else(||invalid("invalid storage RPC receipts"))?;
if records.len()>4096 { return Err(invalid("storage_pressure: receipt transaction bound")); }
let receipts=records.iter().map(|r|Ok(ExactReceipt{namespace:field(r,"namespace")?.into(),key:field(r,"key")?.into(),bytes:bytes(r,"bytes")?})).collect::<Result<Vec<_>>>()?;
let work=match input.get("work") {
None=>vec![],
Some(work)=>work.as_array().ok_or_else(||invalid("invalid outstanding-work list"))?.iter().map(|r|Ok(crate::indexed_work::WorkChange {
collection:field(r,"collection")?.into(),key:field(r,"key")?.into(),
expected_digest:if r["expectedDigest"].is_null() {None} else {Some(bytes(r,"expectedDigest")?)},
bytes:if r["bytes"].is_null() {None} else {Some(bytes(r,"bytes")?)},
})).collect::<Result<Vec<_>>>()?,
};
json!(store.commit_with_work(field(input,"key")?,field(input,"expectedGeneration")?.parse::<crate::indexed_revision::Revision>()?,bytes(input,"bytes")?,receipts,work)?.to_string())
},
_=>return Err(invalid("invalid storage RPC operation")),
})
}
fn work_value(record: crate::indexed_work::WorkRecord) -> Value {
json!({"key":record.key,"bytes":STANDARD.encode(record.bytes),"digest":STANDARD.encode(record.digest)})
}
pub fn serve(store: IndexedStore, input: impl BufRead, mut output: impl Write) -> Result<()> {
let mut input = input;
loop {
let mut line = Vec::new();
loop {
let available = input
.fill_buf()
.map_err(crate::indexed_store::storage_error)?;
if available.is_empty() {
if line.is_empty() {
return Ok(());
}
return Err(invalid("incomplete storage RPC frame"));
}
let count = available
.iter()
.position(|b| *b == b'\n')
.map(|n| n + 1)
.unwrap_or(available.len());
if line.len() + count > 96 * 1024 * 1024 {
return Err(invalid("storage_pressure: storage RPC frame bound"));
}
let complete = available[count - 1] == b'\n';
line.extend_from_slice(&available[..count]);
input.consume(count);
if complete {
break;
}
}
let request: Value =
serde_json::from_slice(&line).map_err(crate::indexed_store::storage_error)?;
let id = request["id"]
.as_u64()
.ok_or_else(|| invalid("invalid storage RPC id"))?;
let close = request["operation"] == "close";
let result = if close {
Ok(Value::Null)
} else {
dispatch(&store, &request)
};
let response = match result {
Ok(value) => json!({"id":id,"value":value}),
Err(error) => {
let message = error.to_string();
let code = if message.contains("stale indexed store generation")
|| message.contains("stale outstanding-work digest")
{
"stale_authority"
} else if message.contains("exact receipt replay conflict") {
"receipt_conflict"
} else if message.contains("storage_pressure:") {
"storage_pressure"
} else {
"storage_unavailable"
};
json!({"id":id,"error":{"code":code,"message":message}})
}
};
serde_json::to_writer(&mut output, &response)
.map_err(crate::indexed_store::storage_error)?;
output
.write_all(b"\n")
.and_then(|_| output.flush())
.map_err(crate::indexed_store::storage_error)?;
if close {
return Ok(());
}
}
}
packages/paperclip-runner/runner/crates/runner-core/src/indexed_session_snapshot.rs
+704
View File
@@ -0,0 +1,704 @@
//! A resumable, exact two-store snapshot at an admitted quiet boundary. The
//! controller must separately fence its authority and provider checkpoint. This
//! operation does not infer process-tree death or authorize a restored session.
use crate::{
durable::{open_private_regular_file, verify_private_directory, DurableRunnerError},
indexed_store::{storage_error, IndexedStore},
};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use sha2::{Digest, Sha256};
use std::{
fs,
io::{Read, Write},
path::{Path, PathBuf},
};
type Result<T> = std::result::Result<T, DurableRunnerError>;
const MANIFEST: &str = "indexed-session-snapshot.json";
const NAMES: [&str; 2] = ["runner-state", "codex-provider-state"];
const KEYS: [&str; 2] = ["runner", "codex-provider"];
const SCHEMAS: [&str; 2] = [
"paperclip.runner.durable.state.indexed.v1",
"paperclip.runner.codex-provider-state.indexed.v1",
];
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct ExpectedState {
pub generation: String,
pub sha256: String,
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
struct Snapshot {
schema: String,
source: PathBuf,
states: [ExpectedState; 2],
locators: [String; 2],
}
fn invalid(s: &str) -> DurableRunnerError {
DurableRunnerError::invalid(s)
}
fn present(path: &Path) -> Result<bool> {
match fs::symlink_metadata(path) {
Ok(_) => Ok(true),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(false),
Err(e) => Err(storage_error(e)),
}
}
fn read(path: &Path) -> Result<Vec<u8>> {
let mut bytes = Vec::new();
open_private_regular_file(path)
.map_err(storage_error)?
.take(16385)
.read_to_end(&mut bytes)
.map_err(storage_error)?;
if bytes.len() > 16384 {
return Err(invalid("snapshot metadata exceeds capacity"));
}
Ok(bytes)
}
fn sync(path: &Path) -> Result<()> {
fs::File::open(path)
.and_then(|f| f.sync_all())
.map_err(storage_error)
}
fn directory(path: &Path) -> Result<()> {
let mut builder = fs::DirBuilder::new();
#[cfg(unix)]
{
use std::os::unix::fs::DirBuilderExt;
builder.mode(0o700);
}
builder.create(path).map_err(storage_error)?;
sync(path.parent().unwrap())
}
fn publish_file(path: &Path, bytes: &[u8]) -> Result<()> {
if present(path)? {
if read(path)? != bytes {
return Err(invalid("snapshot metadata changed"));
}
return Ok(());
}
let temporary = path.with_extension(format!("{}.tmp", uuid::Uuid::new_v4()));
let mut options = fs::OpenOptions::new();
options.create_new(true).write(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
options.mode(0o600);
}
let mut file = options.open(&temporary).map_err(storage_error)?;
let result = (|| {
file.write_all(bytes)
.and_then(|_| file.sync_all())
.map_err(storage_error)?;
fs::hard_link(&temporary, path).map_err(storage_error)?;
sync(path.parent().unwrap())
})();
let _ = fs::remove_file(temporary);
result
}
fn move_once(source: &Path, destination: &Path) -> Result<()> {
if present(destination)? {
if present(source)? {
return Err(invalid("snapshot move destination conflicts"));
}
return Ok(());
}
#[cfg(any(target_os = "linux", target_os = "android", target_vendor = "apple"))]
rustix::fs::renameat_with(
rustix::fs::CWD,
source,
rustix::fs::CWD,
destination,
rustix::fs::RenameFlags::NOREPLACE,
)
.map_err(storage_error)?;
#[cfg(not(any(target_os = "linux", target_os = "android", target_vendor = "apple")))]
return Err(invalid(
"snapshot publication is unqualified on this platform",
));
sync(destination.parent().unwrap())?;
sync(source.parent().unwrap())
}
fn binding(locator: &str, index: usize) -> Result<String> {
let value: Value = serde_json::from_str(locator).map_err(storage_error)?;
if locator.len() > 4096 || value["schema"] != SCHEMAS[index] {
return Err(invalid("snapshot locator is invalid"));
}
value["binding"]
.as_str()
.filter(|s| !s.is_empty() && s.len() <= 1024 && !s.chars().any(char::is_control))
.map(str::to_owned)
.ok_or_else(|| invalid("snapshot binding is invalid"))
}
fn verify(store: &IndexedStore, index: usize, expected: &ExpectedState) -> Result<Value> {
let snapshot = store
.read_state(KEYS[index])?
.ok_or_else(|| invalid("snapshot current state is missing"))?;
if snapshot.generation.to_string() != expected.generation
|| format!("{:x}", Sha256::digest(&snapshot.bytes)) != expected.sha256
{
return Err(invalid("snapshot current state changed"));
}
serde_json::from_slice(&snapshot.bytes).map_err(storage_error)
}
fn verify_backup(directory: &Path, index: usize, snapshot: &Snapshot) -> Result<()> {
verify_private_directory(directory)?;
let path = directory.join(format!("{}.sqlite", NAMES[index]));
for suffix in ["", "-wal", "-shm"] {
let file = PathBuf::from(format!("{}{suffix}", path.display()));
if suffix.is_empty() || present(&file)? {
crate::indexed_store::verify_sqlite_file(&file).map_err(storage_error)?;
}
}
let db = rusqlite::Connection::open_with_flags(
&path,
rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY | rusqlite::OpenFlags::SQLITE_OPEN_NOFOLLOW,
)
.map_err(storage_error)?;
db.execute_batch(
"PRAGMA query_only=ON; PRAGMA trusted_schema=OFF; PRAGMA cache_size=-4096; BEGIN",
)
.map_err(storage_error)?;
let stored: (i64, String) = db
.query_row(
"SELECT schema_version,binding FROM store_binding WHERE singleton=1",
[],
|row| Ok((row.get(0)?, row.get(1)?)),
)
.map_err(storage_error)?;
if stored != (2, binding(&snapshot.locators[index], index)?) {
return Err(invalid("snapshot database binding differs"));
}
let complete: i64 = db
.query_row(
"SELECT complete FROM backup_manifest WHERE singleton=1",
[],
|row| row.get(0),
)
.map_err(storage_error)?;
if complete != 1 {
return Err(invalid("snapshot backup is incomplete"));
}
crate::indexed_partitions::verify_backup_receipts(&db)?;
let state = crate::indexed_store::read_state(&db, KEYS[index])?
.ok_or_else(|| invalid("snapshot current state missing"))?;
if state.generation.to_string() != snapshot.states[index].generation
|| format!("{:x}", Sha256::digest(&state.bytes)) != snapshot.states[index].sha256
{
return Err(invalid("snapshot current state changed"));
}
Ok(())
}
fn verify_completed(directory: &Path, snapshot: &Snapshot) -> Result<()> {
verify_private_directory(directory)?;
let _lifetimes = [
crate::indexed_lifetime::StoreLifetime::existing_exclusive(
&directory.join("runner-state.sqlite"),
)?,
crate::indexed_lifetime::StoreLifetime::existing_exclusive(
&directory.join("codex-provider-state.sqlite"),
)?,
];
if read(&directory.join(MANIFEST))? != serde_json::to_vec(snapshot).map_err(storage_error)? {
return Err(invalid("snapshot publication conflicts"));
}
for i in 0..2 {
if read(&directory.join(format!("{}.json", NAMES[i])))? != snapshot.locators[i].as_bytes() {
return Err(invalid("snapshot locator changed"));
}
verify_backup(directory, i, snapshot)?;
}
Ok(())
}
/// The expected generations and hashes are supplied by current-state admission,
/// never reconstructed from history. Both lifetime fences stay owned throughout
/// the copy. No new provider/runner writer may enter between the two snapshots.
pub fn snapshot_session(
source: &Path,
destination: &Path,
states: [ExpectedState; 2],
) -> Result<Value> {
snapshot_with_hook(source, destination, states, |_| Ok(()))
}
fn snapshot_with_hook(
source: &Path,
destination: &Path,
states: [ExpectedState; 2],
hook: impl Fn(&str) -> Result<()>,
) -> Result<Value> {
verify_private_directory(source)?;
let requested_destination = destination.to_owned();
let parent = destination
.parent()
.ok_or_else(|| invalid("snapshot has no parent"))?;
verify_private_directory(parent)?;
let source = source.canonicalize().map_err(storage_error)?;
let destination = parent.canonicalize().map_err(storage_error)?.join(
destination
.file_name()
.ok_or_else(|| invalid("snapshot has no filename"))?,
);
if destination.starts_with(&source) || source.starts_with(&destination) {
return Err(invalid("snapshot overlaps its source"));
}
for state in &states {
if state
.generation
.parse::<crate::indexed_revision::Revision>()
.ok()
.filter(|n| *n != crate::indexed_revision::Revision::Absent)
.map(|n| n.to_string())
.as_ref()
!= Some(&state.generation)
|| state.sha256.len() != 64
|| !state
.sha256
.bytes()
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
{
return Err(invalid("snapshot expectation is invalid"));
}
}
let snapshot = Snapshot {
schema: "paperclip.runner.indexed-session-snapshot.v1".into(),
source: source.clone(),
states,
locators: [
String::from_utf8(read(&source.join("runner-state.json"))?).map_err(storage_error)?,
String::from_utf8(read(&source.join("codex-provider-state.json"))?)
.map_err(storage_error)?,
],
};
if present(&destination)? {
verify_completed(&destination, &snapshot)?;
return Ok(
serde_json::json!({"snapshot":"complete","destination":requested_destination,"states":snapshot.states}),
);
}
// Always acquire in the same order; failure releases the first fence.
let stores = [
IndexedStore::open_quiescent(
&source.join("runner-state.sqlite"),
&binding(&snapshot.locators[0], 0)?,
)?,
IndexedStore::open_quiescent(
&source.join("codex-provider-state.sqlite"),
&binding(&snapshot.locators[1], 1)?,
)?,
];
let runner = verify(&stores[0], 0, &snapshot.states[0])?;
let provider = verify(&stores[1], 1, &snapshot.states[1])?;
if runner["lifecycle"] != "suspended"
|| runner["ackedSourceSeq"]
.as_u64()
.and_then(|n| n.checked_add(1))
!= runner["nextSourceSeq"].as_u64()
|| runner["nextSourceSeq"].as_u64().is_none()
|| provider["activeProviderTurnId"].as_str().is_some()
|| provider["startupAttempt"].is_object()
|| provider["ambiguousTurnStartPending"] == true
|| ["pendingEvents", "queuedEvents"].iter().any(|key| {
provider[key]
.as_array()
.is_none_or(|items| !items.is_empty())
})
{
return Err(invalid("snapshot requires a settled native session"));
}
for key in ["runnerInstanceId", "normalizedSessionId"] {
if runner[key].as_str().is_none_or(|value| {
value.is_empty() || value.len() > 512 || value.chars().any(char::is_control)
}) {
return Err(invalid("snapshot session identity is invalid"));
}
}
for (index, expected) in [
(
0,
format!(
"runner/{}/{}",
runner["runnerInstanceId"].as_str().unwrap_or(""),
runner["normalizedSessionId"].as_str().unwrap_or("")
),
),
(
1,
format!(
"provider/{}/{}",
runner["runnerInstanceId"].as_str().unwrap_or(""),
runner["normalizedSessionId"].as_str().unwrap_or("")
),
),
] {
if binding(&snapshot.locators[index], index)? != expected {
return Err(invalid("snapshot session binding differs"));
}
}
let staging = destination.with_file_name(format!(
".{}.preparing",
destination.file_name().unwrap().to_string_lossy()
));
if !present(&staging)? {
directory(&staging)?;
}
verify_private_directory(&staging)?;
// A second snapshot process cannot race assembly or publish this job.
let _job_lock = crate::indexed_partitions::PartitionLock::acquire(&staging.join("snapshot"))?;
publish_file(
&staging.join("request.json"),
&serde_json::to_vec(&snapshot).map_err(storage_error)?,
)?;
hook("intent")?;
let merged = staging.join("runner");
let provider_root = staging.join("provider");
let assembling = staging.join("assembling.json");
if !present(&assembling)? {
for (i, target) in [&merged, &provider_root].iter().enumerate() {
if !present(target)? {
stores[i].begin_named_backup(target, &format!("{}.sqlite", NAMES[i]))?;
while !stores[i].backup_step()? {
hook("copy")?;
std::thread::yield_now();
}
} else {
// Release any source pin left after the atomic publication.
stores[i].backup_step()?;
}
verify_backup(target, i, &snapshot)?;
hook(if i == 0 { "runner" } else { "provider" })?;
}
publish_file(&assembling, b"{\"assembling\":true}")?;
} else if read(&assembling)? != b"{\"assembling\":true}" {
return Err(invalid("snapshot assembly marker differs"));
}
for suffix in [
".sqlite",
".sqlite.receipts",
".sqlite.routing",
".sqlite-wal",
".sqlite-shm",
] {
let name = format!("codex-provider-state{suffix}");
if suffix == ".sqlite"
|| suffix == ".sqlite.receipts"
|| suffix == ".sqlite.routing"
|| present(&provider_root.join(&name))?
|| present(&merged.join(&name))?
{
move_once(&provider_root.join(&name), &merged.join(&name))?;
}
hook("merge")?;
}
for i in 0..2 {
publish_file(
&merged.join(format!("{}.json", NAMES[i])),
snapshot.locators[i].as_bytes(),
)?;
}
for name in NAMES {
crate::indexed_partitions::seal_backup_files(&merged.join(format!("{name}.sqlite")))?;
hook("sealed")?;
}
for name in NAMES {
// Read-only inspection uses existing lifetime/operation fences before
// the first execution open of a restored snapshot.
for suffix in [".sqlite.lifetime", ".sqlite.lock"] {
publish_file(&merged.join(format!("{name}{suffix}")), b"")?;
}
}
publish_file(
&merged.join(MANIFEST),
&serde_json::to_vec(&snapshot).map_err(storage_error)?,
)?;
verify_completed(&merged, &snapshot)?;
hook("verified")?;
move_once(&merged, &destination)?;
hook("published")?;
Ok(
serde_json::json!({"snapshot":"complete","destination":requested_destination,"states":snapshot.states}),
)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::indexed_store::ExactReceipt;
use serde_json::json;
use std::sync::atomic::{AtomicUsize, Ordering};
struct Fixture {
root: PathBuf,
source: PathBuf,
destination: PathBuf,
states: [ExpectedState; 2],
}
impl Fixture {
fn new() -> Self {
let root = std::env::temp_dir().join(format!(
"paperclip-session-snapshot-{}",
uuid::Uuid::new_v4()
));
directory(&root).unwrap();
let source = root.join("source");
directory(&source).unwrap();
let mut states = Vec::new();
for i in 0..2 {
let binding = if i == 0 { "runner/r/s" } else { "provider/r/s" };
publish_file(
&source.join(format!("{}.json", NAMES[i])),
&serde_json::to_vec(&json!({"schema":SCHEMAS[i],"binding":binding})).unwrap(),
)
.unwrap();
let store =
IndexedStore::open(&source.join(format!("{}.sqlite", NAMES[i])), binding, true)
.unwrap();
let bytes=serde_json::to_vec(&if i==0 {json!({"lifecycle":"suspended","runnerInstanceId":"r","normalizedSessionId":"s","nextSourceSeq":3,"ackedSourceSeq":2})} else {json!({"lifecycle":"ready","pendingEvents":[],"queuedEvents":[]})}).unwrap();
let generation = store
.commit(
KEYS[i],
0,
bytes.clone(),
(0..270)
.map(|n| ExactReceipt {
namespace: "ancient".into(),
key: format!("{n:08}"),
bytes: vec![i as u8 + 20; 4096],
})
.collect(),
)
.unwrap();
states.push(ExpectedState {
generation: generation.to_string(),
sha256: format!("{:x}", Sha256::digest(&bytes)),
});
}
Self {
destination: root.join("published"),
root,
source,
states: states.try_into().unwrap(),
}
}
fn restore(&self) {
for i in 0..2 {
let store = IndexedStore::open(
&self.destination.join(format!("{}.sqlite", NAMES[i])),
if i == 0 { "runner/r/s" } else { "provider/r/s" },
false,
)
.unwrap();
verify(&store, i, &self.states[i]).unwrap();
for n in [0, 127, 269] {
assert_eq!(
store.receipt("ancient", &format!("{n:08}")).unwrap(),
Some(vec![i as u8 + 20; 4096])
);
}
let partition = store.partitions(None, 1).unwrap();
assert!(partition[0]
.file
.starts_with(&format!("{}.sqlite.receipts/", NAMES[i])));
}
}
}
impl Drop for Fixture {
fn drop(&mut self) {
let _ = fs::remove_dir_all(&self.root);
}
}
#[test]
fn exact_two_store_snapshot_restores_after_relocated_source_volume_is_lost() {
let f = Fixture::new();
let volume = f.root.join("volume");
directory(&volume).unwrap();
let path = f.source.join("codex-provider-state.sqlite");
let store = IndexedStore::open(&path, "provider/r/s", false).unwrap();
store.relocate_partition("", &volume).unwrap();
for _ in 0..100 {
store.maintain().unwrap();
}
assert!(store.partitions(None, 1).unwrap()[0]
.file
.starts_with(volume.canonicalize().unwrap().to_str().unwrap()));
drop(store);
snapshot_session(&f.source, &f.destination, f.states.clone()).unwrap();
snapshot_session(&f.source, &f.destination, f.states.clone()).unwrap();
fs::remove_dir_all(&f.source).unwrap();
fs::remove_dir_all(volume).unwrap();
f.restore();
}
#[test]
fn opaque_revisions_restore_both_stores_and_ancient_receipts_after_source_loss() {
let mut f = Fixture::new();
for i in 0..2 {
let path = f.source.join(format!("{}.sqlite", NAMES[i]));
let db = rusqlite::Connection::open(&path).unwrap();
db.execute(
"UPDATE current_state SET generation=?1 WHERE key=?2",
rusqlite::params![i64::MAX, KEYS[i]],
)
.unwrap();
drop(db);
let store = IndexedStore::open(
&path,
if i == 0 { "runner/r/s" } else { "provider/r/s" },
false,
)
.unwrap();
let current = store.read_state(KEYS[i]).unwrap().unwrap();
let revision = store
.commit(KEYS[i], current.generation, current.bytes, vec![])
.unwrap();
assert!(matches!(
revision,
crate::indexed_revision::Revision::Opaque(_)
));
f.states[i].generation = revision.to_string();
}
snapshot_session(&f.source, &f.destination, f.states.clone()).unwrap();
fs::remove_dir_all(&f.source).unwrap();
f.restore();
}
#[test]
fn every_copy_and_publication_boundary_resumes_without_partial_activation() {
let boundary_count = {
let baseline = Fixture::new();
let calls = AtomicUsize::new(0);
snapshot_with_hook(
&baseline.source,
&baseline.destination,
baseline.states.clone(),
|_| {
calls.fetch_add(1, Ordering::SeqCst);
Ok(())
},
)
.unwrap();
let count = calls.load(Ordering::SeqCst);
assert!(
count >= 10,
"snapshot boundary coverage unexpectedly shrank to {count}"
);
baseline.restore();
count
};
let mut interruptions = 0;
let mut completed = false;
for stop_at in 0..=boundary_count {
let f = Fixture::new();
let calls = AtomicUsize::new(0);
let result = snapshot_with_hook(&f.source, &f.destination, f.states.clone(), |_| {
if calls.fetch_add(1, Ordering::SeqCst) == stop_at {
Err(invalid("injected snapshot interruption"))
} else {
Ok(())
}
});
if result.is_ok() {
assert_eq!(
stop_at, boundary_count,
"snapshot completed before the last observed boundary"
);
completed = true;
break;
}
assert!(result
.unwrap_err()
.to_string()
.contains("injected snapshot interruption"));
interruptions += 1;
if f.destination.exists() {
f.restore();
}
snapshot_session(&f.source, &f.destination, f.states.clone()).unwrap();
f.restore();
}
assert!(
completed,
"fault matrix did not complete after every snapshot boundary"
);
assert_eq!(
interruptions, boundary_count,
"fault matrix did not interrupt every boundary exactly once"
);
}
#[test]
fn rejects_live_connections_changed_generations_and_conflicting_destinations() {
let f = Fixture::new();
for i in 0..2 {
let store = IndexedStore::open(
&f.source.join(format!("{}.sqlite", NAMES[i])),
if i == 0 { "runner/r/s" } else { "provider/r/s" },
false,
)
.unwrap();
assert!(
snapshot_session(&f.source, &f.destination, f.states.clone())
.unwrap_err()
.to_string()
.contains("open connection")
);
assert!(!f.destination.exists());
drop(store);
}
let mut changed = f.states.clone();
changed[0].generation = "2".into();
assert!(snapshot_session(&f.source, &f.destination, changed)
.unwrap_err()
.to_string()
.contains("state changed"));
directory(&f.destination).unwrap();
assert!(snapshot_session(&f.source, &f.destination, f.states.clone()).is_err());
assert!(fs::read_dir(&f.destination).unwrap().next().is_none());
}
#[test]
fn completed_snapshot_retry_rejects_missing_or_corrupted_historical_receipts() {
for missing in [false, true] {
let f = Fixture::new();
snapshot_session(&f.source, &f.destination, f.states.clone()).unwrap();
let db = rusqlite::Connection::open(f.destination.join("runner-state.sqlite")).unwrap();
let relative = crate::indexed_partitions::partitions(&db, None, 1)
.unwrap()
.into_iter()
.next()
.expect("snapshot has a receipt route")
.file;
drop(db);
let partition = f.destination.join(relative);
if missing {
fs::remove_file(partition).unwrap();
} else {
let db = rusqlite::Connection::open(partition).unwrap();
db.execute("UPDATE receipt_rows SET body=x'00' WHERE ordinal=1", [])
.unwrap();
}
assert!(snapshot_session(&f.source, &f.destination, f.states.clone()).is_err());
}
}
#[test]
fn rejects_unsettled_delivery_before_creating_a_backup() {
let mut f = Fixture::new();
let store =
IndexedStore::open(&f.source.join("runner-state.sqlite"), "runner/r/s", false).unwrap();
let mut state: Value =
serde_json::from_slice(&store.read_state("runner").unwrap().unwrap().bytes).unwrap();
state["nextSourceSeq"] = json!(4);
let bytes = serde_json::to_vec(&state).unwrap();
store.commit("runner", 1, bytes.clone(), vec![]).unwrap();
drop(store);
f.states[0] = ExpectedState {
generation: "2".into(),
sha256: format!("{:x}", Sha256::digest(bytes)),
};
assert!(
snapshot_session(&f.source, &f.destination, f.states.clone())
.unwrap_err()
.to_string()
.contains("settled native session")
);
assert!(!f.destination.exists());
}
}
packages/paperclip-runner/runner/crates/runner-core/src/indexed_store.rs
+1267
View File
File diff suppressed because it is too large. Load diff
packages/paperclip-runner/runner/crates/runner-core/src/indexed_work.rs
+111
View File
@@ -0,0 +1,111 @@
//! Outstanding work is separate from immutable history and from the one-row
//! current checkpoint. Exact mutations share the authority commit boundary.
use crate::{durable::DurableRunnerError, indexed_store::storage_error};
use rusqlite::{params, Connection, OptionalExtension};
use sha2::{Digest, Sha256};
type Result<T> = std::result::Result<T, DurableRunnerError>;
#[derive(Clone, Debug)]
pub struct WorkChange {
pub collection: String,
pub key: String,
pub expected_digest: Option<Vec<u8>>,
pub bytes: Option<Vec<u8>>,
}
#[derive(Clone, Debug, PartialEq)]
pub struct WorkRecord {
pub key: String,
pub bytes: Vec<u8>,
pub digest: Vec<u8>,
}
fn invalid(message: &str) -> DurableRunnerError {
DurableRunnerError::invalid(message)
}
pub(crate) fn initialize(db: &Connection) -> Result<()> {
db.execute_batch("CREATE TABLE IF NOT EXISTS current_work(collection TEXT NOT NULL,key TEXT NOT NULL,bytes BLOB NOT NULL,digest BLOB NOT NULL,PRIMARY KEY(collection,key)) WITHOUT ROWID;").map_err(storage_error)
}
pub(crate) fn validate(changes: &[WorkChange]) -> Result<()> {
if changes.len() > 128 {
return Err(invalid(
"storage_pressure: outstanding-work transaction bound",
));
}
for (index, change) in changes.iter().enumerate() {
if change.collection.is_empty()
|| change.collection.len() > 128
|| change.key.is_empty()
|| change.key.len() > 1024
|| change.collection.contains('\0')
|| change.key.contains('\0')
|| change
.expected_digest
.as_ref()
.is_some_and(|d| d.len() != 32)
|| change.bytes.as_ref().is_some_and(|b| b.len() > 16384)
|| changes[..index]
.iter()
.any(|prior| prior.collection == change.collection && prior.key == change.key)
{
return Err(invalid("invalid outstanding-work mutation"));
}
}
Ok(())
}
pub(crate) fn get(db: &Connection, collection: &str, key: &str) -> Result<Option<WorkRecord>> {
let value:Option<(Vec<u8>,Vec<u8>)>=db.query_row("SELECT CASE WHEN length(bytes)<=16384 THEN bytes END,CASE WHEN length(digest)=32 THEN digest END FROM current_work WHERE collection=?1 AND key=?2",params![collection,key],|r|Ok((r.get(0)?,r.get(1)?))).optional().map_err(storage_error)?;
value
.map(|(bytes, digest)| {
if Sha256::digest(&bytes).as_slice() != digest {
return Err(invalid("outstanding-work digest mismatch"));
}
Ok(WorkRecord {
key: key.into(),
bytes,
digest,
})
})
.transpose()
}
pub(crate) fn page(
db: &Connection,
collection: &str,
after: &str,
limit: usize,
) -> Result<Vec<WorkRecord>> {
if !(1..=128).contains(&limit) || collection.len() > 128 || after.len() > 1024 {
return Err(invalid("invalid outstanding-work page"));
}
let mut query = db
.prepare(
"SELECT key FROM current_work WHERE collection=?1 AND key>?2 ORDER BY key LIMIT ?3",
)
.map_err(storage_error)?;
let keys = query
.query_map(params![collection, after, limit as i64], |r| {
r.get::<_, String>(0)
})
.map_err(storage_error)?
.collect::<std::result::Result<Vec<_>, _>>()
.map_err(storage_error)?;
keys.iter()
.map(|key| get(db, collection, key)?.ok_or_else(|| invalid("outstanding work disappeared")))
.collect()
}
pub(crate) fn apply(db: &Connection, changes: &[WorkChange]) -> Result<()> {
validate(changes)?;
for change in changes {
if get(db, &change.collection, &change.key)?.map(|r| r.digest) != change.expected_digest {
return Err(invalid("stale outstanding-work digest"));
}
if let Some(bytes) = &change.bytes {
db.execute("INSERT INTO current_work VALUES(?1,?2,?3,?4) ON CONFLICT(collection,key) DO UPDATE SET bytes=excluded.bytes,digest=excluded.digest",params![change.collection,change.key,bytes,Sha256::digest(bytes).to_vec()]).map_err(storage_error)?;
} else {
db.execute(
"DELETE FROM current_work WHERE collection=?1 AND key=?2",
params![change.collection, change.key],
)
.map_err(storage_error)?;
}
}
Ok(())
}
packages/paperclip-runner/runner/crates/runner-core/src/legacy_indexed_import.rs
+448
View File
@@ -0,0 +1,448 @@
//! Fenced, restartable preparation of legacy local authority. The caller owns
//! the session fence; this module never replaces a live locator or source file.
use crate::indexed_revision::Revision;
use std::fs::{self, File};
use std::io::{self, BufReader, Read, Write};
use std::path::{Path, PathBuf};
use serde::{de::DeserializeOwned, Deserialize, Serialize};
use sha2::{Digest, Sha256};
use crate::durable::{
create_private_temporary_file, open_private_regular_file, DurableRunnerError,
};
use crate::indexed_store::{ExactReceipt, IndexedStore};
type Result<T> = std::result::Result<T, DurableRunnerError>;
const PROGRESS: &str = "legacy-import";
const PAGE_BYTES: usize = 8 * 1024 * 1024;
fn invalid(message: impl Into<String>) -> DurableRunnerError {
DurableRunnerError::invalid(message)
}
struct HashReader<R> {
inner: R,
digest: Sha256,
remaining: u64,
}
impl<R: Read> Read for HashReader<R> {
fn read(&mut self, bytes: &mut [u8]) -> io::Result<usize> {
let count = self.inner.read(bytes)?;
self.remaining = self
.remaining
.checked_sub(count as u64)
.ok_or_else(|| io::Error::other("legacy source exceeded its admission bound"))?;
self.digest.update(&bytes[..count]);
Ok(count)
}
}
fn same_file(a: &fs::Metadata, b: &fs::Metadata) -> bool {
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
a.dev() == b.dev()
&& a.ino() == b.ino()
&& a.len() == b.len()
&& a.mtime() == b.mtime()
&& a.mtime_nsec() == b.mtime_nsec()
&& a.ctime() == b.ctime()
&& a.ctime_nsec() == b.ctime_nsec()
}
#[cfg(not(unix))]
{
a.len() == b.len() && a.modified().ok() == b.modified().ok()
}
}
/// The bound is the old format's admitted pending state, not indexed history.
/// Hash and decode the source once without allocating a second whole-file copy.
pub(crate) struct LegacySource {
pub digest: String,
path: PathBuf,
metadata: fs::Metadata,
}
impl LegacySource {
pub fn verify(&self) -> Result<()> {
let current = open_private_regular_file(&self.path)
.and_then(|file| file.metadata())
.map_err(|e| invalid(e.to_string()))?;
if !same_file(&self.metadata, &current) {
return Err(invalid("legacy source changed during import"));
}
Ok(())
}
}
pub(crate) fn read_legacy<T: DeserializeOwned>(
path: &Path,
maximum: u64,
) -> Result<(T, LegacySource)> {
let file = open_private_regular_file(path).map_err(|e| invalid(e.to_string()))?;
let before = file.metadata().map_err(|e| invalid(e.to_string()))?;
if before.len() > maximum {
return Err(invalid("legacy source exceeds its admission bound"));
}
let mut reader = HashReader {
inner: BufReader::with_capacity(64 * 1024, file),
digest: Sha256::new(),
remaining: maximum,
};
let state = serde_json::from_reader(&mut reader)
.map_err(|e| invalid(format!("invalid legacy source: {e}")))?;
let after = reader
.inner
.get_ref()
.metadata()
.map_err(|e| invalid(e.to_string()))?;
let current = open_private_regular_file(path)
.and_then(|file| file.metadata())
.map_err(|e| invalid(e.to_string()))?;
if !same_file(&before, &after) || !same_file(&before, &current) {
return Err(invalid("legacy source changed during import"));
}
Ok((
state,
LegacySource {
digest: format!("{:x}", reader.digest.finalize()),
path: path.into(),
metadata: before,
},
))
}
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
pub struct PreparedLegacyAuthority {
pub schema: String,
pub binding: String,
pub fence_id: String,
pub source_digest: String,
pub current_digest: String,
pub current_key: String,
pub generation: u64,
pub receipts: u64,
}
#[derive(Deserialize, Serialize)]
#[serde(rename_all = "camelCase")]
struct Progress {
schema: String,
binding: String,
fence_id: String,
source_digest: String,
receipts: u64,
receipt_digest: String,
prepared: Option<PreparedLegacyAuthority>,
}
pub(crate) struct ImportWriter {
pub store: IndexedStore,
progress: Progress,
generation: Revision,
seen: u64,
digest: Sha256,
pending: Vec<ExactReceipt>,
bytes: usize,
}
impl ImportWriter {
pub fn open(path: &Path, binding: &str, fence: &str, source_digest: &str) -> Result<Self> {
if fence.is_empty() || fence.len() > 240 || fence.chars().any(char::is_control) {
return Err(invalid(
"legacy import requires a bounded exclusive fence identity",
));
}
let store = IndexedStore::open(path, binding, true)?;
let saved = store.read_state(PROGRESS)?;
let mut generation = saved.as_ref().map_or(Revision::Absent, |s| s.generation);
let progress: Progress = match saved {
Some(saved) => {
serde_json::from_slice(&saved.bytes).map_err(|e| invalid(e.to_string()))?
}
None => Progress {
schema: "paperclip.runner.local-import.v1".into(),
binding: binding.into(),
fence_id: fence.into(),
source_digest: source_digest.into(),
receipts: 0,
receipt_digest: format!("{:x}", Sha256::digest([])),
prepared: None,
},
};
if progress.schema != "paperclip.runner.local-import.v1"
|| progress.binding != binding
|| progress.fence_id != fence
|| progress.source_digest != source_digest
{
return Err(invalid("legacy import binding, fence, or source changed"));
}
if generation == 0 {
// Never adopt a live store as an import destination.
if store.read_state("runner")?.is_some()
|| store.read_state("codex-provider")?.is_some()
{
return Err(invalid(
"legacy import destination is already an execution authority",
));
}
generation = store.commit(
PROGRESS,
0,
serde_json::to_vec(&progress).map_err(|e| invalid(e.to_string()))?,
vec![],
)?;
}
Ok(Self {
store,
progress,
generation,
seen: 0,
digest: Sha256::new(),
pending: vec![],
bytes: 0,
})
}
pub fn add(&mut self, receipt: ExactReceipt) -> Result<()> {
if receipt.bytes.len() > 32 * 1024 * 1024 {
return Err(invalid("legacy receipt exceeds per-record admission bound"));
}
for part in [
receipt.namespace.as_bytes(),
receipt.key.as_bytes(),
receipt.bytes.as_slice(),
] {
self.digest.update((part.len() as u64).to_be_bytes());
self.digest.update(part);
}
self.seen = self
.seen
.checked_add(1)
.ok_or_else(|| invalid("legacy receipt cursor overflow"))?;
if self.seen <= self.progress.receipts {
if self.seen == self.progress.receipts
&& format!("{:x}", self.digest.clone().finalize()) != self.progress.receipt_digest
{
return Err(invalid("legacy receipt prefix changed"));
}
return Ok(());
}
self.bytes += receipt.bytes.len();
self.pending.push(receipt);
if self.pending.len() >= 128 || self.bytes >= PAGE_BYTES {
self.flush()?;
}
Ok(())
}
fn flush(&mut self) -> Result<()> {
if self.pending.is_empty() {
return Ok(());
}
self.progress.receipts = self.seen;
self.progress.receipt_digest = format!("{:x}", self.digest.clone().finalize());
self.generation = self.store.commit(
PROGRESS,
self.generation,
serde_json::to_vec(&self.progress).map_err(|e| invalid(e.to_string()))?,
std::mem::take(&mut self.pending),
)?;
self.bytes = 0;
Ok(())
}
pub fn finish(mut self, key: &str, current: Vec<u8>) -> Result<PreparedLegacyAuthority> {
if self.seen < self.progress.receipts {
return Err(invalid("legacy receipt source is incomplete"));
}
self.flush()?;
let generation = match self.store.read_state(key)? {
None => self.store.commit(key, 0, current.clone(), vec![])?,
Some(existing) if existing.generation == 1 && existing.bytes == current => {
Revision::from(1)
}
Some(_) => return Err(invalid("legacy import destination has advanced or changed")),
};
let proof = PreparedLegacyAuthority {
schema: "paperclip.runner.prepared-legacy-authority.v1".into(),
binding: self.progress.binding.clone(),
fence_id: self.progress.fence_id.clone(),
source_digest: self.progress.source_digest.clone(),
current_digest: format!("{:x}", Sha256::digest(&current)),
current_key: key.into(),
generation: generation
.legacy()
.ok_or_else(|| invalid("legacy preparation revision is not initial"))?,
receipts: self.seen,
};
if self
.progress
.prepared
.as_ref()
.is_some_and(|old| old != &proof)
{
return Err(invalid("prepared legacy authority changed"));
}
self.progress.prepared = Some(proof.clone());
self.store.commit(
PROGRESS,
self.generation,
serde_json::to_vec(&self.progress).map_err(|e| invalid(e.to_string()))?,
vec![],
)?;
Ok(proof)
}
}
pub(crate) fn require_prepared(store: &IndexedStore) -> Result<()> {
if let Some(saved) = store.read_state(PROGRESS)? {
let progress: Progress =
serde_json::from_slice(&saved.bytes).map_err(|e| invalid(e.to_string()))?;
if progress.prepared.is_none() {
return Err(invalid("legacy import has not prepared its authority"));
}
}
Ok(())
}
pub(crate) fn require_active_directory(directory: &Path) -> Result<()> {
let Some(root) = directory.parent() else {
return Ok(());
};
let path = root.join("indexed-migration.json");
let mut file = match open_private_regular_file(&path) {
Ok(file) => file,
Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(()),
Err(e) => return Err(invalid(e.to_string())),
};
let mut bytes = Vec::new();
Read::by_ref(&mut file)
.take(16 * 1024 + 1)
.read_to_end(&mut bytes)
.map_err(|e| invalid(e.to_string()))?;
let marker: serde_json::Value = serde_json::from_slice(&bytes)
.map_err(|_| invalid("legacy migration marker is invalid"))?;
if bytes.len() > 16 * 1024
|| marker["schema"] != "paperclip.runner.legacy-activation.v1"
|| marker["phase"] != "active"
{
return Err(invalid("legacy migration is pending; execution is fenced"));
}
Ok(())
}
/// Used only inside a staging directory. Existing bytes must be identical.
pub(crate) fn publish_staged_locator(path: &Path, bytes: &[u8]) -> Result<()> {
match open_private_regular_file(path) {
Ok(mut file) => {
let mut existing = Vec::new();
Read::by_ref(&mut file)
.take(8193)
.read_to_end(&mut existing)
.map_err(|e| invalid(e.to_string()))?;
return if existing == bytes {
Ok(())
} else {
Err(invalid(
"staged locator already contains different authority",
))
};
}
Err(e) if e.kind() == io::ErrorKind::NotFound => (),
Err(e) => return Err(invalid(e.to_string())),
}
let (temporary, mut file) = create_private_temporary_file(path)?;
file.write_all(bytes)
.and_then(|_| file.sync_all())
.map_err(|e| invalid(e.to_string()))?;
// Linking is no-clobber, unlike rename over a concurrently-created locator.
fs::hard_link(&temporary, path).map_err(|e| invalid(e.to_string()))?;
fs::remove_file(temporary).map_err(|e| invalid(e.to_string()))?;
File::open(
path.parent()
.ok_or_else(|| invalid("locator lacks parent"))?,
)
.and_then(|f| f.sync_all())
.map_err(|e| invalid(e.to_string()))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn legacy_import_resumes_exact_pages_without_publishing_partial_authority() {
let root = std::env::temp_dir().join(format!("local-import-{}", uuid::Uuid::new_v4()));
let path = root.join("runner-state.sqlite");
let receipt = |n| ExactReceipt {
namespace: "tools".into(),
key: format!("call-{n:04}"),
bytes: vec![n as u8; 512],
};
{
let mut writer = ImportWriter::open(&path, "binding", "fence", "source").unwrap();
for n in 0..150 {
writer.add(receipt(n)).unwrap();
}
assert!(writer.store.read_state("runner").unwrap().is_none());
assert!(require_prepared(&writer.store).is_err());
// Simulated importer loss after a committed page and before the
// buffered remainder. No current authority exists yet.
}
assert!(ImportWriter::open(&path, "binding", "other-fence", "source").is_err());
assert!(ImportWriter::open(&path, "binding", "fence", "other-source").is_err());
let mut writer = ImportWriter::open(&path, "binding", "fence", "source").unwrap();
for n in 0..500 {
writer.add(receipt(n)).unwrap();
}
let proof = writer.finish("runner", b"current".to_vec()).unwrap();
assert_eq!(proof.receipts, 500);
let store = IndexedStore::open(&path, "binding", false).unwrap();
require_prepared(&store).unwrap();
for n in [0, 127, 128, 499] {
assert_eq!(
store.receipt("tools", &format!("call-{n:04}")).unwrap(),
Some(receipt(n).bytes)
);
}
// Commit-before-publication retry is identical, not a new generation.
let mut writer = ImportWriter::open(&path, "binding", "fence", "source").unwrap();
for n in 0..500 {
writer.add(receipt(n)).unwrap();
}
assert_eq!(writer.finish("runner", b"current".to_vec()).unwrap(), proof);
store
.commit("runner", 1, b"advanced".to_vec(), vec![])
.unwrap();
let mut writer = ImportWriter::open(&path, "binding", "fence", "source").unwrap();
for n in 0..500 {
writer.add(receipt(n)).unwrap();
}
assert!(writer.finish("runner", b"current".to_vec()).is_err());
drop(store);
fs::remove_dir_all(root).unwrap();
}
#[test]
fn legacy_source_reader_hashes_exact_bytes_and_rejects_changed_or_oversized_input() {
let root =
std::env::temp_dir().join(format!("local-import-source-{}", uuid::Uuid::new_v4()));
fs::create_dir(&root).unwrap();
let path = root.join("source.json");
let (temporary, mut file) = create_private_temporary_file(&path).unwrap();
let bytes = b"{\"pending\":[1,2,3]}\n";
file.write_all(bytes).unwrap();
drop(file);
fs::rename(temporary, &path).unwrap();
let (value, source): (serde_json::Value, _) = read_legacy(&path, 1024).unwrap();
assert_eq!(value["pending"][2], 3);
assert_eq!(source.digest, format!("{:x}", Sha256::digest(bytes)));
assert!(read_legacy::<serde_json::Value>(&path, 2).is_err());
fs::write(&path, b"{}").unwrap();
assert!(source.verify().is_err());
fs::remove_dir_all(root).unwrap();
}
}
packages/paperclip-runner/runner/crates/runner-core/src/lib.rs
+17
View File
@@ -6,6 +6,7 @@ pub mod acpx_provider_backend;
pub mod acpx_provider_checkpoint;
pub mod acpx_provider_session;
pub mod acpx_provider_state;
mod acpx_receipt_ledger;
pub mod acpx_sidecar_transport;
pub mod aws_agentcore_provider;
pub mod claude_managed_provider;
@@ -14,6 +15,17 @@ mod codex_startup_trust;
pub mod durable;
pub mod fake_harness;
pub mod generated_acpx_sidecar_contract;
pub mod indexed_archive;
mod indexed_commit;
pub mod indexed_inspection;
mod indexed_lifetime;
mod indexed_partitions;
pub mod indexed_revision;
pub mod indexed_rpc;
pub mod indexed_session_snapshot;
pub mod indexed_store;
pub mod indexed_work;
pub mod legacy_indexed_import;
pub mod local_runner;
pub mod managed_provider;
pub mod managed_provider_backend;
@@ -26,6 +38,7 @@ pub mod qualified_launch;
pub mod question_response;
pub mod replay;
mod stable_identity;
mod storage_capacity;
use std::error::Error;
use std::fmt::{self, Display, Formatter};
@@ -362,3 +375,7 @@ mod tests {
);
}
}
mod output_body;
mod process_generation;
packages/paperclip-runner/runner/crates/runner-core/src/managed_provider_backend.rs
+1
View File
@@ -2372,6 +2372,7 @@ mod tests {
fn test_command(sequence: u64, command_type: &str, payload: Value) -> Command {
Command {
controller_epoch: None,
schema: "paperclip.prp.command.v1".to_owned(),
command_id: format!("command-{sequence}"),
controller_seq: sequence,
packages/paperclip-runner/runner/crates/runner-core/src/output_body.rs
+190
View File
@@ -0,0 +1,190 @@
//! Lossless, credential-scrubbed output, carried in independently bounded frames.
//! The preview is display-only. Chunks precede its reference and share the
//! authenticated run/session envelope; neither grants mutation authority.
use crate::durable::{redact_output_text, DurableRunnerError, EventPriority};
use crate::provider_events::NormalizedProviderEvent;
use serde_json::{json, Value};
use sha2::{Digest, Sha256};
pub const CAPABILITY: &str = "history.output_bodies.v1";
pub const CHUNK_BYTES: usize = 32 * 1024 - 2;
const MAX_BODY_BYTES: usize = 4 * 1024 * 1024;
fn digest(bytes: &[u8]) -> String {
format!("{:x}", Sha256::digest(bytes))
}
pub fn add_codex_body(method: &str, params: &Value, events: &mut Vec<NormalizedProviderEvent>) {
let text = match method {
"item/agentMessage/delta" | "item/commandExecution/outputDelta" => params.get("delta"),
"item/started" | "item/completed" => params
.pointer("/item/text")
.or_else(|| params.pointer("/item/aggregatedOutput"))
.or_else(|| params.pointer("/item/output")),
_ => None,
}
.and_then(Value::as_str);
let Some(text) = text.filter(|text| text.len() > 4_000) else {
return;
};
if events.is_empty() {
return;
}
// Admission already bounded the provider frame. Redact before splitting so
// a credential crossing a chunk boundary cannot escape the scrubber.
let safe = redact_output_text(text);
let mut chunks = Vec::new();
let mut rest = safe.as_str();
while !rest.is_empty() {
let mut end = rest.len().min(CHUNK_BYTES);
while !rest.is_char_boundary(end) {
end -= 1;
}
// Include JSON escaping in the per-frame budget (e.g. NUL or controls).
while serde_json::to_vec(&rest[..end])
.expect("text serializes")
.len()
> 32 * 1024
{
end /= 2;
while !rest.is_char_boundary(end) {
end -= 1;
}
}
// Scrubbing is idempotent; apply it to the independently readable chunk
// as well, including credential-like suffixes produced by the split.
chunks.push(redact_output_text(&rest[..end]));
rest = &rest[end..];
}
let mut hash = Sha256::new();
let mut length = 0usize;
for chunk in &chunks {
hash.update(chunk.as_bytes());
length += chunk.len();
}
let sha = format!("{:x}", hash.finalize());
let reference = json!({"schema":"paperclip.output.body.v1", "bodyId":sha,
"sha256":sha, "byteLength":length.to_string(), "mediaType":"text/plain; charset=utf-8"});
events[0].payload["outputBody"] = reference.clone();
let mut prefix = Vec::with_capacity(chunks.len() + events.len());
let mut offset = 0usize;
for chunk in chunks {
prefix.push(NormalizedProviderEvent {
event_type: "output.body.chunk".into(),
priority: EventPriority::P1,
payload: json!({"schema":"paperclip.output.body.chunk.v1", "body":reference,
"offset":offset.to_string(), "sha256":digest(chunk.as_bytes()), "text":chunk}),
});
offset += chunk.len();
}
prefix.append(events);
*events = prefix;
}
pub fn sanitize_chunk(payload: &Value) -> Result<Value, DurableRunnerError> {
let invalid = || DurableRunnerError::invalid("invalid durable output body chunk");
let body = payload.get("body").ok_or_else(invalid)?;
let text = payload
.get("text")
.and_then(Value::as_str)
.ok_or_else(invalid)?;
let integer = |value: Option<&Value>| -> Result<usize, DurableRunnerError> {
let s = value.and_then(Value::as_str).ok_or_else(invalid)?;
let n = s.parse::<usize>().map_err(|_| invalid())?;
if n.to_string() != s {
return Err(invalid());
}
Ok(n)
};
let offset = integer(payload.get("offset"))?;
let length = integer(body.get("byteLength"))?;
let sha = body
.get("sha256")
.and_then(Value::as_str)
.ok_or_else(invalid)?;
if payload["schema"] != "paperclip.output.body.chunk.v1"
|| body["schema"] != "paperclip.output.body.v1"
|| body["mediaType"] != "text/plain; charset=utf-8"
|| body["bodyId"] != sha
|| sha.len() != 64
|| !sha
.bytes()
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
|| text.is_empty()
|| text.len() > CHUNK_BYTES
|| length > MAX_BODY_BYTES
|| offset
.checked_add(text.len())
.is_none_or(|end| end > length)
|| payload["sha256"] != digest(text.as_bytes())
|| redact_output_text(text) != text
|| payload.as_object().is_none_or(|o| o.len() != 5)
|| body.as_object().is_none_or(|o| o.len() != 5)
{
return Err(invalid());
}
Ok(payload.clone())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn preserves_large_command_delta_without_waiting_for_completion() {
let text = "tool-output-🌳".repeat(10_000);
let params = json!({"itemId":"exec-1", "delta":text});
let mut events = crate::provider_events::normalize_codex_notification(
"item/commandExecution/outputDelta",
&params,
);
add_codex_body("item/commandExecution/outputDelta", &params, &mut events);
let last = events.last().unwrap();
assert_eq!(last.event_type, "item.delta");
assert_eq!(last.payload["kind"], "commandExecution");
let mut body = String::new();
for event in &events[..events.len() - 1] {
sanitize_chunk(&event.payload).unwrap();
body.push_str(event.payload["text"].as_str().unwrap());
}
assert_eq!(body, text);
assert_eq!(
last.payload["outputBody"]["sha256"],
digest(body.as_bytes())
);
}
#[test]
fn chunks_preserve_large_unicode_output_and_scrub_boundary_credentials() {
let text = format!(
"{} api_key=super-secret-value {}",
"🌳".repeat(8200),
"z".repeat(25_000)
);
let mut events = vec![NormalizedProviderEvent {
event_type: "item.completed".into(),
priority: EventPriority::P1,
payload: json!({"text":"preview"}),
}];
add_codex_body(
"item/completed",
&json!({"item":{"text":text}}),
&mut events,
);
let mut output = String::new();
for event in &events[..events.len() - 1] {
sanitize_chunk(&event.payload).unwrap();
assert_eq!(event.payload["offset"], output.len().to_string());
assert!(serde_json::to_vec(&event.payload).unwrap().len() < 60_000);
output.push_str(event.payload["text"].as_str().unwrap());
}
assert!(!output.contains("super-secret-value"));
assert_eq!(output.matches('🌳').count(), 8200);
assert_eq!(
events.last().unwrap().payload["outputBody"]["sha256"],
digest(output.as_bytes())
);
let mut altered = events[0].payload.clone();
altered["text"] = json!("altered");
assert!(sanitize_chunk(&altered).is_err());
}
}
packages/paperclip-runner/runner/crates/runner-core/src/process_generation.rs
+164
View File
@@ -0,0 +1,164 @@
//! Provider lifetime identity. Retained small ordinals keep their exact JSON;
//! exhausting that namespace switches permanently to equality-only UUIDs.
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use uuid::Uuid;
pub const LEGACY_PROCESS_GENERATION_MAX: u64 = 9_007_199_254_740_991;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ProcessGeneration {
Legacy(u64),
Opaque(Uuid),
}
impl Default for ProcessGeneration {
fn default() -> Self {
Self::Legacy(0)
}
}
impl From<u64> for ProcessGeneration {
fn from(value: u64) -> Self {
Self::Legacy(value)
}
}
impl PartialEq<u64> for ProcessGeneration {
fn eq(&self, value: &u64) -> bool {
matches!(self, Self::Legacy(n) if n == value)
}
}
impl ProcessGeneration {
pub fn next(self) -> Option<Self> {
if let Self::Legacy(n) = self {
if n < LEGACY_PROCESS_GENERATION_MAX {
return Some(Self::Legacy(n + 1));
}
}
for _ in 0..32 {
let next = Self::Opaque(Uuid::new_v4());
if next != self {
return Some(next);
}
}
None
}
pub fn opaque(self) -> bool {
matches!(self, Self::Opaque(_))
}
pub fn is_successor_of(self, previous: Self) -> bool {
match (previous, self) {
(Self::Legacy(a), Self::Legacy(b)) => a < LEGACY_PROCESS_GENERATION_MAX && b == a + 1,
(Self::Legacy(a), Self::Opaque(_)) => a >= LEGACY_PROCESS_GENERATION_MAX,
(Self::Opaque(a), Self::Opaque(b)) => a != b,
_ => false,
}
}
/// Retained state used ordinal ordering as an additional corruption check.
/// An opaque lifetime is never ordered against another opaque lifetime;
/// completion authority can intentionally belong to an older process.
pub fn rejects_completed(self, completed: Self) -> bool {
match (self, completed) {
(Self::Legacy(a), Self::Legacy(b)) => b > a,
(Self::Legacy(_), Self::Opaque(_)) => true,
_ => false,
}
}
}
impl Serialize for ProcessGeneration {
fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
match self {
Self::Legacy(n) => serializer.serialize_u64(*n),
Self::Opaque(id) => serializer.serialize_str(&format!("p:{id}")),
}
}
}
impl<'de> Deserialize<'de> for ProcessGeneration {
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
#[derive(Deserialize)]
#[serde(untagged)]
enum Wire {
Legacy(u64),
Opaque(String),
}
match Wire::deserialize(deserializer)? {
Wire::Legacy(n) if n <= LEGACY_PROCESS_GENERATION_MAX => Ok(Self::Legacy(n)),
Wire::Legacy(_) => Err(serde::de::Error::custom(
"legacy provider process generation exceeds the safe integer boundary",
)),
Wire::Opaque(s) => {
if let Some(id) = s.strip_prefix("p:").and_then(|s| Uuid::parse_str(s).ok()) {
if id.get_version_num() == 4
&& id.get_variant() == uuid::Variant::RFC4122
&& s == format!("p:{id}")
{
return Ok(Self::Opaque(id));
}
}
Err(serde::de::Error::custom(
"invalid provider process generation",
))
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn opaque_lifetimes_preserve_exact_legacy_bytes_and_never_downgrade() {
let old = ProcessGeneration::from(LEGACY_PROCESS_GENERATION_MAX);
assert_eq!(
serde_json::to_string(&old).unwrap(),
LEGACY_PROCESS_GENERATION_MAX.to_string()
);
let first = old.next().unwrap();
let second = first.next().unwrap();
assert!(first.opaque() && second.opaque());
assert_ne!(first, second);
assert!(first.is_successor_of(old));
assert!(second.is_successor_of(first));
assert!(!old.is_successor_of(second));
assert!(!first.is_successor_of(first));
for generation in [ProcessGeneration::default(), old, first, second] {
assert_eq!(
serde_json::from_str::<ProcessGeneration>(
&serde_json::to_string(&generation).unwrap()
)
.unwrap(),
generation
);
}
assert!(old.rejects_completed(first));
assert!(!second.rejects_completed(first));
for invalid in [
"\"p:bad\"",
"\"p:00000000-0000-0000-0000-000000000000\"",
"-1",
"1.5",
] {
assert!(serde_json::from_str::<ProcessGeneration>(invalid).is_err());
}
}
#[test]
fn legacy_wire_rejects_values_above_the_javascript_safe_integer_boundary() {
let max = LEGACY_PROCESS_GENERATION_MAX.to_string();
assert_eq!(
serde_json::from_str::<ProcessGeneration>(&max).unwrap(),
ProcessGeneration::Legacy(LEGACY_PROCESS_GENERATION_MAX)
);
assert_eq!(
serde_json::to_string(&ProcessGeneration::Legacy(LEGACY_PROCESS_GENERATION_MAX))
.unwrap(),
max
);
assert!(serde_json::from_str::<ProcessGeneration>("9007199254740992").is_err());
assert!(serde_json::from_str::<ProcessGeneration>("18446744073709551615").is_err());
// Defensive in-memory conversion can still advance an old oversized
// value without wrapping; persisted/wire values above the boundary are
// rejected because JavaScript cannot represent them exactly.
let oversized_legacy = ProcessGeneration::from(LEGACY_PROCESS_GENERATION_MAX + 1);
let successor = oversized_legacy.next().unwrap();
assert!(successor.opaque());
assert!(successor.is_successor_of(oversized_legacy));
}
}
packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs
+679 -71
View File
File diff suppressed because it is too large. Load diff
packages/paperclip-runner/runner/crates/runner-core/src/provider_bridge.rs
+239 -10
View File
@@ -2,6 +2,7 @@ use std::collections::{BTreeMap, BTreeSet, VecDeque};
use std::error::Error;
use std::fmt::{self, Display, Formatter};
use crate::indexed_store::{ExactReceipt, IndexedStore};
use serde::de::{self, MapAccess, Visitor};
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use serde_json::Value;
@@ -86,6 +87,15 @@ struct CompletedToolCall {
result: ToolResult,
}
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
#[serde(tag = "kind", rename_all = "snake_case")]
enum IndexedToolReceipt {
Completed { receipt: CompletedToolCall },
// Legacy checkpoints sometimes retained only an identity. Preserve that
// denial; absence of an old result must never authorize executing it again.
LegacyTombstone { call_id: String },
}
#[derive(Clone, Debug, Default, PartialEq)]
struct SettledCallIds {
order: VecDeque<String>,
@@ -187,6 +197,10 @@ impl DurableReplayFilter {
#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)]
#[serde(rename_all = "camelCase")]
pub struct ProviderToolBridge {
#[serde(default, skip_serializing_if = "Option::is_none")]
indexed_receipt_namespace: Option<String>,
#[serde(skip)]
receipt_store: Option<IndexedStore>,
authorized: BTreeMap<String, AuthorizedTool>,
#[serde(default)]
catalog_operations: Vec<AuthorizedTool>,
@@ -310,6 +324,98 @@ fn is_false(value: &bool) -> bool {
}
impl ProviderToolBridge {
pub(crate) fn require_exact_migration_source(&self) -> Result<(), ProviderBridgeError> {
if !self.settled_call_filter.is_empty() {
return Err(ProviderBridgeError::invalid(
"legacy provider has probabilistic receipts; migration requires reconciliation",
));
}
Ok(())
}
pub(crate) fn bind_indexed_receipts(
&mut self,
store: IndexedStore,
namespace: String,
) -> Result<(), ProviderBridgeError> {
if self
.indexed_receipt_namespace
.as_ref()
.is_some_and(|current| current != &namespace)
{
return Err(ProviderBridgeError::invalid(
"provider receipt namespace changed",
));
}
self.indexed_receipt_namespace = Some(namespace);
self.receipt_store = Some(store);
Ok(())
}
fn indexed_receipt(
&self,
call_id: &str,
) -> Result<Option<IndexedToolReceipt>, ProviderBridgeError> {
let Some(namespace) = &self.indexed_receipt_namespace else {
return Ok(None);
};
let store = self
.receipt_store
.as_ref()
.ok_or_else(|| ProviderBridgeError::invalid("provider receipt store is unavailable"))?;
store
.receipt(namespace, call_id)
.map_err(|error| ProviderBridgeError::invalid(error.to_string()))?
.map(|bytes| {
serde_json::from_slice(&bytes).map_err(|_| {
ProviderBridgeError::invalid("provider indexed receipt is malformed")
})
})
.transpose()
}
/// Build a bounded checkpoint candidate. The caller must commit these
/// receipts and the candidate state in ONE store transaction before using
/// the candidate or releasing a provider result. This function does no I/O.
pub(crate) fn checkpoint_indexed_receipts(
&mut self,
) -> Result<Vec<ExactReceipt>, ProviderBridgeError> {
let Some(namespace) = &self.indexed_receipt_namespace else {
return Ok(vec![]);
};
let mut receipts = Vec::new();
for (key, receipt) in self.completed.iter().chain(self.settled_results.iter()) {
receipts.push(ExactReceipt {
namespace: namespace.clone(),
key: key.clone(),
bytes: serde_json::to_vec(&IndexedToolReceipt::Completed {
receipt: receipt.clone(),
})
.map_err(|_| ProviderBridgeError::invalid("cannot encode provider receipt"))?,
});
}
for key in self.settled_call_ids.iter() {
if !self.completed.contains_key(key) && !self.settled_results.contains_key(key) {
receipts.push(ExactReceipt {
namespace: namespace.clone(),
key: key.clone(),
bytes: serde_json::to_vec(&IndexedToolReceipt::LegacyTombstone {
call_id: key.clone(),
})
.map_err(|_| {
ProviderBridgeError::invalid("cannot encode provider tombstone")
})?,
});
}
}
self.completed.clear();
self.settled_results.clear();
self.settled_call_ids.clear();
self.retained_result_bytes = 0;
self.durable_run_receipt_limit_reached = !self.settled_call_filter.is_empty();
Ok(receipts)
}
pub fn prepare(&mut self, tool_set: AuthorizedToolSet) -> Result<(), ProviderBridgeError> {
self.prepare_internal(tool_set, false)
}
@@ -620,15 +726,33 @@ impl ProviderToolBridge {
}
return Ok(Some(completed.result.clone()));
}
Ok(None)
match self.indexed_receipt(call_id)? {
Some(IndexedToolReceipt::Completed { receipt }) => {
if receipt.call.call_id != call_id
|| receipt.call.operation_id != operation_id
|| &receipt.call.input != input
{
return Err(ProviderBridgeError::invalid(
"provider replayed an indexed tool call with different input",
));
}
Ok(Some(receipt.result))
}
Some(IndexedToolReceipt::LegacyTombstone { .. }) => Err(ProviderBridgeError::invalid(
"legacy settled tool result cannot be replayed",
)),
None => Ok(None),
}
}
pub fn has_completed_call(&self, call_id: &str) -> bool {
self.completed.contains_key(call_id) || self.has_settled_call_id(call_id)
pub fn has_completed_call(&self, call_id: &str) -> Result<bool, ProviderBridgeError> {
Ok(self.completed.contains_key(call_id)
|| self.has_settled_call_id(call_id)
|| self.indexed_receipt(call_id)?.is_some())
}
pub(crate) fn has_call_receipt(&self, call_id: &str) -> bool {
self.pending.contains_key(call_id) || self.has_completed_call(call_id)
pub(crate) fn has_call_receipt(&self, call_id: &str) -> Result<bool, ProviderBridgeError> {
Ok(self.pending.contains_key(call_id) || self.has_completed_call(call_id)?)
}
pub fn begin_call(
@@ -682,6 +806,7 @@ impl ProviderToolBridge {
if self.completed.contains_key(&call_id)
|| self.settled_results.contains_key(&call_id)
|| self.has_settled_call_id(&call_id)
|| self.indexed_receipt(&call_id)?.is_some()
{
return Err(ProviderBridgeError::invalid(
"provider reused a completed tool call id",
@@ -783,6 +908,16 @@ impl ProviderToolBridge {
))
};
}
if let Some(receipt) = self.indexed_receipt(&result.call_id)? {
return match receipt {
IndexedToolReceipt::Completed { receipt } if receipt.result == result => {
Ok(result.result)
}
_ => Err(ProviderBridgeError::invalid(
"conflicting or incomplete indexed tool result",
)),
};
}
if self.has_settled_call_id(&result.call_id) {
return Err(ProviderBridgeError::invalid(
"legacy settled tool result cannot be replayed",
@@ -1510,6 +1645,100 @@ mod tests {
use super::*;
use serde_json::json;
#[test]
#[ignore = "explicit 100,000-call storage qualification"]
fn indexed_receipts_allow_one_hundred_thousand_calls_without_a_turn_reset() {
let directory =
std::env::temp_dir().join(format!("paperclip-bridge-indexed-{}", uuid::Uuid::new_v4()));
let path = directory.join("state.sqlite");
let store = IndexedStore::open(&path, "provider-session", true).unwrap();
let operation = AuthorizedTool {
operation_id: "get_task_context".into(),
version: 1,
description: "Read task context".into(),
input_schema: json!({"type":"object"}),
response_schema: json!({"type":"object"}),
};
let mut bridge = ProviderToolBridge::default();
bridge
.prepare(AuthorizedToolSet {
schema: TOOL_SET_SCHEMA.into(),
schema_version: 1,
catalog_digest: authorized_tool_catalog_digest(std::slice::from_ref(&operation))
.unwrap(),
operations: vec![operation],
})
.unwrap();
bridge
.bind_indexed_receipts(store.clone(), "semantic/session-1".into())
.unwrap();
for index in 0..100_000u64 {
let id = format!("call-{index}");
bridge
.begin_call(
id.clone(),
"get_task_context".into(),
json!({"index":index}),
)
.unwrap();
bridge
.apply_result(ToolResult {
call_id: id,
operation_id: "get_task_context".into(),
result: json!({"index":index}),
is_error: false,
})
.unwrap();
let mut candidate = bridge.clone();
let receipts = candidate.checkpoint_indexed_receipts().unwrap();
let bytes = serde_json::to_vec(&candidate).unwrap();
assert!(bytes.len() < 2_048);
store.commit("bridge", index, bytes, receipts).unwrap();
bridge = candidate;
assert!(!bridge.durable_run_receipt_limit_reached());
}
assert!(
bridge.pending.is_empty()
&& bridge.completed.is_empty()
&& bridge.settled_call_ids.is_empty()
);
drop(bridge);
drop(store);
let store = IndexedStore::open(&path, "provider-session", false).unwrap();
let snapshot = store.read_state("bridge").unwrap().unwrap();
assert_eq!(snapshot.generation, 100_000);
let mut recovered: ProviderToolBridge = serde_json::from_slice(&snapshot.bytes).unwrap();
// Missing storage is an error even when the in-memory cache is empty.
assert!(recovered
.replay_result("call-0", "get_task_context", &json!({"index":0}))
.is_err());
recovered
.bind_indexed_receipts(store.clone(), "semantic/session-1".into())
.unwrap();
recovered.attach_existing_run().unwrap();
assert_eq!(
recovered
.replay_result("call-0", "get_task_context", &json!({"index":0}))
.unwrap()
.unwrap()
.result,
json!({"index":0})
);
assert!(recovered
.replay_result("call-0", "get_task_context", &json!({"index":1}))
.is_err());
assert!(recovered
.begin_call(
"call-0".into(),
"get_task_context".into(),
json!({"index":0})
)
.is_err());
drop(recovered);
drop(store);
std::fs::remove_dir_all(directory).unwrap();
}
fn completion_bridge() -> ProviderToolBridge {
let operation = AuthorizedTool {
operation_id: "paperclip_finish".to_owned(),
@@ -1558,7 +1787,7 @@ mod tests {
error.safe_provider_message(),
Some(COMPLETION_INPUT_SCHEMA_HINT)
);
assert!(!over_limit.has_call_receipt("call-over-limit"));
assert!(!over_limit.has_call_receipt("call-over-limit").unwrap());
}
#[test]
@@ -1575,7 +1804,7 @@ mod tests {
assert!(message.contains("/properties/summary/type"), "{message}");
assert!(!message.contains("PRIVATE_FIELD"));
assert!(!message.contains("PRIVATE_VALUE"));
assert!(!bridge.has_call_receipt("bad-summary"));
assert!(!bridge.has_call_receipt("bad-summary").unwrap());
bridge
.begin_call(
"corrected-summary".to_owned(),
@@ -1684,7 +1913,7 @@ mod tests {
})
.unwrap();
assert!(!bridge.has_call_receipt("shared-call"));
assert!(!bridge.has_call_receipt("shared-call").unwrap());
bridge
.begin_call(
"shared-call".to_owned(),
@@ -1692,7 +1921,7 @@ mod tests {
json!({}),
)
.unwrap();
assert!(bridge.has_call_receipt("shared-call"));
assert!(bridge.has_call_receipt("shared-call").unwrap());
}
#[test]
@@ -1826,7 +2055,7 @@ mod tests {
let recovered: ProviderToolBridge =
serde_json::from_str(&serde_json::to_string(&bridge).unwrap()).unwrap();
recovered.validate_recovered().unwrap();
assert!(recovered.has_completed_call("call-0"));
assert!(recovered.has_completed_call("call-0").unwrap());
}
#[test]
packages/paperclip-runner/runner/crates/runner-core/src/provider_events.rs
+30 -4
View File
@@ -1,3 +1,4 @@
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
use sha2::{Digest, Sha256};
@@ -12,7 +13,7 @@ use crate::stable_identity::{
const MAX_TEXT_CHARS: usize = 4_000;
#[derive(Clone, Debug, PartialEq)]
#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)]
pub struct NormalizedProviderEvent {
pub event_type: String,
pub priority: EventPriority,
@@ -741,15 +742,15 @@ pub fn normalize_codex_notification(method: &str, params: &Value) -> Vec<Normali
),
}),
),
"item/agentMessage/delta" => push(
"item/agentMessage/delta" | "item/commandExecution/outputDelta" => push(
&mut events,
"item.delta",
EventPriority::P2,
json!({
"provider": "codex",
"itemId": stable_id(string(params.get("itemId")), "codex-message"),
"kind": "agentMessage",
"channel": "progress",
"kind": if method == "item/commandExecution/outputDelta" { "commandExecution" } else { "agentMessage" },
"channel": if method == "item/commandExecution/outputDelta" { "detail" } else { "progress" },
"providerMethod": method,
"text": bounded_text(string(params.get("delta")), MAX_TEXT_CHARS),
}),
@@ -1470,6 +1471,31 @@ mod tests {
assert!(!events[0].payload.to_string().contains("secret-value"));
}
#[test]
fn preserves_command_output_deltas_as_tool_output() {
// A provider completion snapshot can retain only part of its output.
// Preserve every admitted delta independently; never relabel it as prose.
let events = normalize_codex_notification(
"item/commandExecution/outputDelta",
&json!({"threadId":"thread", "turnId":"turn", "itemId":"exec-1",
"delta":"tool output Authorization: Bearer secret-value"}),
);
assert_eq!(events.len(), 1);
assert_eq!(events[0].event_type, "item.delta");
assert_eq!(events[0].payload["kind"], "commandExecution");
assert_eq!(events[0].payload["channel"], "detail");
assert_eq!(events[0].payload["itemId"], "exec-1");
assert_eq!(
events[0].payload["providerMethod"],
"item/commandExecution/outputDelta"
);
assert!(events[0].payload["text"]
.as_str()
.unwrap()
.starts_with("tool output"));
assert!(!events[0].payload.to_string().contains("secret-value"));
}
#[test]
fn bounds_and_redacts_command_output() {
let events = normalize_codex_notification(
packages/paperclip-runner/runner/crates/runner-core/src/storage_capacity.rs
+122
View File
@@ -0,0 +1,122 @@
//! A bounded, synchronous write can wait for capacity without unwinding its
//! command/effect stack. The owning runner supplies a control-only callback;
//! storage-only callers without one still receive the original error promptly.
use crate::durable::DurableRunnerError;
use std::{cell::RefCell, rc::Rc};
type Result<T> = std::result::Result<T, DurableRunnerError>;
type Callback = Box<dyn FnMut() -> Result<()>>;
/// Use only for an idempotent storage step or an exact token-bound commit.
/// Never wrap a provider/business operation in this retry helper.
pub(crate) fn retry<T>(mut operation: impl FnMut() -> Result<T>) -> Result<T> {
loop {
check_cancelled()?;
match operation() {
Err(error) if wait(&error)? => continue,
result => return result,
}
}
}
struct Waiter {
callback: RefCell<Callback>,
cancelled: RefCell<Option<DurableRunnerError>>,
}
thread_local! { static CURRENT: RefCell<Option<Rc<Waiter>>> = const { RefCell::new(None) }; }
/// The callback owns its transport state, so no borrowed pointer or unsafe
/// lifetime extension crosses this scope. Drop restores the prior scope even
/// when an operation panics. A child storage thread never inherits this hook.
pub(crate) fn with_wait<T>(
callback: impl FnMut() -> Result<()> + 'static,
operation: impl FnOnce() -> Result<T>,
) -> Result<T> {
struct Restore(Option<Rc<Waiter>>);
impl Drop for Restore {
fn drop(&mut self) {
CURRENT.with(|slot| {
slot.replace(self.0.take());
});
}
}
let waiter = Rc::new(Waiter {
callback: RefCell::new(Box::new(callback)),
cancelled: RefCell::new(None),
});
let _restore = Restore(CURRENT.with(|slot| slot.replace(Some(waiter.clone()))));
let result = operation();
let cancelled = waiter.cancelled.borrow().clone();
match cancelled {
Some(error) => Err(error),
None => result,
}
}
pub(crate) fn check_cancelled() -> Result<()> {
CURRENT.with(|slot| {
slot.borrow()
.as_ref()
.and_then(|waiter| waiter.cancelled.borrow().clone())
.map_or(Ok(()), Err)
})
}
pub(crate) fn wait(error: &DurableRunnerError) -> Result<bool> {
check_cancelled()?;
if !error.is_storage_capacity() {
return Ok(false);
}
let waiter = CURRENT.with(|slot| slot.borrow().clone());
let Some(waiter) = waiter else {
return Ok(false);
};
let result = waiter.callback.borrow_mut()();
match result {
Ok(()) => Ok(true),
Err(error) => {
let error = DurableRunnerError::storage_wait_cancelled(format!(
"storage_pressure: control wait stopped: {error}"
));
*waiter.cancelled.borrow_mut() = Some(error.clone());
Err(error)
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn only_physical_capacity_retries_and_cancel_is_sticky() {
let full = DurableRunnerError::storage_capacity("full");
assert!(!wait(&full).unwrap());
with_wait(
|| Ok(()),
|| {
assert!(wait(&full)?);
assert!(!wait(&DurableRunnerError::invalid(
"storage_pressure: record bound"
))?);
let result = with_wait(
|| Err(DurableRunnerError::invalid("revoked")),
|| {
assert!(wait(&full).unwrap_err().is_storage_wait_cancelled());
assert!(check_cancelled().is_err());
Ok(())
},
);
assert!(result.is_err());
assert!(wait(&full)?);
Ok(())
},
)
.unwrap();
assert!(!wait(&full).unwrap());
}
#[test]
fn panic_does_not_leak_a_callback() {
let _ =
std::panic::catch_unwind(|| with_wait(|| Ok(()), || -> Result<()> { panic!("test") }));
assert!(!wait(&DurableRunnerError::storage_capacity("full")).unwrap());
}
}
packages/paperclip-runner/runner/crates/runner-core/tests/acpx_indexed_receipts.rs
+297
View File
@@ -0,0 +1,297 @@
use std::path::PathBuf;
use std::time::Duration;
use paperclip_runner_core::acpx_provider_session::{
AcpxPermissionMode, AcpxProviderSession, AcpxProviderSessionConfig,
};
use paperclip_runner_core::acpx_provider_state::AcpxProviderStateEvent;
use paperclip_runner_core::acpx_sidecar_transport::AcpxSidecarTransportConfig;
use paperclip_runner_core::provider_bridge::{
authorized_tool_catalog_digest, AuthorizedTool, AuthorizedToolSet, ToolResult,
};
use serde_json::json;
fn tool_set() -> AuthorizedToolSet {
let operations = vec![AuthorizedTool {
operation_id: "issues.read".to_owned(),
version: 1,
description: "Read an issue.".to_owned(),
input_schema: json!({"type":"object"}),
response_schema: json!({"type":"object"}),
}];
AuthorizedToolSet {
schema: "paperclip.runner.authorized-tools.v1".to_owned(),
schema_version: 1,
catalog_digest: authorized_tool_catalog_digest(&operations).unwrap(),
operations,
}
}
fn config(mode: &str) -> AcpxProviderSessionConfig {
AcpxProviderSessionConfig {
transport: AcpxSidecarTransportConfig {
command: PathBuf::from(env!("CARGO_BIN_EXE_fake-acpx-sidecar")),
args: vec!["--mode".to_owned(), mode.to_owned()],
verified_launch: None,
request_timeout: Duration::from_secs(1),
shutdown_grace: Duration::from_millis(100),
},
agent: "codex".to_owned(),
model: "gpt-5.6-sol".to_owned(),
run_id: "run-1".to_owned(),
catalog_revision: 1,
runtime_directory: std::env::temp_dir(),
normalized_session_id: "session-1".to_owned(),
working_directory: std::env::temp_dir(),
permission_mode: AcpxPermissionMode::ApproveReads,
permission_mode_pinned: true,
system_instructions: "Complete the supplied task.".to_owned(),
runtime_context: serde_json::Value::Null,
tool_set: tool_set(),
expected_identity: None,
}
}
use paperclip_runner_core::indexed_store::IndexedStore;
use serde_json::Value;
use std::fs;
struct Directory(PathBuf);
impl Directory {
fn new() -> Self {
let path =
std::env::temp_dir().join(format!("paperclip-acpx-indexed-{}", uuid::Uuid::new_v4()));
fs::create_dir(&path).unwrap();
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
fs::set_permissions(&path, fs::Permissions::from_mode(0o700)).unwrap();
}
Self(path)
}
fn path(&self) -> PathBuf {
self.0.join("acpx-provider-receipts.sqlite")
}
}
impl Drop for Directory {
fn drop(&mut self) {
let _ = fs::remove_dir_all(&self.0);
}
}
#[test]
fn retains_exact_receipts_beyond_the_active_turn_limit_without_growing_current_state() {
let directory = Directory::new();
let mut config = config("indexed-many-tools");
let expected_calls: usize = std::env::var("PAPERCLIP_ACPX_RECEIPT_QUALIFICATION")
.map(|value| value.parse().expect("invalid qualification call count"))
.unwrap_or(64);
assert!(expected_calls >= 64);
config
.transport
.args
.extend(["--calls".to_owned(), expected_calls.to_string()]);
let mut session =
AcpxProviderSession::start_with_indexed_receipts(&config, &directory.path()).unwrap();
let pid = session.process_id();
session
.start_turn(
"turn-1",
"Run many sequential calls",
&config.working_directory,
)
.unwrap();
let reader = IndexedStore::open(&directory.path(), "acpx/session-1", false).unwrap();
let initial_current = reader
.read_state("acpx-session")
.unwrap()
.unwrap()
.bytes
.len();
let mut calls = 0;
let mut peak = 0;
while session.state().active_turn_id().is_some() {
let events = session
.poll_event(Duration::from_secs(10))
.unwrap()
.unwrap();
if !events.is_empty() {
let batch = session.pending_event_batch().unwrap();
assert_eq!(session.poll_event(Duration::ZERO).unwrap().unwrap(), events);
assert_eq!(session.pending_event_batch().as_ref(), Some(&batch));
}
// This consumer's acceptance is deliberately explicit. The production
// executor does this only after saving its normalized outbox.
session.acknowledge_event_batch().unwrap();
for event in events {
if let AcpxProviderStateEvent::ToolCall {
call_id,
operation_id,
input,
} = event
{
assert_eq!(input["index"], calls);
session
.deliver_tool_result(&ToolResult {
call_id,
operation_id,
result: json!({"index":calls}),
is_error: false,
})
.unwrap();
calls += 1;
}
}
peak = peak.max(
reader
.read_state("acpx-session")
.unwrap()
.unwrap()
.bytes
.len(),
);
assert_eq!(session.process_id(), pid);
}
assert_eq!(calls, expected_calls);
println!("ACPX indexed receipts: calls={calls}, initial_current_bytes={initial_current}, peak_current_bytes={peak}");
assert!(
peak <= initial_current + 4096,
"current checkpoint grew from {initial_current} to {peak} bytes"
);
let ancient = reader
.receipt("acpx/run-1/turn-1/dynamic", "history-call-0")
.unwrap()
.unwrap();
let ancient: Value = serde_json::from_slice(&ancient).unwrap();
assert_eq!(ancient["receipt"]["result"]["result"], json!({"index":0}));
assert!(reader
.receipt("acpx-turns/run-1", "turn-1")
.unwrap()
.is_some());
let mut recovered_config = config.clone();
recovered_config.expected_identity = Some(session.identity().clone());
session.shutdown("finished").unwrap();
drop(session);
let mut recovered =
AcpxProviderSession::start_with_indexed_receipts(&recovered_config, &directory.path())
.unwrap();
assert!(recovered
.start_turn(
"turn-1",
"Must reject ancient turn",
&config.working_directory
)
.unwrap_err()
.to_string()
.contains("settled turn identity"));
recovered.shutdown("finished").unwrap();
}
#[test]
fn rejects_unacknowledged_history_before_spawning_a_replacement() {
let directory = Directory::new();
let mut config = config("indexed-many-tools");
let mut session =
AcpxProviderSession::start_with_indexed_receipts(&config, &directory.path()).unwrap();
session
.start_turn("turn-1", "One pending call", &config.working_directory)
.unwrap();
let events = session
.poll_event(Duration::from_secs(10))
.unwrap()
.unwrap();
assert!(!events.is_empty());
config.expected_identity = Some(session.identity().clone());
session.shutdown("simulated interrupted consumer").unwrap();
drop(session);
// A valid executable with an observable side effect proves that the
// retained-state guard runs before process creation.
let command = directory.0.join("must-not-spawn");
fs::write(&command, "#!/bin/sh\n: > \"$0.spawned\"\nexit 72\n").unwrap();
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
fs::set_permissions(&command, fs::Permissions::from_mode(0o700)).unwrap();
}
config.transport.command = command;
let error = match AcpxProviderSession::start_with_indexed_receipts(&config, &directory.path()) {
Ok(_) => panic!("unacknowledged batch granted a replacement"),
Err(error) => error,
};
assert!(error.to_string().contains("requires reconciliation"));
assert!(!directory.0.join("must-not-spawn.spawned").exists());
let reader = IndexedStore::open(&directory.path(), "acpx/session-1", false).unwrap();
let current = reader.read_state("acpx-session").unwrap().unwrap();
let value: Value = serde_json::from_slice(&current.bytes).unwrap();
assert!(!value["events"].as_array().unwrap().is_empty());
}
#[test]
fn an_unconfirmed_resolution_retains_its_exact_receipt_and_delivery_intent() {
let directory = Directory::new();
let config = config("resolutions-wrong-ack");
let mut session =
AcpxProviderSession::start_with_indexed_receipts(&config, &directory.path()).unwrap();
session
.start_turn("turn-1", "Resolve one call", &config.working_directory)
.unwrap();
session
.poll_event(Duration::from_secs(10))
.unwrap()
.unwrap();
session.acknowledge_event_batch().unwrap();
let result = ToolResult {
call_id: "call-1".into(),
operation_id: "issues.read".into(),
result: json!({"id":"issue-1"}),
is_error: false,
};
assert!(session.deliver_tool_result(&result).is_err());
let reader = IndexedStore::open(&directory.path(), "acpx/session-1", false).unwrap();
assert!(reader
.receipt("acpx/run-1/turn-1/dynamic", "call-1")
.unwrap()
.is_some());
let value: Value =
serde_json::from_slice(&reader.read_state("acpx-session").unwrap().unwrap().bytes).unwrap();
assert_eq!(value["delivery"], serde_json::to_value(&result).unwrap());
session.shutdown("finished").unwrap();
}
#[test]
fn a_failed_receipt_commit_never_releases_the_tool_result_to_the_provider() {
let directory = Directory::new();
let mut config = config("indexed-many-tools");
let marker = directory.0.join("resolution-received");
config.transport.args.extend([
"--resolution-marker".to_owned(),
marker.to_string_lossy().into_owned(),
]);
let mut session =
AcpxProviderSession::start_with_indexed_receipts(&config, &directory.path()).unwrap();
session
.start_turn("turn-1", "One pending call", &config.working_directory)
.unwrap();
session
.poll_event(Duration::from_secs(10))
.unwrap()
.unwrap();
session.acknowledge_event_batch().unwrap();
let database = rusqlite::Connection::open(directory.path()).unwrap();
database.execute_batch("CREATE TRIGGER reject_checkpoint BEFORE UPDATE ON current_state BEGIN SELECT RAISE(ABORT, 'fixture checkpoint failure'); END;").unwrap();
let error = session
.deliver_tool_result(&ToolResult {
call_id: "history-call-0".into(),
operation_id: "issues.read".into(),
result: json!({"index":0}),
is_error: false,
})
.unwrap_err();
assert!(error.to_string().contains("fixture checkpoint failure"));
assert!(
!marker.exists(),
"tool result escaped before its durable receipt"
);
assert!(session.state().pending_tool("history-call-0").is_some());
session.shutdown("finished").unwrap();
}
packages/paperclip-runner/runner/crates/runner-core/tests/acpx_warm_attachment.rs
+1
View File
@@ -24,6 +24,7 @@ impl Drop for Fixture {
fn command(sequence: u64, kind: &str, payload: Value) -> Command {
Command {
controller_epoch: None,
schema: "paperclip.prp.command.v1".to_owned(),
command_id: format!("command-{sequence}"),
controller_seq: sequence,
packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs
+267 -16
View File
@@ -166,6 +166,7 @@ fn durable_config(directory: &Path) -> DurableRunnerConfig {
fn command(id: &str, sequence: u64, command_type: &str, payload: Value) -> Command {
Command {
controller_epoch: None,
schema: "paperclip.prp.command.v1".to_owned(),
command_id: id.to_owned(),
controller_seq: sequence,
@@ -3282,8 +3283,10 @@ fn durable_backend_replays_pending_tool_calls_without_mutating_the_event_queue()
let resume_count = call_count(&directory, "thread/resume");
let mut next = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
let mut session_resumed_events = 0;
let mut replay_events = Vec::new();
for _ in 0..32 {
let events = poll_and_ack(&mut next).expect("poll exact pending replay");
replay_events.extend(events.iter().cloned());
session_resumed_events += events
.iter()
.filter(|event| event.event_type == "session.resumed")
@@ -3303,6 +3306,7 @@ fn durable_backend_replays_pending_tool_calls_without_mutating_the_event_queue()
);
for _ in 0..4 {
let events = poll_and_ack(&mut next).expect("poll exact pending replay");
replay_events.extend(events.iter().cloned());
session_resumed_events += events
.iter()
.filter(|event| event.event_type == "session.resumed")
@@ -3331,8 +3335,28 @@ fn durable_backend_replays_pending_tool_calls_without_mutating_the_event_queue()
);
assert_eq!(
after["nextProviderEventSeq"].as_u64(),
before["nextProviderEventSeq"].as_u64().map(|sequence| sequence + 3),
"exact restore adds one session.resumed and two durable startup facts, never duplicate tool inputs, during replay {replay}"
before["nextProviderEventSeq"].as_u64(),
"new identities do not consume the retained numeric namespace"
);
assert_eq!(replay_events.len(), 3, "exact restore adds one session.resumed and two durable startup facts during replay {replay}");
assert_eq!(
replay_events
.iter()
.filter(|event| {
event.event_type == "harness.diagnostic"
&& event.payload["code"] == "provider_startup_ownership"
})
.count(),
2
);
let ids: std::collections::BTreeSet<_> = replay_events
.iter()
.map(|event| event.executor_event_id.as_str())
.collect();
assert_eq!(
ids.len(),
3,
"each restored fact has a distinct exact identity"
);
recovered = Some(next);
if replay < 3 {
@@ -4915,21 +4939,58 @@ fn receipt_limit_rejects_the_call_and_keeps_polling_when_interrupt_fails() {
saturate_provider_tool_receipts(&directory);
let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
let resumed = wait_for_executor_event(&mut recovered, "session.resumed");
assert_eq!(resumed.payload["provider"], "codex");
let diagnostic = wait_for_executor_event(&mut recovered, "harness.diagnostic");
assert_eq!(
diagnostic.payload["code"],
"semantic_tool_turn_receipt_limit"
);
// Debug builds can spend the entire two-second courtesy window persisting
// the deliberately saturated legacy JSON ledger. Both outcomes are valid:
// an authoritative interrupt after retries, or conservative failure after
// the exact provider is reaped. Collect the entire acknowledged batch so a
// terminal delivered alongside its diagnostic is not discarded by a helper.
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(45);
let mut resumed = false;
let mut diagnostic = false;
let mut terminal = None;
while std::time::Instant::now() < deadline && terminal.is_none() {
for event in poll_and_ack(&mut recovered).expect("poll receipt-limit settlement") {
resumed |= event.event_type == "session.resumed";
diagnostic |= event.event_type == "harness.diagnostic"
&& event.payload["code"] == "semantic_tool_turn_receipt_limit";
assert_ne!(
event.event_type, "semantic_tool.input",
"overflowing call must never execute"
);
if matches!(
event.event_type.as_str(),
"turn.interrupted" | "turn.failed"
) {
terminal = Some(event);
}
}
if terminal.is_none() {
std::thread::sleep(std::time::Duration::from_millis(1));
}
}
assert!(resumed && diagnostic);
assert_eq!(call_count(&directory, "tool-response:failure"), 1);
assert_eq!(call_count(&directory, "turn/interrupt"), 1);
let interrupted = wait_for_executor_event(&mut recovered, "turn.interrupted");
assert_eq!(
interrupted.payload["provider"], "codex",
"the retry must settle the receipt-exhausted turn"
);
assert_eq!(call_count(&directory, "turn/interrupt"), 3);
let terminal = terminal.expect("receipt-limit interruption must settle before the deadline");
assert_eq!(terminal.payload["provider"], "codex");
let attempts = call_count(&directory, "turn/interrupt");
assert!((1..=3).contains(&attempts));
if terminal.event_type == "turn.failed" {
assert_eq!(
terminal.payload["code"],
"semantic_tool_turn_receipt_limit_interrupt_unconfirmed"
);
assert_eq!(terminal.payload["interruptAccepted"], false);
assert_eq!(terminal.payload["providerTerminalObserved"], false);
assert_eq!(terminal.payload["providerShutdownFailed"], false);
assert_eq!(attempts, 1, "only the configured first RPC is rejected");
} else if attempts < 3 {
assert_eq!(
terminal.payload["code"],
"semantic_tool_turn_receipt_limit_interrupt_deadline"
);
assert_eq!(terminal.payload["interruptAccepted"], true);
assert_eq!(terminal.payload["providerShutdownFailed"], false);
}
recovered.shutdown().expect("stop recovered provider");
fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
@@ -6217,6 +6278,38 @@ fn lightweight_history_repeated_cursor_is_not_idle_evidence() {
fs::remove_dir_all(directory).unwrap();
}
#[test]
fn lightweight_history_settlement_race_does_not_scan_completed_history() {
for settled in [false, true] {
let directory = temporary_directory("history-settlement-race");
let mut args = vec![
"--hold-turn",
"--require-lightweight-history",
"--unbounded-completed-history",
];
if settled {
args.push("--settle-during-history");
}
let config = provider_config(&directory, &args);
let mut provider = CodexProvider::start(&config, None).unwrap();
provider.start_turn("Keep working", &config.cwd).unwrap();
let result = provider.read_thread();
if settled {
assert_eq!(result.unwrap()["thread"]["turns"], json!([]));
} else {
assert!(result
.unwrap_err()
.to_string()
.contains("active turn absent from current history window"));
}
let calls = fs::read_to_string(directory.join("calls.log")).unwrap();
assert_eq!(calls.matches("thread/turns/list").count(), 4);
assert_eq!(calls.matches("thread/read").count(), 2);
provider.shutdown().unwrap();
fs::remove_dir_all(directory).unwrap();
}
}
fn skill_wire_requests(directory: &Path) -> Vec<Value> {
fs::read_to_string(directory.join("requests.ndjson"))
.unwrap()
@@ -6225,6 +6318,164 @@ fn skill_wire_requests(directory: &Path) -> Vec<Value> {
.collect()
}
fn indexed_provider_history_directory(label: &str) -> PathBuf {
let directory = temporary_directory(label);
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
fs::set_permissions(&directory, fs::Permissions::from_mode(0o700)).unwrap();
}
directory
}
#[test]
fn indexed_provider_history_selects_paginated_start_and_verifies_cold_resume() {
let directory = indexed_provider_history_directory("indexed-provider-history");
let log = directory.join("requests.ndjson");
let config = provider_config(
&directory,
&[
"--require-paginated-history",
"--request-log",
log.to_str().unwrap(),
],
);
let runner_config = durable_config(&directory);
let mut executor = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
executor
.execute(&command(
"prepare",
1,
"run.prepare",
json!({
"provider": config, "durability": "durability.indexed_state.v1"
}),
))
.unwrap();
executor
.execute(&command("open", 2, "session.open", json!({})))
.unwrap();
executor.shutdown().unwrap();
drop(executor);
let mut restored = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
restored
.execute(&command(
"turn",
3,
"turn.start",
json!({"text": "Continue"}),
))
.unwrap();
restored.shutdown().unwrap();
drop(restored);
let frames = skill_wire_requests(&directory);
let start = frames
.iter()
.find(|frame| frame["method"] == "thread/start")
.unwrap();
let resume = frames
.iter()
.find(|frame| frame["method"] == "thread/resume")
.unwrap();
assert_eq!(start["params"]["historyMode"], "paginated");
assert_eq!(resume["params"]["threadId"], "codex-thread-1");
assert_eq!(resume["params"]["excludeTurns"], true);
// Resume verifies the persisted contract; it cannot convert it by request.
assert!(resume["params"].get("historyMode").is_none());
fs::remove_dir_all(directory).unwrap();
}
#[test]
fn indexed_provider_history_rejects_unsupported_mode_before_a_turn() {
for switch in ["--legacy-history-mode", "--missing-history-mode"] {
let directory = indexed_provider_history_directory("unsupported-indexed-provider-history");
let config = provider_config(&directory, &[switch]);
let mut executor =
CodexCommandExecutor::with_runner_config(&directory, &durable_config(&directory));
executor
.execute(&command(
"prepare",
1,
"run.prepare",
json!({
"provider": config, "durability": "durability.indexed_state.v1"
}),
))
.unwrap();
assert!(executor
.execute(&command("open", 2, "session.open", json!({})))
.unwrap_err()
.to_string()
.contains("codex_history_mode_unsupported"));
assert_eq!(call_count(&directory, "turn/start"), 0);
drop(executor);
fs::remove_dir_all(directory).unwrap();
}
}
#[test]
fn indexed_provider_history_rejects_a_legacy_cold_resume_without_starting_fresh() {
let directory = indexed_provider_history_directory("unsupported-resumed-provider-history");
let config = provider_config(&directory, &[]);
let runner_config = durable_config(&directory);
let mut executor = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
executor
.execute(&command(
"prepare",
1,
"run.prepare",
json!({
"provider": config, "durability": "durability.indexed_state.v1"
}),
))
.unwrap();
executor
.execute(&command("open", 2, "session.open", json!({})))
.unwrap();
executor.shutdown().unwrap();
drop(executor);
fs::write(directory.join("legacy-history-mode"), "legacy").unwrap();
let mut restored = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
assert!(restored
.execute(&command(
"turn",
3,
"turn.start",
json!({"text": "Continue"})
))
.unwrap_err()
.to_string()
.contains("codex_history_mode_unsupported"));
assert_eq!(call_count(&directory, "thread/start"), 1);
assert_eq!(call_count(&directory, "thread/resume"), 1);
assert_eq!(call_count(&directory, "turn/start"), 0);
drop(restored);
fs::remove_dir_all(directory).unwrap();
}
#[test]
fn legacy_provider_history_preserves_the_provider_default() {
let directory = temporary_directory("legacy-provider-history-default");
let log = directory.join("requests.ndjson");
let config = provider_config(
&directory,
&[
"--legacy-history-mode",
"--request-log",
log.to_str().unwrap(),
],
);
let mut provider = CodexProvider::start(&config, None).unwrap();
provider.shutdown().unwrap();
let frames = skill_wire_requests(&directory);
let start = frames
.iter()
.find(|frame| frame["method"] == "thread/start")
.unwrap();
assert!(start["params"].get("historyMode").is_none());
fs::remove_dir_all(directory).unwrap();
}
#[test]
fn skill_instructions_flag_reaches_start_and_resume_and_preserves_absent_config() {
for flag in [Some(true), Some(false), None] {
packages/paperclip-runner/runner/crates/runner-core/tests/durable_recovery.rs
+1
View File
@@ -36,6 +36,7 @@ fn config(state_dir: PathBuf) -> DurableRunnerConfig {
fn command() -> Command {
Command {
controller_epoch: None,
schema: "paperclip.prp.command.v1".to_owned(),
command_id: "command_1".to_owned(),
controller_seq: 1,
packages/paperclip-runner/runner/crates/runner-core/tests/indexed_history_growth.rs
+166
View File
@@ -0,0 +1,166 @@
use paperclip_runner_core::indexed_store::{ExactReceipt, IndexedStore};
use std::{fs, path::Path, time::Instant};
fn tree_metrics(path: &Path, depth: usize) -> (u64, u64, u64) {
assert!(depth <= 2, "partition directory fanout exceeded the bound");
let root_metadata = fs::symlink_metadata(path).unwrap();
assert!(root_metadata.is_dir() && !root_metadata.file_type().is_symlink());
let mut physical = 0u64;
let mut allocated = 0u64;
let mut largest_file = 0u64;
// Iterate entries as a stream: routing history is a bounded-depth hash
// fanout, so qualification does not materialize the file list in memory.
for entry in fs::read_dir(path).unwrap() {
let entry = entry.unwrap();
let kind = entry.file_type().unwrap();
assert!(!kind.is_symlink(), "partition tree contains a symlink");
if kind.is_dir() {
let (child_physical, child_allocated, child_largest) =
tree_metrics(&entry.path(), depth + 1);
physical += child_physical;
allocated += child_allocated;
largest_file = largest_file.max(child_largest);
continue;
}
assert!(kind.is_file(), "partition entry is not a regular file");
let metadata = fs::symlink_metadata(entry.path()).unwrap();
assert!(metadata.is_file() && !metadata.file_type().is_symlink());
physical += metadata.len();
largest_file = largest_file.max(metadata.len());
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
allocated += metadata.blocks() * 512;
}
#[cfg(not(unix))]
{
allocated += metadata.len();
}
}
(physical, allocated, largest_file)
}
#[test]
#[ignore = "explicit 10 GiB storage qualification; requires local disk space"]
fn ten_gib_history_keeps_resume_and_ancient_replay_indexed() {
let root =
std::env::temp_dir().join(format!("paperclip-history-growth-{}", uuid::Uuid::new_v4()));
let path = root.join("state.sqlite");
let mut store = IndexedStore::open(&path, "growth-qualification", true).unwrap();
let mut payload = vec![0u8; 64 * 1024];
let mut random = 0x12345678u32;
for byte in &mut payload {
random ^= random << 13;
random ^= random >> 17;
random ^= random << 5;
*byte = random as u8;
}
let current = b"bounded-current-state";
let mut generation = paperclip_runner_core::indexed_revision::Revision::Absent;
let mut written = 0u64;
let started = Instant::now();
let gib = std::env::var("PAPERCLIP_HISTORY_QUALIFICATION_GIB")
.unwrap_or_else(|_| "10".into())
.parse::<u64>()
.unwrap();
assert!(
[10, 100].contains(&gib),
"qualification size must be 10 or 100 GiB"
);
let mut targets = vec![
16 * 1024 * 1024u64,
1024 * 1024 * 1024,
10 * 1024 * 1024 * 1024,
];
if gib == 100 {
targets.push(100 * 1024 * 1024 * 1024);
}
for target in targets {
let mut commits = Vec::new();
while written < target {
let first = written / payload.len() as u64;
let receipts = (first..first + 256)
.map(|id| ExactReceipt {
namespace: "history".into(),
key: format!("event-{id:020}"),
bytes: payload.clone(),
})
.collect();
let now = Instant::now();
generation = store
.commit("authority", generation, current.to_vec(), receipts)
.unwrap();
commits.push(now.elapsed().as_secs_f64() * 1000.0);
written += 256 * payload.len() as u64;
}
drop(store);
let now = Instant::now();
store = IndexedStore::open(&path, "growth-qualification", false).unwrap();
let restored = store.read_state("authority").unwrap().unwrap();
let reopen_ms = now.elapsed().as_secs_f64() * 1000.0;
assert_eq!(restored.bytes, current);
assert_eq!(restored.generation, generation);
assert_eq!(
store
.receipt("history", "event-00000000000000000000")
.unwrap()
.unwrap(),
payload
);
let mut reads = Vec::new();
for _ in 0..100 {
let now = Instant::now();
assert_eq!(
store.read_state("authority").unwrap().unwrap().bytes,
current
);
reads.push(now.elapsed().as_secs_f64() * 1000.0);
}
reads.sort_by(f64::total_cmp);
commits.sort_by(f64::total_cmp);
let mut partitions = 0usize;
let mut largest_logical_partition = 0u64;
let mut after: Option<String> = None;
loop {
let page = store.partitions(after.as_deref(), 128).unwrap();
if page.is_empty() {
break;
}
partitions += page.len();
largest_logical_partition = largest_logical_partition.max(
page.iter()
.map(|partition| partition.bytes)
.max()
.unwrap_or(0),
);
after = page.last().map(|partition| partition.lower_key.clone());
if page.len() < 128 {
break;
}
}
let root_database_bytes = fs::metadata(&path).unwrap().len();
let (partition_files_bytes, allocated_partition_bytes, largest_file) =
tree_metrics(&root.join("state.sqlite.receipts"), 0);
let (routing_page_files_bytes, allocated_routing_page_bytes, largest_routing_page_file) =
tree_metrics(&root.join("state.sqlite.routing"), 0);
if target >= 1024 * 1024 * 1024 {
assert!(partitions > 1);
assert!(
largest_logical_partition < 1024 * 1024 * 1024,
"partition grew beyond the maintenance envelope"
);
}
println!(
"history_bytes={written} root_database_bytes={root_database_bytes} receipt_partition_files_bytes={partition_files_bytes} allocated_receipt_partition_bytes={allocated_partition_bytes} receipt_partitions={partitions} largest_receipt_partition_file_bytes={largest_file} routing_page_files_bytes={routing_page_files_bytes} allocated_routing_page_bytes={allocated_routing_page_bytes} largest_routing_page_file_bytes={largest_routing_page_file} largest_logical_partition_bytes={largest_logical_partition} current_bytes={} reopen_ms={reopen_ms:.3} read_p95_ms={:.3} batch_commit_p95_ms={:.3}",
current.len(),
reads[94],
commits[(commits.len() - 1) * 95 / 100]
);
}
drop(store);
fs::remove_dir_all(root).unwrap();
println!(
"qualification_elapsed_seconds={:.3}",
started.elapsed().as_secs_f64()
);
}
packages/paperclip-runner/runner/crates/runner-core/tests/indexed_session_snapshot.rs
+163
View File
@@ -0,0 +1,163 @@
use paperclip_runner_core::indexed_store::{ExactReceipt, IndexedStore};
use rusqlite::Connection;
use serde_json::json;
use sha2::{Digest, Sha256};
use std::{
fs,
path::PathBuf,
process::{Command, Stdio},
time::{Duration, Instant},
};
#[test]
fn sigkill_during_native_snapshot_resumes_exact_receipts_and_restores_without_source() {
let root =
std::env::temp_dir().join(format!("paperclip-snapshot-crash-{}", uuid::Uuid::new_v4()));
fs::create_dir(&root).unwrap();
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
fs::set_permissions(&root, fs::Permissions::from_mode(0o700)).unwrap();
}
let source = root.join("source");
let destination = root.join("snapshot");
let names = ["runner-state", "codex-provider-state"];
let bindings = ["runner/r/s", "provider/r/s"];
let keys = ["runner", "codex-provider"];
let mut hashes = vec![];
for i in 0..2 {
let store = IndexedStore::open(
&source.join(format!("{}.sqlite", names[i])),
bindings[i],
true,
)
.unwrap();
let locator = json!({"schema":if i==0 {"paperclip.runner.durable.state.indexed.v1"} else {"paperclip.runner.codex-provider-state.indexed.v1"},"binding":bindings[i]});
let file = source.join(format!("{}.json", names[i]));
fs::write(&file, serde_json::to_vec(&locator).unwrap()).unwrap();
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
fs::set_permissions(file, fs::Permissions::from_mode(0o600)).unwrap();
}
let state = if i == 0 {
json!({"lifecycle":"suspended","runnerInstanceId":"r","normalizedSessionId":"s","nextSourceSeq":2,"ackedSourceSeq":1})
} else {
json!({"lifecycle":"ready","pendingEvents":[],"queuedEvents":[]})
};
let bytes = serde_json::to_vec(&state).unwrap();
hashes.push(format!("{:x}", Sha256::digest(&bytes)));
store
.commit(
keys[i],
0,
bytes,
(0..2000)
.map(|n| ExactReceipt {
namespace: "ancient".into(),
key: format!("{n:08}"),
bytes: vec![(n % 251) as u8; 16 * 1024],
})
.collect(),
)
.unwrap();
}
let args = vec![
"storage".to_owned(),
"snapshot-session".into(),
"--directory".into(),
source.to_string_lossy().into_owned(),
"--destination".into(),
destination.to_string_lossy().into_owned(),
"--runner-generation".into(),
"1".into(),
"--runner-sha256".into(),
hashes[0].clone(),
"--provider-generation".into(),
"1".into(),
"--provider-sha256".into(),
hashes[1].clone(),
];
let mut child = Command::new(env!("CARGO_BIN_EXE_paperclip-runnerd"))
.args(&args)
.stdout(Stdio::null())
.stderr(Stdio::null())
.spawn()
.unwrap();
let deadline = Instant::now() + Duration::from_secs(30);
let mut copied = false;
while Instant::now() < deadline {
let db = Connection::open_with_flags(
source.join("runner-state.sqlite"),
rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY,
)
.unwrap();
let staging = db.query_row(
"SELECT staging FROM receipt_backup WHERE phase='copying'",
[],
|row| row.get::<_, String>(0),
);
if let Ok(staging) = staging {
if let Ok(target) = Connection::open_with_flags(
PathBuf::from(staging).join("runner-state.sqlite"),
rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY,
) {
copied = target
.query_row(
"SELECT EXISTS(SELECT 1 FROM backup_copy WHERE cursor>0)",
[],
|row| row.get::<_, bool>(0),
)
.unwrap_or(false);
}
}
if copied {
break;
}
if let Some(status) = child.try_wait().unwrap() {
panic!("snapshot exited before crash boundary: {status}");
}
std::thread::sleep(Duration::from_millis(1));
}
// Kill only the exact child owned by this test, and reap it before reopening.
child.kill().unwrap();
let status = child.wait().unwrap();
assert!(!status.success());
assert!(copied, "no durable partial copy observed");
assert!(
!destination.exists(),
"partial snapshot must not be published"
);
let resumed = Command::new(env!("CARGO_BIN_EXE_paperclip-runnerd"))
.args(&args)
.output()
.unwrap();
assert!(
resumed.status.success(),
"{}",
String::from_utf8_lossy(&resumed.stderr)
);
fs::remove_dir_all(&source).unwrap();
for i in 0..2 {
let restored = IndexedStore::open(
&destination.join(format!("{}.sqlite", names[i])),
bindings[i],
false,
)
.unwrap();
assert_eq!(
format!(
"{:x}",
Sha256::digest(restored.read_state(keys[i]).unwrap().unwrap().bytes)
),
hashes[i]
);
for n in [0, 127, 1024, 1999] {
assert_eq!(
restored.receipt("ancient", &format!("{n:08}")).unwrap(),
Some(vec![(n % 251) as u8; 16 * 1024])
);
}
}
fs::remove_dir_all(root).unwrap();
}
packages/paperclip-runner/runner/crates/runner-core/tests/indexed_sqlite_locks.rs
+101
View File
@@ -0,0 +1,101 @@
use std::{fs, path::PathBuf, process::Command};
use paperclip_runner_core::indexed_store::{read_only_receipt, ExactReceipt, IndexedStore};
use rusqlite::{Connection, OpenFlags};
const TEST: &str = "preserves_sqlite_locks_across_same_process_opens_and_external_readers";
#[test]
fn preserves_sqlite_locks_across_same_process_opens_and_external_readers() {
let role = std::env::var("PAPERCLIP_SQLITE_LOCK_TEST_ROLE").ok();
let root = std::env::var_os("PAPERCLIP_SQLITE_LOCK_TEST_ROOT").map(PathBuf::from);
if role.as_deref() == Some("reader") {
let db = Connection::open_with_flags(
fs::canonicalize(root.unwrap().join("authority.sqlite")).unwrap(),
OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NOFOLLOW,
)
.unwrap();
assert_eq!(
db.query_row("SELECT count(*) FROM current_state", [], |r| r
.get::<_, i64>(0))
.unwrap(),
1
);
return;
}
if role.as_deref() == Some("writer") {
let root = root.unwrap();
let path = root.join("authority.sqlite");
let store = IndexedStore::open(&path, "lock-regression", true).unwrap();
let mut generation = paperclip_runner_core::indexed_revision::Revision::Absent;
for sequence in 0..24 {
generation = store
.commit(
"authority",
generation,
serde_json::to_vec(
&serde_json::json!({"sequence":sequence,"padding":"x".repeat(64 * 1024)}),
)
.unwrap(),
vec![ExactReceipt {
namespace: "receipts".into(),
key: sequence.to_string(),
bytes: sequence.to_string().into_bytes(),
}],
)
.unwrap();
// These public paths used to open and close SQLite's database/SHM
// descriptors outside its VFS, silently cancelling POSIX locks.
let second = IndexedStore::open(&path, "lock-regression", false).unwrap();
assert_eq!(
second.read_state("authority").unwrap().unwrap().generation,
generation
);
drop(second);
assert_eq!(
read_only_receipt(&path, "lock-regression", generation, "receipts", "0").unwrap(),
Some(b"0".to_vec())
);
let result = Command::new(std::env::current_exe().unwrap())
.args(["--exact", TEST, "--nocapture"])
.env("PAPERCLIP_SQLITE_LOCK_TEST_ROLE", "reader")
.env("PAPERCLIP_SQLITE_LOCK_TEST_ROOT", &root)
.output()
.unwrap();
assert!(
result.status.success(),
"external reader failed: {}",
String::from_utf8_lossy(&result.stderr)
);
}
assert_eq!(
store.read_state("authority").unwrap().unwrap().generation,
24
);
return;
}
let root =
std::env::temp_dir().join(format!("paperclip-sqlite-locks-{}", uuid::Uuid::new_v4()));
fs::create_dir(&root).unwrap();
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
fs::set_permissions(&root, fs::Permissions::from_mode(0o700)).unwrap();
}
// A regression can SIGBUS on macOS or corrupt state on other systems. Keep
// it in a child so the test reports the failure instead of losing its suite.
let result = Command::new(std::env::current_exe().unwrap())
.args(["--exact", TEST, "--nocapture"])
.env("PAPERCLIP_SQLITE_LOCK_TEST_ROLE", "writer")
.env("PAPERCLIP_SQLITE_LOCK_TEST_ROOT", &root)
.output()
.unwrap();
assert!(
result.status.success(),
"writer exited {:?}; retained evidence at {}: {}",
result.status,
root.display(),
String::from_utf8_lossy(&result.stderr)
);
fs::remove_dir_all(root).unwrap();
}
packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs
+1
View File
@@ -150,6 +150,7 @@ fn assert_valid_terminal(payload: &Value) {
fn command(sequence: u64, command_type: &str, payload: Value) -> Command {
Command {
controller_epoch: None,
schema: "paperclip.prp.command.v1".to_owned(),
command_id: format!("command-{sequence}"),
controller_seq: sequence,
packages/paperclip-runner/runner/crates/runner-core/tests/process_supervisor.rs
+3 -1
View File
@@ -110,7 +110,9 @@ fn verified_launch_uses_open_command_and_script_after_atomic_path_replacement()
.unwrap();
assert_eq!(
process
.receive_stdout_line(Duration::from_secs(1))
// This checks the authenticated artifact, not a one-second process
// startup SLA. Loaded hosts can delay the first scheduled output.
.receive_stdout_line(Duration::from_secs(5))
.unwrap()
.as_deref(),
Some("old-command")
packages/paperclip-runner/runner/crates/runner-core/tests/provider_bridge.rs
+11 -11
View File
@@ -758,8 +758,8 @@ fn exact_identity_overflow_saturates_the_durable_run() {
.settle_turn("provider_turn_terminated")
.expect("the controlled turn stop retains replay protection");
assert!(recovered.durable_run_receipt_limit_reached());
assert!(recovered.has_completed_call("settled-0"));
assert!(recovered.has_completed_call("last-call"));
assert!(recovered.has_completed_call("settled-0").unwrap());
assert!(recovered.has_completed_call("last-call").unwrap());
let stopped_turn_receipt = recovered
.replay_result("last-call", "get_task_context", &json!({}))
.unwrap()
@@ -780,8 +780,8 @@ fn exact_identity_overflow_saturates_the_durable_run() {
.replay_result("last-call", "get_task_context", &json!({}))
.unwrap()
.is_none());
assert!(recovered.has_completed_call("last-call"));
assert!(recovered.has_completed_call("settled-0"));
assert!(recovered.has_completed_call("last-call").unwrap());
assert!(recovered.has_completed_call("settled-0").unwrap());
assert!(recovered.durable_run_receipt_limit_reached());
let saturation = recovered
.begin_call("next-call".into(), "get_task_context".into(), json!({}))
@@ -791,8 +791,8 @@ fn exact_identity_overflow_saturates_the_durable_run() {
let encoded = serde_json::to_string(&recovered).unwrap();
let mut recovered: ProviderToolBridge = serde_json::from_str(&encoded).unwrap();
recovered.attach_existing_run().unwrap();
assert!(recovered.has_completed_call("settled-0"));
assert!(recovered.has_completed_call("last-call"));
assert!(recovered.has_completed_call("settled-0").unwrap());
assert!(recovered.has_completed_call("last-call").unwrap());
assert!(recovered.durable_run_receipt_limit_reached());
recovered.attach_run(tools("computed")).unwrap();
@@ -857,7 +857,7 @@ fn settled_result_byte_exhaustion_recovers_after_turn_cleanup() {
.replay_result("large-settled-0", "get_task_context", &json!({}))
.unwrap()
.is_none());
assert!(recovered.has_completed_call("large-settled-0"));
assert!(recovered.has_completed_call("large-settled-0").unwrap());
assert!(!recovered.durable_run_receipt_limit_reached());
recovered
.begin_call(
@@ -1034,9 +1034,9 @@ fn full_identity_ledger_fails_closed_across_turn_and_recovery() {
.expect_err("a full exact ledger must stop fresh work");
assert!(saturation.is_active_turn_receipt_limit());
assert!(recovered.durable_run_receipt_limit_reached());
assert!(recovered.has_completed_call("settled-65535"));
assert!(recovered.has_completed_call("settled-00000"));
assert!(!recovered.has_completed_call("current-call"));
assert!(recovered.has_completed_call("settled-65535").unwrap());
assert!(recovered.has_completed_call("settled-00000").unwrap());
assert!(!recovered.has_completed_call("current-call").unwrap());
assert!(recovered
.begin_call("settled-00000".into(), "get_task_context".into(), json!({}))
.is_err());
@@ -1054,7 +1054,7 @@ fn full_identity_ledger_fails_closed_across_turn_and_recovery() {
let mut recovered_again: ProviderToolBridge = serde_json::from_value(round_trip).unwrap();
recovered_again.attach_existing_run().unwrap();
assert!(recovered_again.durable_run_receipt_limit_reached());
assert!(recovered_again.has_completed_call("settled-00000"));
assert!(recovered_again.has_completed_call("settled-00000").unwrap());
recovered_again.attach_run(tools("computed")).unwrap();
assert!(!recovered_again.durable_run_receipt_limit_reached());
packages/paperclip-runner/scripts/generate-protocol-schema-module.mjs
+1
View File
@@ -13,6 +13,7 @@ const validatorsOutputPath = resolve(
"src/protocol/generated/standalone-validators.ts",
);
const schemaNames = [
"output-body-reference",
"identity",
"capabilities",
"capabilities-v2",
packages/paperclip-runner/scripts/stage-runner-binary.mjs
+25 -11
View File
@@ -1,5 +1,6 @@
import { execFile } from "node:child_process";
import { chmod, copyFile, mkdir } from "node:fs/promises";
import { chmod, copyFile, mkdir, rename, rm } from "node:fs/promises";
import { randomUUID } from "node:crypto";
import path from "node:path";
import { promisify } from "node:util";
import { fileURLToPath } from "node:url";
@@ -8,17 +9,30 @@ const execFileAsync = promisify(execFile);
const packageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
const executable = process.platform === "win32" ? "paperclip-runnerd.exe" : "paperclip-runnerd";
const source = path.join(packageRoot, "runner", "target", "release", executable);
// Match Cargo's explicit target directory so isolated qualification builds
// cannot accidentally stage an older default-target executable.
const targetDirectory = process.env.CARGO_TARGET_DIR
? path.resolve(process.cwd(), process.env.CARGO_TARGET_DIR)
: path.join(packageRoot, "runner", "target");
const source = path.join(targetDirectory, "release", executable);
const destinationDirectory = path.join(packageRoot, "dist", "bin");
const destination = path.join(destinationDirectory, executable);
await mkdir(destinationDirectory, { recursive: true });
await copyFile(source, destination);
if (process.platform !== "win32") await chmod(destination, 0o755);
// Rust's linker emits an ad-hoc Mach-O signature. Copying that executable to
// its package location preserves the bytes but can leave the kernel rejecting
// the new inode with SIGKILL. Re-sign the staged inode so local packaged-runner
// evals execute the same artifact that was just built.
if (process.platform === "darwin") {
await execFileAsync("codesign", ["--force", "--sign", "-", destination]);
}
// Existing agents can keep this executable mapped for days. Never overwrite
// their inode (including during code signing); publish a new one atomically.
const temporary = `${destination}.${randomUUID()}.tmp`;
try {
await copyFile(source, temporary);
if (process.platform !== "win32") await chmod(temporary, 0o755);
// Rust's linker emits an ad-hoc Mach-O signature. Copying that executable to
// its package location preserves the bytes but can leave the kernel rejecting
// the new inode with SIGKILL. Re-sign the staged inode so local packaged-runner
// evals execute the same artifact that was just built.
if (process.platform === "darwin") {
// The staging filename is random, but the signed identifier must be
// stable: identical code must retain its artifact digest across rebuilds.
await execFileAsync("codesign", ["--force", "--sign", "-", "--identifier", "paperclip-runnerd", temporary]);
}
await rename(temporary, destination);
} finally { await rm(temporary, { force: true }); }
packages/paperclip-runner/src/protocol/generated/schema-bundle.ts
+317
View File
@@ -1,5 +1,40 @@
// Generated by scripts/generate-protocol-schema-module.mjs. Do not edit.
export const outputBodyReferenceSchema = {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://paperclip.dev/schemas/prp/v1/output-body-reference.schema.json",
"title": "Immutable output body reference",
"type": "object",
"required": [
"schema",
"bodyId",
"sha256",
"byteLength",
"mediaType"
],
"properties": {
"schema": {
"const": "paperclip.output.body.v1"
},
"bodyId": {
"type": "string",
"pattern": "^[a-f0-9]{64}$"
},
"sha256": {
"type": "string",
"pattern": "^[a-f0-9]{64}$"
},
"byteLength": {
"type": "string",
"pattern": "^[1-9][0-9]{0,6}$"
},
"mediaType": {
"const": "text/plain; charset=utf-8"
}
},
"additionalProperties": false
} as const;
export const identitySchema = {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://paperclip.dev/schemas/prp/v1/identity.schema.json",
@@ -343,6 +378,11 @@ export const commandSchema = {
"type": "integer",
"minimum": 1
},
"controllerEpoch": {
"type": "string",
"pattern": "^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$",
"description": "Requires negotiated transport.command_epochs.v1; absent is the retained numeric namespace."
},
"type": {
"enum": [
"run.prepare",
@@ -437,6 +477,11 @@ export const commandV2Schema = {
"type": "integer",
"minimum": 1
},
"controllerEpoch": {
"type": "string",
"pattern": "^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$",
"description": "Requires negotiated transport.command_epochs.v1; absent is the retained numeric namespace."
},
"type": {
"enum": [
"run.prepare",
@@ -1142,6 +1187,9 @@ export const providerEventSchema = {
"null"
],
"pattern": "^sha256:[a-f0-9]{64}$"
},
"outputBody": {
"$ref": "https://paperclip.dev/schemas/prp/v1/output-body-reference.schema.json"
}
},
"additionalProperties": false
@@ -3454,6 +3502,10 @@ export const eventSchema = {
"minLength": 1,
"maxLength": 160
},
"sourceEpoch": {
"type": "string",
"pattern": "^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$"
},
"sourceSeq": {
"type": "integer",
"minimum": 1,
@@ -3552,6 +3604,7 @@ export const eventSchema = {
"turn.failed",
"turn.interrupted",
"turn.cancelled",
"output.body.chunk",
"item.started",
"item.delta",
"item.completed",
@@ -3641,6 +3694,52 @@ export const eventSchema = {
"debug": {
"type": "object",
"additionalProperties": true
},
"sourceEpochTransition": {
"type": "object",
"additionalProperties": false,
"required": [
"schema",
"runId",
"transitionId",
"fromEpoch",
"nextEpoch",
"finalOrdinal"
],
"properties": {
"schema": {
"const": "paperclip.prp.event-epoch.v1"
},
"runId": {
"type": "string",
"minLength": 1,
"maxLength": 240
},
"transitionId": {
"type": "string",
"pattern": "^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$"
},
"fromEpoch": {
"anyOf": [
{
"type": "string",
"pattern": "^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$"
},
{
"type": "null"
}
]
},
"nextEpoch": {
"type": "string",
"pattern": "^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$"
},
"finalOrdinal": {
"type": "integer",
"minimum": 1,
"maximum": 9007199254740991
}
}
}
},
"allOf": [
@@ -4123,6 +4222,79 @@ export const eventSchema = {
}
}
}
},
{
"if": {
"properties": {
"eventType": {
"const": "output.body.chunk"
}
}
},
"then": {
"properties": {
"payload": {
"type": "object",
"required": [
"schema",
"body",
"offset",
"sha256",
"text"
],
"properties": {
"schema": {
"const": "paperclip.output.body.chunk.v1"
},
"body": {
"type": "object",
"required": [
"schema",
"bodyId",
"sha256",
"byteLength",
"mediaType"
],
"properties": {
"schema": {
"const": "paperclip.output.body.v1"
},
"bodyId": {
"type": "string",
"pattern": "^[a-f0-9]{64}$"
},
"sha256": {
"type": "string",
"pattern": "^[a-f0-9]{64}$"
},
"byteLength": {
"type": "string",
"pattern": "^[1-9][0-9]{0,6}$"
},
"mediaType": {
"const": "text/plain; charset=utf-8"
}
},
"additionalProperties": false
},
"offset": {
"type": "string",
"pattern": "^(0|[1-9][0-9]{0,6})$"
},
"sha256": {
"type": "string",
"pattern": "^[a-f0-9]{64}$"
},
"text": {
"type": "string",
"minLength": 1,
"maxLength": 32766
}
},
"additionalProperties": false
}
}
}
}
],
"additionalProperties": true
@@ -4155,6 +4327,10 @@ export const eventV2Schema = {
"minLength": 1,
"maxLength": 160
},
"sourceEpoch": {
"type": "string",
"pattern": "^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$"
},
"sourceSeq": {
"type": "integer",
"minimum": 1,
@@ -4256,6 +4432,7 @@ export const eventV2Schema = {
"turn.failed",
"turn.interrupted",
"turn.cancelled",
"output.body.chunk",
"item.started",
"item.delta",
"item.completed",
@@ -4327,9 +4504,75 @@ export const eventV2Schema = {
"debug": {
"type": "object",
"additionalProperties": true
},
"sourceEpochTransition": {
"type": "object",
"additionalProperties": false,
"required": [
"schema",
"runId",
"transitionId",
"fromEpoch",
"nextEpoch",
"finalOrdinal"
],
"properties": {
"schema": {
"const": "paperclip.prp.event-epoch.v1"
},
"runId": {
"type": "string",
"minLength": 1,
"maxLength": 240
},
"transitionId": {
"type": "string",
"pattern": "^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$"
},
"fromEpoch": {
"anyOf": [
{
"type": "string",
"pattern": "^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$"
},
{
"type": "null"
}
]
},
"nextEpoch": {
"type": "string",
"pattern": "^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$"
},
"finalOrdinal": {
"type": "integer",
"minimum": 1,
"maximum": 9007199254740991
}
}
}
},
"allOf": [
{
"if": {
"properties": {
"eventType": {
"enum": [
"tool.execution.started",
"tool.execution.progressed",
"tool.execution.completed"
]
}
}
},
"then": {
"properties": {
"payload": {
"$ref": "https://paperclip.dev/schemas/prp/v1/provider-event.schema.json#/$defs/toolExecution"
}
}
}
},
{
"if": {
"properties": {
@@ -4389,6 +4632,79 @@ export const eventV2Schema = {
}
}
}
},
{
"if": {
"properties": {
"eventType": {
"const": "output.body.chunk"
}
}
},
"then": {
"properties": {
"payload": {
"type": "object",
"required": [
"schema",
"body",
"offset",
"sha256",
"text"
],
"properties": {
"schema": {
"const": "paperclip.output.body.chunk.v1"
},
"body": {
"type": "object",
"required": [
"schema",
"bodyId",
"sha256",
"byteLength",
"mediaType"
],
"properties": {
"schema": {
"const": "paperclip.output.body.v1"
},
"bodyId": {
"type": "string",
"pattern": "^[a-f0-9]{64}$"
},
"sha256": {
"type": "string",
"pattern": "^[a-f0-9]{64}$"
},
"byteLength": {
"type": "string",
"pattern": "^[1-9][0-9]{0,6}$"
},
"mediaType": {
"const": "text/plain; charset=utf-8"
}
},
"additionalProperties": false
},
"offset": {
"type": "string",
"pattern": "^(0|[1-9][0-9]{0,6})$"
},
"sha256": {
"type": "string",
"pattern": "^[a-f0-9]{64}$"
},
"text": {
"type": "string",
"minLength": 1,
"maxLength": 32766
}
},
"additionalProperties": false
}
}
}
}
],
"additionalProperties": true
@@ -4651,6 +4967,7 @@ export const fixtureSchema = {
} as const;
export const prpSchemaBundle = {
"output-body-reference": outputBodyReferenceSchema,
"identity": identitySchema,
"capabilities": capabilitiesSchema,
"capabilities-v2": capabilitiesV2Schema,
packages/paperclip-runner/src/protocol/generated/standalone-validators.ts
+1 -1
View File
@@ -4,4 +4,4 @@
import equalModule from "ajv/dist/runtime/equal.js";
import ucs2LengthModule from "ajv/dist/runtime/ucs2length.js";
"use strict";export const fixtureValidator: unknown = validate20;const schema31 = {"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"https://paperclip.dev/schemas/prp/v1/fixture.schema.json","title":"PRP scripted replay fixture","type":"object","required":["schema","fixtureVersion","protocolVersion","name","description","identity","capabilities","commands","events","result"],"properties":{"schema":{"const":"paperclip.prp.fixture.v1"},"fixtureVersion":{"const":1},"protocolVersion":{"enum":[1,2]},"name":{"type":"string","minLength":1,"maxLength":120},"description":{"type":"string","minLength":1,"maxLength":1000},"identity":{"$ref":"https://paperclip.dev/schemas/prp/v1/identity.schema.json"},"capabilities":{"oneOf":[{"$ref":"https://paperclip.dev/schemas/prp/v1/capabilities.schema.json"},{"$ref":"https://paperclip.dev/schemas/prp/v2/capabilities.schema.json"}]},"commands":{"type":"array","items":{"oneOf":[{"$ref":"https://paperclip.dev/schemas/prp/v1/command.schema.json"},{"$ref":"https://paperclip.dev/schemas/prp/v2/command.schema.json"}]}},"events":{"type":"array","minItems":1,"items":{"oneOf":[{"$ref":"https://paperclip.dev/schemas/prp/v1/event.schema.json"},{"$ref":"https://paperclip.dev/schemas/prp/v2/event.schema.json"}]}},"requests":{"type":"array","items":{"$ref":"https://paperclip.dev/schemas/prp/v1/request.schema.json"}},"result":{"$ref":"https://paperclip.dev/schemas/prp/v1/result.schema.json"}},"additionalProperties":true};const schema41 = {"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"https://paperclip.dev/schemas/prp/v1/capabilities.schema.json","title":"PRP negotiated capabilities","type":"object","required":["schema","sessionReusePolicy","driver","steer","interrupt","resume","runtimeRequests","structuredResult","typedEvents"],"properties":{"schema":{"const":"paperclip.prp.capabilities.v1"},"sessionReusePolicy":{"enum":["new_per_run","reuse_per_issue","reuse_per_workspace"]},"driver":{"type":"object","required":["kind","version"],"properties":{"kind":{"type":"string","minLength":1,"maxLength":80},"version":{"type":"string","minLength":1,"maxLength":80}},"additionalProperties":true},"steer":{"type":"boolean"},"interrupt":{"type":"boolean"},"resume":{"type":"boolean"},"runtimeRequests":{"type":"boolean"},"structuredResult":{"type":"boolean"},"typedEvents":{"type":"boolean"},"typedEventFamilies":{"type":"array","maxItems":64,"items":{"type":"object","required":["family","version","availability","detailLevel"],"properties":{"family":{"type":"string","minLength":1,"maxLength":160,"pattern":"^[a-z][a-z0-9_.-]*$"},"version":{"const":1},"availability":{"enum":["available","unsupported","policy_disabled"]},"detailLevel":{"enum":["summary","structured"]}},"additionalProperties":false}},"semanticTools":{"type":"object","required":["schema","schemaVersion","operations"],"properties":{"schema":{"const":"paperclip.prp.semantic_tools.v1"},"schemaVersion":{"const":1},"operations":{"type":"array","items":{"type":"object","required":["operationId","version","availability","redactionDisposition"],"properties":{"operationId":{"type":"string","minLength":1,"maxLength":160,"pattern":"^[A-Za-z0-9][A-Za-z0-9._:-]*$"},"version":{"const":1},"availability":{"enum":["available","denied","unsupported"]},"redactionDisposition":{"const":"digest_only"},"requiredClaims":{"type":"array","items":{"type":"string","minLength":1,"maxLength":160},"uniqueItems":true},"reasonCode":{"type":"string","minLength":1,"maxLength":160,"pattern":"^[a-z][a-z0-9_.:-]*$"}},"additionalProperties":true}}},"additionalProperties":true},"unsupported":{"type":"array","items":{"type":"string","minLength":1},"uniqueItems":true}},"additionalProperties":true};const schema45 = {"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"https://paperclip.dev/schemas/prp/v1/command.schema.json","title":"PRP runner command","type":"object","required":["schema","commandId","controllerSeq","type","issuedAt","payload"],"properties":{"schema":{"const":"paperclip.prp.command.v1"},"commandId":{"type":"string","minLength":1,"maxLength":160},"controllerSeq":{"type":"integer","minimum":1},"type":{"enum":["run.prepare","run.attach","session.open","turn.start","turn.steer","turn.interrupt","turn.stop","request.resolve","interaction.receipt","semantic_tool.result","session.snapshot","session.close","session.budget.increase","session.destroy","run.cancel","runner.drain","runner.suspend","runner.shutdown"]},"issuedAt":{"type":"string","format":"date-time"},"deadlineAt":{"type":"string","format":"date-time"},"precondition":{"type":"object","properties":{"runnerState":{"type":"array","items":{"type":"string"}},"runState":{"type":"array","items":{"type":"string"}},"sessionState":{"type":"array","items":{"type":"string"}},"activeTurnId":{"type":["string","null"]}},"additionalProperties":true},"payload":{"type":"object","additionalProperties":true}},"additionalProperties":true};const schema46 = {"$schema":"https://json-schema.org/draft/2020-1 Line truncated
"use strict";export const fixtureValidator: unknown = validate20;const schema31 = {"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"https://paperclip.dev/schemas/prp/v1/fixture.schema.json","title":"PRP scripted replay fixture","type":"object","required":["schema","fixtureVersion","protocolVersion","name","description","identity","capabilities","commands","events","result"],"properties":{"schema":{"const":"paperclip.prp.fixture.v1"},"fixtureVersion":{"const":1},"protocolVersion":{"enum":[1,2]},"name":{"type":"string","minLength":1,"maxLength":120},"description":{"type":"string","minLength":1,"maxLength":1000},"identity":{"$ref":"https://paperclip.dev/schemas/prp/v1/identity.schema.json"},"capabilities":{"oneOf":[{"$ref":"https://paperclip.dev/schemas/prp/v1/capabilities.schema.json"},{"$ref":"https://paperclip.dev/schemas/prp/v2/capabilities.schema.json"}]},"commands":{"type":"array","items":{"oneOf":[{"$ref":"https://paperclip.dev/schemas/prp/v1/command.schema.json"},{"$ref":"https://paperclip.dev/schemas/prp/v2/command.schema.json"}]}},"events":{"type":"array","minItems":1,"items":{"oneOf":[{"$ref":"https://paperclip.dev/schemas/prp/v1/event.schema.json"},{"$ref":"https://paperclip.dev/schemas/prp/v2/event.schema.json"}]}},"requests":{"type":"array","items":{"$ref":"https://paperclip.dev/schemas/prp/v1/request.schema.json"}},"result":{"$ref":"https://paperclip.dev/schemas/prp/v1/result.schema.json"}},"additionalProperties":true};const schema41 = {"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"https://paperclip.dev/schemas/prp/v1/capabilities.schema.json","title":"PRP negotiated capabilities","type":"object","required":["schema","sessionReusePolicy","driver","steer","interrupt","resume","runtimeRequests","structuredResult","typedEvents"],"properties":{"schema":{"const":"paperclip.prp.capabilities.v1"},"sessionReusePolicy":{"enum":["new_per_run","reuse_per_issue","reuse_per_workspace"]},"driver":{"type":"object","required":["kind","version"],"properties":{"kind":{"type":"string","minLength":1,"maxLength":80},"version":{"type":"string","minLength":1,"maxLength":80}},"additionalProperties":true},"steer":{"type":"boolean"},"interrupt":{"type":"boolean"},"resume":{"type":"boolean"},"runtimeRequests":{"type":"boolean"},"structuredResult":{"type":"boolean"},"typedEvents":{"type":"boolean"},"typedEventFamilies":{"type":"array","maxItems":64,"items":{"type":"object","required":["family","version","availability","detailLevel"],"properties":{"family":{"type":"string","minLength":1,"maxLength":160,"pattern":"^[a-z][a-z0-9_.-]*$"},"version":{"const":1},"availability":{"enum":["available","unsupported","policy_disabled"]},"detailLevel":{"enum":["summary","structured"]}},"additionalProperties":false}},"semanticTools":{"type":"object","required":["schema","schemaVersion","operations"],"properties":{"schema":{"const":"paperclip.prp.semantic_tools.v1"},"schemaVersion":{"const":1},"operations":{"type":"array","items":{"type":"object","required":["operationId","version","availability","redactionDisposition"],"properties":{"operationId":{"type":"string","minLength":1,"maxLength":160,"pattern":"^[A-Za-z0-9][A-Za-z0-9._:-]*$"},"version":{"const":1},"availability":{"enum":["available","denied","unsupported"]},"redactionDisposition":{"const":"digest_only"},"requiredClaims":{"type":"array","items":{"type":"string","minLength":1,"maxLength":160},"uniqueItems":true},"reasonCode":{"type":"string","minLength":1,"maxLength":160,"pattern":"^[a-z][a-z0-9_.:-]*$"}},"additionalProperties":true}}},"additionalProperties":true},"unsupported":{"type":"array","items":{"type":"string","minLength":1},"uniqueItems":true}},"additionalProperties":true};const schema45 = {"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"https://paperclip.dev/schemas/prp/v1/command.schema.json","title":"PRP runner command","type":"object","required":["schema","commandId","controllerSeq","type","issuedAt","payload"],"properties":{"schema":{"const":"paperclip.prp.command.v1"},"commandId":{"type":"string","minLength":1,"maxLength":160},"controllerSeq":{"type":"integer","minimum":1},"controllerEpoch":{"type":"string","pattern":"^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$","description":"Requires negotiated transport.command_epochs.v1; absent is the retained numeric namespace."},"type":{"enum":["run.prepare","run.attach","session.open","turn.start","turn.steer","turn.interrupt","turn.stop","request.resolve","interaction.receipt","semantic_tool.result","session.snapshot","session.close","session.budget.increase","session.destroy","run.cancel","runner.drain","runner.suspend","runner.shutdown"]},"issuedAt":{"type":"string","format":"date-time"},"deadlineAt":{"type":"string","format":"date-time"},"precondition":{"type":"object","properties":{"runnerState":{"type":"array","items":{"type":"string"}},"runState":{"type":"array","items":{"type":"string"}},"sessionState":{"type":"array","items":{"type":"string"}Line truncated
packages/paperclip-runner/src/protocol/replay-contract.ts
+3
View File
@@ -1,3 +1,4 @@
import type { EventEpochTransition } from "../control-plane/event-epochs.js";
import type { ErrorObject, ValidateFunction } from "ajv/dist/2020.js";
import type { FromSchema } from "json-schema-to-ts";
@@ -70,6 +71,8 @@ export interface PrpEvent {
schema: PrpEventV1["schema"] | PrpEventV2["schema"];
sourceEventId: string;
sourceSeq: number;
sourceEpoch?: string;
sourceEpochTransition?: EventEpochTransition;
sourceInstanceId: string;
sourceKind: PrpEventV1["sourceKind"] | PrpEventV2["sourceKind"];
runId: string;
packages/paperclip-runner/test/protocol-contract.test.mjs
+1 -1
View File
@@ -27,7 +27,7 @@ async function fixture(relativePath) {
test("all schema IDs are unique and all external references resolve", async () => {
const schemas = await loadSchemaCatalog(resolve(protocolRoot, "schemas"));
assert.equal(schemas.length, 25);
assert.equal(schemas.length, 26);
assert.doesNotThrow(() => compileProtocolValidators(schemas));
});