Merge pinned production for the post-custody chat pilot

Preserve frozen 244-scenario YAML intent and 191-capability coverage. Production baseline: 39e811c9c5.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-10-27 21:03:19 -05:00
commit 0195fcb016
398 files changed
+90564 -66016

No files matched your search

+1
View File
@@ -89,6 +89,7 @@ packages/grok-acp/bin
!packages/paperclip-runner/test-fixtures/pi-acp/profile-v28-identity.json
!packages/paperclip-runner/test-fixtures/pi-acp/profile-v29-identity.json
!packages/paperclip-runner/test-fixtures/pi-acp/profile-v30-identity.json
!packages/paperclip-runner/test/fixtures/copilot-profile-v23-identity.json
@@ -0,0 +1,12 @@
import test from 'node:test';
import { verifyControllerRuntime } from '../../packages/paperclip-runner/scripts/controlled-evals/verify-runtime.mjs';
test('the uploaded controller preserves every inventoried runtime file', {
skip: !process.env.PAPERCLIP_CONTROLLER_TRANSPORT_DIRECTORY,
}, async () => {
await verifyControllerRuntime({
directory: process.env.PAPERCLIP_CONTROLLER_TRANSPORT_DIRECTORY,
sourceRevision: process.env.GITHUB_SHA,
receiptSha256: process.env.PAPERCLIP_CONTROLLER_RECEIPT_SHA256,
});
});
+10
View File
@@ -164,6 +164,16 @@ jobs:
with:
node-version: 24
package-manager-cache: false
- name: Apply reviewed qualification lock corrections
if: inputs.release_run_id == '' && (inputs.build_eval_image || inputs.publish_eval_image)
env:
TARGET_SHA: ${{ needs.authorize_manual.outputs.target_sha }}
run: |
set -euo pipefail
if [ "$TARGET_SHA" = "$GITHUB_SHA" ]; then
node tests/runner-e2e/prepare-report-lock.mjs
fi
- name: Resolve source dependencies without lifecycle scripts
if: inputs.release_run_id == ''
run: |
@@ -187,6 +187,18 @@ jobs:
echo '::notice title=Paid runner routing::Using a standard GitHub-hosted runner'
fi
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: ${{ github.sha }}
persist-credentials: false
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
- name: Preflight reviewed report dependencies before paid admission
run: node tests/runner-e2e/prepare-report-lock.mjs
target_lock:
name: Resolve target pnpm lockfile
needs: authorize
@@ -1273,6 +1285,9 @@ jobs:
with:
version: 9.15.4
- name: Apply reviewed reporting lock corrections
run: node tests/runner-e2e/prepare-report-lock.mjs
- name: Install pinned reporting dependencies
run: pnpm install --frozen-lockfile --ignore-scripts --store-dir "${{ runner.temp }}/eval-report-store"
@@ -1500,6 +1515,9 @@ jobs:
# Use a job-local store and disable lifecycle scripts throughout install.
# OIDC is available to every step, even before the AWS credential exchange.
- name: Apply reviewed publication lock corrections
run: node tests/runner-e2e/prepare-report-lock.mjs
- name: Install pinned publication dependencies
run: pnpm install --frozen-lockfile --ignore-scripts --store-dir "${{ runner.temp }}/eval-publisher-store"
@@ -513,10 +513,25 @@ jobs:
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: runner-protocol-runtime-${{ github.run_id }}-${{ github.run_attempt }}
include-hidden-files: true
path: ${{ runner.temp }}/runner-protocol-runtime/
retention-days: 30
if-no-files-found: error
- name: Download controller artifact for a credential-free transport test
if: needs.authorize.outputs.accounting_mode == 'controlled'
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
artifact-ids: ${{ steps.upload.outputs.artifact-id }}
path: ${{ runner.temp }}/runner-controller-roundtrip
- name: Verify the real uploaded controller inventory before private preparation
if: needs.authorize.outputs.accounting_mode == 'controlled'
env:
PAPERCLIP_CONTROLLER_TRANSPORT_DIRECTORY: ${{ runner.temp }}/runner-controller-roundtrip
PAPERCLIP_CONTROLLER_RECEIPT_SHA256: ${{ steps.build.outputs.runtime_receipt_sha256 }}
run: node --test .github/tests/runner-controller-artifact-transport.test.mjs
prepare_controller:
name: Prepare one shared cost and trace manifest
needs: [authorize, catalog, build_runner, build_controller]
+12
View File
@@ -49,6 +49,18 @@ and leaves all existing cases passive. Source identities are not grants.
### Task query performance
Native session resume selects its newest progress barrier within the existing
32-run task/session window in one SQL snapshot. Only the chosen checkpoint source
returns its saved native input, checkpoint, tool fingerprint, wake envelope, and
canonical task brief. Unused bootstrap candidates do not return full context or
profiles. Wrong runtime, nonterminal status, process identity, provider events,
invalid but non-null checkpoints, and future checkpoint barriers retain their
existing selection rules. Stored run payloads and history reads stay complete.
Run the synthetic PostgreSQL comparison with a pinned pre-change revision:
`pnpm --filter @paperclipai/server exec tsx ../scripts/benchmark-native-session-resume-reads.ts --baseline-ref=<commit>`.
An optional `PAPERCLIP_SESSION_RESUME_BENCHMARK_DATABASE_URL` must name a dedicated
test database ending in `_test`; otherwise the harness starts disposable PostgreSQL.
Agent request authentication uses a compact run-identity read. It omits saved
execution payloads, results, and log excerpts from the query response. The read
still locks the task before the run and reconciles pending steering from durable
+81 -2
View File
@@ -838,6 +838,41 @@ Set `SEARCH_EVAL_SCALE=1` to additionally measure a disposable 10,000-task,
30,000-comment dataset. `SEARCH_EVAL_REPORT=/tmp/search-quality.json` saves
per-query results and latency measurements; scale measurements are opt-in.
### Native skill preparation benchmark
Measure native skill preparation through the production service, real PostgreSQL,
and an in-memory OpenTelemetry exporter:
```sh
pnpm --filter @paperclipai/server exec tsx scripts/benchmark-native-skill-preparation.ts
```
The default fixture has a 208-file repository of about 10.8 MB. The report includes
cold and warm resolution, a fresh service instance, detailed phase timings, and
regular-pool SQL statement counts (excluding the separate cache-lock connection).
Every delivered file is checked outside the timed section. No provider is started.
`--snapshot-file=/absolute/path/snapshot.json --entrypoint=skills/example/SKILL.md`
uses a captured repository file inventory instead. `--network-delay-ms=25` adds
25 ms in each direction through a local TCP relay; this is simulated transport
latency, not a measurement of a hosted database.
`--actor=agent` adds a real agent, run, and responsible user, then records version
pins through the same callback boundary as execution. `--staging-shape=true`
seeds 14 projects, 10 sources, 8 bindings, and 156 other skills. These match one
observed staging inventory's counts; their content is synthetic. Combine these
options to include authorization and unselected library rows in the measurement.
By default, spans stay in memory. `--export-otlp=true` also exports them using
the operator's `OTEL_EXPORTER_OTLP_ENDPOINT`, protocol, and headers. Keep header
values out of command arguments and reports. The report records source hashes,
the export mode, and trace IDs. Exported traces identify this local diagnostic
in their service version; they are not evidence of staging performance.
If creating another embedded cluster is impractical, `--postgres-port=55448
--postgres-user=paperclip` uses an existing **local** server. It creates and migrates
a uniquely named disposable database and drops that database afterward; it does
not reuse an application database.
### Repository workspace admission benchmark
Measure a fresh checkout with and without the repository object cache:
@@ -1853,6 +1888,17 @@ TLS and protocol metadata. To use an isolated external database, set
`PAPERCLIP_CONTINUATION_BENCHMARK_DATABASE_URL` to a disposable database ending
in `_test`. The harness applies migrations there and adds synthetic fixtures.
Continuation history reads select completed tool receipts and the typed
conversation/workspace recovery fields used to choose the prior terminal run.
Every completed receipt keeps its full result. Receipt order, company/agent/task
scope, and the complete retained history stay unchanged. Saved results remain
available to detail, recovery, and execution paths.
Use `pnpm --filter @paperclipai/server exec tsx ../scripts/benchmark-continuation-history.ts --baseline-ref=<pre-fix-commit>`
to compare generated SQL, complete continuation envelopes, and DataRow bytes.
The benchmark includes a large required-receipt control. Its external database
option is `PAPERCLIP_CONTINUATION_HISTORY_BENCHMARK_DATABASE_URL`, with the same
disposable `_test` database restriction.
Retry scheduling is in `server/src/services/heartbeat/retries.ts`. It owns bounded
retry schedules, connection and workspace contention deferrals, shared-workspace
holder checks, due retry promotion, and retry-now requests. `createHeartbeatRetries`
@@ -1984,7 +2030,7 @@ agent conversion are outside this playbook.
## Paperclip Runner Adapter Conversion
Paperclip Runner offers native Codex, OpenCode, **ACPX Claude**, and **ACPX Pi**. New
Paperclip Runner offers native Codex, OpenCode, **ACPX Claude**, **ACPX Grok Build**, and **ACPX Pi**. New
Claude Code agents default to ACPX Claude on Apple Silicon, Intel Mac, Linux x64
and Linux ARM64 with supported glibc. New OpenCode and Pi agents default to their native
runtimes on Apple Silicon, Intel Mac, and Linux x64 with supported glibc;
@@ -1995,11 +2041,15 @@ requires matching remote assets. The runner override is in Advanced. Ordinary Cl
and provider routing remain authoritative. Existing agents keep their saved runner. Converting an existing Claude, Codex, OpenCode, or Pi agent selects its
corresponding provider, preserves compatible models, credentials, workspace,
and instructions, and resets execution sessions while retaining run history.
Grok Build uses its qualified macOS ARM64 and Linux x64 targets. Its conversion
preserves the selected model, account, and explicit reasoning effort through ACP.
Custom commands, extra arguments, legacy permission modes, search flags, and
nonzero turn limits require Legacy; conversion rejects them before saving.
Other adapters require an explicit provider choice. Legacy ACPX Codex agent
settings normalize to native Codex on configuration updates and before fresh
runs; immutable run descriptors remain readable. The **Paperclip Runner**
experimental setting applies to other native providers. Native Codex, OpenCode,
ACPX Claude, and ACPX Pi do not require it. Company access checks still apply.
ACPX Claude, ACPX Grok Build, and ACPX Pi do not require it. Company access checks still apply.
Native Codex preserves an explicit `adapterConfig.search: false` when converting
or saving an agent. The disabled hosted-search policy is recorded in the native
@@ -3332,6 +3382,35 @@ pnpm --filter @paperclipai/server exec tsx ../scripts/benchmark-workspace-prepar
It compares actual preparation guards with synthetic history. Local timings are
separate from Git materialization, environment acquisition, and provider startup.
### Sidebar badge run-query benchmark
The badge query keeps the newest 200 runs of nonterminated same-company agents.
It selects each agent's first observed run before status or Mine filters. An
observation ordinal preserves equal-time input order. Per-agent ranking orders
by full creation time first, then that ordinal for exact ties. SQL returns only failure
IDs, issue bindings, and creation times. Dismissals and fresh issue authorization
still run in the service.
Run the synthetic PostgreSQL comparison with a pinned pre-change revision:
```sh
PAPERCLIP_LOG_LEVEL=error SIDEBAR_REQUIRE_BYTE_REDUCTION=1 pnpm --filter @paperclipai/server exec tsx ../scripts/benchmark-sidebar-badges.ts --baseline-ref=bf3ef12f3a78d0b15a32ad57f7e16520fbf6cd5e
```
The harness starts disposable embedded PostgreSQL by default. To use an existing
isolated test database with current migrations, set
`PAPERCLIP_SIDEBAR_TEST_DATABASE_URL` to a dedicated database whose name ends in
`_test`. The harness removes only its own synthetic company fixtures. Do not use
a development or hosted company database.
The report includes service-generated SQL, complete badge equality, candidate
bindings, alternating warm samples, plans, and UTF-8 response-value bytes before
Drizzle decoding. This byte count excludes protocol framing. The mixed fixture
covers shared agents, superseding successes, timestamp ties, Mine users,
dismissal equality, private and deleted issues, and more than 200 runs. The
all-failed, all-succeeded, and empty controls expose neutral or slower paths.
Timings describe local SQL work and do not establish hosted request latency.
### Budget accounting performance
Budget admission and settlement share the company accounting lock. They evaluate
+1 -1
View File
@@ -49,7 +49,7 @@ start execution. This preference does not change the org chart or permissions.
### Agent Execution
Existing Claude and Codex agents can upgrade from the legacy runner on their
Existing Claude, Codex, and Grok Build agents can upgrade from the legacy runner on their
Harness / Runtime page when Paperclip Runner is the qualified default for their
environment. One action saves pending edits and translates compatible settings
while keeping the harness, selected model, account bindings, instructions, and
+1 -1
View File
@@ -1356,7 +1356,7 @@ Invites tab does not hide this Cloud action.
## 10.3 Agents
The Harness / Runtime configuration page offers **Upgrade runner** for existing
legacy Claude and Codex agents when the target environment selects Paperclip
legacy Claude, Codex, and Grok Build agents when the target environment selects Paperclip
Runner by default. The action uses the normal authorized agent PATCH with
`runner: "paperclip"`, includes pending configuration edits, and retains the
model and compatible settings. A rejected conversion leaves saved configuration
+37 -3
View File
@@ -2,14 +2,48 @@
Select **Grok Build** in the native runner provider selector. The stored contract is
`adapterType: "paperclip_runner"` with `provider: "acpx"`, `acpxAgent: "grok"`,
and `model: "grok-4.7"`. Existing `grok_local` agents keep their legacy adapter.
and `model: "grok-4.7"`. Existing `grok_local` agents keep their legacy adapter
until an explicit upgrade. On a qualified macOS ARM64 or Linux x64 computer,
**Harness / Runtime → Upgrade runner** preserves the agent, model, account,
instructions, and history. It starts fresh execution sessions. Advanced also
offers an explicit Runner choice. Unsupported legacy settings are rejected
before saving; select compatible settings or keep Legacy.
New Grok runner agents default to **Full auto (approve all)**
(`acpxPermissionMode: "approve-all"`) in setup and the configuration form.
API configurations that omit the permission mode use the same default. No
additional permission setting is needed for unattended execution. Explicitly
saved restrictions remain unchanged.
On Cloud, an operator must enable `enableNativeRunner` for the instance before
the new-agent picker or direct setup page offers the native runner.
Grok Build's qualified harness does not require the `enableNativeRunner` flag.
Explicit thinking effort uses `reasoningEffort` (`minimal`, `low`, `medium`,
`high`, or `xhigh`). The immutable v10 execution input, provider descriptor,
and recovery identity bind this value as `grokEffort`. The Grok provider boundary
adds native `_meta.modelId` and `_meta.reasoningEffort` to session create, load,
and resume requests before transport writes and observation. Grok 1.0.13 reports
the selected effort in `_meta["x.ai/sessionConfig"].options`; it does not support
the standard `session/set_config_option` effort API. The Runner checks that native
acknowledgement and the effective model before a prompt. A lazy restored handle
must connect through read-only native session inspection first. Reapplying the
bound metadata on restore prevents Grok from resetting effort to its default.
Missing configuration, a rejected value, or effort drift stops admission.
Admission failures keep a closed Grok reason code through the sidecar boundary.
Other startup failures retain their closed admission stage when no more specific
known error code is available. These diagnostics contain no native payload,
credential, arbitrary provider message, or workspace path. They do not change
admission or retry policy.
Omitting effort retains Grok's provider default. This setting remains outside
the model-visible task prompt. Historical execution inputs retain their schemas.
The upgrade resolves an unset model or the legacy `grok-build` default to the
native `grok-4.7` default. Task-level model and effort overrides use the Grok
catalog and keep the account and tool policy unchanged.
Migration also preserves the effective `disableWebSearch` setting, whose legacy
default is `true`. Native input v10 carries it as `grokDisableWebSearch` through
startup and recovery. Grok's private `requirements.toml` enforces disabled search
above compatible project settings. Explicit `false` retains enabled search;
unset native configurations retain their provider default. See Grok's
[configuration reference](https://github.com/xai-org/grok-build/blob/main/crates/codegen/xai-grok-pager/docs/user-guide/26-config-reference.md).
Grok Build speaks [ACP over stdio](https://github.com/xai-org/grok-build/blob/main/crates/codegen/xai-grok-pager/docs/user-guide/15-agent-mode.md).
The runner owns `grok agent --no-leader stdio` through ACPX, including session
+231
View File
@@ -0,0 +1,231 @@
# Approval queue seeding without retention polling
## Obligation and stack dependency
The attention maintenance scan does two different jobs: retention and automatic
queue membership. Its call to `materializeSeededQueues` adds pending approvals to
the PR queue when the first linked issue has a pull-request work product. Queue
API readers use saved membership; they do not first build the attention feed.
Removing approvals from maintenance therefore lost automatic queue population.
A real PostgreSQL regression reproduced an empty queue with a pending linked
approval and PR work product, without an attention reader.
This slice owns automatic PR-queue membership for approvals. It depends on the
approval producer boundaries introduced in #17497 and is required with that
retention migration before treating the combined behavior as complete. Join
requests have no issue association or matching seed rule. Other attention-source
seed discovery remains in the existing feed scan and must migrate with an
explicit owner before those sources leave that scan.
## Source of truth and ownership
Existing approvals, issue approvals, issue work products, queues, and queue items
are the only durable state. No new table, outbox, external API, or provider branch
is introduced. `approvalQueueSeedingService` selects missing memberships for:
- A pending approval in an active company.
- Its first linked issue, ordered by issue ID, in the same company and eligible
under the attention feed's execution-issue visibility predicate.
- A pull-request work product belonging to that issue and company.
- A missing PR queue or an enabled seed rule without this approval's membership.
Disabled rules, nonmatching approvals, completed approvals, and existing seeded
history are passive. A pending approval itself prevents its linked Slack/Speko
conversation from projecting as idle, so this source does not depend on every
other conversation-state producer. Hidden/harness/conversation-container issue
changes retain their explicit signals.
The queue service owns seed matching, creation, membership uniqueness, and audit
writes. The feed and worker use the same writer through a small normalized source
contract. The candidate lookup and writer share the pull-request product SQL
predicate, so the matching policy has one owner. A worker does not construct a presentation-layer AttentionItem. Queue
reads keep their existing source authorization checks. A queue-rule update and
seeding serialize on the existing queue row. Existing memberships are not pruned
when an approval resolves. This preserves current behavior; pruning is a separate
change, including the proposal in #13364.
## Producer audit
| Producer | Commit boundary and effect |
| --- | --- |
| Approval create, cancel, revision, resubmit, resolve | `writeApprovalChange` notifies retention and company-scoped queue seeding before writes in the owning transaction. Approve/reject/cancel retain their explicit tool-review notification; ordinary mutations do not wake those queues. |
| Shared approval records, budget approvals, formal tool approvals | The same approval notification covers their existing transactions. |
| Issue approval link, unlink, link-many, tool-created links | Existing approval transaction helper. Link removal can expose a different first issue. |
| Issue visibility changes and issue deletion | Existing approval notification before the update or cascading delete. Routine failure cleanup retains this signal. |
| Company activation, pause, archive, deletion and budget pause/resume | Existing company notification wakes both approval consumers. |
| Work-product create, type/issue update, import batch, delete | The owning work-product transaction notifies seeding when a pull-request row is involved. |
| PR queue creation, rule update and manual approval membership removal | The queue mutation transaction notifies seeding. An enabled rule can re-add a still-matching manually removed item, as the old scan did. |
Other direct work-product writes only create artifacts or update title, metadata,
or trust. They cannot create a pull-request match. Supported writes use these
in-process service boundaries. Independent SQL writers and multiple server
replicas are not covered by local transaction signals.
Retention-only keep/archive/revive changes do not create queue projection debt.
The old seed writer ran before retention filtering, so archived or kept approvals
still follow the same seeding rule. User-specific dismissals do not filter the
unscoped maintenance feed. No new deadline is inferred from those display states.
## Worker lifecycle
Startup reconstructs missing memberships across the instance. Each later committed
producer carries its company ID and wakes only that company's reconciliation.
Pending scopes coalesce in memory; one company's ordinary mutations cannot restart
a global scan. Scope rotation prevents one failing company from starving another. Each sweep processes at most 100 approvals, with an ID cursor and a fixed upper ID per scan cycle. The cursor advances past
failures and finishes that finite cycle before revisiting them; new arrivals cannot
extend it indefinitely. Each row is rechecked under
an approval lock before the queue writer runs. One bad row cannot permanently
hide later candidates. Atomic existing membership uniqueness prevents duplicate
rows and duplicate insertion audit events.
After a sweep, another lookup decides whether actual projection debt remains.
Unprocessed pages and newer committed signals retain a one-second continuation.
Failed scopes back off exponentially from one second to one minute. A successful
nonempty batch resets consecutive failures; an empty cursor rollover while debt
remains preserves outage backoff. New signals
do not reset a failing scope's backoff; other companies can proceed independently.
The next retry is registered with the shared scheduler, with no SQL during the
wait. A failed scope remains tracked until its owed memberships are reconciled. The generation check follows the final awaited lookup, so a commit arriving
during an empty read cannot be erased. Once reconciled, there is no deadline and
no recurring worker SQL. The shared coordinator preserves requested work during
drain. Passive pending checks inspect memory only; startup reconstructs from rows. Shutdown aborts between rows
and waits for an admitted transaction; unprocessed rows remain recoverable.
Registration follows the existing heartbeat-maintenance enablement switch.
This slice does not extend Cloud alarm coverage or loosen host sleep admission.
It proves a worker's quiet/reactivation behavior, not deployed host or database
suspension. The remaining attention maintenance interval is still present for
other source owners.
## Verification
- Before the fix, the PostgreSQL regression found no PR queue after the normal
maintenance path excluded approvals. After the fix, the queue API finds it
without a full attention read.
- Twenty-one real-database worker tests cover empty-to-active-to-quiet behavior,
startup and completed history, actual approval/link/work-product producers,
batch import, first-link selection, company isolation, hidden targets, queue
rule enablement, company activation, resubmission, outer commit and rollback,
first write during drain, database failure through drain, multiple pages,
failed-page progress and retry, concurrent resolution, archived/kept history, shutdown/restart, and an admitted transaction at shutdown.
- 115 existing tests pass across queue routes, attention, approval retention,
approval service, work products, and startup wiring. The shared writer still
passes existing plan/question/PR seed tests and authorization tests.
- Workspace typechecks and the production build pass. Fresh PR checks are
tracked in the PR and must pass before readiness is claimed.
- The actual pending lookup on 40,000 completed approvals plus one pending PR
approval uses the existing partial pending-retention index. The probe runs in
a rolled-back transaction; execution took 0.094 ms on this local fixture. No new index or migration is needed for this case.
The first expanded test run caught two fixture errors: a hand-inserted queue
omitted required creator attribution, and a failure test expected no idle hold
while committed projection work was still unresolved. The corrected test waits
for the active attempt to finish and checks passive SQL directly while preserving
that required hold. Neither required weakening the worker's recovery contract.
Review found a queue/item lock inversion between seeded insertion and manual
membership changes. A two-connection PostgreSQL regression reproduced `deadlock
detected`. Manual add/remove now lock the queue before touching its items, matching
the seed writer. The regression and the existing queue route suite pass (27 cases).
Server typecheck and build pass after the repair. Fresh-head CI/review is required.
Security review found that normal producer wakes could restart a cross-company
scan. The regression reproduced another company's queue being populated by an
unrelated company write. Producers now include company hints, including batch
imports and nested transactions. A separate controlled PostgreSQL test holds the
final empty read while a real producer commits and verifies the retained wake.
After company scoping, all 169 selected cases pass across approval queue seeding,
approval retention, queue routes, approval services, work products, and issue
comment routes. Server typechecking passes. The preceding run suffered timeouts
and a consequent cleanup foreign-key error; it is not counted as passing. The
route suite separately exposed cold imports inside timed tests. Moving those
imports to suite loading preserves assertions and prevents timed-out requests
from leaking into later mock state. The watchdog fixture collision from CI uses
monotonic identifiers instead; its 19 real-PostgreSQL cases pass.
The next CI run passed the security gate and the two originally failing shards.
Its remaining server shard caught a historical cleanup test checking 33 provider
inspections after only 32 had completed. The test advanced fake time before the
previous asynchronous page had necessarily published its next deadline. It now
waits for all 33 inspections, then retains the zero-further-work assertions. All
14 environment cleanup cases pass on real PostgreSQL; production code is unchanged.
Further review reproduced 120 failed transactions in two simulated minutes for a
persistently failing projection. Per-scope backoff now bounds that test to at most
eight attempts, even with new signals every second. Removing the fault recovers
the membership. A real coordinator/producer case proves thirty seconds without
SQL during backoff, another company's successful queue seeding during that wait,
and eventual recovery without losing the original work. All 25 worker cases pass.
The first consumer command omitted the external DB setting for queue routes and
skipped those six cases; the corrected command executes and passes all six.
The shared PR predicate is exercised by both those writer tests and worker tests.
Server typecheck and build pass; fresh remote review/checks remain required.
The final startup, approval-retention, and attention consumer run passes all 93
cases. With the 25 worker and six queue cases, 124 affected tests pass.
## Current retention-parent integration
The seeding worker reuses `firstApprovalIssueId` from the approval-visibility
owner rather than retaining its own first-link SQL. Company IDs are part of the
approval notification contract because projection reconciliation is scoped.
Generic approval writes notify retention and seeding only; the existing terminal
approve/reject/cancel owners alone notify tool-review consumers. Current precise
issue-deletion target notifications, unseen retention controls, finite retention
cycles and the real source-row lock regressions are preserved.
All 522 cases in fifteen complete suites pass at `4e725e2055`, with server
typecheck and production build. This includes seeding, queue routes, work products,
issue comments, both retention workers, approvals, attention, retention controls,
budgets, tool gateway, startup, scoped deadlines, environment cleanup and task
watchdogs. Four new disposable fixtures were cloned from the already migrated
parent fixture; this is not a fresh-empty-schema claim. The other suites used
separate existing disposable databases. No database was shared with another
concurrent local test run. The subsequent parent merge changes only two documents;
all tested runtime and acceptance sources remain byte-identical.
The first integration typecheck caught two incorrect gateway company references.
They now use the existing session company, and typecheck plus the complete gateway
suite pass. Earlier deadlock, cross-company wake, backoff, skipped-fixture and
teardown failures remain in the historical sections above. The parent's attention
cleanup timeout and expiry-test CI assertion are recorded in its document; this
larger current run passes without changing their assertions or timeouts. Fresh
exact-head reviews and CI remain required before this dependent PR can merge.
## Final master and review repairs
Join retention merged as `8406047b4e`. The normal master reconciliation first
proved that landed tree equals the existing parent. Conflict resolution preserved
the complete tested seeding tree; it also removed a duplicate helper introduced
by automatic merge. The PR again differs from master in only its original 25
seeding paths. Database schema and migration metadata are identical to master.
Three retry regressions failed before repair: ongoing arrivals skipped a recovered
membership, a partial successful batch retained the old sixty-second failure
penalty, and resetting that counter on an empty cursor rollover weakened outage
backoff. The worker now fixes the upper ID per cycle and resets consecutive
failures only on real successful progress or complete reconciliation. Exact
one-second recovery and two-second consecutive-failure assertions pass; repeated
signals still cannot reset a failing scope's delay. No new table or retry wrapper
is introduced.
The external seeding, queue-route and watchdog fixtures now apply pending
migrations before creating their client. Each of three empty-database regressions
failed with missing tables before this setup repair. All 55 cases then passed on
a second set of genuinely empty databases, with the suites applying their own
migrations. The final additional rollover guard brings the three complete suites
to 56 passing cases (28 seeding, 6 queue-route, 22 watchdog), using those now
migrated fixtures. Server types/build pass at `7948136ef8`. The earlier 522-case
integration run remains evidence for its stated revision and unchanged consumers;
it is not relabeled as a current full-suite run.
The first expanded fresh run passed 54 cases and failed the new backoff test's
cleanup expectation: after an entirely failed full page, a separate empty pass
ends the cycle before the next pass revisits rows. The corrected fixture permits
that explicit continuation and still checks complete recovery and the exact retry
deadline. Earlier failures remain recorded. Fresh final-head CI and reviews are
required before merge.
+164
View File
@@ -0,0 +1,164 @@
# Approval retention without idle polling
This slice removes approval retention from the recurring attention-feed sweep.
It does not migrate the other attention source kinds or establish host sleep.
## Obligation and owner
A pending approval in an active company is eligible for automatic archiving
90 days after its last approval activity, unless kept or already archived.
This is a presentation/attention archive: it does not approve, reject, cancel,
or delete the approval. The deadline and policy are unchanged.
Approval rows are the source of truth, including those never viewed in the
attention UI. Scheduling only existing `decision_retention` rows would miss
unseen approvals because the old feed reader materializes those rows lazily.
`approval-retention.ts` discovers eligible approvals directly and registers
the earliest actual deadline with the existing delivery coordinator.
The query retains the attention feed's first-linked-issue rule and consumes
`executionIssueCondition` from the issue visibility owner. Hidden, harness and
agent-conversation tasks remain excluded. A waiting external conversation with
a linked pending approval is active under that same shared visibility policy;
its chat state alone cannot hide this approval. Tests exercise the Slack case.
Each pass visits at most 100 candidates in activity/id order. The cursor keeps
the database timestamp as text so sub-millisecond precision cannot repeat a
failed page. A failed row does
not prevent later pages from progressing. The failed durable record remains
eligible for recovery. Each candidate is re-read and locked before archiving,
so a resubmission committed while the worker waited cannot archive the new
activity using an old selection. Existing retention version, keep and archive
compare-and-set checks remain authoritative. This does not add cross-table
serializability for concurrent link/visibility/company policy changes.
## Producer coverage
| Supported producer | Notification boundary |
| --- | --- |
| Approval create, cancel, revision, resubmit | SQL write transaction in approval records |
| Approval resolution, including hire lifecycle | SQL write, retaining the caller's outer transaction |
| Hard budget approval and budget resolution | Existing accounting transaction |
| Formal tool approval and issue link | SQL write transaction; provider work is outside it |
| Link, unlink and bulk link | Issue approval write transaction |
| Issue visibility changes and deletion | Existing issue mutation transaction |
| Routine creation rollback cleanup | Existing transaction before cascading issue deletion |
| Company activation, pause, archive and deletion | Existing company/accounting transaction |
| Keep, manual archive, revive and accepted archive manifest | Existing retention mutation transaction |
| Retention materialization or source activity change | Signal only when inserting or changing actual state |
Approval comments do not change approval activity under the existing policy.
Company portability imports do not write approvals; restored durable records
are reconstructed at startup. Signals belong to the application DB owner and
its dedicated child pools. Independent SQL writers or another active process
need explicit integration; this change does not claim to observe them.
## Shared contracts and lifecycle
The main maintenance feed excludes the approval source before collecting it.
Public attention reads keep their existing default behavior. Other attention
sources remain in the aggregate sweep until separately migrated.
The retention service accepts the source fields it actually uses rather than
requiring fabricated full UI attention items. Its existing versioned state,
audit writes, authorization and archive-notification outbox are reused. No new
queue table is introduced. Transaction-aware notifications retain intent until
the outer transaction settles, including nested writes and rollback.
The deadline worker uses the shared scheduler's generation and drain behavior.
Startup reconstructs pending work; idle drain retains requests without passive
SQL; resume reconciles them. Failed work retains a 30-second recovery spacing.
No eligible approval means no recurring deadline or query. Completed history,
kept/archived approvals, and inactive companies are quiet until a supported
mutation changes eligibility. Shutdown joins admitted work and stops between
candidates. Notification delivery remains owned by the separate archive worker.
A pending-only `(updated_at, id)` index bounds deadline lookup independently of
resolved approval history. The migration uses concurrent, idempotent creation
and the existing migration runner's nontransactional execution list.
## Verification and limits
Real-Postgres lifecycle tests cover unseen sources, future deadlines, retained
history, keep/unkeep, archive/revive, revision/resubmission, company activity,
linked issue visibility/deletion/order, normal versus maintenance attention
reads, nested commits/rollback, startup and first-write drain, failed database
writes through drain, bounded progress past failure, shutdown, and a concurrent
resubmission. The waiting-Slack case also checks durable notification creation.
Existing approval, attention, retention, budget, tool-gateway and startup suites
are relevant consumers. Track final results and any failures in the PR and the
[stack ledger](plans/2026-10-24-recurring-server-sleep-stack.md).
The Cloud sleep guards are unchanged. An in-process deadline cannot wake stopped
compute, and this approval deadline has not been added to Cloud's durable wake
contract. This proves only the named worker's quiet/reactivation behavior.
## Initial review repair verification
The feed and retention worker now share `approval-visibility.ts` for the first
linked issue and its execution visibility. The feed still applies actor-specific
read access before including a linked approval. Both link orders are covered:
hidden first/visible later stays excluded; visible first/hidden later stays
eligible. The worker's current-state recheck remains under the approval row lock.
Manual keep, archive and revive materialize an unseen approval on demand inside
the existing retention write transaction. They lock the company-scoped approval
row in the same order as automatic archival, read its real activity timestamp,
and use the existing versioned `syncItems` operation. Missing or unauthorized
sources retain their existing rejection; other source kinds are unchanged. This
adds no eager startup materialization or recurring scan.
Four real regressions failed before repair: unseen young keep/archive/revive
returned 404, and the atomic materialization test could not reach the keep write.
All pass after repair. A failing keep update rolls back the inserted retention
row and emits no audit; the next successful keep persists exactly one audit.
All 26 lifecycle cases pass, including the existing timestamp-precision failure
rotation. The complete eight-suite consumer selection passes 311 cases at
`063b61584d`: approval worker/service, attention, retention, budget, tool gateway,
startup and scoped deadlines. Server typecheck and production build pass.
Normal parent reconciliation preserves every historical parent SQL file and all
390 parent journal entries. The approval index is regenerated through the normal
workflow as concurrent/idempotent `0392_small_frightful_four.sql`; five generated
snapshots remain (0388–0392). Schema drift, safety/numbering and actual fresh
migration apply/replay on five isolated PostgreSQL databases pass. Earlier
transaction-stub, budget-import, query-probe and CI failures remain recorded in
the stack ledger. Fresh current-head CI and both reviews are required.
## Wake amplification repair
Ordinary creation, links, revision requests and resubmission signal retention
only. Tool-review queues are notified by the actual terminal transition owners:
approve/reject in approval resolution, and cancel in the approval service. These
are the same terminal transitions that notified tool reviews before this PR.
Cancellation must remain a producer because it settles parked tool actions as
unavailable. No extra wrapper or scope argument is threaded through those calls.
The aggregate retention worker now coalesces sequential notifications for thirty
seconds after a successful reconciliation. This is a single instance-wide read
budget, stricter than granting each company an independent thirty-second budget,
and matches the former default maintenance cadence. Known business deadlines
still preempt the coalescing window. The window schedules no work on its own;
without a notification or saved deadline the worker remains quiet. Newly changed
eligibility can wait up to that window, as under the old default maintenance.
Three new real regressions failed before repair: approve and reject cases
observed unwanted tool-review signals from ordinary mutations, and four
sequential tenant writes caused eight global reads rather than one two-read
reconciliation. Terminal approve/reject/cancel signals are now tested explicitly.
With another company's pending approval present, sequential writes cause no extra
SQL within the window and one reconciliation when the window ends.
The first coalescing fixture incorrectly expected the pending-work hold to clear
before the requested reconciliation; it now lets the deferred wake remain owed
and measures SQL directly. The first broad repair passed 313 cases and failed
the existing parked-review cancellation case. Restoring the original cancel
producer fixed that regression without weakening its outcome assertion. All
315 cases in eight full suites now pass at `dd844f4e8c`, including 30 approval
worker cases; server types/build pass. The earlier security check at `5991f257bb`
was action-required and Greptile reported service error reference
`8fb1835c-e564-476b-b802-d5be164ea915`. Neither is claimed as passing; the repaired
head requires fresh security, CI and both bot reviews.
+381
View File
@@ -0,0 +1,381 @@
# Approved tool-action recovery
Approved tool actions recover at startup and after committed review changes.
The aggregate maintenance tick no longer scans approved requests. The existing
request and invocation rows remain authoritative; this change adds no queue table.
## Work and quiet states
The gateway owns the approved-work predicate. A signed agent action with
`executionOnApprove: true` owes execution through the normal approval path.
A legacy signed action without that flag remains deliberately inert. A Test-tab
approval can owe execution even when no agent run exists. Recovery uses the saved
origin and current authorization/target checks; the scheduler sees only whether
another attempt is needed.
Startup discovers companies with saved approved requests. Committed review
signals carry the affected company ID; subsequent discovery and recovery reads
filter by that company. The shared scoped worker coalesces changes and dispatches
one company per pass with a one-second minimum spacing. A company writing reviews
does not scan or lock another company's approved backlog.
Recovery pages that company's approved requests in ID order using a partial index. It does not
let a full batch of unavailable approvals starve later work. A failed read or
unresolved executable approval retains a 30-second retry. New writes preserve
that company's retry deadline, while other companies can run independently.
A company pass still drains all its pages; this change does not cap provider
calls within that pass or alter the provider's existing execution timeout. Once a pass leaves no
such obligation, there is no recurring approved-recovery query. An empty system,
configured idle tools, completed history, and deliberately inert approvals are
quiet after reconciliation.
## Producer and transaction ownership
The existing tool-review notification boundary now signals both pending expiry
and approved recovery. This covers human review transactions, gateway approval
and accepted-interaction inference, administrative interaction resolution, review
queue invalidation, and the gateway's approval claims and managed-argument
invalidation. The notification is awaited before SQL in the existing transaction;
caller-owned outer transactions retain commit/rollback ownership.
A pending-only change can cause one empty company-scoped discovery read. It does not
create a periodic scan. No per-route recovery bookkeeping or database listener
is added. Signals use the existing database owner's single-active-writer model.
Independent SQL writers or another process need a separate notification contract.
## Single execution and outcome ownership
Agent actions retain their existing approved-to-executing claim. Test-tab
foreground approval and recovery now share an equivalent claim before provider
dispatch. The invocation then starts only if the request still holds that claim.
Provider calls and target discovery remain outside SQL transactions. Current rate-policy admission and its receipt commit together in a short database transaction.
The Test-tab invocation outcome and terminal request status commit together.
Claim timestamps fence late settlement. A later audit failure cannot overwrite
an already committed outcome. Recovery repairs legacy approved Test-tab requests
from a saved terminal invocation without decoding old arguments or calling the
provider. A legacy executing invocation moves to the existing interrupted-work
path; it is not dispatched again.
If the provider may have run but its result cannot be persisted, the executing
claim remains authoritative. The existing stale-execution recovery quarantines
unknown outcomes. This slice does not promise exactly-once external effects or
replay an uncertain provider call.
## Lifecycle and remaining duties
The shared deadline worker coalesces notifications, preserves changes during a
pass, and retains skipped work across idle drain/resume. Startup reconstructs
saved approvals. Company deletion invalidates cached recovery and retries through
the same transactional notification boundary. Shutdown stops its coordinator registration. With heartbeat
scheduling disabled, startup keeps the existing one-off recovery behavior.
Pending expiry is owned by the preceding slice. Interrupted execution and
interaction projection repair remain in the old sweep until their own migrations.
The public combined sweep retains all duties for manual/recovery callers.
No Cloud eligibility or external-wake proof changes here; worker quiet does not
establish deployed database or host suspension.
## Verification
Real PostgreSQL service/startup tests cover producer commit and rollback,
startup reconstruction, empty/configured/history quiet states, concrete retries,
first-write drain/resume, legacy inert approvals, and multi-page recovery.
Controlled Test-tab provider tests cover both foreground/recovery race winners,
completed legacy outcomes, interrupted legacy calls, target changes, and a failed
atomic outcome write followed by quarantine without another provider call.
The service/startup selection passes 123 tests. The existing gateway acceptance
suite passes 92 tests. The strengthened paging and outcome-failure cases pass
again after their final edits. Workspace typecheck and build pass. Full CI and
review remain separate PR gates; no real external provider was invoked by these
fixtures.
Review found that a one-shot success-write failure could fall into the provider
error handler and save a false failure on its second attempt. The added sequence-
based PostgreSQL fault reproduces that bug before the fix. Provider execution,
pre-dispatch validation, and outcome persistence now have separate error
boundaries. Both persistent and one-shot result-write failures retain the claim
for quarantine without repeating the call. The complete 123-test selection and
92 gateway acceptance cases pass after the repair, as do typecheck and build.
The existing tool-access service regression now expects a completed Test-tab
request to be executed and verifies exactly one provider call before asking for
a new approval. Its full 469-test suite passes. The initial local command skipped
its database cases; the suite now honors its supplied disposable database through
the standard test-database guard independently of embedded Postgres support.
Company-scoping regression: a real signed/reviewed action in company A previously
caused an unfiltered approved-request SELECT over company B's inert legacy action.
The captured-SQL test fails before this repair and passes with scoped reads. Added
lifecycle cases check retry preservation across committed writes, independent
company progress, and disarming after the real company-deletion producer. These
checks supplement the earlier execution, transaction, and paging evidence above.
After company scoping, the complete gateway-service and startup selection passes
149 tests against a disposable PostgreSQL database, and server typecheck passes.
Stack integration, workspace checks, and fresh remote CI remain separate gates.
## Preparation failures retain an approved Test-tab action
A real PostgreSQL policy-read timeout reproduced a permanent-failure bug after
claiming a Test-tab action: the provider had never been called, but both request
and invocation were saved as failed, so approved recovery could not retry them.
The preparation boundary now distinguishes explicit validation/access rejections
from unexpected dependency errors and transient HTTP errors. The existing
`tool-approval-execution` owner releases only its exact claim: invocation and request
are locked in the established order, their link/company/claim timestamp must
match, and the invocation must still await approval with no recorded start.
The request returns to `approved` with a transaction-aware notification. Signed
arguments and the human approval remain unchanged; the error propagates so the
existing recovery worker retains its retry obligation.
This boundary ends before `runTestToolInvocation`. Provider dispatch and outcome
persistence keep their existing conservative no-replay behavior. A changed claim,
started invocation or prior-dispatch evidence cannot be released by this path.
If the requeue transaction itself fails, the error propagates and no terminal
failure is fabricated; existing interrupted-execution reconciliation remains the
fallback. This change does not infer replay safety from a failed requeue.
Before the fix, both foreground and recovery timeout regressions failed. They
now observe the committed claim under a real PostgreSQL table lock, wait for
`55P03`, verify unchanged approval and no provider call, then prove automatic
recovery invokes the provider once and becomes quiet. Additional real-row cases
cover changed ownership, mismatched invocation, a started call, prior-dispatch
history, and outer-transaction rollback. A released claim cannot start execution.
All 117 gateway-service cases pass on the owning slice, including existing target
invalidation, provider-outcome persistence failures and unknown-outcome no-replay
cases. Integrated consumer checks, workspace verification and remote PR gates
must still be recorded after propagation.
## Agent-origin preparation and dispatch ownership
Agent-origin approvals use the same pre-dispatch retry contract as Test-tab
approvals. Previously, a policy or identity dependency failure after claiming an
approved action permanently failed it; earlier issue/signing failures could
leave the claim executing. Preparation now has one owning error boundary around
issue, signature, identity, target, policy, managed-argument, and content checks.
Validation and nontransient access rejections retain their terminal handling.
Unexpected dependency failures and transient HTTP errors release only the exact,
still-undispatched claim to the existing approved queue and publish its existing
transactional work signal. No table, maintenance scan, or retry wrapper per
producer is added.
`tool-approval-execution.ts` owns the shared claim token, release, and dispatch
transition. Its Test-tab claim/outcome functions remain specific to that consumer.
Both origins use the same invocation-then-request locks and verify invocation,
company, request linkage, claim timestamp, awaiting-approval state, and absence
of an earlier start before dispatch. An agent that loses its claim during
preparation cannot call the provider or overwrite its successor. The Test-tab
unapproved direct-call path retains its original behavior.
Only preparation is retryable here. Recording the invocation start, provider
execution, and outcome persistence remain outside that catch. A provider error,
unknown outcome, or failed durable release does not establish replay safety.
Existing interruption handling remains the conservative fallback. Closed-issue
expiry, signature/target invalidation, identity authority, and policy rejection
remain enforced, including the final issue check before recording dispatch.
Real PostgreSQL lock-timeout cases cover foreground approval and recovery:
observe the committed executing claim, block the policy read, then require the
same signed request to remain approved with no invocation start and zero provider
calls. Recovery after releasing the table lock executes once and becomes quiet.
A separate contention case replaces the claim while preparation is blocked and
proves the old owner cannot dispatch; the surviving approval later executes once.
A provider-error case proves that an already-started call is not requeued as a
preparation failure. The existing Test-tab timeout and ownership cases exercise
the other consumer of the shared boundary.
## New or edited rate rules during approval
A reviewed invocation already has durable `consumedRateAdmissions` in its policy
explanation. The original read-only recheck retained unchanged rules correctly,
but new or edited rules only inspected their counters. Two approved actions could
therefore both execute against one available slot. The real recovery regression
failed for both a newly added limit and an increased limit before this repair.
The policy service now owns `admitInvocation`: lock the company-scoped invocation,
validate its exact actor/task/tool/argument binding, recheck current policy, spend
only admissions not already recorded for that policy generation, and save the
receipt atomically. The gateway supplies its execution claim; that claim is locked
and checked by ID, invocation, company, executing status and timestamp. This also
supports historical approved requests whose invocation still says pending, without
accepting a stale claimant. Existing claim/start/settlement owners still control
provider execution. No queue table, periodic query, or per-route wrapper is added.
Counter updates preserve the number of already spent slots when the limit changes.
A rejection by a later policy or failure to save the receipt rolls back new charges.
A retry of a committed admission retains the saved receipt, including concurrent
rechecks. Read-only approval preflight and browser authority probes stay read-only;
trust-rule usage is not counted again during execution revalidation. The policy
service and its metric helper accept only the database capabilities they use, so
transaction clients do not require a cast to a root connection.
The owning gateway and policy suites pass all 181 cases against disposable real
PostgreSQL databases. Coverage includes two competing invocations, concurrent
retries of one invocation, changed limits, failed receipt writes, rejection after
an earlier rate charge, unchanged admissions, historical approvals, scope mismatch,
claim loss, quiet recovery and unknown provider outcomes. Server typechecking
passes after preparing current runner dependencies.
Earlier verification failures remain part of this evidence: the first full run
was stopped because an old claim-loss fixture held the policy table while trying
to revoke the newly locked invocation. It now pauses discovery before admission
and retains the no-provider-dispatch assertion. A new receipt assertion was fixed
to permit the intended additional receipt while preserving original fields. The
next full run exposed an overly strict historical-approval check; exact execution-
claim validation fixed it. Typechecking then exposed root-client-only signatures,
which were narrowed to their actual database capabilities. The final complete
suites and TypeScript check pass; stack integration and current-head CI remain
separate gates. This proves worker and admission behavior, not deployed host sleep.
## Managed-argument expiry ordering
Latest-head CI exposed a preparation-order regression in
`tool-gateway.test.ts`: a valid historical Shopify approval whose managed UCP
arguments were obsolete became `failed` instead of `expired`. The new consuming
admission compared the current invocation identity before the existing managed
argument expiry owner ran. It prevented dispatch, but bypassed the correct saved
expiry outcome.
Approved execution now verifies target/signed/managed arguments and content
before consuming admission. Exact claim fencing, fresh policy checks, atomic
rate receipts, and the final issue-open check remain before dispatch. No check
was removed and stale payloads are not permitted to execute.
The existing regression reproduced the CI failure on this owner in
`/tmp/sleep-approved-managed-expiry-red.log`. All 92 gateway acceptance cases
pass after the ordering change (`/tmp/sleep-approved-managed-expiry-full.log`),
including the historical approval's expired request, failed invocation with
`approvalState: expired`, and zero provider requests. Server typecheck passes
in `/tmp/sleep-approved-managed-expiry-types.log`. Latest-head remote CI and
combined-stack acceptance still need their own results.
## Delegated Test-tab admission after policy changes
A delegated Test-tab call has an originating agent but no gateway token. The
named-gateway admission path therefore did not evaluate its caller policy. If a
rate rule was added while approval was pending, approval charged only the target.
A shared company/tool bucket then rejected the caller against that same charge;
a caller-specific rule instead allowed execution without recording its charge.
The Test-tab owner now supplies its caller/target policy evaluation to the existing
claim-transaction admission API. Both decisions use the claimed invocation's
receipts. The target can share the caller's charge only when the rule and bucket
match. Both receipts commit together. A denial rolls back new charges and receipts.
Management reauthorization and credential identity checks remain with their owners,
and provider execution stays outside the transaction. This adds no queue or timer.
Three actual HTTP/PostgreSQL cases cover a newly added shared rate rule, separate
caller and target rules, and a target denial after caller admission. The unchanged
implementation failed the first two cases: the shared rule rejected execution,
and separate rules left only one charge. The denial case already passed. After
the repair, all three pass, including duplicate approval, both saved receipts,
one provider execution, and rejection of a subsequent authorized call.
The full connection-management suite passes all 78 cases against a fresh database.
All 92 existing gateway tests and the server production typecheck also pass.
Intermediate fixture failures remain accounted for: the new assertion initially
used a nonexistent counter count field instead of remaining; the subsequent-call
fixture first reused an ended run and then lacked responsible-user authority.
The final fixture uses a new live run with the original responsible user. A reused
full-suite database also retained a conflicting Slack bot identity; final acceptance
uses a fresh isolated database. These are local checks, not current-head CI or a
host-suspension qualification.
## Historical argument identity compatibility repair
Earlier invocations stored the hash of their redacted argument preview. Modern
admission correctly requires the complete request identity, including a keyed hash
for sensitive arguments. The gateway already verified signed historical arguments,
but admission rejected their saved preview hash even with no rate policy. That
permanently failed an otherwise valid approval before the provider call.
Both agent-origin and Test-tab preparation now bind the verified complete identity
for a historical invocation before current policy admission. Signature, reviewed
target and managed-argument checks still run first. The approval execution owner
performs the identity transition in an invocation-then-request lock transaction.
It requires the exact current claim, company and invocation relationship, an
undispatched awaiting-approval invocation, and the expected reviewed/preview hash.
Unrelated saved hashes are not accepted. Modern matching identities do not add
this transaction. Current policies and rate charges still apply afterward.
Four real-PostgreSQL regressions reproduced the defect: valid sensitive historical
arguments, with and without a newly configured rate rule, through each of the two
origins. Four controls keep invalid signatures and unrelated invocation hashes
rejected without dispatch or identity changes. The provider receives the original
verified arguments exactly once, rather than their redacted preview. A separate
real-transaction case proves a replaced claim cannot change the saved identity.
The first draft fixtures used a nonexistent request-level tool name and then
non-sensitive arguments whose preview and complete hashes were equal. Those
fixture failures are not credited as product regressions. The corrected four
positive cases fail before the repair and all eight parameter cases pass after.
The complete gateway, policy and startup selection passes 230 cases with no skips;
server types pass. The complete assembled-server connection-management suite also
passes all 78 cases, including delegated approval and rate-policy behavior. Fresh
remote CI and reviews remain required after publication.
The repair was then integrated with the master-based parent, including #17918's
tracing changes, at `81231824d773a79fc50c835eccf7d9b3977e79fc`. Server types and the
production server build pass. The gateway, policy, startup and preparation-timing
suites pass all 242 cases. The complete HTTP management suite passes all 78 cases
on a fresh dedicated database, for 320 passing cases across the accepted runs.
The first combined integrated run passed 319 cases and failed the Slack setup
case because its reused HTTP database still held the previous run's bot identity.
The existing ownership guard correctly rejected the duplicate connection. A fresh
dedicated fixture resolved that environment collision; no product behavior or
assertion changed for the rerun. That failed run remains part of the evidence.
## Gateway admission uses its transaction connection
Review of the master-targeted branch exposed a delegated gateway deadlock: the
admission transaction held the sole pool connection while issuer/token authority
reads used the root client. Foreground approval and approved recovery each failed
the real one-connection regression; a revoked-issuer control already passed.
The admission callback now receives its transaction and passes it through the
existing issuer authority checks. Both connection authorization checks and the
token read use that connection. Ordinary checks retain their root client, and
current authority, claim, company and rate enforcement remain in place.
The three constrained-pool cases pass, as do all 81 HTTP integration cases and
190 gateway, policy, startup and deadline consumer cases. The initial consumer
command selected an incorrect policy filename and omitted the gateway database
override; it was interrupted and is not counted as passing evidence.
## Pending historical approvals retain their original admission
A second review finding concerned the normal foreground approval gate, before
execution has claimed and rebound historical arguments. A real PostgreSQL
regression with signed sensitive arguments and an already-consumed limit of one
failed with `rate_limited`; the Test-tab and invalid-signature controls passed.
The gateway now supplies the preview digest only after verifying the signed
payload and matching either supported form of the review's canonical digest. Policy may match that digest
only in a read-only preflight, with every other invocation-context field and
stored rate-rule receipt still checked. The consuming admission continues to
require the complete keyed identity, rebound under the exact execution claim.
New or changed rate rules still apply. Historical recovery, invalid signatures,
foreign invocation hashes, stale claims, and provider argument preservation stay
covered. This adds no schema, recurring query or additional connection.
The preceding CI head also failed E2E shard 3/8. Its failure is retained in the
validation record; local gateway checks do not erase it. Fresh-head CI and both
required automated reviews remain merge gates.
The five full gateway/policy/startup/deadline suites passed 329 cases before the
additional mixed historical identity control. The first combined attempt is
retained: a Test-tab failure-result assertion incorrectly expected a thrown
error, and reusing the earlier HTTP database left formal-approval foreign keys
that prevented gateway acceptance setup. The assertion now checks the existing
failed-result contract, with unchanged no-provider, hash and counter checks;
the complete acceptance rerun used an independently migrated gateway database.
Server production typechecking and build passed. The earlier E2E failure was a
500 during the agent-chat project-plan fixture restoration, with a PostgreSQL
deadlock in cancellation; it remains a fresh-head CI gate.
+127
View File
@@ -0,0 +1,127 @@
# Decision-archive notification delivery
Auto-archive already writes `decision_archive_notification_outbox` in the same
transaction that archives its retention row. The old retention sweep called
notification delivery on every maintenance tick, including an empty outbox.
Delivery is now its own coordinator worker. Retention scanning remains a separate
migration; this PR does not change what is archived or when.
## Ownership and producer audit
`decision-retention.ts` owns eligibility, current-version checks, source authority,
origin resolution, and archive writes. Its sole outbox producer is `autoArchive`.
It now awaits the delivery notification immediately before inserting the outbox
row in the existing transaction. Caller-owned transactions and savepoints retain
outer commit/rollback ownership through the shared database signal mechanism.
Manual archive/revive and reviewed-manifest archive do not create these
notifications today; this change does not add notifications to those actions.
`decision-archive-delivery.ts` owns claim recovery, selection, grouping, delivery,
and settlement. `decisionRetentionService.deliverNotifications` remains a
compatible delegate. Its notification callback and batch type retain their
existing contract. The scheduler only sees the delivery operation and next
work deadline. Claim and acknowledgement writes belong to this worker; they do
not broadcast self-generated wakes. No table is added. A concurrent partial index supports the pending delivery
order on the existing outbox.
The server registers the worker in enabled-heartbeat startup housekeeping and
removes notification delivery from `runRetentionSweep`. Registration scans saved
work before the retention pass can enqueue more. The disabled-heartbeat branch
continues not to dispatch agent notifications. A service without a callback
leaves outbox rows intact and does not attempt notification delivery.
## Work and deadlines
Pending outbox rows are due for delivery. Each pass claims up to 500 rows and
groups them by company and origin agent, preserving company scope and the
existing notification payload. Failed or remaining pending rows retain a
30-second retry. Never-attempted rows sort first, then the oldest attempts, so
one failing batch cannot repeatedly hide later rows.
A delivering row retains its existing five-minute claim deadline. Startup
reconstructs that deadline from `lastAttemptAt`; a legacy missing timestamp is
immediately recoverable. A worker waiting only for a future claim deadline does
not repeatedly query until that deadline. Settlement and failure reset require
the same claim timestamp, so a delayed old result cannot settle or release a
replacement claim. Delivered counts include only rows actually settled.
An empty outbox or retained delivered history has no worker deadline and no
recurring SQL. The coordinator coalesces writes and prevents overlapping passes.
The shared deadline worker preserves first enqueue during drain, changes during
delivery, and failed retries across drain/resume. Coordinator shutdown joins
in-flight work. These guarantees do not replace the durable rows.
## Recovery and limits
A failed enqueue rolls back both the archive mutation and its outbox insertion.
An uncertain transaction retains the shared intent until its outcome is known.
A notification failure returns the same rows to pending. A process stopping
while delivering leaves the claim for deadline recovery on restart.
Delivery remains at least once. A wake accepted before a lost acknowledgement
can be retried. This change does not promise exactly-once agent wakeups or add
new deduplication storage. The five-minute stale threshold is unchanged.
Supported producers share the process-local database signal owner. Independent
SQL or another active server needs a separate wake contract. The existing Cloud
sleep proof still blocks this outbox conservatively; no external-wake coverage
or sleep guard is relaxed. Worker quiet does not prove actual database or host
suspension. Retention scanning and decision continuations remain separate duties.
## Verification
The original migration selection passed 54 tests: 15 real-PostgreSQL retention/delivery cases
and 39 startup cases. Twelve new cases cover configured idle, retained delivered
history, real enqueue, restart, exact stale-claim timing, legacy missing timestamps,
first enqueue during drain, failed-delivery retry through drain, outer commit and
rollback, enqueue failure rollback, writes during delivery, more than 500 rows,
and stale-result fencing. Quiet assertions observe no reads or writes across an
hour of controlled time. Workspace typecheck and build pass.
The first 52-case selection passed. Adding fault/concurrency tests then exposed
two fixture errors: duplicate globally unique issue identifiers across two seeded
companies, and asserting quiet before rollback reconciliation had run. Seeded
identifiers now use their company prefix; quiet checks advance the retained retry
and await authoritative settlement. All 54 cases passed on rerun. No production
hold, recovery deadline, or timeout was weakened. Full repository CI and fresh
review are required before this PR is ready.
## Review repairs: startup, claim results and backlog selection
Startup regressions now exercise `startServer` itself. Enabled startup must
register `decisionArchiveDelivery` and invoke the delivery owner; calling the
captured heartbeat interval still performs retention reads without another
notification delivery. Calling the registered worker invokes delivery again.
The disabled-heartbeat case explicitly preserves the existing no-notification
contract. An initial expanded test incorrectly expected that disabled mode to
register the worker and failed; inspection confirmed the documented disabled
contract, so the test now asserts it rather than enabling new background work.
The real stale-claim regression first returned `notifiedAgents: 1` after the
replacement claim prevented any settlement. The counter now increments only
for a group that actually settles rows, while the existing timestamp fence and
at-least-once delivery contract remain intact. Failure reset uses one timestamp
for `lastAttemptAt` and `updatedAt`.
Generated migration `0387_boring_professor_monster.sql` creates a concurrent
partial index on `(last_attempt_at NULLS FIRST, created_at, id)` for pending rows.
The existing top-level migration executor handles concurrent creation and
invalid-index recovery. No history migration or existing journal entry changes;
the generated snapshots retain the five newest versions.
A real migration and query-plan check used a separate database with 50,000
outbox rows, including 30,000 pending. The existing claim-selection query returned
500 rows in both cases. Without the new index PostgreSQL sorted the pending
backlog with a top-N heapsort and touched 944 shared buffers; with the index it
used an index-only scan without sorting and touched 505 buffers. The baseline
index removal was rolled back inside that dedicated database, and the migrated
index was confirmed valid afterwards. These local measurements establish the
access path, not production latency or cost savings.
All 58 cases across real PostgreSQL delivery/retention, startup and the shared
deadline worker pass. Generated snapshot drift, migration numbering/safety,
server types and build pass. The earlier stale-counter failure and invalid
scheduler-disabled expectation are retained above. The existing 500-row batch
limit and thirty-second retry for remaining pending work are unchanged. Fresh
current-head CI, Greptile and Ponytail remain required before merge.
+193
View File
@@ -0,0 +1,193 @@
# Decision continuation delivery while idle
A terminal decision can owe an agent wake. Its existing row stores that debt:
`continuationPolicy = wake_origin_agent`, `metadata.continuationPending = true`,
and either expired/cancelled status or a decided status with succeeded/partial/
failed execution. Open decisions and running effects are not ready to deliver.
This migration adds no queue table and changes no wake payload.
## Ownership and producers
`decisions.ts` owns decision creation, authorization, signatures, effects, and
state changes. Its ten update sites now use a narrow transaction-aware write
helper before the SQL statement. These cover the running claim; ordinary effect
completion; archive-proposal signature, manifest, and execution failures; archive
success; TTL rejection on use; cancel; direct dismissal; and background expiry.
The helper acquires intent in the existing outer transaction or its savepoint.
It does not include effect execution, provider calls, activity publication, or
agent wake callbacks in a new database transaction.
Creation and bundle creation insert open decisions, so they do not create ready
continuation debt. Acknowledgement only clears debt and belongs to the delivery
owner; it does not signal itself. The production decision services are created
by server startup and routes with the application's DB handle. Independent SQL
writers and additional active processes require a separate notification and
ownership contract. This change does not make wake-producing service methods
safe to call inside an uncommitted caller transaction; only their SQL mutation
boundary has the existing outer-commit guarantee.
`decision-continuations.ts` owns the readiness predicate, delivery, and recovery.
Each worker pass attempts at most 100 candidates. A saved in-memory cursor
advances past failed candidates between passes and wraps after the last page.
Remaining durable work retains another pass. This avoids starving later rows behind a failing first page.
A partial ID index contains ready pending continuations only. The predicate uses
fixed SQL literals so a prepared generic plan can still use that index.
The shared DB package exposes an opaque work-scope identity. Raw DB object
identity is insufficient because dedicated and liveness child pools already
share their owner's signals. The key is shared by those children and nested
transactions; independent roots have different keys. No connection credentials,
private transaction state, or decision policy are exposed. Existing signal
consumers and transactions that do not register work keep their behavior.
## Foreground, recovery, and failure
Foreground delivery remains immediate. All service instances sharing the DB
owner coalesce a simultaneous wake for the same decision into one promise.
Delivery re-reads durable readiness after claiming that local flight, so a late
foreground call cannot resend a continuation already acknowledged by recovery.
The promise is removed on success or failure. Effect execution and its final
state write also share one local flight per decision. This prevents concurrent
archive callers from turning a successful archive into a stale-manifest failure. This is process-local coalescing,
not a new durable claim or an exactly-once guarantee.
Effect completion requires the row to remain running. A competing completion
cannot restore pending metadata after another caller has completed and delivered
it. Effects retain their own existing durable claims and authorization checks.
The delivery worker never executes effects.
A wake failure leaves pending metadata intact. A failed acknowledgement also
leaves debt, so a later attempt can send the wake again. A crash after accepting
the wake but before acknowledgement retains the existing at-least-once contract.
An uncertain producer transaction remains held until the shared work-signal
protocol establishes settlement; one empty scan cannot clear that uncertainty.
## Lifecycle and scope
Startup reconstructs pending delivery. Committed changes wake the worker through
the existing coordinator. Failures or remaining ready debt retain a 30-second
retry. Open/running decisions, no decisions, and delivered history have no
recurring continuation scan or deadline. Changes during a pass retain another
request. Drain/standby defer dispatch without consuming that request; shutdown
waits for the current admitted delivery and stops before starting another.
The shared deadline adapter forwards its existing AbortSignal to task execution;
existing callbacks that do not consume the signal keep their behavior. A future process-local retry is not a Cloud alarm.
Enabled-heartbeat startup registers this worker before the combined decision
sweep. The main tick skips that sweep's independent continuation scan. Its public
API retains combined behavior by default. Expiry or effect recovery can still
deliver their newly produced continuations immediately. Disabled-heartbeat
startup keeps its existing behavior and does not dispatch this new worker.
Decision expiry, invalidated-target scanning, interrupted effect recovery, and
retention remain separate migrations. Existing Cloud safety blockers are not
relaxed. This establishes worker quiet under the named conditions, not deployed
host/database suspension or measured fleet savings.
## Verification
The selected run passes 93 tests: 50 real-PostgreSQL decision cases, 39 startup
cases, and four database work-signal unit cases. Fifteen new continuation cases
cover real decide/cancel/dismiss/expiry producers, running-to-terminal completion,
foreground contention, dedicated child pools, stale caller re-read, restart,
failed acknowledgement, outer rollback/commit, first write during drain, failed
retry through drain, changes during delivery, shutdown, and paging past a failed
candidate. Quiet checks observe no SQL calls across an hour of controlled time.
Nine pre-existing embedded-PostgreSQL signal tests are skipped locally because
the embedded runtime is unavailable; CI must run that supported environment.
Workspace typecheck and build pass. The final literal-predicate adjustment also
passes all 49 decision tests before the additional archive-race regression.
The original continuation index applied and replayed without error. After parent integration it is generated migration `0388_ambitious_micromax.sql`; the fresh apply and replay also pass. A rolled-back synthetic
40,000-row history with one pending continuation uses the partial index under
`force_generic_plan`, reads two shared buffers, and executes in 0.015 ms on the
local PostgreSQL server. This is query evidence, not a production latency claim.
The first test selection had 90 passes and one failure: the injected PostgreSQL
acknowledgement error is wrapped by Drizzle, but the assertion expected its
message at the top level. The assertion now checks the original cause. No
production retry, hold, or timeout was weakened. A recursive unit-fixture type
initially prevented migration generation; an explicit fixture type fixed it.
Full CI and fresh review remain required before PR readiness.
Review found two real regressions. Before repair, a controlled concurrent archive
reported failed after archiving successfully, and shutdown sent a second wake
after stop was requested. Both regressions failed before the fix. Shared local
effect ownership fixes archive completion; bounded passes and abort propagation
fix backlog shutdown. The 93-case selection passes after repair. The paging test
also proves a failed first candidate cannot hide the 101st candidate across
bounded passes. A result-type alias initially made the known decision ID optional;
the alias now matches the existing outcome contract.
The existing tool-review and archive-delivery consumers also pass 124 real-DB
tests after AbortSignal propagation. Together with the decision/startup/signal
selection, 217 selected tests pass; the same nine older embedded-only cases
remain locally skipped. Workspace typecheck passes after the outcome type fix.
## Explicit archive-failure delivery
The three archive-proposal failure paths now explicitly set
`continuationPending: true` for `wake_origin_agent`, rather than assuming the
pre-read running row already carries that metadata. The existing running-state
fence remains, so stale settlement cannot re-arm an acknowledged result.
Invalid signatures, manifest mismatches and archive execution failures each
failed a real-PostgreSQL recovery regression before repair. Each now saves the
failed result, wakes once, acknowledges delivery and remains quiet on replay.
Normal parent integration preserves the projection interaction-link index and
archive pending-order index. The unmerged continuation index was regenerated
as 0388 from the parent's generated metadata; every parent journal entry is
unchanged, five snapshots remain, the SQL uses the existing top-level concurrent
executor, and snapshot/schema drift passes. No generated snapshot was hand-edited.
The final affected selection passes 111 cases: 53 decisions, 15 retention/archive,
42 startup and one scoped-deadline case, with all four suites exercised. The
first combined command used the wrong retention DB variable and skipped those
15 cases; the corrected external-PostgreSQL retention run passes all 15. Server
typecheck and build pass. The historical nine embedded-only work-signal skips
above are not reclassified as executed by this selection. Fresh CI and reviews
remain required.
## Bounded notification-driven recovery
Security review identified that sequential decision writes could each start an
instance-wide continuation scan. A real-producer regression reproduced six
sweeps for six sequential cancellations. The worker now uses the existing
`notificationCoalesceMs: 30_000` option. The first wake after a quiet window still
reconciles promptly, and subsequent notifications in that window share its end
deadline. Foreground continuation delivery is unchanged and immediate. Recovery
can wait up to thirty seconds, matching the prior maintenance cadence. This
bounds global scan frequency; it does not make queries company-scoped or add a
cross-process notification contract.
The regression verifies one scan during repeated real create/cancel writes,
retention and eventual delivery of a later pending row, and no recurring SQL
after settlement. Its first green attempt wrongly expected zero active work
while the coalesced request was pending. The existing intent hold correctly
remains until reconciliation; the test now checks that hold's actual terminal
boundary through the existing quiet assertion. No production hold was removed.
All 293 integrated cases across decisions, archive, gateway/projection, startup
and scoped-deadline suites pass without skips; server types/build pass. Fresh
security review, Greptile, Ponytail and exact-head CI remain merge gates.
The archive contention regression no longer waits a fixed 100 ms. A dedicated
single-connection contender signals its decision read; two FIFO SQL round trips
then establish that it reached shared execution while the first archive remains
held. The test requires one contender decision read and one archive execution.
Temporarily removing the effect-flight guard made this test fail (two reads);
restoring the production guard passes. The mutation was restored before commit.
This verifies actual cross-pool contention without a timing delay or a new
production-only test hook.
Archive #17475 is merged. Normal reconciliation with its exact squash
`5785090ff1` preserves all 31 independent incoming master files, including
Runner controller-authority recovery. All 97 decision/startup/deadline cases and
server types/build pass after that integration. The Runner vendor rebuilt from
its changed inputs. The preceding full five-suite selection passed 293 cases;
the continuation owner's production behavior is unchanged by reconciliation.
Fresh exact-head reviews and CI remain required on the master-target candidate.
+255
View File
@@ -0,0 +1,255 @@
# Idle decision-effect recovery
Interrupted decision effects now have a dedicated deadline worker. The main
maintenance tick no longer scans running decisions. Target invalidation remains
in that tick; TTL expiry and continuation delivery have their own workers.
## Obligation and activation
The existing decision row is durable truth. A row with `status = decided` and
`executionStatus = running` owes effect recovery. Its next deadline is
`updatedAt + PAPERCLIP_DECISIONS_RECOVERY_GRACE_MS`, default 60 seconds. Zero is
supported; invalid or negative values retain the existing default. Reads do not
move the timestamp. Open decisions and completed history create no recovery debt.
The existing `writeDecisionChange` transaction boundary signals effect recovery
for running claims, effect/archive settlement, cancellation, dismissal and expiry.
The generic `notifyDecisionChange` helper signals continuation and expiry only;
open-decision creation does not owe effect recovery. Notifications are awaited
before writes. Outer commit activates work; rollback does not publish a claim.
The additional topic does not wrap provider work or agent wakes in SQL transactions
and does not introduce a new queue table.
Committed notification bursts coalesce for up to one second after a successful
pass. This bounds sequential tenant writes to at most one notification-driven
global pass per window, rather than only coalescing overlapping scans. A saved
recovery deadline can preempt that window. New obligations discovered by a signal
can wait up to one second; foreground effects still execute immediately. The
window alone never schedules work when there is no notification or saved debt.
The supported writer model is the application DB signal owner and its child pools.
Independent SQL clients and multiple active writers need a separate notification
contract. Startup reconstructs committed claims; it does not observe future writes
from unsupported writers.
## Ownership and recovery
`decision-effect-recovery.ts` owns selection, grace, bounded paging, and earliest
deadline lookup. `decisions.ts` owns execution, current membership, effect claims,
audit, archive policy, and continuation delivery. The worker adapter only binds
that domain contract to the shared coordinator. Startup registers it only when
heartbeat scheduling is enabled. The public combined sweep retains its default
recovery behavior for existing callers.
Each pass honors the existing configured batch size (default 100, finite values
clamped to at least one integer). Its timestamp/ID cursor advances past failed
candidates within a frozen final timestamp/ID. Newer due claims wait for the next
rotation, so a continuous stream cannot postpone an earlier failure forever.
Both cursor positions retain PostgreSQL precision. Genuine execution or settlement
failures retain their retry obligation.
A missing saved decider or chosen option is a deterministic malformed outcome.
Recovery rechecks it under a row lock, records an audited failed outcome (partial
if a completed effect receipt exists), and preserves all effect receipts. It
creates no authority and executes no effect for the malformed claim. The failure
reason and continuation obligation are saved in the same transaction as the audit.
An audit or settlement failure rolls back and leaves the claim pending. A claim
repaired before the lock is acquired remains eligible for normal recovery.
The decision service re-reads current status before recovery. Foreground and
recovery execution share the existing per-decision flight scoped to the DB owner.
Individual effects keep their durable records and transaction advisory locks.
Completed effects are not repeated after a failed final decision status write.
Recovery derives authority from the saved decider's current active membership.
It preserves the current authorization intersection and current target checks.
The scheduler has a 30-second minimum retry spacing for pending or failed work.
A first claim or failed retry during drain retains its wake without passive SQL.
Shutdown stops subscriptions and checks cancellation between candidates. An
already admitted effect/continuation is awaited; later candidates remain durable
for restart. No periodic pass remains after all claims settle.
## Existing guarantees and limits
Regular effects preserve their existing per-effect transaction boundaries.
Continuation delivery retains its separate at-least-once wake/acknowledgement
contract. This migration does not establish exactly-once external delivery.
Archive proposals have a separate versioned manifest contract. Recovery still
verifies the saved signature/manifest and current authority. A crash after an
archive commits but before the decision settles can encounter the changed
manifest version on restart. The existing result is a failed decision with an
archive error, while the archive remains committed once. The new worker preserves
and tests that outcome; it does not claim to repair this preexisting atomicity gap.
Cloud sleep guards remain conservative. The internal next deadline is not a
persisted external alarm and does not authorize host sleep with running decisions.
This change establishes worker quiet, not actual instance/database suspension.
## Verification
The new real-PostgreSQL suite covers 17 cases: empty/open/completed-history quiet,
real foreground claim and failed settlement, exact grace and zero override,
restart from a claimed effect, active foreground/recovery overlap, outer commit
and rollback, first claim and failed retry through drain, revoked authority,
continuation failure, aggregate exclusion, fair paging past missing-decider and
malformed claims, archive recovery before and after an archive commit, a claim
created during delivery, and shutdown between candidates. Quiet assertions observe
SQL methods through one hour of fake time after settling.
The combined selection passes 212 tests: the 17 new cases, 50 existing decision
service cases, 14 TTL cases, 39 startup cases, and 92 agent-history route cases.
The first 15-case recovery run also passed. Workspace typecheck/build and the
final server typecheck pass. No failure was suppressed or timeout increased.
The current regenerated migration `0390_salty_killraven.sql` adds one partial index concurrently, with `IF NOT EXISTS`, through
the migration owner's top-level statement path. Generation retains the newest
five schema snapshots. Fresh application, replay, safety, and numbering checks
pass. With 40,000 completed decisions and one running claim, both due selection
and earliest-deadline lookup use the partial index and read two buffers (0.012 ms
and 0.003 ms in the local probe). The probe transaction is rolled back.
Full PR CI and review must pass separately before this slice is ready.
A follow-up regression seeds a legacy invalid claim with a PostgreSQL-generated
microsecond timestamp ahead of a valid claim. With a one-row page, the original
Date cursor selected the invalid claim again and starved the valid effect. That
regression failed before repair. Recovery now retains the timestamp as database
text for cursor comparisons, leaving business grace deadlines unchanged. The
unresolved claim remains durable debt and is revisited after paging wraps; the
valid claim progresses without repeating its completed effect. The regression
now holds a candidate unresolved through the dispatch callback, because known
malformed claims receive a durable failed outcome as described above.
All 18 effect-recovery lifecycle cases and server typechecking pass after repair.
### Foreground join test timing
CI exposed a timing assumption in the foreground/recovery overlap test: it
expected exactly two lifetime recovery sweeps. A controlled PostgreSQL replay
waited for the committed claim's deadline discovery before advancing time. It
recorded two completed, zero-resumption scans before the deadline (startup and
claim notification), followed by the legitimate deadline scan. The old assertion
then failed deterministically with three calls. No recurring idle poll or duplicate
effect was found in that sequence.
The test now waits for discovery, measures the deadline recovery relative to that
point, and proves that its promise stays pending while foreground execution holds
the effect lock. Advancing through several retry periods cannot start another
sweep or effect. After release, recovery completes, one comment and one continuation
wake exist, and the worker performs no SQL through the existing hour-long quiet
check. Counting actual lock attempts also distinguishes joining from a second
execution blocked on the same PostgreSQL advisory lock. A temporary mutation that
removed flight joining made this stronger assertion fail; production code was
restored before final verification. No runtime code, retry policy, timeout, or
business deadline changed.
All 123 tests across the four complete effect-recovery, decision-service, expiry,
and startup suites passed, followed by server typechecking. Evidence:
`/tmp/sleep-effect-join-red.log`, `/tmp/sleep-effect-join-green.log`,
`/tmp/sleep-effect-join-mutation.log`, `/tmp/sleep-effect-join-owning-tests.log`,
and `/tmp/sleep-effect-join-root-types.log` with their status files. The initial
failure and mutation failure are expected evidence, not passing runs. Remaining
review findings and fresh stack CI remain separate acceptance requirements.
### Recovery review repairs (October 26)
Two regressions failed before repair: an old failure was not revisited while newer
due claims kept each page full, and creating an open decision caused an unnecessary
effect-recovery scan. Frozen rotation boundaries now preserve both retry fairness
and forward progress. Creation still signals its continuation/expiry consumers;
the effect queue is signaled by the existing claim/settlement write boundary.
Four malformed-outcome cases also failed before repair. Recovery now records
`effectRecoveryError` and a `decision.recovery_failed` activity atomically. Live
activity publication follows commit. Existing completed effect receipts remain
unchanged, and the normal continuation owner delivers the result. A real audit
insert failure leaves both status and audit uncommitted, then later recovery
settles successfully. A real PostgreSQL row-lock test repairs a claim while
settlement waits; the locked recheck leaves that claim running, and its valid
effect executes once on the next pass.
The shared execution flight now loads the current claim and derives recovery
authority together. Recovery no longer performs a redundant full-row lookup
before entering the flight. Foreground overlap, current membership checks,
per-effect locks, continuation behavior, and the resumption count are preserved.
Only the two known malformed states are terminalized; other failures retain their
existing retry policy. No schema, queue, or external-wake contract is added.
All 24 recovery lifecycle cases and server typechecking pass. Controlled failures
and final evidence are retained in `/tmp/sleep-effect-fairness-red.log`,
`/tmp/sleep-effect-invalid-red.log`, `/tmp/sleep-effect-owner-tests.log`, and
`/tmp/sleep-effect-owner-types.log`. Broader stack and remote acceptance remain
separate requirements; these results do not prove host suspension.
The owning branch also passes all 129 cases across the four complete recovery,
decision-service, expiry, and startup suites. Database-backed suites use separate
migrated disposable databases. See `/tmp/sleep-effect-review-owner-acceptance.log`.
### Bounded notification review repair
Ten sequential real cancellation transactions reproduced ten extra recovery
scans. After enabling the existing shared notification-coalescing capability,
the same transactions cause one follow-up scan and then return to SQL quiet.
A separate real-database case preserves an already loaded deadline 500 milliseconds
away. The initial deadline fixture observed the scheduler while its SQL pass was
still active; waiting for that read removed the fixture race before the baseline
was recorded. The cancellation amplification still failed before the production fix.
All 131 tests in the four complete recovery, decision-service, expiry and startup
suites pass at the owning branch, as does server typechecking. The helper's optional
coalescing capability is promoted from the later plugin scheduling slice into this
PR; consumers that omit it keep their previous behavior. The integrated stack is
verified separately before publication. The original full-suite and external-wake
limitations remain.
## Current parent integration
The expiry and continuation review repairs are integrated normally. The running
claim index is regenerated as 0390 after parent 0388/0389, with concurrent creation
registered in the existing executor. All 388 parent journal entries are unchanged;
the five newest generated snapshots remain. Schema drift, numbering/safety and
fresh migration apply/replay pass. No generated snapshot was hand-edited.
All 141 cases in the five complete effect-recovery, decision/continuation, expiry,
startup and scoped-deadline suites pass. This includes all 26 recovery cases and
the parent's deterministic cross-pool archive contention test. Server types and
build pass. The first integrated run passed 140 cases and failed an old scheduler
assertion: the coordinator now also holds an earlier continuation-coalescing
deadline. The overlap test services that earlier registered wake, then still
requires the original effect grace deadline, one active effect flight, two total
advisory-lock attempts (test gate plus foreground), one committed effect and one
continuation. No production timing or retry was changed to pass the test.
Existing one-second effect-notification coalescing and thirty-second failure
retry remain. Continuation notification coalescing is owned separately by the
parent. Extra upstream embedded-only tests attempted during parent integration
could not initialize locally; fresh current-head CI must cover that environment.
Full CI and both automated reviews remain merge gates.
After normal integration of master-target expiry parent `f7c61e9912`, the
effect-recovery owner, worker, decision execution and notification implementations
are unchanged. All 69 current recovery/startup/deadline cases and server types/
build pass with the updated Runner vendor. The earlier full 141-case selection
remains evidence at its stated revision. Parent migration history is an exact
prefix and five generated snapshots remain. Fresh head CI and both automated
reviews are required after this parent integration.
## Landed expiry integration and CI interruption
Expiry-only creation notification from `aa03d2c4c2` is integrated. All 143 cases
in the complete recovery, decision, expiry, startup and scoped-deadline suites
pass, including single/bundle creation without continuation signals. The normal
merge of landed expiry `3421366903` also preserves all 21 independently merged
master files. The same 143 cases pass after that reconciliation.
The previous candidate `f54a707d74` had both reviews at 5/5, but CI run
40597921712 failed because server shard 6 was interrupted. Its test step was
cancelled and orphan cleanup ran; no failing test assertion or suite summary was
reported. The workflow eventually completed with that shard and its dependent
verify gate failed. It was not retried or presented as passing. The necessary
expiry parent repair creates a new candidate with fresh CI and reviews required.
+177
View File
@@ -0,0 +1,177 @@
# Decision TTL expiry while idle
Open decisions already contain an `expiresAt` deadline. The expiry worker now
uses that timestamp instead of scanning TTL candidates on each maintenance tick.
This slice owns time-based expiry only. Target invalidation, interrupted effect
recovery, and retention remain separate duties.
## Producers and transaction ownership
Creation and bundle creation call `notifyDecisionCreated` before the decision
insert in their existing transaction. It signals expiry only: an open row cannot
owe continuation delivery yet. Existing decision updates share `notifyDecisionChange`, which now
signals expiry as well as continuation delivery. Cancellation, dismissal, and
execution claims therefore remove obsolete expiry deadlines. All production
inserts and updates are in the decision owner; acknowledgement only changes
terminal delivery metadata and cannot change an open expiry. No supported API
currently edits an existing decision's expiry timestamp. A future such writer
must use this same notification boundary.
The helper adds no recovery wrappers to routes. Outer transactions and nested
savepoints retain the shared commit/rollback and uncertain-outcome protocol.
Creation without a committed row cannot publish a business deadline. Startup
reconstructs saved deadlines, including already-overdue work. Independent SQL
writers and another active server process need their own wake contract.
`decision-expiry.ts` owns due selection, the next deadline, and expiry settlement.
`decisions.ts` delegates expiry-on-use and the existing target-driven expiration
transition to that owner. Target selection remains outside the TTL worker.
Startup registers TTL work before the combined decision sweep. The periodic
combined sweep opts out of TTL selection and cannot expire a row merely because
its TTL elapsed. Its public default still includes TTL and preserves the
configured combined-sweep batch limit.
## Deadline, settlement, and recovery
The next deadline is the earliest open `expiresAt`. A partial index on
`(expires_at, id)` covers open decisions. The query selects one indexed row;
retained terminal history does not enlarge the deadline lookup. Expiry is due
when `expiresAt <= now`, including equality. The old combined path skipped the
exact equality despite selecting that row; this migration fixes that boundary.
Each worker pass honors `PAPERCLIP_DECISIONS_SWEEP_BATCH_SIZE` (default 100),
ordered by expiry and ID.
Its in-memory cursor advances across failures and wraps after the last page.
Remaining overdue work retains a one-second continuation through the saved
earliest deadline after a successful pass.
Failures retain the existing 30-second retry throttle. New earlier creation or
resolution of the earliest decision causes deadline reconciliation. When only a
future expiry remains, the worker does not query again until that deadline or a
committed change. No open decisions means no recurring expiry work.
Settlement requires the row to remain open. TTL settlement also rechecks that
the saved deadline is due. Metadata is merged into the current row. The existing
expiry activity is persisted in the same transaction as the terminal transition.
Publication and origin continuation follow commit. If the audit insert fails,
the expiry and continuation obligation both roll back. A deadline-driven expiry
records `ttl`; target-driven expiration retains its own reason. When both causes
are possible, the first successful terminal transition determines the reason.
Continuation failures belong to the continuation worker after the decision is
expired. Neither owner replays decision effects.
Drain and standby defer dispatch while retaining the request. The worker checks
the coordinator's AbortSignal between candidates. Shutdown finishes the admitted
expiry and leaves untouched candidates for restart. It does not process the
whole backlog before stopping. There is no new queue table or permanent DB
listener, and no provider call is held inside a new transaction.
## Limits
The disabled-heartbeat branch keeps its existing behavior and does not register
this worker. Expiry-on-use remains enforced independently of background work.
The in-process deadline is not an external Cloud alarm. Existing conservative
sleep blockers remain; this migration does not permit an instance to sleep with
unsupported future obligations. Worker quiet is not proof of host or database
suspension.
## Verification
The original combined selection passed 103 tests: 14 initial real-PostgreSQL expiry worker
cases, 50 decision service/continuation cases, and 39 startup cases. It covered
empty and terminal-history quiet, actual create and bundle producers, exact
expiry, moving the next deadline earlier, cancellation, restart, outer rollback
and commit, first creation during drain, failed retry through drain, failed wake
delivery, separate aggregate/TTL ownership, 101 candidates with a failed first
row, creation during delivery, and shutdown/restart with outstanding candidates.
The parent archive race and bounded continuation shutdown regressions also pass.
The first selection had 97 passes and one fixture failure. The fault test cleared
its error collector before an already-admitted failed query had finished. It now
checks all collected causes after recovery settles. The next selection passed
101 cases; after the parent review repairs and bounded expiry/shutdown coverage,
all 103 pass. No production hold, retry, or timeout was weakened.
The original index applied and replayed successfully; after normal parent
integration it is regenerated as `0389_brave_hulk.sql`. Fresh apply also passes. With 40,000 synthetic decisions
and one open row, the next-deadline query uses the partial index, reads two shared
buffers, and executes in 0.022 ms locally. The probe transaction rolls back. This
is query evidence, not a production latency or savings claim. Workspace
typecheck and build pass on the combined branch. Full PR CI and fresh review
remain required before readiness.
A follow-up paging regression uses 101 overdue rows sharing a PostgreSQL
microsecond timestamp, with the first 100 writes failing. The original Date-based
cursor rounded the timestamp to milliseconds and repeatedly selected that failed
page, starving the last valid row. The cursor now carries PostgreSQL timestamp
text into its tuple comparison; business deadline calculations still use Date.
The regression failed before the fix and verifies progress past the failed page,
then recovery of all failed rows after the fault is removed.
The repaired expiry suite passes all 15 cases, and server typechecking passes.
## Current review repairs and verification
Three review findings failed real PostgreSQL regressions before repair: the
standalone worker ignored the configured batch limit, a successful overdue
backlog waited thirty seconds between batches, and a failed audit write left the
row expired without durable activity. The worker now honors its existing batch
configuration and continues successful overdue work after one second. The
shared deadline adapter accepts an optional `continuationMs`; omitted values
preserve every existing consumer's retry interval. Errors still retain the
existing thirty-second retry. Expiry and durable activity commit atomically,
then publish activity and deliver the origin continuation. Provider calls stay
outside that transaction.
All 129 selected cases pass: 18 expiry, 53 decision/continuation, 15 archive, 42
startup and one scoped-deadline case. The archive selection exercises an existing
shared-deadline consumer with the new option absent. Server typecheck/build and
generated snapshot/schema drift pass. Parent journal entries remain unchanged
and five generated snapshots remain. No snapshot was hand-edited.
The first complete selection passed 113 cases and failed the future-startup
assertion because it inspected the scheduler before an additional committed
notification reconciliation finished. Coordinator `ready` awaits the first
admitted pass, while a notification arriving during that pass can schedule
another immediate pass. The assertion now waits for reconciliation to expose
the exact saved future deadline, then still proves the decision does not expire
early. The three production regressions and this fixture failure remain
recorded; no error retry or business deadline was relaxed to pass the tests.
Fresh current-head CI and reviews remain merge gates.
After normal integration of continuation recovery's final coalescing and
deterministic contention tests and pinned master `8161f2a813`, all 130 selected
cases pass: 76 expiry/archive/startup/deadline cases plus all 54 decision cases.
Server types/build pass. Existing notification coalescing remains thirty
seconds for continuation recovery; expiry's successful backlog continuation is
one second, and failure retry remains thirty seconds. Migration replay reports
no pending migrations and the parent journal remains an exact prefix. Extra
upstream embedded-only tests attempted during parent integration could not
initialize locally; fresh CI must cover that environment, as recorded in the
archive parent PR.
Normal integration with master-target continuation parent `18f682863c` preserves
expiry, deadline and producer implementations exactly. All 61 expiry/startup/
scoped-deadline cases and server types/build pass with the rebuilt Runner vendor.
The previous full 130-case selection remains evidence at its stated revision.
No migration SQL changed, and five generated snapshots remain. Fresh head CI
and reviews are required after this parent integration.
## Creation notification review repair
Single and bundle creation previously notified continuation recovery even though
the inserted decision was open. Two real producer regressions failed before
repair because continuation received intent and settlement. Creation now uses the
existing central notification file's expiry-only `notifyDecisionCreated` helper.
Update and terminal transitions keep `notifyDecisionChange`, including the normal
continuation signal. No optional flags, per-route bookkeeping or new table is
needed. Both regressions verify the exact future expiry, no early wake, and a
subsequent cancellation that still signals and delivers continuation normally.
All 117 current expiry/decision/startup/deadline cases pass, including 20 expiry
cases. Server types/build pass. The preceding f7 candidate's identical-tree
merge proof remains history; this subsequent creation-signal repair changes
runtime behavior and has its own acceptance above. Fresh current-head CI and
both reviews are required.
+164
View File
@@ -0,0 +1,164 @@
# Idle decision target invalidation
Open decisions no longer require a periodic target scan. The target worker
reconciles committed issue changes and decision creation. Startup reconstructs
open decisions in bounded pages. After reconciliation, valid open decisions and
completed history cause no recurring SQL. TTL expiry, continuation delivery,
interrupted effects, and retention remain separate responsibilities.
## Domain policy
Only strict effect references participate. The existing shared
`decisionEffectTargetIssueIds` helper includes primary and secondary references.
A missing or cancelled target expires a decision with `target_gone`. If all
strict targets are done and at least one was not done in the proposal snapshot,
it expires with `target_completed`. Lenient decisions and proposals intentionally
acting on already-done targets remain open. A target that reopened before
reconciliation is checked in its current state; the notification is not authority
to expire it. The existing open-status compare-and-set arbitrates expiry against
other decision transitions.
`decision-targets.ts` owns this policy and scoped reconciliation. It delegates
expiry, activity, and continuation to the existing expiry owner. Startup
registers the target worker when scheduling is enabled. The combined internal
`sweepExpired` method is removed after the final duty is split. No new table or
migration is added.
## Producer audit
| Producer | Intent before write | Scope |
| --- | --- | --- |
| Decision creation, including bundles and caller-owned transactions | Existing creation transaction, before insertion | Decision ID |
| `issueService.update` entering/reasserting done or cancelled | Existing issue update transaction, before the patch | Issue ID |
| Issue removal | Lock the issue and capture linked open decision IDs in the existing transaction, before deletion | Affected decision IDs; no signal when none |
| Routine dispatch failure deleting its created issue | Same locked pre-delete lookup in the existing dispatch transaction | Affected decision IDs; usually no signal |
The existing target link table supports ordinary issue scopes. Before a target
is deleted, the decision owner locks its issue row, reads distinct linked open
decisions in the same company, and signals their IDs. The links may then cascade
without losing notification scope. The row lock closes the race in which a new
proposal could add its target reference after discovery but before deletion.
This remains part of the caller's existing transaction; no provider operation
is wrapped in a new transaction. Production hints require an issue or decision
ID, so routine cleanup and repeated removals do not restart full startup paging.
Company deletion no longer signals target work. With no decision rows it cannot
create a same-company invalidation obligation. Pre-existing restrictive origin
run/issue foreign keys still prevent deleting some companies with decisions;
that separate limitation is unchanged. This migration does not claim to implement
complete decision-data deletion for those companies.
Other direct issue writers were inspected: checkout/adoption enter in-progress;
conversation writers enter todo/in-progress/in-review; recovery enters blocked;
release preserves existing terminal status; administrative release changes locks
and assignment. Tree restoration excludes terminal snapshots. Route aliases and
raw privacy SQL do not write terminal status. These paths do not create new
invalidation obligations. They can restore a target before the worker runs; the
worker reads its current status.
The initial decision feature (0197 / commit `9c1f8e7887`) already stored primary
and secondary target links in the creation transaction. The later shared-helper
extraction `ef33c1d9ed` preserved that behavior. There is no newly assumed legacy
secondary-link backfill. Direct SQL writers or damaged/incomplete link data are
outside the supported live writer contract; startup still examines all open rows.
## Scheduling and lifecycle
The owning service holds pending scopes and cursors, not a second durable queue.
The database decisions, options, snapshots, and target rows remain truth. Each
pass examines at most 100 rows/scopes, ordered by expiry and ID using the existing
open-decision index. The cursor retains PostgreSQL timestamp text, including
microseconds, rather than rounding through a JavaScript Date. An empty scope consumes one unit of the pass budget. It
rotates unfinished scopes so a long startup scan does not starve new changes.
A settled notification replaces its scope with a fresh cursor. An admitted pass
cannot delete that replacement when it completes. Failed rows receive individual
retry scopes while later rows continue; query failures retain their scope.
Retries use the coordinator's existing 30-second cadence only while reconciliation
is unfinished. Unknown hints request a bounded full scan. Once all scopes settle,
`hasPending` returns from memory and no future target scan is scheduled.
Intent registration is awaited before writes. Nested writes belong to the outer
transaction. Rollback does not publish a change; ambiguous outcomes retain the
existing XID reconciliation contract. Notifications cover the application's DB
owner and child pools, not arbitrary SQL writers or independent active processes.
Drain preserves pending scopes without passive target SQL. Resume processes them.
Shutdown checks cancellation between rows and awaits admitted expiry/delivery.
Startup reconstructs anything left after process exit. Cloud's conservative
sleep guards are unchanged; this establishes worker quiet, not permission to
sleep with unsupported obligations or proof of host suspension.
## Current verification
All 248 cases in seven complete suites pass: 25 target, 80 routine, 54 decision,
26 effect-recovery, 20 expiry, 42 startup and one scoped-deadline case. PostgreSQL
suites use separate migrated external fixtures. Server typecheck and build pass.
No new migration or table is added. The normal parent merge preserves landed
expiry and all 21 independent master files. Server types and build use the
updated Runner vendor (1,524 inputs and 1,563 outputs).
Four real regressions failed before repair: sub-millisecond paging, deletion
behind a full valid startup page, unrelated deletion notifications and successful
empty-company deletion notifications. They all pass. The new concurrency test
observes the proposal's actual PostgreSQL lock wait while deletion is held.
Temporarily removing the issue-row lock made the test fail because the proposal
was accepted instead of waiting; restoring it passes. The mutation was restored
before commit. No timeout or production hold was weakened.
The combined internal method had no production caller after this split. The
server package exposes only its startup module; neither its exports nor the
plugin SDK exposes the decision service. Its tests now call the dedicated TTL,
target, effect and continuation workers, preserving their outcome assertions.
The obsolete cursor, scan options and exported target-policy method are removed.
HTTP and plugin SDK contracts are unchanged.
Fresh current-head CI and both automated reviews remain required. Historical
fixture and CI failures below remain part of the verification record. Extra
upstream embedded-only tests attempted during parent integration could not
initialize locally; CI must cover that environment.
## Earlier verification
The initial 17-case real-PostgreSQL target suite passed. The combined selection
passed 233 tests: those 17, 17 effect-recovery cases, 14 TTL cases, 50 decision
service cases, 39 startup cases, 80 routine-service cases, and 16 routine-route
cases. The final target run passed all 18 cases, including a change to the same scope
during reconciliation. In total, 234 selected cases pass across these runs.
Coverage includes actual issue update/removal, decision creation after cancellation,
secondary targets, all-target completion, intentional done targets, lenient history,
outer commit/rollback, startup after deleted links, first change and failed expiry
through drain, reopened targets, scope isolation, more than one page of valid
open rows, failure fairness, creation during delivery, and shutdown. The original routine
failure test observed intent and settlement around its cleanup transaction. The
current test requires no target signal when that created issue has no linked
decisions.
Quiet assertions observe SQL through one hour of fake time after settlement.
Workspace typecheck/build passed before the final test-only addition. No runtime
test failure was suppressed or timeout increased. Existing warnings from startup
fixture signal listeners remain visible. The routine suite now supports the
validated disposable external test database path, so local embedded PostgreSQL
unavailability does not silently skip its 80 cases.
A local query probe with 40,000 completed decisions, one open decision, and 40,001
target links used existing indexes: scoped lookup read five buffers (0.012 ms),
full open-page lookup two (0.003 ms). The probe was rolled back. No new schema
index is needed for this change. Full PR CI and fresh review remain required.
## Earlier review and CI repair
Review found that an absent deletion hint was dropped when the outer transaction
also carried a scoped hint. Both delete-before-update and update-before-delete
regressions reproduced the missing expiry after the target links cascaded away.
That revision sent the explicit `all` hint alongside scoped hints; the current
pre-delete decision-ID contract below supersedes the global hint.
The repaired target suite passes 20 cases; the TTL suite passes 14 cases.
The initial CI run failed the TTL worker's drain assertion. It measured SQL
before already-admitted sweeps had finished. The test now joins those specific
sweep promises before measuring passive SQL; unresolved-intent holds remain
intact. Waiting for the entire idle count to reach zero was tested and rejected:
that count correctly includes the retained recovery holds. No production drain
or retry behavior changed for this test repair.
+45
View File
@@ -101,3 +101,48 @@ moves here. Other maintenance duties, external probes, and connected transports
still affect idle traffic. No Cloud alarm coverage or host-sleep exemption changes.
These tests prove the changed workers become quiet and recover; they do not prove
deployed suspension or fleet savings.
## Custom-image setup expiry
Setup expiry uses a separate `custom-image-expiry` worker. The custom-image
service owns the active-status predicate, the earliest saved `expiresAt`, and
provider cancellation. Startup registers the worker in the existing enabled
heartbeat housekeeping path; the maintenance tick no longer calls its sweep.
All supported setup producers live in `environment-custom-images.ts`: initial
creation, provider refresh, cancellation/failure status, and promotion. They
register intent in their owning transaction. A provider refresh can move the
deadline earlier; a terminal transition removes it. Historical verification
metadata does not change active expiry eligibility and does not need this signal.
Environment deletion can only remove obligations; an already scheduled pass
observes the removal. Independent SQL and other DB roots are outside the existing
process-local notification contract.
No active dated session means no recurring reads or provider calls. Overdue
batches and failures retain a bounded retry, while first writes and failed
reconciliation survive drain. A missing expiry retains the existing behavior
(no inferred expiration); sleep safety still treats unsupported live resources
conservatively. Provider cancellation, failure state, and historical resource
verification keep their existing semantics. No Cloud alarm coverage changes.
## Historical retry test observation
The historical verification worker intentionally does not delay server readiness.
Its provider sweep finishes before the following database pending-state read
registers a retry deadline. The regression test now waits for that future
deadline before inspecting it; sweep completion alone is not scheduler settlement.
The runtime worker, retry interval, and readiness contract are unchanged.
A controlled 150ms delay before the real pending-state query reproduced the
current CI failure: the immediate assertion observed a null deadline. The
corrected assertion passed with the same delay. The temporary delay was removed,
and all 14 cleanup tests passed against an isolated PostgreSQL database. The
existing external test-database hook is carried at the stack foundation so
intermediate branches can use this same verification path.
An initial attempt at the foundation could not start embedded PostgreSQL because
that earlier branch lacked the external fixture hook. A broader fixture wait for
all idle holds to clear then failed three cases that legitimately retain pending
cleanup obligations. That wait was removed. Only the historical deadline assertion
changed; its requirement is now a deadline strictly in the future. The complete
suite passes with the original readiness and idle-hold behavior preserved.
+141
View File
@@ -0,0 +1,141 @@
# Gateway credential expiry and idle behavior
Gateway sessions and named gateway tokens already persist their expiry times.
`cleanupExpiredSessions` remains the owner of physical deletion. Authentication
continues to check expiry, revocation, company scope, run ownership, and active
run state on use. No table, authorization rule, or external alarm is added.
## Producer inventory
- `createSession` registers intent before inserting a run-bound session in a
transaction. Its existing audit and returned credential remain unchanged.
- `createNamedGatewayToken` registers intent before inserting an expiring token
in a transaction. A token without an expiry owes no expiry cleanup and does
not notify this worker.
- Last-use updates do not change expiry. Token/session revocation, including
heartbeat completion and connection removal, also leaves the saved expiry
intact. Revoked rows retain the same physical deletion deadline.
- Gateway/company deletion removes dependent rows and can leave one harmless
stale deadline. It cannot create overdue work. No supported service changes
a saved credential expiry after creation.
The database signal layer owns outer commit, rollback, savepoints, and uncertain
outcomes. The worker supports producers sharing that signal owner. Independent
SQL writers and another process need a separate live ingress contract. Startup
reconstructs existing rows but does not provide continuous replica discovery.
## Worker contract
`nextCredentialExpiryAt` reads one earliest non-null expiry from each table using
expiry-first indexes, then compares those at most two values. The migration adds
the session expiry index; the token expiry index already exists.
An empty credential store or retained non-expiring tokens returns no deadline.
`registerGatewayCredentialExpiryWork` supplies cleanup and deadline selection to
the shared `registerDeadlineWork` helper. Provider and authentication behavior
stay in the gateway service; startup only registers the worker.
The worker starts where the old startup cleanup ran. With heartbeat scheduling
enabled, committed changes and actual expiry times replace the recurring cleanup
call. With heartbeat scheduling disabled, the existing startup-only cleanup is
preserved. Tool action-review recovery is a separate obligation and is unchanged.
Token and session deletion are each bounded to 500 rows per pass. Session
selection orders by expiry and rechecks expiry when deleting the selected IDs.
A remaining overdue expiry schedules another pass after 30 seconds. Failed reconciliation also
retries at that bound. No unfinished reconciliation or future expiry means no
recurring SQL. A generation retains writes arriving during reconciliation.
Startup restores deadlines; drain retains first writes without passive SQL;
resume completes overdue cleanup. The coordinator owns shutdown and waits for
in-flight work.
Sequential committed writes are coalesced for up to 30 seconds after a
successful cleanup pass, matching the former heartbeat cleanup cadence. A
previously loaded expiry can preempt that window. A newly reported credential
may physically remain until that bounded follow-up; authentication still rejects
it immediately at expiry. A successful pass with no future expiry and no new
notifications creates no timer. This uses the shared optional notification
coalescing capability, promoted from the later plugin/decision recovery slices;
workers that omit it retain their existing behavior.
## Evidence and limits
The concurrent session index runs through the migration executor's existing
per-file top-level path. Initial application and repeat application were tested.
A PostgreSQL plan with 20,000 sessions and 20,000 tokens, including 19,000
non-expiring tokens, used both expiry indexes and read one row from each.
This establishes bounded lookup on that fixture, not production latency.
The new PostgreSQL suite exercises real session/token creation, configured idle
state, non-expiring history, expiry ordering, revoked-token cleanup, outer commit
and rollback, startup reconstruction, a backlog larger than one cleanup batch,
and drain/resume. It tests expired authentication before deferred physical
cleanup, through both session and named-gateway entry points. Existing gateway
acceptance and server-startup tests cover the changed integration boundaries.
Use only disposable migrated databases for the test URL overrides. Their names
must end in `_test`; the suites delete their fixtures. Ordinary CI uses embedded
PostgreSQL. Test credentials are synthetic and no external provider is contacted
by the expiry suite.
The migration proves worker behavior, not host suspension. Existing Cloud idle
proof remains conservative. An in-process deadline cannot wake stopped compute;
external wake coverage must be verified before changing sleep eligibility.
## Related unmerged token-lifetime work
At implementation time, PRs #17221 and #17274 change run-scoped token lifetime
and cleanup eligibility. This slice preserves the current master policy and
changes scheduling only. When combining those changes, the deadline query and
producer signals must use the same token eligibility as cleanup: run-token idle
markers must not schedule the operator-token expiry worker. Keep the run-liveness
reaper with its own owner and retain both regression suites. A conflict-free
text merge alone does not prove this integration.
## Notification amplification review repair (2026-10-27)
Ten sequential real session/token creation transactions reproduced ten extra
instance-wide cleanup passes. With coalescing they cause one follow-up pass.
A separate 501-session fixture reproduced the unbounded delete, then passed
with 500 removed initially and the remaining overdue row removed at the saved
30-second retry. A still-valid session survives both passes. The tests also
preserve an already loaded expiry only 500 milliseconds away and verify that a
new one-second session is rejected while physical cleanup waits for coalescing.
The initial full run exposed four outer-transaction tests whose fake-clock wait
expected the intent hold to release before the new coalescing follow-up. Their
settlement helper now advances the clock through that window while preserving
the original commit/rollback assertions. The complete gateway expiry, gateway
acceptance, custom-image deadline, and startup suites pass: 147 tests in four
files. Server typechecking also passes on the owning branch. This is local
behavioral evidence; combined-stack and fresh remote checks remain required.
The full local test:run remains unverified because of embedded PostgreSQL
startup/capacity limits; the two real gateway suites used separate migrated,
task-owned disposable PostgreSQL databases.
A later combined run exposed a test-clock overshoot: repeated blind 30-second
advances could expire the future session before its intermediate assertion.
The settlement helper now lets real SQL settle and advances only to an actually
armed follow-up within the coalescing window. All 12 credential expiry tests pass
on the owning gateway branch with the existing exact-deadline and transaction
assertions intact. This follow-up changes the test fixture only.
## One-shot cleanup preserves complete backlog removal
Review found that bounding worker passes also truncated startup cleanup when the
heartbeat scheduler was disabled. That path calls cleanup only once, so it could
leave every expired token or session after the first 500 rows indefinitely.
`cleanupExpiredSessions` now drains the existing backlog in 500-row SQL batches
by default, using one fixed expiry cutoff for the invocation. The deadline worker
opts into `singleBatch: true` and retains its existing coalescing and next-page
deadline. Future and nonexpiring credentials survive both paths. The return value
still counts deleted sessions, as before. No periodic fallback was added.
Three real-PostgreSQL regressions reproduced the truncation with 1,001 expired
sessions, tokens, or both, without registering a worker. They pass after the fix.
All 55 expiry/startup cases pass, including the existing worker's 501-row bounded
pass and commit/rollback tests. The original explicit-cleanup compatibility case
and server typecheck also pass. Evidence is in `/tmp/sleep-master-gateway-manual-`
logs: `red`, `green`, `compatibility`, and `types`. This fixes one-shot behavior;
it does not establish provider suspension or complete the stack's CI acceptance.
+121
View File
@@ -0,0 +1,121 @@
# Idle Grok routine dispatch
The Grok dispatcher previously started a one-second timer whenever public MCP
was configured. Each tick read experimental settings, even when Grok was
otherwise disabled. Enabled ticks scanned prepared dispatches regardless of
whether any work existed. The disabled baseline regression observed three
settings queries in 3.2 seconds.
The dispatcher now registers with the existing delivery coordinator and shared
scheduler. It performs one startup reconciliation. The worker becomes quiet
when Grok is disabled or when no prepared dispatch remains. Quiet means no
recurring worker SQL or provider calls, including with retained terminal history.
This does not establish whole-instance or database suspension.
## Ownership and producers
`grokRoutineService.reserve` is the only supported creator of prepared dispatches.
It signals intent before its insert in the existing transaction. Route verification,
manual retry of a confirmed rejection, and native-broker assignment admission all
use this service. The native broker passes its transaction, so the wake follows
the outer commit. Savepoint release is not publication; rollback does not send.
`instanceSettingsService.updateExperimental` signals the same queue before its
existing transactional update. This covers enablePublicMcp, enableNativeRunner,
and enableGrokBot changes, including caller-owned outer transactions. It permits
prepared work retained while disabled to resume when enabled. Managed settings
and environment overlays are process configuration and are reconstructed on restart.
Other settings writers update general/default-environment fields, not these flags.
All other dispatch writers move rows away from prepared: send claims, delivery
settlement, acknowledgement, explicit recovery, assignment completion, and
revocation. None reopens a row as prepared. They cannot introduce hidden new work.
A previously scheduled backlog pass may observe cancellation and disarm.
Signals cover the current process's shared database owner and child pools.
Independent SQL writers or another process need explicit ingress integration;
this change does not claim cross-process notifications. Startup reconstructs
committed prepared rows from the existing dispatch table. No queue table is added.
## Delivery and recovery
The Grok service continues to own current authority checks, binding locks,
prepared-to-sending compare-and-set, secret access, provider delivery, and outcome
settlement. The worker does not interpret provider payloads or relocate policy
into app wiring. A foreground send and background recovery share the durable
claim; only the winner contacts the provider.
Each pass processes at most ten prepared rows. The next lookup uses the same
predicate and schedules the existing one-second cadence only while prepared work
remains, or after a reconciliation failure. New commits wake the shared worker;
its deadline adapter preserves requested generations through concurrent changes
and drain. The partial created_at/id index keeps both selection and pending-work
lookup independent of retained completed history. The migration builds it
concurrently, is replay-safe, and uses the normal snapshot-retention workflow.
The existing send-error policy is preserved: if sending throws and the row is
still prepared, the dispatcher cancels it. A row already claimed as sending is
never reset or replayed. Accepted, sending, unknown, acknowledged, rejected, and
cancelled rows do not become automatic retry work. Explicit operator retry still
creates a new intent only for confirmed rejection. This migration does not claim
exactly-once delivery or resolve uncertain provider outcomes automatically.
## Lifecycle and sleep boundary
The app registers only when public MCP is configured, after the shared coordinator
exists. Registration does not delay readiness for a provider send. The coordinator
tracks startup work and joins admitted sends on shutdown. Abort prevents the next
row from starting; existing HTTP timeouts bound a current transport operation.
Later prepared rows remain durable for restart.
Drain and warm standby retain pending requests without passive worker queries.
Resume uses the same coordinator retry path; a first write or failed sweep cannot
be forgotten merely because dispatch was paused. A disabled feature may retain
prepared rows without scheduling a scan; a committed settings change reactivates
it. Disabled feature state is not permission to discard durable work or relax
Cloud sleep guards. Existing binding/assignment/hold blockers remain unchanged.
## Verification
- The baseline failed the disabled-feature quiet assertion with three SQL reads.
- 21 real-PostgreSQL lifecycle cases cover disabled/configured idle, first real
reservation, prepared startup recovery, all six non-prepared states, re-enable,
outer commit/rollback, drain/startup, failed scan through drain, concurrent
enqueue, two-page draining, foreground claim overlap, expired unsent work,
settings transaction rollback/commit, and shutdown joining a send.
- The existing 45 Grok qualification cases and 39 startup cases pass together
with the new suite: 105 cases total.
- The four affected settings suites pass 154 cases (259 selected cases total).
- Workspace typecheck, production build, migration checks and replay pass. A
generic-plan probe with 40,000 completed dispatches and one prepared row uses
the partial index for both queries (six shared buffers including the routine
join; 0.032ms batch and 0.025ms next-work execution in this local probe).
- Remote PR review and CI must pass before the slice is ready.
The tests use the existing real OAuth/agent/routine fixture and a controlled
provider response. A validated disposable external PostgreSQL option permits
local execution where embedded PostgreSQL is unavailable; CI retains embedded
PostgreSQL by default. Provider-host suspension is outside this local proof.
## Current parent reconciliation
The normal merge of target parent `85ce6545bf` preserves its complete tree outside
this duty's explicit files. Grok dispatch, reservation, settings and app wiring
are unchanged from the preceding candidate. The index is regenerated as
`0391_third_roulette.sql`, with concurrent/idempotent execution. All 389 parent
journal entries are unchanged; five generated snapshots remain (0387–0391).
Schema drift, migration safety/numbering and fresh apply/replay on two isolated
PostgreSQL databases pass.
All 273 cases in nine complete suites pass at `6d81746fe2`: dispatch lifecycle,
Grok qualification, server startup, scoped deadlines and all five instance-settings
suites. Server typecheck and production build pass with the current Runner vendor.
The earlier 259-case and populated-plan measurements above remain historical
evidence; fresh exact-head reviews and CI are required.
An earlier browser CI run failed during agent-chat fixture restoration with a
lock timeout/HTTP 500; the subsequent case observed the experiment left enabled.
Those failures were not dispatcher assertions and are retained in the evidence.
The later pre-reconciliation `a74305e2b4` checks were terminal and green. Neither
that older result nor local acceptance substitutes for this candidate's CI.
+116
View File
@@ -0,0 +1,116 @@
# Interrupted tool-action recovery
An executing tool-action request has an existing ten-minute stale deadline from
its `updatedAt` claim timestamp. Recovery now registers the earliest actual
claim deadline with the shared scheduler. It no longer scans executing requests
on every aggregate maintenance tick.
## Ownership and producers
The gateway owns execution claims, provider calls, and result settlement. The
new interruption service owns only the deadline query and stale-row settlement.
The worker uses the existing deadline registration and exposes no provider
protocol or signed argument representation to the scheduler.
The review write helper signals interruption alongside approved recovery and
pending expiry. All three agent claim paths and the Test-tab claim use that
helper. Test-tab outcomes already commit both rows inside it. Agent pre-dispatch
and execution failure settlement uses the same notification transaction, and
successful or failed direct execution request updates now notify before writing.
Administrative review transitions continue to use the same existing boundary.
Unrelated pending review changes can cause a bounded empty reconciliation. They
do not create a periodic deadline. Process-local signals cover the existing
single-active-writer database owner; independent writers require a wake contract.
## Settlement and quiet states
The query uses a partial executing-request index and reads one earliest row.
There is no deadline after all executing claims settle. Retained terminal
requests and configured-but-idle tools produce no recurring worker queries.
At the deadline, settlement locks invocation then request, matching the existing
execution-failure lock order. It rechecks the claim status and timestamp after
locking. A changed or completed claim cannot be settled from an earlier scan.
Unknown outcomes atomically mark both rows failed with
`tool_execution_outcome_unknown`. Recovery never calls the external provider.
A saved terminal invocation is authoritative. Recovery repairs its request to
executed or failed without replacing the saved result, failure reason, or
completion time with an unknown-outcome failure. This covers a crash between
legacy invocation completion and request settlement. Interaction projection
runs after the transaction through the existing gateway owner. A failed
projection still relies on the separate projection-repair duty.
A database failure rolls back both outcome rows and retains a 30-second retry.
Startup reconstructs saved deadlines. Drain pauses dispatch without forgetting
the first claim or failed retry; resume processes overdue claims. Completion
notifications cancel unnecessary future scans. The scheduler-disabled path keeps
one startup pass. The public combined review sweep retains its existing duties.
## Verification and limits
Sixteen new real-PostgreSQL cases cover real foreground execution activation
and completion, configured idle and terminal history, restart/deadline timing,
six saved terminal invocation states, first-write drain/resume, outer commit and
rollback, transactional failure/retry, concurrent claim change, and multi-page
settlement. The service/startup selection passes 136 tests on the combined parent-review
head. Workspace typecheck/build and migration safety checks pass on that head.
Migration application and reapplication passed. The combined run also exercised
469 tool-access tests: 468 passed and an unchanged connection-archive case timed
out under concurrent build load. That case passes in isolation. The separate full tool-access suite passes all
469 tests and gateway acceptance passes all 92 (697 across the final selections).
Fresh full CI/review remains required.
The first typecheck caught optional approval IDs losing narrowing in transaction
callbacks; capturing them before the callback fixes it. Two new fault/transaction
cases initially waited for idle before advancing their next fake-clock retry.
They now advance time while waiting for the authoritative outcome, and the full
selection passes. These are fixture corrections, not permission to clear an
unresolved work hold.
The ten-minute policy is unchanged. Active provider execution, unknown external
outcomes, and other worker obligations still prevent unsafe sleep. No Cloud
eligibility or external-alarm coverage changes here. Worker quiet is not evidence
of whole-instance or database suspension.
## Sequential-write cost correction (October 27)
Review found that sequential tenant writes could each request a global
interruption pass, even when no execution was stale. A real-SQL regression
created ten tool review requests through the gateway and allowed each preceding
pass to finish. It measured 11 stale-claim scans including startup. Concurrent
notification coalescing alone did not cover this sequence.
The worker now uses the existing deadline helper's 30-second notification
coalescing window, matching the former default maintenance cadence. The same
regression measures one startup scan and one follow-up scan for those ten
requests, then no worker SQL through an additional idle hour. A loaded stale
claim deadline only 500 milliseconds away still preempts the window and
quarantines the unknown outcome without invoking the provider. Foreground
execution and result settlement remain immediate. Newly discovered work can
wait up to this window; the ten-minute stale-claim policy is unchanged.
This is the bounded-frequency alternative raised in review. Discovery is still
instance-wide, and real stale backlogs still drain through the existing pages.
It does not claim company-isolated scans or constant cost for a real backlog.
No notification and no saved execution deadline means no periodic scan. No new
table, index, or scheduler is added. Startup, drain, retry and shutdown retain
the existing owners.
The first full verification passed 175 cases and failed three fake-clock waits:
settlement writes leave a notification pending for its coalesced follow-up, so
waiting for the idle hold without advancing that armed deadline cannot finish.
An initial shared clock adjustment then advanced an unrelated five-second
expiry test too far (177 passed, one failed). The adjustment is now confined to
the interruption tests. It lets real SQL finish and advances only to an actually
armed short follow-up. The imminent-deadline test checks quarantine before that
30-second window and separately waits for eventual quiet. Product assertions
were retained.
The final complete gateway-service and startup-wiring suites pass all 178 cases
without skips, and server typechecking passes, on owner source based on
`e255d8eff3363cf0bb63541bdc7ef055d4321f83`. The attempted third test selector
`deadline-work.test.ts` did not exist at this owner; no separate helper-suite
pass is claimed. The shared helper is exercised by the full gateway suites.
Fresh combined-source checks and remote review remain required.
+183
View File
@@ -0,0 +1,183 @@
# Join request retention while idle
Pending join requests already have durable activity timestamps. Retention now
uses their next 90-day archive deadline instead of including them in the main
maintenance tick's attention-feed scan. The regular attention UI still includes
join requests. This changes automatic retention discovery, not access approval.
## Owner and eligibility
`join-request-retention.ts` owns candidate eligibility, its locked recheck and
deadline. `decision-retention-sweep.ts` shares bounded paging, per-row retention
transactions and failure handling with the approval-retention owner. The shared
sweep consumes normalized candidates; each owner retains its table/query policy.
A candidate is a pending_approval request belonging to an active company, with
an invite whose company matches the request. Kept and already archived items do
not schedule work. Invite expiration or revocation does not remove the existing
attention item and therefore does not disable its retention. No provider policy
is interpreted by the worker or shared scheduler.
The next deadline is the earliest eligible updatedAt plus 90 days, rounded up
in SQL to the scheduler's millisecond resolution. It must not wake a fraction of
a millisecond early and then wait thirty seconds for recovery. The partial
pending index orders updated_at and id. Resolved request history is excluded by
the index predicate. Each pass attempts at most 100 due rows. The private cursor
preserves database timestamp precision so a failed page cannot starve later work.
Errors keep the shared 30-second retry; successful full pages continue until
work is exhausted. No eligible rows means no scheduled SQL or network scan.
Each candidate is re-read under its request row lock before retention changes.
A writer that wins that lock first is rechecked before archival. If archival
commits first, a later activity write leaves the item archived until explicit
revival; updating activity does not clear existing retention choices. The tests
cover the writer-first race and the archive-first committed outcome. Supported
invite replay expires before the 90-day cutoff and Slack access replay deduplicates;
a future producer that needs to reappear after archival must request revival.
The shared retention owner still controls keep, archive/revive, version checks,
activity logging, and notification creation. Archiving does not approve or reject a join.
Join requests have no origin agent, so their archival creates no agent delivery.
Company and invite joins preserve existing visibility policy; this does not add
cross-table serializability beyond the existing retention contract.
## Producer audit
| Producer | Notification boundary |
| --- | --- |
| Invite acceptance creates a pending request | Existing acceptance transaction, before the request insert |
| Supported gateway invite replay changes activity | Small writeJoinRequestChange transaction around the existing SQL update |
| Automatic human approval after acceptance | Same helper around the status update |
| Explicit approval and rejection | Same helper around the status update |
| Slack self-service access request | Existing platform transaction, before the request insert |
| Company pause, resume, archive, or deletion | Company-level retention notification inside the existing transaction |
| Budget-driven company pause or resume | Same company notification in the accounting transaction |
| Keep/unkeep, manual archive/revive, accepted archive proposal | Existing typed retention notification now routes join_request to its worker |
Provider calls, membership grants, and unrelated route behavior stay outside the
small SQL mutation helper. Nested callers retain the existing outer transaction
settlement and uncertain-outcome protocol. The access router itself requires a
root database connection; it is not wrapped in a caller transaction.
API-key consumption only permits an already-approved request. It cannot create
pending retention work and does not acquire a new wrapper or signal. Invite
acceptance that reuses a human request does not update that request. Revocation,
expiry, and acceptedAt changes do not change the feed predicate. Invite/company
identity is immutable in supported routes; company deletion is the invite delete
owner. A future producer that changes these facts must join the same contract.
Process-local signals do not cover independent SQL writers or another instance.
## Lifecycle and limits
Startup reconstructs pending deadlines. The worker is registered only when the
existing heartbeat maintenance switch is enabled. Writes during drain retain a
wake without passive SQL. Failed work retains its retry across drain. Shutdown
joins an admitted archive and stops before subsequent candidates; restart picks
up the durable remainder. There is no new queue table or permanent DB listener.
The in-process deadline is not a Cloud alarm. Existing unsupported-future-work
sleep blockers remain in place.
## Earlier verification
Seventeen new real-PostgreSQL cases pass. They cover actual invite acceptance,
rejection, human approval, gateway replay, and Slack access requests; empty and
resolved-history quiet; saved deadlines; startup reconstruction; keep and manual
archival controls; company pause/resume; unchanged expired/revoked invite policy;
normal versus maintenance feeds; nested commit/rollback; first writes and failed
work during drain; concurrent source updates; 101-row microsecond paging with a
full failed page; and shutdown/restart.
Another 139 consumer cases pass: 71 invitation/startup cases, 21 approval worker
cases, and 47 shared retention/attention/budget cases. This is 156 selected tests,
not a full local suite claim. The first consumer command used a nonexistent
retention test filename and ran only the approval suite; the corrected selection
ran the actual retention suite against its dedicated external database.
Initial test failures exposed fixture assumptions: the access router cannot run
on a transaction handle; unresolved recovery cannot be expected to become idle;
Slack links require actual endpoint/principal ownership rows; and shutdown tests
must establish admission and deterministic candidate order. Those fixtures were
corrected without weakening assertions or production checks. The first replay
fixture lacked required gateway credentials. The final fixture supplies test-only
credentials. A reused transaction spy retained admission calls during the quiet
assertion; restoring it before the quiet measurement fixed the measurement.
Migration 0387 applies, the full migration set replays, and the concurrent index
statement replays idempotently. Migration safety and numbering checks pass; the
five newest schema snapshots are retained. The actual next-deadline query with
40,000 resolved requests and one pending request uses the pending index and
executes in 0.017 ms locally. The probe rolls back its data. This is query-plan
evidence, not production latency or measured savings. Workspace typechecking and the production build pass. Fresh PR CI/review remain
required before readiness.
## Current review repair and verification
The duplicate approval/join archive loop is now one `createDecisionRetentionSweep`
implementation. Each owner supplies its candidate ordering boundary, page, locked current-state
lookup and next deadline. This keeps eligibility, company/invite joins and
source-row locks with their owner while sharing cursor/error/retention behavior.
The existing 100-row budget, PostgreSQL text cursor and failure rotation remain.
Both workers coalesce notification-driven global reads under a thirty-second
aggregate budget; saved deadlines preempt it and no idle scan is scheduled.
Manual keep/archive/revive can materialize an unseen join request in the existing
retention write transaction, as approvals already can. The company-scoped source
is locked, its real activity is read, and existing versioned state is synchronized.
Authorization, archive choices and other source kinds retain their contracts.
No route bookkeeping, queue table, permanent listener or Cloud exemption is added.
Five real regressions failed before repair: three unseen-request controls returned
404 and both owners rounded a saved microsecond deadline down. All pass. The first
two-owner run passed 50 cases but failed two old producer expectations that used
a truncated JavaScript Date. They now compute the expected scheduler millisecond
independently from integer database microseconds; exact assertions remain.
The first missing-lock mutation exposed a weak test: its timing-only writer gate
released before proving the sweep had reached the lock, so it still passed without
`FOR UPDATE`. The strengthened test observes the actual PostgreSQL lock wait before
releasing the writer. It passes with the production lock and fails with zero waiters
when that lock is removed. The production source was restored in `finally` before
commit. A separate archive-first committed-order case preserves the existing
archived-stays-archived behavior and verifies explicit revival.
All 340 cases in nine complete suites pass at `405c7dcdef`, including all 31
approval and 22 join worker cases. Server typecheck and production build pass.
Normal integration of parent `0b0e844b77` changes only four independently merged
Runner E2E accounting files and the ledger; runtime and acceptance sources remain
byte-identical. Existing results retain their tested revision.
Generated concurrent/idempotent migration `0393_open_grandmaster.sql` adds only the
pending-request index. All 391 parent journal entries and historical SQL files are
unchanged; five generated snapshots remain (0389–0393). Migration safety/numbering,
schema drift and actual apply/replay on six isolated PostgreSQL fixtures pass.
Earlier setup, timing, mutation and CI failures remain recorded here and in the
stack ledger. Fresh final-head CI, security and both 5/5 reviews remain required.
## Retry progress under continuing arrivals
Each scan cycle saves the final due `(updated_at, id)` tuple, at full database
precision. Its pages stop at that boundary. Later due rows cannot keep extending
the cycle and postpone failed rows forever. The cursor still advances past an
individual failure, so one bad request—or an entire failed page—does not block
later healthy rows. Once that finite cycle ends, the next cycle revisits failures.
There is no new durable queue or per-row retry registry. Restart reconstructs
eligibility from existing rows.
Two real PostgreSQL regressions failed before this repair: continued newer overdue
arrivals left a recovered approval and join request behind the cursor. Both now
archive in the next cycle. All 55 worker cases pass, including the existing
100-failure-page progress and microsecond-cursor cases. The nine-suite run passed
341 cases but failed the attention suite's cleanup hook after its 16,400-run case:
`delete from heartbeat_runs` exceeded thirty seconds. After vacuum/analyze of only
the disposable attention fixture, all 31 attention cases pass without code or
timeout changes. The other eight suites passed 311 cases in the original run;
342 distinct cases pass across those complete-suite executions. This demonstrates
recovery, not a proven explanation of the cleanup slowdown. Server typecheck and
production build pass at `df1815fbba`; the subsequent master integration has an
identical complete source tree.
Earlier CI at `df1d4d0` also failed the unchanged decision-expiry restart test:
its earliest wake was a thirty-second retry rather than the saved expiry. The
complete unchanged expiry suite passes all twenty cases locally. Its CI timing
cause remains unproven; it is not labeled infrastructure or silently discarded.
Fresh final-head CI and reviews must pass before merging.
+17
View File
@@ -95,3 +95,20 @@ cover the surrounding login contracts.
The fake-clock database fixture warms its connections and disables pool idle/lifetime
expiry so time jumps test the worker rather than racing socket connection timers.
Database suspension and reconnect behavior require separate qualification.
## CI fixture settlement across the migration stack
The setup-token consume fixture can finish its initial fake-clock jump while
cleanup is still performing database work. A write during that pass can request
another generation whose coalesced deadline lies beyond the jump. Waiting for
zero tracked work without advancing to that registered deadline then times out,
even though cleanup has a valid wake and can finish.
The existing fixture repair from `59f8c8b3e5` is now carried at the earliest stack
branch so every intermediate PR receives it. The helper advances to registered
deadlines only while work remains tracked, preserving its existing five-second
observation limit and all production behavior. All 16 login-cleanup cases passed
on this branch against a dedicated PostgreSQL database; the prior later-branch
verification and failed CI history remain documented in the GitHub review-check
migration audit. No runtime change or weakened quiet assertion is included.
Evidence: `/tmp/sleep-login-settlement-owning-tests.log` and its zero status file.
+90
View File
@@ -0,0 +1,90 @@
# Secret-proposal expiry and idle behavior
Secret proposals already store `status`, `expiresAt`, and encrypted pending
values. This worker uses those records; it adds no queue table or migration.
## Ownership and activation
`secret-proposals.ts` still owns quotas, authority checks, approval, expiry,
ciphertext scrubbing, dependent binding rejection, audit records, and interaction
reflection. `expiryDeadlines(companyId?)` returns one saved deadline per pending
company. Normal write notifications reload only the affected company. Startup
reconstruction loads all companies once. The manual `nextExpiryAt()` query remains
available to existing callers.
Terminal history does not create a deadline. Approval still rejects an expired
proposal synchronously, even when maintenance is paused.
The supported producers register `secretProposalExpiry` intent before their
writes in the owning transaction:
- `createWithinQuota`: both secret and binding creation, after quota admission.
- `markApproved`: ordinary approval and prerequisite approval during a cascade.
- `transition`: rejection, withdrawal, expiry, and dependent binding rejection.
- Interaction resolution: rejection, withdrawal/cancellation, expiry, and failed
acceptance execution that resolves a linked pending binding.
- Company deletion: a company hint removes its cached deadline after deletion commits.
Each producer includes its company ID. The remaining proposal write only attaches
an interaction ID and does not change expiry eligibility. Independent SQL writers
or another DB root are not supported live producers of these process-local signals;
startup reconstructs their persisted rows, but it is not a live replica contract.
The shared database signal boundary defers notification until the outer transaction
settles. Nested savepoints, rollback, and uncertain commit outcomes retain the
existing protocol. Callers do not pass recovery counters or reconnect the database.
## Scheduling and lifecycle
`registerSecretProposalExpiryWork` runs once at startup while heartbeat scheduling
is enabled. It replaces the secret-proposal sweep in the aggregate maintenance
tick. Disabled heartbeat scheduling keeps its existing behavior.
The worker reloads the affected company's saved minimum expiry after a committed
change. It does not sweep future work. At a due deadline it sweeps at most 100
expired proposals for that company, then reads that company's next deadline.
Companies rotate fairly between bounded passes. Retained history creates no work.
No pending proposal, unresolved write, or failed reconciliation means no recurring
query. Unscoped legacy hints still reconstruct all companies.
`scoped-deadline-work.ts` owns scope invalidation, cached deadlines, fair dispatch,
and failure backoff. It owns no SQL or proposal policy. Failed loads and failed
company sweeps back off independently from 30 seconds to five minutes. A new
write reloads that company without erasing its failed sweep's backoff. Other
companies keep their own deadlines, including already-due work. The coordinator's
optional `nextRunAtOnError` contract permits immediate eligible work only after
the worker has recorded per-item backoffs. Failed transaction reconciliation and
failed pending-state inspection retain the coordinator's normal retry.
Secret expiry opts out of the shared helper's default one-second pass spacing so
newly learned short deadlines are not clamped. Other scoped consumers retain that
default. Custom-image expiry continues to use `deadline-work.ts` unchanged. No
shared scheduler or startup wiring interprets secret-proposal policy.
A request stays pending until reconciliation succeeds. Changes arriving during a
pass, first writes during drain, and failures followed by drain retain a request.
Passive drain performs no proposal SQL. Resume reconciles retained requests;
restart reconstructs deadlines. Coordinator shutdown unsubscribes and waits for
active work. Deferred physical cleanup does not extend approval authorization.
## Verification and limits
The PostgreSQL route suite exercises actual service producers, nested outer commit
and rollback, quiet empty and terminal states, quiet future deadlines, automatic
ciphertext scrubbing and dependent-card resolution, startup reconstruction,
withdrawal, interaction rejection/failed execution, failure retry, and drain/resume.
It also verifies that expired approval fails while cleanup is paused. The existing
coordinator suite verifies fallback retry and uncertainty boundaries. New cases
record actual SQL to prove normal writes and due expiry stay company-scoped; they
also cover five-second deadlines, earlier replacement deadlines, company deletion,
and another company making progress while one sweep fails. A deterministic
shared-worker test covers a 250ms replacement deadline.
The worker tests use a controlled clock and a real PostgreSQL database. Existing
route tests cover quotas, isolation, authorization, concurrency, and lifecycle
semantics. The test-only database URL must refer to a disposable migrated database:
the suite deletes its fixture tables. Ordinary CI uses its isolated embedded DB.
This is worker quiet, not proof of host or database suspension. No Cloud alarm or
sleep-eligibility exemption is added. Existing sleep-safety checks remain in force;
a local expiry timer cannot wake a stopped host. Tool review expiry and gateway
session cleanup are separate migrations.
+103
View File
@@ -0,0 +1,103 @@
# Terminal workspace scan and retry deadline precision
Terminal workspace maintenance already uses the shared delivery coordinator and
retains deadlines only for outstanding work. Its bounded scan must progress past
ineligible rows so that later eligible work can finish and the worker can park.
The source query orders candidates by `(updated_at, id)`. PostgreSQL timestamps
can contain microseconds. The former cursor used a JavaScript Date, which loses
that precision. A skipped row could then compare greater than its own rounded
cursor. Repeated full pages kept recovery active and prevented later cleanup.
The query now carries the database timestamp as text alongside each workspace.
Only the private scan cursor uses that value. Workspace data, business deadlines,
and the frozen rotation boundary keep their existing types and behavior. The
cursor comparison uses the same timestamp/ID tuple as the ordering. The source
service continues to own cleanup eligibility, authorization, concurrency fences,
page limits, abort handling, failed-page recovery, and rotation under churn.
No scheduler API, queue, migration, producer signal, or external alarm changes.
## Verification
The existing full-page starvation regression now gives its skipped rows actual
PostgreSQL microsecond timestamps. Before the repair, the eligible workspace was
never archived. After the repair, the full workspace service and terminal worker
selection passes all 87 tests, including bounded rotation under continuous churn,
changed rows behind the cursor, failed-page recovery, quiet workers, and shutdown.
The service suite accepts the repository's validated disposable external
PostgreSQL URL. Setup applies pending migrations before opening the test client,
then clears fixture tables before the first test as well as after each test. This
supports a fresh database and reuse after an interrupted test run. The database
connection closes during teardown. Server typechecking passes.
This fixes recovery progress. It does not establish provider suspension or change
which active or unresolved resources prevent instance sleep.
Workspace-wide typechecking and the production build also pass. Full sharded CI
and fresh review remain required before this PR is ready.
Review follow-up reproduced the missing-schema failure against a newly created,
unmigrated disposable database. The setup repair migrates that same database
inside the suite; the full workspace-service selection then runs without manual
schema preparation. The production cursor implementation is unchanged.
## Preserve one retry deadline through early inspection
Nested-clone migration and isolated-source rehoming kept a retry guard and a
separate advertised scheduler deadline. Each value called the clock separately.
When the clock advanced between those calls, an early inspection copied the
earlier guard back into the advertised deadline. CI observed a one-millisecond
backward change in an otherwise unchanged retry.
Each path now computes one retry timestamp and stores that value in all of its
retry and scheduling state. The cadence, cleanup eligibility, quiet conditions,
rotation, and wake contract do not change. No new timer, queue, or table is added.
The existing nested-clone regression and the isolated-rehome dry-run case now
use a clock that advances one millisecond per read. Both failed before the fix
with the same one-millisecond drift. Both exact-deadline assertions remain. The
complete migration suites passed all 49 cases afterward, using real PostgreSQL
and Git, including preserved files/history and interrupted migration recovery.
The complete workspace-service and terminal-worker suites passed another 87
cases; server typechecking passed. The two focused pre-fix failures used a name
filter intentionally; the reported final 136 cases are four complete suites
without skips. Validated external test database hooks keep each migration suite
on its own disposable database and retain the existing embedded fixture fallback.
These are local correctness and scheduling results, not deployed host-sleep or
external-wake proof. Remote acceptance must be checked at the published head.
## External fixture migrations (October 26 review repair)
Review found that the two migration suites opened external test databases without
applying migrations. The previous verification used prepared databases, which
hid this setup gap. Both selected retry cases failed against new empty disposable
databases with missing `companies` and restore-point tables. Setup now applies
pending migrations before creating its test client, matching the workspace
service fixture. The embedded path is unchanged.
All 49 cases in the two complete suites then passed against those same databases,
including Git/filesystem migration, recovery, and exact retry-deadline checks.
The fixture itself prepared the databases. This repair changes no production
recovery code. Evidence: `/tmp/sleep-workspace-fresh-red.log` and
`/tmp/sleep-workspace-fresh-green.log`.
## Current parent verification
Approval-retention parent `3efda33812` is integrated with normal merges. All
workspace precision, retry and fixture implementations remain unchanged from the
preceding candidate. All 136 cases in the complete workspace-service, terminal
worker, isolated-rehome and nested-clone suites pass at `9d846c84e7`; server types
and production build pass with the current Runner vendor.
The preceding `f0f0de1f6f` run also passed all 136 cases with three newly created,
unmigrated external databases: the suites themselves applied the current
migrations. The `9d846c84e7` run reused those clean fixtures after the parent's
approval wake repair. This retains actual proof of fixture self-preparation
without mislabeling the reused run as a fresh-database test. No new migration,
queue, wake contract or workspace policy change is added by this PR.
Earlier microsecond starvation, one-millisecond retry drift and missing-schema
failures remain documented above. Fresh exact-head CI and both 5/5 reviews remain
required in stack order; the parent is not claimed merged.
+155
View File
@@ -0,0 +1,155 @@
# Pending tool-review expiry and idle behavior
Pending tool action requests already save their review deadline in `expiresAt`.
This worker replaces the recurring pending-expiry scan with committed changes,
startup reconstruction, and that saved deadline. It does not add a queue table
or change the one-hour review policy.
## Ownership and producer inventory
The tool gateway owns request signing and authorization. `toolReviewExpiryService`
owns pending-expiry selection and settlement. `registerToolReviewExpiryWork`
supplies company-scoped operations to the existing shared scoped deadline worker. Startup wires
that contract without interpreting tool or provider behavior.
The supported writers are:
- `tool-access-policy.ts` inserts an unsigned pending request without an expiry.
It creates no expiry obligation yet. The existing signing grace remains intact.
- The normal agent gateway signing path and the board test-call path assign the
saved deadline. Each signals in its SQL transaction before that update.
- Gateway signing failures, competing-call invalidation, invalid signatures,
and legacy pending-to-approved/expired transitions signal before removing a
pending obligation.
- `commitToolActionReview` signals inside its existing human-resolution
transaction. The issue/interaction locks, human authority checks, and delivery
enqueue remain owned there.
- Administrative interaction resolution signals in the caller-owned transaction
before resolving linked requests. Review-queue invalidation signals before
invalidating unreadable or unsigned requests.
- Company deletion signals its company before removing saved review obligations.
- The expiry worker signals before its own settlement. A compare-and-set against
pending status and the observed update time protects concurrent resolution.
Every notification requires the company ID. Normal commits reload only that
company's saved minimum expiry; due cleanup selects only that company. Startup
reconstructs all saved company deadlines once. A company with only unsigned
null-expiry requests has no deadline.
Only the SQL mutation is wrapped. Signing, provider requests, telemetry, and
interaction publication are not moved inside an enclosing recovery transaction.
The database signal layer handles nested savepoints, outer commit/rollback, and
uncertain transaction outcomes. No recovery object is passed through callers.
Independent SQL writers and other database owners/replicas still require a live
notification contract. Startup reconstructs saved rows but is not ongoing
cross-process discovery.
## State and recovery
Expiry updates the authoritative request and invocation in one transaction. If
invocation settlement fails, the request remains pending and the worker retries.
After commit, the gateway reflects the result in the linked interaction. The
existing terminal projection-repair duty remains in `sweepActionReviews` and
repairs a crash or failure between authoritative settlement and projection.
That duty is a separate migration; this worker must not claim it has completed.
The public/manual `sweepActionReviews()` still includes expiry by default. Server
startup and the recurring legacy review sweep pass `includeExpiry: false` when
expiry has its own startup pass or worker. Approved-action recovery, interrupted
execution quarantine, and terminal projection repair retain their behavior.
An unknown external outcome is never replayed by this expiry worker.
The next deadline comes from pending rows only. Startup uses the existing
partial expiry index to reconstruct company deadlines. A company notification
uses an ordered, single-row lookup backed by the partial `(company_id,
expires_at)` index, rather than aggregating that company's pending backlog.
Null-expiry unsigned requests and retained resolved history do not keep this
worker active. The concurrent index migration uses the existing top-level
executor and is safe to apply again after interruption.
The shared coordinator coalesces changes and preserves a write arriving during
reconciliation. Each due pass expires at most 100 rows for one company; companies
rotate between passes until the backlog drains. Failed scopes retry independently
with exponential backoff from 30 seconds to five minutes. Other companies retain
their own deadlines. New writes do not erase a failed sweep's backoff.
Once reconciliation completes with no deadline, it has no recurring SQL. Drain
retains first writes without passive database scans; resume processes overdue
reviews. Startup reconstructs persisted deadlines. Shutdown stops subscription
and timer ownership and joins active work.
Approval still rejects an expired request synchronously before physical cleanup.
With heartbeat scheduling disabled, startup-only expiry remains as before.
## Verification scope
Real PostgreSQL tests record SQL to prove signing, human decisions, and due expiry
remain company-scoped. They also verify five-second and earlier replacement
deadlines, company deletion, and bounded multi-pass draining. Existing cases cover
normal gateway signing and human decisions,
configured idle and terminal history, outer commit/rollback, startup/restart,
drain/resume, concurrent producer/read, unsigned rows, multi-page expiry, and
injected invocation-write failure with atomic rollback and retry. Existing
service tests cover approval/execution, interrupted outcomes, and projection
repair through the compatible manual entry point. Startup tests cover wiring.
The external test database override requires a disposable database ending in
`_test`; the suite deletes fixtures and stops the worker before database cleanup.
This is one worker migration, not proof that the instance can sleep. Other tool
review duties remain periodic. Existing Cloud sleep-safety gates and external
alarm coverage are unchanged. An in-process deadline cannot wake a stopped host.
## Scoped lookup review repair (2026-10-27)
The original scoped aggregate examined 12,004 rows to return one deadline in a
real PostgreSQL regression. The repaired lookup returns the earliest non-null
pending expiry for the requested company with `ORDER BY expires_at LIMIT 1`.
The grouped startup inventory remains unchanged. Existing company signals,
authority checks, expiry settlement, paging, backoff, and sleep gates remain
owned by their existing services.
The additive company-expiry index is migration `0383_tool_review_company_expiry`.
It builds the index concurrently and idempotently through the existing top-level
migration executor. Initially reserving number 0402 passed per-branch checks but
caused CI to reject descendants whose migrations preceded their target branch.
A local 0380 correction fixed that ordering, but master had meanwhile added its
own 0378 and 0379 migrations. The integrated stack preserves master's history,
renumbers its unmerged migrations after 0379 in PR dependency order, and preserves
every migration's SQL bytes. Exact filename references and journal tags/indices
follow the new names. Stack journal timestamps remain ordered after master's
latest entry. Each generated snapshot includes both master and that PR's schema.
Each PR must pass the exact target-branch migration-order gate, in addition to
the existing numbering, safety, five-snapshot retention, and schema-drift checks.
Local branch checks alone did not establish the stack's append-only ordering.
Fresh application, upgrades from the previous numbered task databases, and
replay also require verification. These unmerged stack migrations have not
changed the SQL of migrations already on master.
Two failures shaped the repair. The first migration attempt omitted its explicit
top-level registration and PostgreSQL rejected the concurrent build inside a
transaction; adding the registration fixed application and replay. An initial
three-column index also lost to the global expiry index in the planner, which
filtered 12,001 foreign-company rows before returning a result. The final index
omits the unused request ID. With 12,000 additional pending rows in each of two
companies, including earlier foreign deadlines, all three measured reloads
examine one row. The assertion counts filtered rows as well as returned rows.
This is evidence for the recorded fixture and query plan, not a production
latency guarantee or a claim that PostgreSQL can never choose another plan.
The complete gateway service, gateway acceptance, credential expiry, startup,
custom-image deadline, and schema snapshot suites pass: 223 tests across six
files. The run also exposed a credential test-clock overshoot; its correction
belongs to the earlier gateway PR and waits for real SQL before advancing to
an actually armed follow-up. Commit/rollback and exact expiry assertions remain.
Server typechecking passes. Combined-stack migration upgrades, full build, and
fresh remote checks remain required. Full local test:run is still unverified
because of embedded PostgreSQL startup/capacity limits; real gateway suites used
isolated, migrated test databases in the task-owned PostgreSQL instance.
A later review found an unbound `query` name in the SQL-capture assertion.
The due-row predicate short-circuited before that expression in the passing
fixture, and server typechecking excludes test sources. The assertion now
independently requires both the due expiry scan and the non-null deadline lookup
through each captured `read.query`. The earlier passing run did not detect this
name error; acceptance must include the service suite with both assertions.
+229
View File
@@ -0,0 +1,229 @@
# Tool-review interaction repair
The tool-action request and invocation are authoritative. The interaction card
is a projection of their outcome. Previously the aggregate maintenance tick
joined these tables repeatedly, even when every card already matched.
The projection worker reconstructs affected companies at startup and reconciles
after committed changes. Producer hints carry the company ID. Subsequent
candidate discovery and paginated repair queries filter by that company; review
writes cannot drive a scan of another company's terminal history. The shared
scoped worker dispatches one company per pass with one-second minimum spacing.
It retains a 30-second retry only for a mismatch or failed reconciliation.
Configured idle tools, pending reviews without terminal outcomes, and matching
terminal history have no recurring projection SQL or provider calls. This is
worker quiet, not whole-instance suspension.
## Ownership and trigger contract
`tool-review-projection.ts` owns candidate discovery and pagination. The gateway
retains card interpretation, current-state checks, locks, activity publication,
and the existing continuation callback. The scheduler consumes company discovery,
`sweep(companyId)`, and the returned retry obligation. No provider payload enters shared scheduling.
No table is added. A concurrent company/interaction lookup index supports the
existing link checks and linked-request updates.
The existing review notification helper now also signals projection repair.
Notifications are awaited before writes and inherit the outer transaction's
commit, rollback, and uncertainty tracking. Unrelated review changes can cause
one empty company-scoped reconciliation; they do not create a recurring deadline.
Interaction-only writes first check for an actual linked tool request using the
company and affected card IDs. Generic confirmation kinds alone do not signal
any tool-review queue. This also preserves recovery for legacy linked cards
without current tool payload flags. These reads happen in the mutation owner,
not a background poll. Sibling selection shares the update predicate and runs
under the existing issue lock before publishing the notification.
| Producer | Notification ownership |
| --- | --- |
| Gateway signing, approval inference, execution claims, provider outcomes, and invalidation | Existing `writeToolReviewChange`; the closed-issue expiry path now uses it too |
| Human approval/rejection | `commitToolActionReview` signals in its existing transaction |
| Pending expiry, approved recovery, interrupted claims, Test-tab settlement | Existing review-write boundary |
| Review-queue invalidation | Existing review-write boundary |
| Interaction withdrawal, skip, terminal-issue expiry | Linked-request resolution signals before the mutation |
| Linked tool-card accept/reject, deleted-addressee cancellation | Projection notification before replacing card state in the owning transaction; unlinked confirmations do not notify |
| Sibling supersession, startup duplicate cleanup, new/historical comment supersession | Projection notification before card replacement, including cases where the request is already terminal |
| Comment-based confirmation resolution | Calls the interaction owner inside the same outer transaction; its later comment-ID update shares that signal |
The audited other interaction writers do not create this mismatch: native bridge
and status-committer summary/status-only updates preserve the projection; native
review retirement owns separate completion-review cards; connection and secret
flows own their separate card types. Document-staleness and plan-resume paths
use document targets, while tool cards use `tool-action:` custom targets.
Reflection consent uses its instruction/profile/skill target keys, not tool-action
keys. A future writer that replaces tool-card result state must use the same
transaction notification boundary. A future writer that changes authoritative
tool outcomes must signal the review boundary.
## Repair, concurrency, and lifecycle
The mismatch predicate preserves the previous policy: terminal executed, failed,
expired, or cancelled requests with a linked interaction whose displayed tool
status differs. Cancelled requests display `expired`. Invocation error and result
summaries come from saved state. Repair never repeats a provider action.
Candidate queries page in request-ID order, 100 rows at a time. A card that cannot
yet be projected does not starve later pages. Per-row failures do not prevent
attempting the remaining candidates; failures keep the retry. A final mismatch
check retains retries for no-op repairs rather than silently declaring them done.
Failed companies retain their own backoff across notifications while other
companies progress. An unresolved no-op projection retains a 30-second deadline;
a new relevant company change can make it actionable sooner. A company pass
still drains all its pages, preserving progress past an unprojectable card.
The joins inspect that company's history on activation. This change removes repeated
idle scans; it does not introduce an indexed projection-debt table or claim that
an activated reconciliation has constant cost independent of history size.
The gateway locks the interaction and request, then checks the current request
status. If another repair already wrote the same status and outcome fields,
it avoids a second write and duplicate activity. Different saved details still
use the existing update path. Missing projection state remains repairable.
The shared deadline worker retains first writes during drain, failed retries,
and changes arriving during reconciliation. Shutdown joins coordinator work.
Startup reconstructs debt from existing rows. With heartbeat scheduling disabled,
startup still performs one pass, preserving the existing disabled-mode contract.
The public/manual combined review sweep remains available; server maintenance
no longer calls it periodically.
## Continuation and sleep limits
Card state is not proof that continuation delivery finished. Existing
`tool_action_deliveries` receipts and their coordinator remain responsible for
that work. Activity publication still occurs after the card transaction; this
slice does not add exactly-once activity delivery or a new outbox.
Signals remain process-local to the supported database owner. Independent SQL
writers and another active server require a separate notification/reconciliation
contract. Cloud eligibility still conservatively blocks retained tool-action
requests. No external wake or sleep-safety guard is relaxed here.
## Verification
Twelve new real-PostgreSQL cases exercise normal approval and single execution,
quiet configured and terminal-history states, four persisted outcome states,
outer commit/rollback, first write during drain, failure retry through drain,
foreground/worker repair contention, multi-page progress past an unprojectable
card, and an administrative card overwrite that reactivates repair. Quiet checks
observe absence of database reads across an hour of controlled time.
The first focused run caught a missing-projection null access in the new
idempotency check; optional access fixes it. The pagination fixture also attempted
to change a referenced request ID; it now orders the additional candidates without
mutating that key. The focused rerun passed all eleven then-existing tests.
After adding the concurrent repair case, all 148 service/startup tests and 92
gateway acceptance tests passed. Workspace typechecking and build passed. All 469 tool-access and 106 interaction
service tests also pass (815 tests across the final selections). The first
interaction run passed 105 cases but timed out in one cleanup hook during a
53-minute local run; the isolated full rerun passed in nine seconds. No production
change or timeout increase was made to force that result. Full repository tests
will run in CI; the local selections do not claim a complete repository test run.
Fresh PR CI/review remains required.
Company-scoping follow-up reproduced two regressions: captured recovery SQL lacked
a company predicate, and ordinary confirmation creation/replacement/cancellation
published three unnecessary tool-projection wakes. Both regressions now pass
through real producers. Additional coverage keeps failed-company retry timing
independent, proves ordinary mutations still complete without recovery scans,
and exercises administrative repair of legacy cards identified by their saved
link rather than payload flags.
Company-scoping follow-up verification: all 180 gateway and startup cases and all 152 interaction and secret-proposal route cases passed against dedicated PostgreSQL test databases. Server typechecking passed. These are local checks; stack CI and deployed sleep qualification remain separate.
## Indexed interaction links
Review found that the new link check had no supporting interaction index. An
ordinary unlinked confirmation could scan all tool-action history for its company.
The generated `0386_steady_epoch.sql` migration adds the non-unique
`(company_id, interaction_id)` index concurrently, and the existing top-level
migration executor handles its creation and interrupted-build recovery. The
schema and generated snapshot describe the same index; older SQL and journal
entries are preserved and snapshot pruning retains the five newest snapshots.
A real PostgreSQL check applied the migration to a fresh dedicated test database
and seeded 30,000 tool-action rows. The ordinary missing-link query returned no
rows in both treatments: without the index it performed a sequential scan,
filtered 30,000 rows and touched 429 shared buffers; with the index it used the
new index and touched two buffers. These are local planner measurements, not a
fleet latency or cost claim. The baseline index removal was rolled back in that
dedicated database, and the index remained valid afterwards. An initial probe
failed while binding a JSON fixture parameter before any comparison; correcting
the parameter encoding made the complete probe pass. That failed fixture is not
counted as a product regression.
Migration numbering/safety checks, generated snapshot drift, server types and
server build pass. All 214 cases in the full gateway/projection/startup/scoped-
deadline selection pass against a separate freshly migrated database, preserving
notification quiet, company isolation, transaction settlement, and interruption
behavior. Latest-head CI and automated reviews remain required merge gates.
## Bounded projection and lifecycle interruption
A review found that one company could drain an unlimited backlog in a single
worker pass. Five real PostgreSQL regressions reproduced 501 rows being consumed
instead of the 500-row budget, missing abort/drain handling, and the absence of
continuation past a full page of unprojectable or failed cards.
The worker now gives its existing service a 500-row budget, the coordinator's
abort signal, and its admission predicate. The service checks admission between
rows and reads at most 100 candidates per page. It retains the last attempted
request ID for each unfinished company, including after a failed pass, so a
poisoned early page cannot hide later healthy cards. A truncated pass schedules
a one-second continuation; the existing shared per-company fairness and failure
backoff remain in force. A completed pass that still has an unprojectable card
backs off from thirty seconds to five minutes. Fresh committed company changes
reset that delay and wake the affected scope promptly. These are real unresolved projections,
not an unconditional empty-history scan.
Reconstruction clears continuation state for companies that no longer have
mismatches. A sixth real regression reproduced an abandoned cursor skipping the
first new earlier card after an empty reconstruction; clearing that cursor makes
the first new pass repair it immediately. The maps contain no durable truth:
restart reconstructs from the existing rows, and completed rows stop matching.
There is no cursor table, independent timer or per-producer recovery wrapper.
Legacy manual/global and scheduler-disabled one-shot repair calls retain their
complete-sweep behavior unless they request a budget. The normal coordinator
worker always supplies the bounded options. Cancellation is observed between
rows and pages; an already issued database query or reflection finishes before
the service stops. A sweep never replays a provider tool action.
All 220 cases across the full gateway/projection/interruption, startup and scoped
deadline suites pass, with no skipped cases. The five initial regressions and
sixth cursor regression failed before their fixes; server types and build pass.
The earlier 219-case acceptance remains evidence for the preceding revision.
Fresh integrated-head CI and both automated reviews remain merge gates.
After normal reconciliation with master `f6bc653f7f`, all 320 integrated cases
pass, including the 81 authenticated gateway HTTP cases and the upstream narrowed
identity-read tests. Server types and build pass. Thirty-five independent master
performance files are preserved exactly; the gateway's narrowed identity read
is preserved alongside the projection extraction. The pre-integration 220-case
selection above remains evidence for its stated revision.
## Review efficiency follow-up
A company-scoped hint now enters the bounded owner directly rather than first
repeating company discovery. Startup/global reconstruction still discovers
companies from existing rows. An empty sweep with no saved cursor returns after
one candidate query; a resumed cursor or attempted reflection still needs the
final query to detect unresolved earlier cards. The reflection callback returns
no settlement result, so an attempted reflection cannot establish that a card
was repaired.
Returned unresolved work now backs off through 30, 60, 120, 240 and 300 seconds,
matching the shared error-backoff cap. Truncated backlog continuation remains
one second. Settlement removes the retry; a new committed hint resets backoff.
No pending row is discarded, and no independent timer or table is added.
The fixed-cadence regression failed before repair and now verifies the full
backoff cap, producer reactivation and quiet after settlement. An initial scan
assertion incorrectly assumed that a complete approval emitted only one wake;
its separate legitimate writes caused two scans even after optimization. The
corrected case isolates one committed hint and verifies one candidate scan with
matching history, followed by an empty direct sweep with one query. All 222
cases in the full gateway, startup and scoped-deadline suites pass without skips;
server typecheck and build pass. Fresh-head CI and both reviews are still required.
+51
View File
@@ -35,6 +35,13 @@ maintains as a dependency; once you install the packages below, they become
normal dependencies of **your own** project, and your own Dependabot updates
them.
An initialized SDK does not confirm ingestion. Verify a fresh trace from the
deployed revision at the configured destination. A rejected export can leave
the application healthy. For Honeycomb, configure the ingest key in the
`x-honeycomb-team` entry of `OTEL_EXPORTER_OTLP_HEADERS`; see
[Honeycomb authentication](https://docs.honeycomb.io/api/query-annotations/authorization).
Keep credential values out of logs and diagnostic reports.
## Budget admission, accounting, and enforcement
The operator-gated `paperclip.budgets` tracer records
@@ -313,6 +320,9 @@ attempt or owning phase:
| `runtime_selection.*` | Completion contracts, session/recovery reads, profiles, interaction materialization, backend discovery, bootstrap, run/coordinator writes, transaction entry/settlement, and workspace synchronization. |
| `runtime_context.instructions`, `.skills`, `.mcp` | The three concurrent runtime-context branches. Their durations overlap. |
| `runtime_context.asset_*` | Source reads, manifest hashing, cache lookup, cold materialization, and exact file verification on cold and warm paths. |
| `skills.inventory`, `.wait`, `.bundled`, `.releases`, `.folders`, `.local_sources` | Company skill reconciliation, waiting for an existing company refresh, and its bundled imports, release verification, folder, and local-source work beneath runtime skill preparation. Inventory children overlap their parent. |
| `skills.cache.acquire`, `skills.rows`, `skills.version.read`, `skills.repository.read`, `skills.version.materialize`, `skills.version.verify` | Dedicated connection/transaction startup plus advisory-lock acquisition, authorized skill rows, frozen version and repository reads, and exact runtime materialization/validation. Lock acquisition ends before the protected work; version verification is inside materialization. Only bounded file counts are added. |
| `skills.consume` | The caller's delivery and durable version-pinning callback while the cache lock remains held. |
| `workspace_sync.prepare_host_current`, `.prepare_durable_seed`, `.prepare_adopt_remote` | Runtime preparation under the selected inbound mode, including snapshot and staging work. A warm mismatch may produce a second preparation; it is real repeated work. |
| `workspace_sync.source_root`, `.repositories`, `.ignore_rules`, `.snapshot_git`, `.snapshot_baseline` | Physical source capture, managed repositories, ignore rules, Git metadata and baseline content snapshot. |
| `workspace_sync.pack`, `.git_clone`, `.git_archive`, `.overlay_copy`, `.overlay_archive`, `.deleted_manifest` | Temporary shallow Git clone, its archive callback, selected working-tree overlay copy/archive, and the deleted-file manifest. The archive is inside the clone's lifetime; do not sum parent and child. |
@@ -334,6 +344,41 @@ attempt or owning phase:
| `response.resolve`, `.authorize`, `.add_comment`, `.transaction*`, `.row_lock`, `.write`, `.publish` | Response selection, existing authorization, comment transaction including commit or rollback, row wait, insert, and external publication enqueue. |
| `execution.realtime` | Synchronous live-event and plugin publication dispatch. It does not measure browser receipt or paint. |
For bounded native OpenCode diagnosis, set
`PAPERCLIP_OPENCODE_PERFORMANCE_DIAGNOSTICS=1` as an agent/environment variable.
The existing configured-environment projection carries it into the native Runner.
It defaults off. The Runner creates an authenticated loopback OTLP sink and enables
the pinned OpenCode runtime's internal tracing. The sink discards logs, resource
attributes, span attributes, events, and unknown span names. It retains only closed
phase names, wall timestamps, and success/failure, with a 1 MiB request limit,
128 retained measurements, and 16 measurements per phase. No operator OTLP endpoint
or ingestion credential is passed to OpenCode.
For fresh remote diagnostic runs, the selected pack must contain the current
build's OpenCode proxy. An otherwise compatible image pack can predate the
collector. Such a pack cannot satisfy the diagnostic request. The existing
verified pack preparation path selects or stages the current proxy; it can add
startup time or fail admission when matching runtime assets are unavailable.
Ordinary runs retain compatible image-pack reuse. Recovery retains the pack
identity already admitted for that session.
Available phases cover context/instruction loading, skill discovery, tool and model
resolution, model streaming (`ai.streamText`, including stream consumption), Git
discovery/default branch, snapshots, dependency installation, and plugin startup.
These intervals overlap; do not sum them as elapsed time. At each terminal turn,
already-received measurements travel in one `harness.diagnostic` event through the
existing durable PRP channel. Late exports may be absent; collection never waits
for a provider flush or delays terminal publication to complete the diagnostic.
Late spans from a prior completed turn are discarded at the next boundary.
The server validates the closed fields again and records them under a foldable
`provider.runtime` group inside `native.session.execute`. The normal operator
endpoint gate controls Honeycomb export. The single diagnostic batch stays in the instance run log. Its OTel projection
does not write duplicate phase rows or await additional database work before
terminal consumption. Without an endpoint, no provider child spans are exported. Raw OpenCode OTLP must
never be sent directly to the operator endpoint: it can contain prompts and
authorization attributes before the loopback sink removes them.
Detail phases emit at most 16 spans of each name per attempt. Larger receipt and
asset inventories continue under the containing group without unbounded per-row
or per-file spans. Adding detail spans does not itself change work, query order,
@@ -1954,6 +1999,12 @@ It carries no skill contents, identifiers, locations, or credentials. It uses th
existing run performance events and operator-configured OpenTelemetry endpoint;
no first-party Telemetry event is added.
The `skills.*` detail spans above distinguish inventory reconciliation and lock
acquisition from repository transfer and file validation. A repository-backed
skill's authored `SKILL.md` is not its complete runtime payload: the generated
discovery entrypoint includes the pinned repository under `.paperclip-repository`.
Measure the delivered bundle's file count and bytes when reproducing its cost.
Runtime preparation refreshes the company inventory once per listing. Local and
catalog directories remain direct sources, so edits are visible on the next
preparation. Explicit version selections still use their stored snapshots.
@@ -634,10 +634,11 @@ Longer polling intervals alone do not complete an item.
[idle-login-cleanup.md](../idle-login-cleanup.md) for producer coverage,
verification and remaining provider-orphan boundaries.
- [ ] **JOB-19 — Custom-image setup expiry.** Current: **30s** active-session
expiry scans. Change: register `expiresAt` and cleanup attempts. **Complete
when:** no sessions means no SQL and expired provider resources are released.
Source: [environment-custom-images.ts](../../server/src/services/environment-custom-images.ts).
- [x] **JOB-19 — Custom-image setup expiry.** Uses committed setup-session
changes and the earliest existing active-session expiry. Startup reconstructs
deadlines; empty and terminal history disarm. Failed reconciliation and first
writes retain their wake through drain. Provider teardown and historical cleanup
remain with the existing services. See [idle-environment-cleanup.md](../idle-environment-cleanup.md#custom-image-setup-expiry).
- [ ] **JOB-20 — Tool connection health.** Current: **30s** DB inventory;
provider probes are conditional. Change: check on use/configuration change,
@@ -646,8 +647,12 @@ Longer polling intervals alone do not complete an item.
visible within the agreed freshness contract. Source:
[index.ts](../../server/src/index.ts), `tools.sweepConnectionHealth`.
- [ ] **JOB-21 — Secret proposals and expired action tokens.** Current: **30s**
expiry/cleanup queries. Change: enforce expiry synchronously on use and
- [ ] **JOB-21 — Secret proposals and expired action tokens (partially migrated).**
Secret-proposal expiry now uses transaction signals and saved expiry deadlines;
see [the lifecycle contract](../idle-secret-proposal-expiry.md). Gateway token
and session expiry now also uses saved deadlines and producer signals; see
[its lifecycle contract](../idle-gateway-credential-expiry.md). Tool review
recovery still uses **30s** queries. Change: enforce expiry synchronously on use and
register required scrub/cleanup deadlines. **Complete when:** idle cleanup
does not extend authorization or secret-retention periods. Sources:
[secret-proposals.ts](../../server/src/services/secret-proposals.ts),
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,53 @@
# Retained workspace recovery: fill a free slot across the cursor
This is a shared CI repair carried at the sleep-stack root. It fixes the existing
retained-workspace owner; it does not add another background duty or migrate the
remaining run/task recovery inventory. No PR is merged by this repair.
## Failure and ownership
The retained-workspace worker permits two physical exports. A held export stays
in the durable unfinished candidates, even though this process already owns it.
Discovery paginates by recovery-action ID and wraps only when its query returns
no rows. A page containing only a held export was nonempty, but its only row was
skipped in memory. Newly committed work before the cursor therefore waited for
another 15-second recovery pass despite a free slot.
The CI failure at PR #17446, run 40423925338 / job 121925042581, leaves the third
export `scheduled` at the existing 20-second assertion. The older #17450 failure
has the same observable result. A deterministic real-PostgreSQL/filesystem
regression reproduces it by starting the highest action ID, holding its provider
resume, completing a middle action, then requesting the lowest action. The
before-fix test failed at that same third-export assertion.
## Change and preserved boundaries
The existing candidate query accepts the locally in-flight run IDs and excludes
them before pagination on both unfinished and passive discovery paths. The
coordinator reads this set after its existing two-slot guard, so the exclusion
contains at most one run for an admitted discovery pass. An empty page can now
wrap in the same pass and find new work before the cursor. The in-memory duplicate
guard remains to handle changes while a query is running.
The retained-workspace module owns candidate selection. Its callers, SQL source
eligibility, physical ownership lock, retry policy, concurrency limit, completion
callbacks, and shutdown joins retain their existing contracts. Other-process
locks are still handled by the existing bounded cursor rotation. No new timer,
queue, table, provider branch, or retry authority is added. Held work remains
tracked as active and is not treated as settled or safe to sleep through.
## Verification
At the integrated pre-repair tip, the deterministic test failed with `scheduled`
instead of `complete`. After the repair, that regression and the existing
other-worker physical-lock rotation test both pass with real files and database
state. The free-slot test keeps the original 20-second completion bound, verifies
preserved files, no extra agent run, at most two concurrent resume operations,
and shutdown waiting for the held export before its final completion.
At the stack root, all 56 retained-workspace recovery cases pass on a dedicated
migrated PostgreSQL test database. The full suite also exercises retries,
exhaustion, stop verification, cleanup authorization, source bindings and saved
file recovery. Integrated workspace typecheck/build and fresh remote checks must
be recorded in the PR after parent propagation. This is not a full repository or
deployed-sleep claim.
@@ -0,0 +1,127 @@
# Connection-grants batch verification
Date: 2026-10-27. Implements the
[selected design](2026-10-27-connection-grants-capability-batch.md).
## Change and authority boundary
`connectionManagementService.describeAgentInstallCapabilities` prepares a direct
saved connection once, checks its retained installation targets once, and checks
each additional candidate in order. It uses the same private target evaluator as
ordinary authorization. The grants GET replaces only its editable-agent loop.
The optimized shape is board actors, non-AI saved connections, canonical retained
and candidate IDs, and no contextual or proposed-resource selectors. Delegated
agents and
complex resources finish the first already prepared evaluation and then use the
ordinary evaluator. Preparation is not restarted. Configure/use capabilities,
grant identity filtering, audience disclosure, writes, and human continuations
retain their existing authority paths.
Reuse lasts one descriptive call. It is not an atomic snapshot: a retained
permission or target removed mid-call can leave a stale displayed choice. The
next description and actual authorization must read fresh authority. No context,
mutation authority, or reusable cache escapes the batch.
## Reproduction and measurements
Pinned baseline: `f6bc653f7f745adaafa26393852fb2f5a99256a4`.
Treatment service SHA-256:
`f5fca127e8a669d343b61455187e102dc6705d87389544ff4edfa5a066185656`.
The benchmark loads the actual baseline service from Git, verifies that its other
runtime dependencies and lockfile match, and compares it with the new production
batch. Every positional capability, reason, and operation must match.
```sh
node --import ./server/node_modules/tsx/dist/esm/index.mjs \
scripts/benchmark-connection-install-capabilities.ts \
--baseline-ref=f6bc653f7f745adaafa26393852fb2f5a99256a4
```
The fixture is a Cloud board member who is not the creator, with a target-scoped
grant covering the synthetic roster. Company installs retain the whole roster;
partial installs retain one quarter, with at least one retained agent. Each arm
has two warmups and five measured samples in alternating order. Counts below are
per invocation. Full positional equality passed in every iteration. Individual
SQL shapes are unchanged; the benchmark saves statement counts, parameters, rows,
and treatment `EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON)` outside timed samples.
| Agents / installation | Old median ms (range) | Batch median ms (range) | SQL calls old → batch | Target lookups old → batch |
| --- | ---: | ---: | ---: | ---: |
| 1 / company | 5.13 (4.53–7.45) | 4.82 (4.38–5.08) | 8 → 8 | 1 → 1 |
| 1 / partial | 5.16 (4.01–5.54) | 5.26 (4.83–6.13) | 7 → 7 | 1 → 1 |
| 8 / company | 231.79 (221.63–238.96) | 31.11 (24.79–35.86) | 344 → 43 | 64 → 8 |
| 8 / partial | 128.84 (114.42–160.09) | 43.07 (39.36–45.69) | 126 → 42 | 22 → 8 |
| 32 / company | 2,804.04 (2,516.50–4,928.42) | 77.19 (75.56–132.76) | 5,216 → 163 | 1,024 → 32 |
| 32 / partial | 697.51 (676.38–732.57) | 73.92 (73.21–77.78) | 1,464 → 162 | 280 → 32 |
| 100 / company | 24,753.29 (23,849.21–25,019.72) | 246.39 (234.94–278.72) | 50,300 → 503 | 10,000 → 100 |
| 100 / partial | 6,338.58 (6,239.68–6,526.36) | 236.37 (228.40–288.54) | 13,075 → 502 | 2,575 → 100 |
At 100 agents, returned Drizzle rows fell from 60,200 to 602 for company installs
and from 15,475 to 526 for partial installs. The one-agent controls are neutral,
including a small overlapping timing regression for the partial case. The table
retains every sample range; do not extrapolate a latency curve from these medians.
The host could not start native PostgreSQL because SysV semaphore allocation
failed, including an unsandboxed low-resource initdb attempt. Local proof used
the existing test constructor seam with a scratch Docker provider: PostgreSQL
18.6 Debian/aarch64 Linux, Node 26.4.0, loopback-only networking, tmpfs storage,
and no simulated database delay. Each provider instance removed only its own
container. Existing database creation, migrations, clients, service code and
cleanup remained in use. The normal embedded package targets Darwin PostgreSQL
18.1; minor version, platform, networking, and compute differ. Defaults included
128 MiB shared buffers, 4 MiB work memory, and JIT enabled.
## Correctness evidence and limits
- Focused real-policy regressions cover scoped grants without installs, retained
missing/denied targets, mixed and duplicate candidates, 403 versus reached
409/500 faults, earlier denial masking an unreachable fault, forced mutation
mode, fresh calls and reauthorization, actor/company isolation,
deliberate within-call staleness, and ordinary fallback preparation counts.
- Real PostgreSQL differential coverage includes company and partial installs,
empty/duplicate candidates, foreign targets, uppercase UUID comparisons,
profile/proposed-provider fallback, and an injected SQL division-by-zero fault
that must escape only when reached.
- The assembled grants GET compares complete old-loop and batch responses across
creator, manager, member, viewer, inactive, and administrator fixtures. It
verifies personal identity filtering, audience disclosure, ordered editable
IDs, and an actual allowed install PUT followed by revocation, fresh denied
capabilities, and a denied PUT with saved installs unchanged.
- Independent review compared the pinned and extracted ordinary evaluator in
18 context/query/lock-order cases and 18 complete batch cases. Additional
checks covered delegated fallback, recursive profiles/gateways/AI pools, run
capture reuse, a succeeded-run continuation, and fresh revocation. This
supplemental proof uses synthetic database transport, not real lock contention.
Initial test failures were incorrect fixture expectations: inactive access hides
the resource with 404, database roster order is not insertion order, and the
existing final install mutation gate requires an unscoped agent-configuration
grant. These were corrected in tests. Complete old/new GET equality passed before
these extra assertions. No product behavior was changed to satisfy them.
Final guard review also found a real defect before merge: saved target IDs are
text, and an empty saved target makes the legacy check depend on the current
candidate. An executable regression returned `[false, true]` in the original
service and `[false, false]` in the first batch. Real PostgreSQL reproduced it.
The guard now requires canonical, nonempty retained IDs as well as candidates;
unusual saved selectors keep ordinary evaluation. The fixed path matches both
the complete capabilities and a later reached infrastructure fault. No stored
target is dropped or normalized, and legacy selector semantics stay unchanged.
The bound is retained-plus-candidate target evaluations, not total CPU, SQL rows,
or endpoint latency. Scoped grant arrays and company hierarchy reads can still
grow inside each policy decision. Recursive and delegated fallback retain their
existing costs. The benchmark measures the descriptive service component;
hosted request attribution and production latency improvement remain unverified.
Local coverage contains 33 batch regressions and 83 connection-management
integration cases, using disposable Docker PostgreSQL.
The required workspace typecheck and build passed using pinned pnpm 9.15.4.
The standard full Vitest workflow was attempted, then stopped after native
database fixtures failed to initialize. A separately isolated failing native
runner case reproduced the same PostgreSQL startup failure. This is not a full
local test pass; the critical connection suites use the disposable Docker
provider described above, and the complete CI suite remains a separate gate.
Final test/CI outcomes are recorded in [PR #17945](https://github.com/paperclipai/paperclip/pull/17945); this document does not
substitute for a final-head check or deployed acceptance.
@@ -0,0 +1,220 @@
# Connection-grants capability fan-out: design and safety proof
Date: 2026-10-27. This records the selected design before implementation.
See [implementation verification](2026-10-27-connection-grants-batch-verification.md)
for measured results and completed acceptance checks.
## Recommendation
Add one narrow, service-owned batch operation for describing which agents may
receive a connection installation. Factor the common installed-agent checks out
of the per-candidate loop. Keep authorization policy in its existing owner and
keep actual mutations on the existing fresh authorization path.
Start with board actors describing a direct, saved, non-AI connection. Preserve
the ordinary evaluator for delegated agents and complex resources until their
additional obligations have executable parity evidence. This is a bounded first
fix, not a general authorization cache or a rewrite of connection permissions.
The endpoint currently asks, for each candidate agent, whether the actor may
change the connection's installations. Each answer checks **all retained
installed agents**, followed by the candidate if it is not already installed.
A company installation expands to the company roster. For 100 candidates with
100 retained targets, this means 10,000 target evaluations before the other route
work. The proposed eligible path needs 100 retained-target evaluations, with
extra evaluations only for candidates outside that set.
Those retained checks are real policy obligations. The fix removes their repeated
execution; it must not remove their authority requirements.
## What the evidence establishes
The prior observation window contains two successful grants GETs lasting
56.506 and 66.388 seconds. There is no matching request trace attributing their
duration to this loop. See [the measurement record](2026-10-27-next-bottlenecks.md).
The source and local probes establish amplification, not the fraction of hosted
latency it causes or the eventual improvement.
The production path is:
- `server/src/routes/tool-access.ts`, `describeConnectionCapabilities`: selects
the company roster and calls `connections:profiles` once per agent, with
mutation authority and `readOnly: false`.
- `server/src/services/connection-management.ts`, `authorize`: resolves the
saved connection, expands its retained installations, then checks the ordered
target set. `capability` converts only errors with status 403 into denials;
other faults escape.
- `server/src/services/authorization.ts`: owns membership and scoped permission
decisions. Some subtree grants can read the whole company hierarchy for each
target, so linear target evaluations do not guarantee linear rows read.
Three independent designs were compared: structural factoring, general lazy
request caching, and a narrow guarded batch. A separate judge selected the narrow
batch. The final design incorporates the structural candidate's preparation
continuation and differential test approach, plus the cache candidate's careful
first-visit ordering and separate work counters.
| Local experiment | Observed result | Limit |
| --- | --- | --- |
| Existing exported service, all-installed fixtures | 4 agents caused 16 target reads; 8 caused 64 | Synthetic database transport, not PostgreSQL |
| Handwritten factored terminal phase compared with the current service | Six static output/fault cases matched; target reads fell 16 to 4 and 64 to 8 | Treatment omits preparation and is not the proposed production implementation |
| General read-reuse experiment, 32 agents | Physical transport reads fell 3,168 to 36, but 1,024 target invocations remained | Demonstrates why caching alone can retain quadratic work |
| Existing policy with a grant scoped to one agent and no installations | Named candidate allowed; targetless request denied | Rules out an extra targetless authorization preflight |
| Existing service with an unavailable retained target | Normal capability denied; `mutation: false` shortcut allowed | Rules out bypassing retained installation checks |
| Existing continuation after membership removal | Fresh `reauthorize` denied | Does not prove a future batch's freshness |
All three local proof scripts were rerun successfully by the primary agent and
the independent judge. They execute unchanged exported services over synthetic
Drizzle transport. They are level-4 evidence for the exercised current-code
facts, not proof of the unimplemented batch, assembled HTTP equivalence,
PostgreSQL performance, or delegated-actor parity.
## Owning API and eligibility
The new operation belongs in `connectionManagementService`, beside `capability`.
Its only initial consumer replaces the `editableAgentIds` loop. Use a fixed
`connections:profiles` action and mutation-mode semantics rather than exposing
general action or read-only knobs. Accept the original actor, base connection
resource, ordered candidate IDs, and existing read context. Return positional
capability descriptions; the route derives the existing editable-ID list.
Do not return an authorization context, audit authority, reusable session, cache
handle, or continuation. Do not change the wire schema. Keep manager checks,
configure/use capabilities, membership gates, grant identity filtering, and
audience disclosure on their existing paths.
The initial optimized shape is deliberately explicit:
- Board actor; one direct saved connection whose actual saved purpose is not AI.
- No non-null profile, gateway, routine, project, or issue selector.
- No own `requestedProvider` or `requestedInstallCompanyIds` property, and no
`selfServiceAgentInstalls: true` request.
- Validated, nonempty roster IDs in their original spelling and input order.
Retained target IDs must also be canonical and nonempty. Unusual identifiers
use the ordinary path rather than normalization: a saved empty target is a
candidate-dependent selector in the legacy evaluator.
Use the saved connection's provider, purpose, and creator derivation, including
its ordinary personal-grant handling. Caller labels cannot establish eligibility.
Unsupported shapes retain the existing per-candidate evaluator. In particular,
AI pools, profile bindings and selectors, current/proposed gateway resources,
and delegated actor contexts keep all recursive checks.
## Evaluation algorithm
Extract private preparation, target evaluation, and completion primitives from
the existing evaluator. Both the ordinary and batch paths must consume these
same primitives; do not copy policy into the route or implement a second policy
engine. Ordinary authorization always starts with fresh private state.
1. An empty candidate list returns an empty result without authorization, as
the current loop does.
2. Begin preparation for the first actual candidate in the current order.
Discover eligibility during the normal saved-connection lookup. The private
preparation frame is consumed exactly once. If ineligible, finish that same
evaluation through the ordinary path, then evaluate the remaining candidates
normally. Do not restart preparation to fall back.
3. For eligible inputs, form the same ordered set of retained installation
targets, including the existing company-install expansion. Evaluate each
retained target once, using the original check order: connection policy,
applicable proposed-provider/resource trust checks, target existence, and
principal-specific target authority. The shared primitive preserves these
obligations even where the initial eligibility guard excludes some branches.
4. A retained-target 403 denies every candidate with the same current denial.
A reached non-403 fault aborts the operation. Do not inspect later targets to
discover a fault the original evaluator would not reach.
5. After the retained prefix succeeds, visit candidates in input order. A
candidate already in the retained set uses the final retained decision,
matching the current ordered-set behavior and success reason. An additional
candidate uses the shared target primitive. Its 403 denies that candidate;
a reached non-403 fault aborts. Return one result per input position.
6. With no retained installations, the candidate is the policy target. Do not
add a targetless preflight: a grant scoped to that candidate may correctly
allow it while denying an unspecified target.
For N candidate positions and S distinct retained targets, the target phase is
bounded by S + N evaluations, and often fewer when candidates are retained.
This is not a promise of S + N SQL statements. Authorization may still read
membership, grants, and hierarchy for each distinct target; complex fallback
keeps its present cost. Measure those separately before widening the change.
Avoid eager prefetch and `Promise.all` here. A denied retained target currently
prevents later target lookups. Loading those targets first changes which faults
are reached. Parallelizing the old loop also retains its total work.
## Two safety facts and the blast radius
**1. A displayed permission must retain every existing installation obligation
and its own target scope.** The batch cannot turn a denied or absent retained
target into editable siblings, substitute a targetless decision, or reuse an
allowed candidate's decision for a different candidate. Preserve positional
results, reason, operation, ordered deduplication, and the 403/fatal-fault boundary.
Existing-code counterexamples prove these shortcuts are unsafe. Actual old/new
service and route comparisons remain required to prove the proposed change.
**2. A description must never become reusable mutation authority.** Batch state
exists only inside one call and cannot cross actors, companies, connections,
requests, operations, or human continuations. Actual writes and `reauthorize`
start fresh. The batch returns descriptions only; identity disclosure does not
consume them as authority.
There is a deliberate consistency change: reused facts and retained decisions
are observed once during the description rather than reread for every candidate.
This is neither a database snapshot nor an atomic revocation boundary. A revoke
or target deletion during the call can leave a briefly stale displayed choice;
a second GET and an actual mutation must observe fresh authority. Sequential
first visits preserve static-fact failure order, not every possible failure or
revocation between removed duplicate reads. Document and test this honestly.
Delegated agents remain on the ordinary path initially. `captureRunIdentity`
locks and can reconcile pending steering identity; it is not a pure read. A
future extension must prove run ownership, work mode, API-key status, responsible
human and external-human ceilings, and pending-identity behavior. Preserve the
original actor reference because existing read-context reuse checks identity by
reference. Do not memoize capture or attach batch state to that context.
The intended change touches the connection-management owner, its route consumer,
and focused tests. Existing connection policy is the authority. There is no
schema migration, provider protocol change, background worker, persistent cache,
UI contract change, or blanket creator/manager allow rule in this design.
## Implementation and acceptance gates
1. **Shared-helper refactor:** preserve ordinary evaluation before enabling the
batch. Prove preparation runs exactly once for the first candidate on both
eligible and fallback paths. Keep its continuation private and consumed.
2. **Actual service differential:** compare the new production batch against the
old full evaluator with a disposable PostgreSQL fixture. Compare every
positional `allowed`, `reason`, and `operation`, not just editable counts.
Cover empty/duplicate candidates; no, partial, and company installations;
scoped grants with no installations; missing/foreign retained and candidate
targets; local board and Cloud creator, owner, administrator, operator,
ordinary member, viewer, and inactive membership.
3. **Failures and fallback:** retained denial before an unreachable fault;
candidate denial followed by a reached later fault; preparation/target 500
and relevant reconciliation 409; only 403 converted. Exercise every excluded
shape, including empty AI pools, explicit null selectors, proposed providers,
company-install changes, profile selectors, and linked gateway obligations.
4. **Full grants GET:** compare complete assembled responses, including order,
editable IDs, configure/use flags, personal identity filtering, manager
visibility, audience members, and route access gates. Exercise the existing
connection Permissions and agent installation UI consumers.
5. **Freshness and isolation:** controlled mid-batch revoke/delete; fresh second
GET; GET then revoke then real mutation; delayed human/OAuth continuation;
concurrent actors, companies, and connections. Agent fast-path enablement
additionally requires real run/key/steering and lock-contention fixtures.
6. **Performance:** at roster sizes 1, 8, 32, and 100 or larger, count preparation,
target evaluations, policy decisions, SQL calls, returned rows, and wall time
separately. Test subtree grants and unrelated-request responsiveness. Measure
assembled route phases and database/pool delay before attributing savings.
After implementation, run the relevant regression suite and the repository's
required typecheck, tests, and build before PR hand-off. Require passing final-head
CI and review before hosted rollout. Repeat acceptance on the exact serving
revision, verify both response behavior and latency with enough samples, and
retain a way to restore the ordinary evaluator if parity fails.
This design pass changed no application behavior and performed no hosted mutation.
It establishes a preferred implementation and executable current-code safety
constraints. Complete batch parity and production latency improvement remain
unproved until the implementation and acceptance gates above pass.
+254
View File
@@ -0,0 +1,254 @@
# Next bottleneck batch
## Scope and observation
Find the next bounded batch after the three merged slow-query fixes. This is a
diagnosis pass: rank fresh costs, identify actual callers and existing work,
and define the smallest performance and correctness experiments. No application
code, schema, hosted configuration or deployment changes are part of this pass.
The reported workspace's Cloud, Neon database/runtime role, and Railway
deployment were revalidated from the live registry. The staging app served
`8548173fc9efdc0fa07644fb92dcc0b6851a95e5` throughout the query window. Upstream
`master` was refreshed to `50b932a142216331b8d476042b454bc123739c40`. Private
resource mapping, normalized SQL and raw logs remain outside Git.
After that stable measurement window, the final serving check at 18:49:08 UTC
observed revision `6e781e0ab20cdaeb6570cebdb5f14dbb8557ba04`.
The grants route/policy/authorization paths, wake-notice owner and skills
route/service are byte-identical between that source and the measured source.
The counts below describe the earlier window, not a post-rollout measurement.
Re-baseline the assembled application before accepting a later performance fix.
The fresh query window is 2026-10-27 **18:32:51.270–18:38:07.464 UTC**, or
316.194 seconds. Both snapshots read every statement for the exact application
database OID and runtime-role OID through the existing monitoring view, using
bounded read-only transactions. There is no top-100 or query-family filter.
- Statistics reset stayed at 14:21:16.446 UTC; eviction count stayed zero.
- There were 4,573 starting and 4,590 ending entries, with no missing entries or
invalid matched counters. Entry-level epochs were checked.
- The 1,147 matched entries with positive call deltas account for 121,299 calls
and 596.418 seconds of summed SQL execution time. They are all top-level
entries, so this total does not add nested statement timings.
- Seventeen new entries contribute 25 calls and 75.941 ms as separate lower
bounds. They are excluded from the matched ranking.
- I/O timing is disabled. Zero timing fields do not establish zero I/O latency.
Point-in-time wait samples cannot establish sustained pressure or its absence.
- The probe comparator passed eight controlled cases, including global/entry
resets, decreasing counters, missing/new entries, evictions and scope changes.
An initial diagnostic assertion about a pooler's default read-only setting
was rejected before capture; the successful probe checks the actual transaction.
The matched HTTP read contains 997 completed log records from the same deployment
and exact time window, below its 2,000-record cap. One WebSocket connection is
excluded from ordinary request latency. The remaining 996 include 603 HTTP 304
responses and one HTTP 500 from issue summaries; this is an observed traffic
sample, not a clean acceptance run. Duration units follow
[Railway's log contract](https://docs.railway.com/observability/logs).
SQL execution time can overlap across requests and background workers. Its sum
is neither elapsed time nor endpoint latency. Honeycomb queries using the
verified host, forwarded host and stack fields found no matching request spans
in the checked dataset/window; the SQL share of these HTTP requests is unproved.
## Fresh rankings
| SQL caller | Calls | Mean SQL time | Total SQL time | Returned/affected rows | Disposition |
| --- | ---: | ---: | ---: | ---: | --- |
| Chat lifecycle milestone candidates | 167 | 1,497.65 ms | 250.108 s | 0 | Exact caller overlaps open draft #17606 |
| Terminal workspace last activity | 245 | 591.64 ms | 144.951 s | 245 | No qualified open fix for these reads found |
| Inbound wake-notice candidates | 647 | 48.99 ms | 31.694 s | 0 | Exact duty overlaps open draft #17647 |
| GitHub stale-review repair UPDATE | 12 | 1,658.12 ms | 19.897 s | 0 | Exact duty overlaps open draft #17664 |
| GitHub review source-run fallback | 82 | 230.25 ms | 18.880 s | 1 | Distinct query in GitHub maintenance owner |
| GitHub stopped-response recovery | 12 | 1,340.53 ms | 16.086 s | 120 | Exact duty overlaps open draft #17661 |
The first two families still account for **66.24%** of matched SQL time after
the earlier query improvements. Their existing optimized SQL shapes are active.
The historical pre-fix fingerprints had no new calls in this window.
Guarded membership upserts used **41.660 ms across 84 calls**. The advisory-lock
statement with a large historical maximum used **0.950 ms across 174 calls**.
Neither historical cumulative total belongs on the new bottleneck shortlist.
| Ordinary HTTP route | Requests | Mean | Sample p95 | Maximum | Relevant result |
| --- | ---: | ---: | ---: | ---: | --- |
| GET connection grants | 2 | 61.447 s | 66.388 s | 66.388 s | Both 200; minimum 56.506 s; very small sample |
| GET company skills | 20 | 2.733 s | 6.015 s | 13.945 s | All 304 |
| GET company issues | 135 | 1.079 s | 2.957 s | 9.671 s | 112 returned 304 |
| GET sidebar badges | 197 | 482 ms | 1.440 s | 5.851 s | All 304 |
| GET run events | 203 | 346 ms | 1.063 s | 4.273 s | 137 returned 304 |
Percentiles use nearest rank within the observed sample. Upstream request
duration is almost identical to total duration for these routes. That locates
the observed delay behind the edge, but does not distinguish application CPU,
database round trips, pool wait or other dependencies.
## Recommended next work
### 1. Connection-grants capability fan-out
This is the largest newly observed individual request delay. The route is
`server/src/routes/tool-access.ts:1754`. Listing grants itself uses batched
database reads. The route also describes connection capabilities by inspecting
every company agent. Each inspection asks the connection-management policy for
a capability; its default mutation-authority calculation expands existing
company-wide installations to their agents and checks each target serially.
This creates quadratic authorization fan-out in the reproduced service path. No provider HTTP call
was found in this listing path. The relevant route and policy paths are
unchanged between the measured source, later serving source and refreshed
upstream master. The per-agent helper is `tool-access.ts:665–692`; the repeated
install expansion and target checks are `connection-management.ts:143–158` and
`:225–249`. It deliberately checks retained installed targets as well as the
candidate target.
The fresh SQL window contains thousands of individually fast membership,
project, issue and run-authority reads. Their counts alone do not attribute them
to these two requests. A scratch local reproduction called the real exported
`connectionManagementService.capability` → `authorize` → authorization service
with a synthetic database transport and an implicit local board actor. Four
agents caused 16 real target probes / 28 transport queries; eight agents caused
64 probes / 88 queries. The outer iteration follows the private route helper's
source; the helper itself was not exported or executed by the harness. This
proves the policy amplification through real code (blast-radius evidence level
4), not its share of the observed hosted request latency or a faster treatment.
The Cloud connection-creator case also reproduced 16 → 64 target probes, with
60 → 216 transport queries and 32 → 128 membership reads. A missing installed
target negative case denied all capabilities. These are synthetic transport
counts through real policy code, not real PostgreSQL or HTTP measurements.
Next measure the assembled route with representative installed scopes and
controlled database round-trip latency. The fix should make capability description share equivalent
request-local facts while retaining each target's authorization decision.
**Safety fact to prove:** any reused authority fact belongs to the same actor,
company, connection and request; a denied or absent target cannot inherit an
allowed sibling's capability. Cover board, user and agent callers, personal and
company installations, project/resource restrictions, removed grants, missing
agents and mixed allowed/denied targets. Preserve current membership/revocation
reads between requests. Do not make a one-minute route appear fast by skipping
its checks. Do not bypass installed-target checks by passing `mutation:false` or
`readOnly:true`; a missing installed target currently denies every candidate.
### 2. Qualify existing inbound wake-notice work
The measured query is owned by
`server/src/services/chat-channels/inbound/wakeup-notices.ts`, in
`createInboundWakeNotices.enqueueInboundWakeupPublications`. This selects
historical notice eligibility; it is not the execution-admission gate. It ran
647 times at approximately 2.05 calls per second and returned no rows. Its
31.694 seconds of SQL time included 12,973,945 shared-buffer hits and no
shared-block reads or temporary blocks in this window. The deployed and current
upstream query paths are identical.
Direct overlap is confirmed in
[#17647](https://github.com/paperclipai/paperclip/pull/17647), draft head
`ca8f2d17c64fc7cecff4526817d1ac6e1d479ff4`. It changes this exact selector and
removes its normal publication-maintenance call. Its coordinator reconstructs
durable scopes at startup, retains cursors, and consumes committed-change
notifications. Explicit/manual publication drains still call the selector, so
eliminating all 647 observed calls is not established. The PR is stacked on
`codex/sleep-cancel-contention` rather than master. Advance and qualify this
existing work, its dependencies and remaining callers before creating a
competing implementation. This pass did not run its tests or qualify its CI.
For remaining query work, capture the actual SQL and parameters from this service and
reproduce long retained action/publication/comment history with no pending
notice, followed by a single real eligible notice. Compare plans and all output
columns before deciding whether to narrow the query or wake it from concrete
committed obligations. A quiet path must still reconstruct unresolved work at
startup and recover missed notifications.
**Safety fact to prove:** a narrower candidate path still finds every notice
authorized by the original action/publication relationship, with unchanged
company scope and deduplication. Cover legacy/native lineage, malformed or
missing context, delayed commits, resolved/expired actions, failed publication,
duplicate delivery and drain/resume. Removing the recurring call requires proof
for every producer and restart/recovery path, not only an empty-queue test.
### 3. Skills-list reconciliation and repeated resolution
`server/src/routes/company-skills.ts:467` calls the service list at
`server/src/services/company-skills.ts:3269`. Every request first awaits
`ensureSkillInventoryCurrent` at line 3235: bundled imports, bundled-release
verification, folder reconciliation and local-source inspection. The shared
promise coalesces concurrent work but is removed when it finishes, so a later
request repeats the refresh. A GET can therefore perform filesystem and
database reconciliation before listing any skills.
The list then resolves every agent's desired skills again for each displayed
skill (`company-skills.ts:3351`). Reference resolution performs searches over
the skill list at line 2152. This is a concrete repeated-work lead, but its share
of the measured 2.733-second HTTP mean remains unproved. These service/route
files are identical in the deployed source, local integration and current
upstream master.
First profile the real service with representative skill and agent counts,
separating inventory refresh, database reads, desired-skill resolution and
serialization. Compute each agent's desired-key set once per request as the
first small candidate, comparing complete results. Treat any longer-lived
inventory cache as a separate change requiring freshness proof.
**Safety fact to prove:** each agent contributes at most once to each selected
skill, with identical exact-ID/key/slug resolution and ambiguous/missing alias
handling. Preserve visibility, folder/subtree/search filtering, sort order,
last-editor results and changed local-source discovery. A cached authorized
inventory cannot be reused for another principal's visibility.
## Existing work and deferred candidates
- [#17606](https://github.com/paperclipai/paperclip/pull/17606) owns the exact chat
lifecycle milestone path. Its draft branch moves lifecycle work to committed
changes and reconstructs pending work. Audit its producer/recovery proof and
drive its checks before starting a competing polling rewrite. Its draft
status and source design are not proof that it is safe or merge-ready.
- [#17664](https://github.com/paperclipai/paperclip/pull/17664) and
[#17661](https://github.com/paperclipai/paperclip/pull/17661) overlap the two
measured GitHub maintenance duties. The source-run fallback is a distinct
delivery/link/run lookup in `chat-github-checks.ts:181–202`, used when the
review lacks a resolvable run. It still merits a local plan/equality experiment,
coordinated with those owner changes. A previous scalar delivery lookup fix
in #17669 is already deployed and must not be proposed again.
- Workspace activity is called at
`server/src/services/execution-workspaces.ts:3576` through the shared terminal
workspace worker, already migrated by #16282. Its query is at
`server/src/services/git-workspace-expiration.ts:153–160`.
[#17505](https://github.com/paperclipai/paperclip/pull/17505) adjusts cursor and
retry deadlines but leaves this query/call intact. A second pass could batch
repeated activity reads; fresh destructive eligibility, live leases and
concurrent reopen checks must remain authoritative. No qualified open fix
reducing these activity reads was found in the scoped check.
- [#17896](https://github.com/paperclipai/paperclip/pull/17896),
[#17894](https://github.com/paperclipai/paperclip/pull/17894), and
[#17893](https://github.com/paperclipai/paperclip/pull/17893) are already merged
after the measured deployment. They narrow explicit-continuation run reads,
avoid duplicate GitHub webhook body transfer and narrow review-wait JSON.
The final 18:49:08 serving source contains all three. Evaluate their impact in
a new post-rollout window before duplicating those changes.
- [#17907](https://github.com/paperclipai/paperclip/pull/17907) keeps the skill
cache lock alive during slow preparation. Its documented scope does not
explain why preparation is slow or remove the skills-list work above.
The prior Slack continuation's 353.059-second delay is confirmed by an exact
deployed-version `heartbeat.queue` span without children. Executor startup begins
after that wait. A periodic recovery chain stayed in flight for at least
240.086 seconds and suppressed eight ticks, but an earlier recovery pass reached
its post-resume stage while the continuation was still queued. Source cleanup
had completed before the answer arrived. This narrows the failure to queue /
admission or recovery scheduling; it does not establish the blocking gate, a
lost wake, or SQL as the cause. Instrument gate decisions and recovery stages
before choosing a repair. No task was replayed or re-woken for diagnosis.
## Finish conditions for the next implementation batch
Take at most the three priorities above, advancing existing work where it
overlaps. Pin the deployed baseline, prove
the suspected cost with the real caller, define the performance criterion, then
compare complete behavior and adversarial authority/recovery cases. Keep
independent changes in independent PRs with green final-head checks. No migration
or boot script is justified by this discovery pass. Repeat hosted acceptance and
a fresh comparable counter/request window after an authorized rollout.
The private evidence set retains all raw inputs, comparator validation, exact
source inventories, request summaries and query snapshots. This document is the
sanitized discovery handoff. Correctness of proposed optimizations is unproved
until their real-code comparisons and regressions run.
+35
View File
@@ -0,0 +1,35 @@
# Run identity reads
`captureRunIdentity` locks the task before the run. It validates company, agent,
and active status. Pending steering still requires its durable acknowledgement.
The accepted requester is read for each operation. No scope adds a cache or skips
these checks.
Callers choose a typed read scope:
| Scope | Retained run data | Consumers |
| --- | --- | --- |
| `identity` | Run and accepted requester identifiers, status | Gateway, native connection tools, connection intents, project skill setup |
| `task_authorization` | Identity, task bindings, wake request, conversation generation, skill-test marker | Projects, workspace repositories, Slack, voice, connection management |
| `token_broker` | Identity, invocation source, task/project/routine and requester attribution | Connection token broker |
| `project_execution` | Task authorization plus runtime mode and the complete saved native execution input | Muse project admission |
Narrow return types omit unselected columns. JSON projections preserve the types
of retained values. The default full read remains available for compatibility and
explicit inspection. Existing GitHub-credential and request-authentication scopes
keep their current contracts. Execution preparation and recovery retain their
separate complete-input reads.
The synthetic real-PostgreSQL benchmark captures the actual locked query and
compares every retained column, context and identity result. It keeps the same
five-query reconciliation transaction, checks the unchanged stored run, records
plans and alternating timings, and measures raw PostgreSQL DataRow bytes:
```sh
pnpm --filter @paperclipai/server exec tsx ../scripts/benchmark-run-identity-projections.ts --baseline-ref=623f1dc289a14a699ebbe3f09b94179d38fc2ae2
```
The default uses disposable embedded PostgreSQL. `PAPERCLIP_TEST_DATABASE_URL`
can select a disposable local database whose name ends in `_test`. DataRow bytes
exclude response metadata, TLS and TCP. Byte reduction does not establish a
production latency or billing change.
+4 -2
View File
@@ -139,9 +139,11 @@ environment, and consent checks as other edits. It adds no agent permission or
credential fallback. Pool and externally managed agents use their existing
configuration surfaces. Board browser verification does not qualify agent access.
The existing-agent upgrade notice uses this configuration PATCH. It does not
The existing-agent upgrade notice for Claude, Codex, and Grok Build uses this configuration PATCH. It does not
grant configuration, account, or environment access. The server resolves the
saved settings under normal company/configuration authorization. Obsolete Claude
saved settings under normal company/configuration authorization. Grok migration preserves
the effective legacy search setting. Task model/effort overrides retain the
agent's account binding and tool policy without granting extra account access. Obsolete Claude
tuning fields are removed with `configurationWarnings`; security incompatibilities
remain errors. Unqualified sandbox upgrades first use the agent-scoped
`POST /api/agents/{id}/qualify-runner` under the same `agent_config:update`
+10
View File
@@ -189,6 +189,16 @@ conversational resolution with a genuine later same-task user message supplies
acceptance; an agent-only resolution does not. A plan approval must name the
current same-task revision. Newer pending/rejected proposals stop new creation.
Already admitted task replay and ordinary delegation retain their existing rules.
Native `paperclip_finish` does not grant completion authority from an opening
answer. The onboarding parent requires a trusted saved human decision on its
latest proposal. Acceptance permits completion under the existing policy;
rejection permits acknowledgment and closure without authorizing new work.
The native status commit checks this decision again under its existing task/run
locks. This adds no new agent endpoint or cross-company authority. The
[native completion controls](../../server/src/services/native-runtime/paperclip-control-plane-port.test.ts)
check persisted outcomes on disposable PostgreSQL; they are not live Cloud
credential or onboarding qualification.
Source: [creation admission](../../server/src/services/onboarding-first-task-admission.ts).
Controls: [native/REST PostgreSQL regressions](../../server/src/__tests__/onboarding-first-task-admission.test.ts).
These controls do not qualify a live Cloud provider journey.
+8 -1
View File
@@ -99,6 +99,9 @@ For an agent-owned plugin environment, verify preparation before connecting,
same-agent probes, account preservation after an environment change, failed
selection-save retry, and switching back while the computer stops. Confirm the
setup hold still blocks ordinary work and API key creation before finishing.
Start through the name/adapter dialog with a managed-only or forced Kubernetes
policy and a draft that inherits its environment. Once preparation completes,
confirm the auth choices, Advanced gateway, and saved subscription remain usable.
The [setup lifecycle tests](../../server/src/modules/agent-lifecycle/__tests__/setup.test.ts)
exercise durable transitions with synthetic plugin resources; they do not prove
live VM provisioning.
@@ -167,12 +170,16 @@ Selecting another runner must not silently discard security settings or change a
recorded execution. Keep runner choice in Advanced and verify the next eligible
run separately from historical transcripts.
Existing legacy Claude and Codex agents on qualified default environments, or
Existing legacy Claude, Codex, and Grok Build agents on qualified default environments, or
probe-capable Daytona snapshots / immutable sandbox images, show **Upgrade runner** above their configuration. Verify a company owner with `agents:configure` and no instance-admin role can qualify the saved computer and upgrade. A caller without configuration permission must be denied before a probe starts. Exercise that action with
pending edits and an environment-variable draft. An unsaved change to a different
effective computer must disable the action and explain that the computer change
must be saved first; restoring the saved target must re-enable it. Reopen the page and compare the
saved harness, model, account bindings, instructions, and run policy. Check a
Grok upgrade with an unset or `grok-build` model: it must use `grok-4.7`.
Compare effective search policy before and after, including the implicit disabled
default and explicit enabled search. Verify task model and effort changes use the
Grok catalog and reach execution while retaining the account and search policy. Check a
saved `maxTurnsPerRun: 1000`: upgrade must warn and remove it. Test an initially
unqualified Daytona Paperclip Computer: click must probe, record qualification,
refresh availability, and only then save. A failed probe or changed selection must
+9
View File
@@ -142,6 +142,15 @@ admit new work. [Admission controls](../../server/src/__tests__/onboarding-first
exercise native and REST calls against disposable PostgreSQL; live model and
Cloud onboarding remain separate evidence.
Native completion also requires a saved human proposal decision. An opening
answer alone must leave the first task in review. Verify that premature native
completion is refused, including a result already saved before finalization.
Accepted proposals and acknowledged human rejections can close the first task;
rejection still does not authorize new tasks. The
[native completion controls](../../server/src/services/native-runtime/paperclip-control-plane-port.test.ts)
exercise these boundaries against disposable PostgreSQL. These controls do not
identify a historical Cloud failure or establish live onboarding success.
For accepted file-producing work, verify exactly one self-assigned child with a
self-contained description. The onboarding run must yield through the existing
dependency/block disposition so the child's own execution can publish its file.
+10
View File
@@ -74,6 +74,16 @@ mutations. Qualify a useful model task separately from saving the account.
### `session`
For a repository project that inherits storage-plugin defaults, run a task and
send two follow-ups. Start with unset task workspace settings, then repeat with
an explicit isolated mode or strategy. Binding must save `reuse_existing` and
the resolved mode without changing the effective policy,
session fingerprint, and workspace identity. An actual project-policy change
must still reset the session and preserve the stored workspace owner. The
[session admission regression](../../server/src/__tests__/heartbeat-session-preparation.test.ts)
checks this with PostgreSQL and synthetic provider execution; it is not live
warm-process or latency evidence.
For `runs-adapters/session`, complete a task in a disposable warm sandbox and
retain its saved workspace file. Let the warm session close normally and verify
that the retained lease has a complete, lease-bound harness backup before
@@ -44,6 +44,7 @@ describe("Paperclip Runner permission defaults", () => {
["codex_local", "codexPermissionMode", "never"],
["opencode_local", "opencodePermissionMode", "allow"],
["pi_local", "acpxPermissionMode", "approve-all"],
["grok_local", "acpxPermissionMode", "approve-all"],
])("uses full auto when converting %s to the new runner", (adapter, key, value) => {
expect(paperclipRunnerTransitionConfig(adapter, undefined)).toMatchObject({ [key]: value });
});
@@ -53,6 +54,15 @@ describe("Paperclip Runner permission defaults", () => {
expect(paperclipRunnerTransitionConfig("pi_local", "openrouter/provider/model")).toMatchObject({ provider: "acpx", acpxAgent: "pi", model: "openrouter/provider/model" });
});
it("keeps Grok's harness and model when converting its adapter", () => {
expect(paperclipRunnerTransitionConfig("grok_local", "grok-4.7"))
.toMatchObject({ provider: "acpx", acpxAgent: "grok", model: "grok-4.7" });
});
it.each([undefined, "", "grok-build"])("resolves Grok's legacy default %s to the native model", model => {
expect(paperclipRunnerTransitionConfig("grok_local", model)).toMatchObject({ model: "grok-4.7" });
});
it("recognizes only exact provider identifiers", () => {
expect(isPaperclipRunnerProvider("codex")).toBe(true);
expect(isPaperclipRunnerProvider("opencode")).toBe(true);
@@ -205,7 +205,7 @@ export function paperclipRunnerTransitionConfig(
providerOverride?: unknown,
): Record<string, unknown> {
const previousProvider =
(previousAdapterType === "claude_local" || previousAdapterType === "pi_local")
(["claude_local", "pi_local", "grok_local"].includes(previousAdapterType))
? "acpx"
: previousAdapterType === "opencode_local"
? "opencode"
@@ -218,13 +218,15 @@ export function paperclipRunnerTransitionConfig(
: previousProvider;
return {
provider,
model: provider === "acpx" && previousAdapterType === "pi_local"
model: provider === "acpx" && previousAdapterType === "grok_local"
? typeof previousModel === "string" && previousModel.trim() && previousModel.trim() !== "grok-build" ? previousModel.trim() : "grok-4.7"
: provider === "acpx" && previousAdapterType === "pi_local"
? typeof previousModel === "string" ? previousModel.trim() : ""
: resolvePaperclipRunnerModel(
provider,
provider === previousProvider ? previousModel : undefined,
),
...(provider === "acpx" ? { acpxAgent: previousAdapterType === "pi_local" ? "pi" : "claude" } : {}),
...(provider === "acpx" ? { acpxAgent: previousAdapterType === "grok_local" ? "grok" : previousAdapterType === "pi_local" ? "pi" : "claude" } : {}),
[PAPERCLIP_RUNNER_PERMISSION_CAPABILITIES[provider].configKey]:
PAPERCLIP_RUNNER_PERMISSION_CAPABILITIES[provider].defaultMode,
lifecycleMode: "per_turn",
+15
View File
@@ -501,6 +501,21 @@ const MIGRATIONS_WITH_TOP_LEVEL_STATEMENTS = new Set([
"0332_little_night_nurse.sql",
"0345_luxuriant_carmella_unuscione.sql",
"0369_late_logan.sql",
"0381_mean_scream.sql",
"0382_narrow_the_executioner.sql",
"0385_open_madame_web.sql",
"0386_steady_epoch.sql",
"0387_boring_professor_monster.sql",
"0388_ambitious_micromax.sql",
"0389_brave_hulk.sql",
"0390_salty_killraven.sql",
"0391_third_roulette.sql",
"0392_small_frightful_four.sql",
"0393_open_grandmaster.sql",
"0384_funny_adam_warlock.sql",
"0383_tool_review_company_expiry.sql",
]);
async function applyPendingMigrationsManually(
+1 -1
View File
@@ -45,4 +45,4 @@ export { issueRelations } from "./schema/issue_relations.js";
export { issueReferenceMentions } from "./schema/issue_reference_mentions.js";
export * from "./schema/index.js";
export { signalDatabaseWork, subscribeDatabaseWork, databaseWorkPending, reconcileDatabaseWork } from "./work-signals.js";
export { databaseWorkScopeKey, signalDatabaseWork, subscribeDatabaseWork, databaseWorkPending, reconcileDatabaseWork } from "./work-signals.js";
@@ -0,0 +1 @@
CREATE INDEX CONCURRENTLY IF NOT EXISTS "tool_gateway_sessions_expiry_idx" ON "tool_gateway_sessions" USING btree ("expires_at");
@@ -0,0 +1 @@
CREATE INDEX CONCURRENTLY IF NOT EXISTS "tool_action_requests_pending_expiry_idx" ON "tool_action_requests" USING btree ("expires_at","id") WHERE "tool_action_requests"."status" = 'pending';
@@ -0,0 +1 @@
CREATE INDEX CONCURRENTLY IF NOT EXISTS "tool_action_requests_company_pending_expiry_idx" ON "tool_action_requests" USING btree ("company_id","expires_at") WHERE "tool_action_requests"."status" = 'pending';
@@ -0,0 +1 @@
CREATE INDEX CONCURRENTLY IF NOT EXISTS "tool_action_requests_approved_idx" ON "tool_action_requests" USING btree ("id") WHERE "tool_action_requests"."status" = 'approved';
@@ -0,0 +1 @@
CREATE INDEX CONCURRENTLY IF NOT EXISTS "tool_action_requests_executing_deadline_idx" ON "tool_action_requests" USING btree ("updated_at","id") WHERE "tool_action_requests"."status" = 'executing';
@@ -0,0 +1 @@
CREATE INDEX CONCURRENTLY IF NOT EXISTS "tool_action_requests_company_interaction_idx" ON "tool_action_requests" USING btree ("company_id","interaction_id");
@@ -0,0 +1 @@
CREATE INDEX CONCURRENTLY IF NOT EXISTS "decision_archive_notification_outbox_delivery_idx" ON "decision_archive_notification_outbox" USING btree ("last_attempt_at" NULLS FIRST,"created_at","id") WHERE "decision_archive_notification_outbox"."status" = 'pending';
@@ -0,0 +1 @@
CREATE INDEX CONCURRENTLY IF NOT EXISTS "decisions_pending_continuation_idx" ON "decisions" USING btree ("id") WHERE "decisions"."continuation_policy" = 'wake_origin_agent' AND "decisions"."metadata" ->> 'continuationPending' = 'true' AND ("decisions"."status" IN ('expired', 'cancelled') OR ("decisions"."status" = 'decided' AND "decisions"."execution_status" IN ('succeeded', 'partial', 'failed')));
@@ -0,0 +1 @@
CREATE INDEX CONCURRENTLY IF NOT EXISTS "decisions_open_expiry_idx" ON "decisions" USING btree ("expires_at","id") WHERE "decisions"."status" = 'open';
@@ -0,0 +1 @@
CREATE INDEX CONCURRENTLY IF NOT EXISTS "decisions_running_recovery_idx" ON "decisions" USING btree ("updated_at","id") WHERE "decisions"."status" = 'decided' AND "decisions"."execution_status" = 'running';
@@ -0,0 +1 @@
CREATE INDEX CONCURRENTLY IF NOT EXISTS "grok_dispatch_prepared_idx" ON "grok_bot_dispatches" USING btree ("created_at","id") WHERE "grok_bot_dispatches"."status" = 'prepared';
@@ -0,0 +1 @@
CREATE INDEX CONCURRENTLY IF NOT EXISTS "approvals_pending_retention_idx" ON "approvals" USING btree ("updated_at","id") WHERE "approvals"."status" = 'pending';
@@ -0,0 +1 @@
CREATE INDEX CONCURRENTLY IF NOT EXISTS "join_requests_pending_retention_idx" ON "join_requests" USING btree ("updated_at","id") WHERE "join_requests"."status" = 'pending_approval';
File diff suppressed because it is too large. Load diff
@@ -1,6 +1,6 @@
{
"id": "bfda87fa-662b-47bb-87c0-dabc814aebd7",
"prevId": "2a9439e3-4a2b-4a20-ab2b-2aee3db8cb03",
"id": "394e85a6-7fc2-4ab1-80a3-33bb4c7003aa",
"prevId": "4f21a636-735f-4dce-9ab8-0583f3bc7b79",
"version": "7",
"dialect": "postgresql",
"tables": {
@@ -22110,6 +22110,34 @@
"concurrently": false,
"method": "btree",
"with": {}
},
"decision_archive_notification_outbox_delivery_idx": {
"name": "decision_archive_notification_outbox_delivery_idx",
"columns": [
{
"expression": "last_attempt_at",
"isExpression": false,
"asc": true,
"nulls": "first"
},
{
"expression": "created_at",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"where": "\"decision_archive_notification_outbox\".\"status\" = 'pending'",
"concurrently": false,
"method": "btree",
"with": {}
}
},
"foreignKeys": {
@@ -24030,6 +24058,44 @@
"method": "btree",
"with": {}
},
"decisions_open_expiry_idx": {
"name": "decisions_open_expiry_idx",
"columns": [
{
"expression": "expires_at",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"where": "\"decisions\".\"status\" = 'open'",
"concurrently": false,
"method": "btree",
"with": {}
},
"decisions_pending_continuation_idx": {
"name": "decisions_pending_continuation_idx",
"columns": [
{
"expression": "id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"where": "\"decisions\".\"continuation_policy\" = 'wake_origin_agent' AND \"decisions\".\"metadata\" ->> 'continuationPending' = 'true' AND (\"decisions\".\"status\" IN ('expired', 'cancelled') OR (\"decisions\".\"status\" = 'decided' AND \"decisions\".\"execution_status\" IN ('succeeded', 'partial', 'failed')))",
"concurrently": false,
"method": "btree",
"with": {}
},
"decisions_bundle_idx": {
"name": "decisions_bundle_idx",
"columns": [
@@ -52812,6 +52878,88 @@
"method": "btree",
"with": {}
},
"tool_action_requests_pending_expiry_idx": {
"name": "tool_action_requests_pending_expiry_idx",
"columns": [
{
"expression": "expires_at",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"where": "\"tool_action_requests\".\"status\" = 'pending'",
"concurrently": false,
"method": "btree",
"with": {}
},
"tool_action_requests_company_pending_expiry_idx": {
"name": "tool_action_requests_company_pending_expiry_idx",
"columns": [
{
"expression": "company_id",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "expires_at",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"where": "\"tool_action_requests\".\"status\" = 'pending'",
"concurrently": false,
"method": "btree",
"with": {}
},
"tool_action_requests_executing_deadline_idx": {
"name": "tool_action_requests_executing_deadline_idx",
"columns": [
{
"expression": "updated_at",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"where": "\"tool_action_requests\".\"status\" = 'executing'",
"concurrently": false,
"method": "btree",
"with": {}
},
"tool_action_requests_approved_idx": {
"name": "tool_action_requests_approved_idx",
"columns": [
{
"expression": "id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"where": "\"tool_action_requests\".\"status\" = 'approved'",
"concurrently": false,
"method": "btree",
"with": {}
},
"tool_action_requests_invocation_idx": {
"name": "tool_action_requests_invocation_idx",
"columns": [
@@ -52827,6 +52975,27 @@
"method": "btree",
"with": {}
},
"tool_action_requests_company_interaction_idx": {
"name": "tool_action_requests_company_interaction_idx",
"columns": [
{
"expression": "company_id",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "interaction_id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"concurrently": false,
"method": "btree",
"with": {}
},
"tool_action_requests_issue_idx": {
"name": "tool_action_requests_issue_idx",
"columns": [
@@ -55222,6 +55391,21 @@
"method": "btree",
"with": {}
},
"tool_gateway_sessions_expiry_idx": {
"name": "tool_gateway_sessions_expiry_idx",
"columns": [
{
"expression": "expires_at",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"concurrently": false,
"method": "btree",
"with": {}
},
"tool_gateway_sessions_run_idx": {
"name": "tool_gateway_sessions_run_idx",
"columns": [
@@ -1,6 +1,6 @@
{
"id": "2a9439e3-4a2b-4a20-ab2b-2aee3db8cb03",
"prevId": "a0fcd9b8-f125-4985-8795-f567466b9aef",
"id": "f38ffbf3-c0ce-435e-aad8-9e5afa6b9417",
"prevId": "394e85a6-7fc2-4ab1-80a3-33bb4c7003aa",
"version": "7",
"dialect": "postgresql",
"tables": {
@@ -22110,6 +22110,34 @@
"concurrently": false,
"method": "btree",
"with": {}
},
"decision_archive_notification_outbox_delivery_idx": {
"name": "decision_archive_notification_outbox_delivery_idx",
"columns": [
{
"expression": "last_attempt_at",
"isExpression": false,
"asc": true,
"nulls": "first"
},
{
"expression": "created_at",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"where": "\"decision_archive_notification_outbox\".\"status\" = 'pending'",
"concurrently": false,
"method": "btree",
"with": {}
}
},
"foreignKeys": {
@@ -24030,6 +24058,66 @@
"method": "btree",
"with": {}
},
"decisions_running_recovery_idx": {
"name": "decisions_running_recovery_idx",
"columns": [
{
"expression": "updated_at",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"where": "\"decisions\".\"status\" = 'decided' AND \"decisions\".\"execution_status\" = 'running'",
"concurrently": false,
"method": "btree",
"with": {}
},
"decisions_open_expiry_idx": {
"name": "decisions_open_expiry_idx",
"columns": [
{
"expression": "expires_at",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"where": "\"decisions\".\"status\" = 'open'",
"concurrently": false,
"method": "btree",
"with": {}
},
"decisions_pending_continuation_idx": {
"name": "decisions_pending_continuation_idx",
"columns": [
{
"expression": "id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"where": "\"decisions\".\"continuation_policy\" = 'wake_origin_agent' AND \"decisions\".\"metadata\" ->> 'continuationPending' = 'true' AND (\"decisions\".\"status\" IN ('expired', 'cancelled') OR (\"decisions\".\"status\" = 'decided' AND \"decisions\".\"execution_status\" IN ('succeeded', 'partial', 'failed')))",
"concurrently": false,
"method": "btree",
"with": {}
},
"decisions_bundle_idx": {
"name": "decisions_bundle_idx",
"columns": [
@@ -52812,6 +52900,88 @@
"method": "btree",
"with": {}
},
"tool_action_requests_pending_expiry_idx": {
"name": "tool_action_requests_pending_expiry_idx",
"columns": [
{
"expression": "expires_at",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"where": "\"tool_action_requests\".\"status\" = 'pending'",
"concurrently": false,
"method": "btree",
"with": {}
},
"tool_action_requests_company_pending_expiry_idx": {
"name": "tool_action_requests_company_pending_expiry_idx",
"columns": [
{
"expression": "company_id",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "expires_at",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"where": "\"tool_action_requests\".\"status\" = 'pending'",
"concurrently": false,
"method": "btree",
"with": {}
},
"tool_action_requests_executing_deadline_idx": {
"name": "tool_action_requests_executing_deadline_idx",
"columns": [
{
"expression": "updated_at",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"where": "\"tool_action_requests\".\"status\" = 'executing'",
"concurrently": false,
"method": "btree",
"with": {}
},
"tool_action_requests_approved_idx": {
"name": "tool_action_requests_approved_idx",
"columns": [
{
"expression": "id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"where": "\"tool_action_requests\".\"status\" = 'approved'",
"concurrently": false,
"method": "btree",
"with": {}
},
"tool_action_requests_invocation_idx": {
"name": "tool_action_requests_invocation_idx",
"columns": [
@@ -52827,6 +52997,27 @@
"method": "btree",
"with": {}
},
"tool_action_requests_company_interaction_idx": {
"name": "tool_action_requests_company_interaction_idx",
"columns": [
{
"expression": "company_id",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "interaction_id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"concurrently": false,
"method": "btree",
"with": {}
},
"tool_action_requests_issue_idx": {
"name": "tool_action_requests_issue_idx",
"columns": [
@@ -55222,6 +55413,21 @@
"method": "btree",
"with": {}
},
"tool_gateway_sessions_expiry_idx": {
"name": "tool_gateway_sessions_expiry_idx",
"columns": [
{
"expression": "expires_at",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"concurrently": false,
"method": "btree",
"with": {}
},
"tool_gateway_sessions_run_idx": {
"name": "tool_gateway_sessions_run_idx",
"columns": [
@@ -55967,6 +56173,12 @@
"notNull": true,
"default": "''"
},
"responsible_user_id": {
"name": "responsible_user_id",
"type": "text",
"primaryKey": false,
"notNull": false
},
"owner_note": {
"name": "owner_note",
"type": "text",
@@ -1,6 +1,6 @@
{
"id": "7832df38-fa1b-4687-bd06-eccbc29bd28c",
"prevId": "3e17203b-ec95-41e9-8363-685982450199",
"id": "e080fc1b-af8e-4012-8ce5-3b5065b65143",
"prevId": "f38ffbf3-c0ce-435e-aad8-9e5afa6b9417",
"version": "7",
"dialect": "postgresql",
"tables": {
@@ -22110,6 +22110,34 @@
"concurrently": false,
"method": "btree",
"with": {}
},
"decision_archive_notification_outbox_delivery_idx": {
"name": "decision_archive_notification_outbox_delivery_idx",
"columns": [
{
"expression": "last_attempt_at",
"isExpression": false,
"asc": true,
"nulls": "first"
},
{
"expression": "created_at",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"where": "\"decision_archive_notification_outbox\".\"status\" = 'pending'",
"concurrently": false,
"method": "btree",
"with": {}
}
},
"foreignKeys": {
@@ -24030,6 +24058,66 @@
"method": "btree",
"with": {}
},
"decisions_running_recovery_idx": {
"name": "decisions_running_recovery_idx",
"columns": [
{
"expression": "updated_at",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"where": "\"decisions\".\"status\" = 'decided' AND \"decisions\".\"execution_status\" = 'running'",
"concurrently": false,
"method": "btree",
"with": {}
},
"decisions_open_expiry_idx": {
"name": "decisions_open_expiry_idx",
"columns": [
{
"expression": "expires_at",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"where": "\"decisions\".\"status\" = 'open'",
"concurrently": false,
"method": "btree",
"with": {}
},
"decisions_pending_continuation_idx": {
"name": "decisions_pending_continuation_idx",
"columns": [
{
"expression": "id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"where": "\"decisions\".\"continuation_policy\" = 'wake_origin_agent' AND \"decisions\".\"metadata\" ->> 'continuationPending' = 'true' AND (\"decisions\".\"status\" IN ('expired', 'cancelled') OR (\"decisions\".\"status\" = 'decided' AND \"decisions\".\"execution_status\" IN ('succeeded', 'partial', 'failed')))",
"concurrently": false,
"method": "btree",
"with": {}
},
"decisions_bundle_idx": {
"name": "decisions_bundle_idx",
"columns": [
@@ -32019,6 +32107,28 @@
"method": "btree",
"with": {}
},
"grok_dispatch_prepared_idx": {
"name": "grok_dispatch_prepared_idx",
"columns": [
{
"expression": "created_at",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"where": "\"grok_bot_dispatches\".\"status\" = 'prepared'",
"concurrently": false,
"method": "btree",
"with": {}
},
"grok_dispatch_company_status_idx": {
"name": "grok_dispatch_company_status_idx",
"columns": [
@@ -47782,6 +47892,18 @@
"primaryKey": false,
"notNull": false
},
"git_authorization_grant_id": {
"name": "git_authorization_grant_id",
"type": "uuid",
"primaryKey": false,
"notNull": false
},
"git_authorized_by_user_id": {
"name": "git_authorized_by_user_id",
"type": "text",
"primaryKey": false,
"notNull": false
},
"pause_reason": {
"name": "pause_reason",
"type": "text",
@@ -47920,6 +48042,19 @@
],
"onDelete": "no action",
"onUpdate": "no action"
},
"projects_git_authorization_grant_id_connection_grants_id_fk": {
"name": "projects_git_authorization_grant_id_connection_grants_id_fk",
"tableFrom": "projects",
"tableTo": "connection_grants",
"columnsFrom": [
"git_authorization_grant_id"
],
"columnsTo": [
"id"
],
"onDelete": "set null",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
@@ -52787,6 +52922,88 @@
"method": "btree",
"with": {}
},
"tool_action_requests_pending_expiry_idx": {
"name": "tool_action_requests_pending_expiry_idx",
"columns": [
{
"expression": "expires_at",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"where": "\"tool_action_requests\".\"status\" = 'pending'",
"concurrently": false,
"method": "btree",
"with": {}
},
"tool_action_requests_company_pending_expiry_idx": {
"name": "tool_action_requests_company_pending_expiry_idx",
"columns": [
{
"expression": "company_id",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "expires_at",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"where": "\"tool_action_requests\".\"status\" = 'pending'",
"concurrently": false,
"method": "btree",
"with": {}
},
"tool_action_requests_executing_deadline_idx": {
"name": "tool_action_requests_executing_deadline_idx",
"columns": [
{
"expression": "updated_at",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"where": "\"tool_action_requests\".\"status\" = 'executing'",
"concurrently": false,
"method": "btree",
"with": {}
},
"tool_action_requests_approved_idx": {
"name": "tool_action_requests_approved_idx",
"columns": [
{
"expression": "id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"where": "\"tool_action_requests\".\"status\" = 'approved'",
"concurrently": false,
"method": "btree",
"with": {}
},
"tool_action_requests_invocation_idx": {
"name": "tool_action_requests_invocation_idx",
"columns": [
@@ -52802,6 +53019,27 @@
"method": "btree",
"with": {}
},
"tool_action_requests_company_interaction_idx": {
"name": "tool_action_requests_company_interaction_idx",
"columns": [
{
"expression": "company_id",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "interaction_id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"concurrently": false,
"method": "btree",
"with": {}
},
"tool_action_requests_issue_idx": {
"name": "tool_action_requests_issue_idx",
"columns": [
@@ -55197,6 +55435,21 @@
"method": "btree",
"with": {}
},
"tool_gateway_sessions_expiry_idx": {
"name": "tool_gateway_sessions_expiry_idx",
"columns": [
{
"expression": "expires_at",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"concurrently": false,
"method": "btree",
"with": {}
},
"tool_gateway_sessions_run_idx": {
"name": "tool_gateway_sessions_run_idx",
"columns": [
@@ -55942,6 +56195,12 @@
"notNull": true,
"default": "''"
},
"responsible_user_id": {
"name": "responsible_user_id",
"type": "text",
"primaryKey": false,
"notNull": false
},
"owner_note": {
"name": "owner_note",
"type": "text",
@@ -1,6 +1,6 @@
{
"id": "a0fcd9b8-f125-4985-8795-f567466b9aef",
"prevId": "7832df38-fa1b-4687-bd06-eccbc29bd28c",
"id": "6974dd25-b8e9-4143-b4b7-ee5a5c3ff54a",
"prevId": "e080fc1b-af8e-4012-8ce5-3b5065b65143",
"version": "7",
"dialect": "postgresql",
"tables": {
@@ -5721,6 +5721,28 @@
}
},
"indexes": {
"approvals_pending_retention_idx": {
"name": "approvals_pending_retention_idx",
"columns": [
{
"expression": "updated_at",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"where": "\"approvals\".\"status\" = 'pending'",
"concurrently": false,
"method": "btree",
"with": {}
},
"approvals_company_status_type_idx": {
"name": "approvals_company_status_type_idx",
"columns": [
@@ -22110,6 +22132,34 @@
"concurrently": false,
"method": "btree",
"with": {}
},
"decision_archive_notification_outbox_delivery_idx": {
"name": "decision_archive_notification_outbox_delivery_idx",
"columns": [
{
"expression": "last_attempt_at",
"isExpression": false,
"asc": true,
"nulls": "first"
},
{
"expression": "created_at",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"where": "\"decision_archive_notification_outbox\".\"status\" = 'pending'",
"concurrently": false,
"method": "btree",
"with": {}
}
},
"foreignKeys": {
@@ -24030,6 +24080,66 @@
"method": "btree",
"with": {}
},
"decisions_running_recovery_idx": {
"name": "decisions_running_recovery_idx",
"columns": [
{
"expression": "updated_at",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"where": "\"decisions\".\"status\" = 'decided' AND \"decisions\".\"execution_status\" = 'running'",
"concurrently": false,
"method": "btree",
"with": {}
},
"decisions_open_expiry_idx": {
"name": "decisions_open_expiry_idx",
"columns": [
{
"expression": "expires_at",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"where": "\"decisions\".\"status\" = 'open'",
"concurrently": false,
"method": "btree",
"with": {}
},
"decisions_pending_continuation_idx": {
"name": "decisions_pending_continuation_idx",
"columns": [
{
"expression": "id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"where": "\"decisions\".\"continuation_policy\" = 'wake_origin_agent' AND \"decisions\".\"metadata\" ->> 'continuationPending' = 'true' AND (\"decisions\".\"status\" IN ('expired', 'cancelled') OR (\"decisions\".\"status\" = 'decided' AND \"decisions\".\"execution_status\" IN ('succeeded', 'partial', 'failed')))",
"concurrently": false,
"method": "btree",
"with": {}
},
"decisions_bundle_idx": {
"name": "decisions_bundle_idx",
"columns": [
@@ -32019,6 +32129,28 @@
"method": "btree",
"with": {}
},
"grok_dispatch_prepared_idx": {
"name": "grok_dispatch_prepared_idx",
"columns": [
{
"expression": "created_at",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"where": "\"grok_bot_dispatches\".\"status\" = 'prepared'",
"concurrently": false,
"method": "btree",
"with": {}
},
"grok_dispatch_company_status_idx": {
"name": "grok_dispatch_company_status_idx",
"columns": [
@@ -47782,6 +47914,18 @@
"primaryKey": false,
"notNull": false
},
"git_authorization_grant_id": {
"name": "git_authorization_grant_id",
"type": "uuid",
"primaryKey": false,
"notNull": false
},
"git_authorized_by_user_id": {
"name": "git_authorized_by_user_id",
"type": "text",
"primaryKey": false,
"notNull": false
},
"pause_reason": {
"name": "pause_reason",
"type": "text",
@@ -47920,6 +48064,19 @@
],
"onDelete": "no action",
"onUpdate": "no action"
},
"projects_git_authorization_grant_id_connection_grants_id_fk": {
"name": "projects_git_authorization_grant_id_connection_grants_id_fk",
"tableFrom": "projects",
"tableTo": "connection_grants",
"columnsFrom": [
"git_authorization_grant_id"
],
"columnsTo": [
"id"
],
"onDelete": "set null",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
@@ -52787,6 +52944,88 @@
"method": "btree",
"with": {}
},
"tool_action_requests_pending_expiry_idx": {
"name": "tool_action_requests_pending_expiry_idx",
"columns": [
{
"expression": "expires_at",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"where": "\"tool_action_requests\".\"status\" = 'pending'",
"concurrently": false,
"method": "btree",
"with": {}
},
"tool_action_requests_company_pending_expiry_idx": {
"name": "tool_action_requests_company_pending_expiry_idx",
"columns": [
{
"expression": "company_id",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "expires_at",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"where": "\"tool_action_requests\".\"status\" = 'pending'",
"concurrently": false,
"method": "btree",
"with": {}
},
"tool_action_requests_executing_deadline_idx": {
"name": "tool_action_requests_executing_deadline_idx",
"columns": [
{
"expression": "updated_at",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"where": "\"tool_action_requests\".\"status\" = 'executing'",
"concurrently": false,
"method": "btree",
"with": {}
},
"tool_action_requests_approved_idx": {
"name": "tool_action_requests_approved_idx",
"columns": [
{
"expression": "id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"where": "\"tool_action_requests\".\"status\" = 'approved'",
"concurrently": false,
"method": "btree",
"with": {}
},
"tool_action_requests_invocation_idx": {
"name": "tool_action_requests_invocation_idx",
"columns": [
@@ -52802,6 +53041,27 @@
"method": "btree",
"with": {}
},
"tool_action_requests_company_interaction_idx": {
"name": "tool_action_requests_company_interaction_idx",
"columns": [
{
"expression": "company_id",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "interaction_id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"concurrently": false,
"method": "btree",
"with": {}
},
"tool_action_requests_issue_idx": {
"name": "tool_action_requests_issue_idx",
"columns": [
@@ -55197,6 +55457,21 @@
"method": "btree",
"with": {}
},
"tool_gateway_sessions_expiry_idx": {
"name": "tool_gateway_sessions_expiry_idx",
"columns": [
{
"expression": "expires_at",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"concurrently": false,
"method": "btree",
"with": {}
},
"tool_gateway_sessions_run_idx": {
"name": "tool_gateway_sessions_run_idx",
"columns": [
@@ -55942,6 +56217,12 @@
"notNull": true,
"default": "''"
},
"responsible_user_id": {
"name": "responsible_user_id",
"type": "text",
"primaryKey": false,
"notNull": false
},
"owner_note": {
"name": "owner_note",
"type": "text",
File diff suppressed because it is too large. Load diff
+91
View File
@@ -2654,6 +2654,97 @@
"when": 1793060791724,
"tag": "0380_broken_aqueduct",
"breakpoints": true
},
{
"idx": 381,
"version": "7",
"when": 1793060791725,
"tag": "0381_mean_scream",
"breakpoints": true
},
{
"idx": 382,
"version": "7",
"when": 1793060791726,
"tag": "0382_narrow_the_executioner",
"breakpoints": true
},
{
"idx": 383,
"version": "7",
"when": 1793060791727,
"tag": "0383_tool_review_company_expiry",
"breakpoints": true
},
{
"idx": 384,
"version": "7",
"when": 1793060791728,
"tag": "0384_funny_adam_warlock",
"breakpoints": true
},
{
"idx": 385,
"version": "7",
"when": 1793060791729,
"tag": "0385_open_madame_web",
"breakpoints": true
},
{
"idx": 386,
"version": "7",
"when": 1793135146893,
"tag": "0386_steady_epoch",
"breakpoints": true
},
{
"idx": 387,
"version": "7",
"when": 1793136115698,
"tag": "0387_boring_professor_monster",
"breakpoints": true
},
{
"idx": 388,
"version": "7",
"when": 1793137983484,
"tag": "0388_ambitious_micromax",
"breakpoints": true
},
{
"idx": 389,
"version": "7",
"when": 1793138457589,
"tag": "0389_brave_hulk",
"breakpoints": true
},
{
"idx": 390,
"version": "7",
"when": 1793139659776,
"tag": "0390_salty_killraven",
"breakpoints": true
},
{
"idx": 391,
"version": "7",
"when": 1793142971791,
"tag": "0391_third_roulette",
"breakpoints": true
},
{
"idx": 392,
"version": "7",
"when": 1793143372644,
"tag": "0392_small_frightful_four",
"breakpoints": true
},
{
"idx": 393,
"version": "7",
"when": 1793145214502,
"tag": "0393_open_grandmaster",
"breakpoints": true
}
]
}
+2
View File
@@ -1,3 +1,4 @@
import { sql } from "drizzle-orm";
import { pgTable, uuid, text, timestamp, jsonb, index } from "drizzle-orm/pg-core";
import { companies } from "./companies.js";
import { agents } from "./agents.js";
@@ -19,6 +20,7 @@ export const approvals = pgTable(
updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(),
},
(table) => ({
pendingRetentionIdx: index("approvals_pending_retention_idx").on(table.updatedAt, table.id).where(sql`${table.status} = 'pending'`),
companyStatusTypeIdx: index("approvals_company_status_type_idx").on(
table.companyId,
table.status,
@@ -248,6 +248,9 @@ export const decisionArchiveNotificationOutbox = pgTable(
table.originAgentId,
),
pendingIdx: index("decision_archive_notification_outbox_pending_idx").on(table.status, table.createdAt),
pendingDeliveryIdx: index("decision_archive_notification_outbox_delivery_idx")
.on(table.lastAttemptAt.asc().nullsFirst(), table.createdAt, table.id)
.where(sql`${table.status} = 'pending'`),
statusCheck: check(
"decision_archive_notification_outbox_status_check",
sql`${table.status} IN ('pending', 'delivering', 'delivered')`,
+6
View File
@@ -69,6 +69,12 @@ export const decisions = pgTable(
table.status,
table.expiresAt,
),
runningRecoveryIdx: index("decisions_running_recovery_idx").on(table.updatedAt, table.id)
.where(sql`${table.status} = 'decided' AND ${table.executionStatus} = 'running'`),
openExpiryIdx: index("decisions_open_expiry_idx").on(table.expiresAt, table.id)
.where(sql`${table.status} = 'open'`),
pendingContinuationIdx: index("decisions_pending_continuation_idx").on(table.id)
.where(sql`${table.continuationPolicy} = 'wake_origin_agent' AND ${table.metadata} ->> 'continuationPending' = 'true' AND (${table.status} IN ('expired', 'cancelled') OR (${table.status} = 'decided' AND ${table.executionStatus} IN ('succeeded', 'partial', 'failed')))`),
bundleIdx: index("decisions_bundle_idx").on(table.bundleId),
originIssueIdx: index("decisions_origin_issue_idx").on(table.originIssueId),
companyIdempotencyUq: uniqueIndex("decisions_company_idempotency_uq")
+2
View File
@@ -1,3 +1,4 @@
import { sql } from "drizzle-orm";
import { index, integer, jsonb, pgTable, text, timestamp, uniqueIndex, uuid } from "drizzle-orm/pg-core";
import { companies } from "./companies.js";
import { agents } from "./agents.js";
@@ -49,6 +50,7 @@ export const grokBotDispatches = pgTable("grok_bot_dispatches", {
updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(),
}, t => [
uniqueIndex("grok_dispatch_event_uq").on(t.bindingId, t.bindingGeneration, t.routineRevision, t.sourceEventId),
index("grok_dispatch_prepared_idx").on(t.createdAt, t.id).where(sql`${t.status} = 'prepared'`),
index("grok_dispatch_company_status_idx").on(t.companyId, t.status),
index("grok_dispatch_assignment_kind_idx").on(t.assignmentId, t.kind),
]);
+2
View File
@@ -31,6 +31,8 @@ export const joinRequests = pgTable(
updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(),
},
(table) => ({
pendingRetentionIdx: index("join_requests_pending_retention_idx").on(table.updatedAt, table.id)
.where(sql`${table.status} = 'pending_approval'`),
inviteUniqueIdx: uniqueIndex("join_requests_invite_unique_idx").on(table.inviteId),
companyStatusTypeCreatedIdx: index("join_requests_company_status_type_created_idx").on(
table.companyId,
+8
View File
@@ -714,6 +714,7 @@ export const toolGatewaySessions = pgTable(
uniqueIndex("tool_gateway_sessions_token_hash_uq").on(table.tokenHash),
index("tool_gateway_sessions_company_agent_idx").on(table.companyId, table.agentId),
index("tool_gateway_sessions_company_expires_idx").on(table.companyId, table.expiresAt),
index("tool_gateway_sessions_expiry_idx").on(table.expiresAt),
index("tool_gateway_sessions_run_idx").on(table.companyId, table.runId),
index("tool_gateway_sessions_issue_idx").on(table.companyId, table.issueId),
index("tool_gateway_sessions_gateway_idx").on(table.companyId, table.gatewayId),
@@ -831,7 +832,14 @@ export const toolActionRequests = pgTable(
},
(table) => [
index("tool_action_requests_company_status_idx").on(table.companyId, table.status),
index("tool_action_requests_pending_expiry_idx").on(table.expiresAt, table.id)
.where(sql`${table.status} = 'pending'`),
index("tool_action_requests_company_pending_expiry_idx").on(table.companyId, table.expiresAt)
.where(sql`${table.status} = 'pending'`),
index("tool_action_requests_executing_deadline_idx").on(table.updatedAt, table.id).where(sql`${table.status} = 'executing'`),
index("tool_action_requests_approved_idx").on(table.id).where(sql`${table.status} = 'approved'`),
index("tool_action_requests_invocation_idx").on(table.invocationId),
index("tool_action_requests_company_interaction_idx").on(table.companyId, table.interactionId),
index("tool_action_requests_issue_idx").on(table.companyId, table.issueId),
],
);
+21 -1
View File
@@ -3,7 +3,7 @@ import { drizzle } from "drizzle-orm/postgres-js";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import { createDb, withDedicatedDbConnection } from "./client.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./test-embedded-postgres.js";
import { databaseWorkPending, installDatabaseWorkSignals, reconcileDatabaseWork, signalDatabaseWork, subscribeDatabaseWork } from "./work-signals.js";
import { databaseWorkScopeKey, databaseWorkPending, installDatabaseWorkSignals, reconcileDatabaseWork, signalDatabaseWork, subscribeDatabaseWork } from "./work-signals.js";
const support = await getEmbeddedPostgresTestSupport();
(support.supported ? describe : describe.skip)("database work signals", () => {
@@ -187,3 +187,23 @@ describe("work signal query boundaries", () => {
await expect(signalDatabaseWork(connection, "bad")).rejects.toThrow("inside a transaction");
});
});
describe("database work scope identity", () => {
type TestConnection = { execute: () => Promise<unknown[]>; transaction: (callback: (tx: TestConnection) => unknown) => Promise<unknown> };
function connection(): TestConnection {
return { execute: vi.fn(async () => []), transaction: async callback => callback(connection()) };
}
it("shares owner identity through child pools and nested transactions, but isolates independent owners", async () => {
const first = installDatabaseWorkSignals(connection());
const second = installDatabaseWorkSignals(connection());
const child = installDatabaseWorkSignals(connection(), first);
const key = databaseWorkScopeKey(first);
expect(databaseWorkScopeKey(second)).not.toBe(key);
expect(databaseWorkScopeKey(child)).toBe(key);
await child.transaction(async tx => {
expect(databaseWorkScopeKey(tx)).toBe(key);
await tx.transaction(async nested => { expect(databaseWorkScopeKey(nested)).toBe(key); });
});
});
});
+6 -2
View File
@@ -5,7 +5,7 @@ import { sql } from "drizzle-orm";
// an XID, so unrelated transactions keep their existing SQL behavior.
type Listener = (event: "intent" | "settled", hints?: ReadonlySet<string>) => void;
type Transaction = { topics: Set<string>; hints: Map<string, Set<string>>; xid?: Promise<string>; uncertain: boolean };
type Scope = { listeners: Map<string, Set<Listener>>; transactions: Set<Transaction>; root: Connection };
type Scope = { key: object; listeners: Map<string, Set<Listener>>; transactions: Set<Transaction>; root: Connection };
type Context = { scope: Scope; transaction?: Transaction };
type Connection = { transaction: (...args: any[]) => Promise<any>; execute: (...args: any[]) => Promise<any> };
const contexts = new WeakMap<object, Context>();
@@ -19,7 +19,7 @@ function publish(scope: Scope, topic: string, event: "intent" | "settled" = "set
/** Installed once by createDb; dedicated child pools share their owner's scope. */
export function installDatabaseWorkSignals<T extends Connection>(db: T, owner?: object): T {
const scope = (owner && contexts.get(owner)?.scope) || { listeners: new Map(), transactions: new Set(), root: db };
const scope = (owner && contexts.get(owner)?.scope) || { key: {}, listeners: new Map(), transactions: new Set(), root: db };
install(db, { scope });
return db;
}
@@ -112,3 +112,7 @@ export function subscribeDatabaseWork(owner: object, topic: string, listener: Li
}
return () => { listeners.delete(listener); };
}
export function databaseWorkScopeKey(connection: object): object {
return contexts.get(connection)?.scope.key ?? connection;
}
+7
View File
@@ -388,6 +388,13 @@ support disabling reasoning; other runner providers reject an explicit selection
before starting work. Native sessions expose support through the
`perTurnReasoning` capability.
For the native `openrouter/` route, the parent-held provider proxy retains the
upstream OpenRouter key. The child config keeps the selected `openrouter` provider
and exact model, using a random loopback token restricted to that model's chat
completions endpoint. The proxy and token expire when the session closes. The
local token remains a credential and is redacted from diagnostics; this does not
claim isolation for every child credential or persisted value.
OpenCode's proxy sends agent instructions once per turn through the system prompt. The
task envelope retains task-specific constraints and the completion contract,
without repeating the system instructions. This applies to all users of the
+4 -4
View File
@@ -14,12 +14,12 @@
"requiresProviderPolicy": false
},
"pi": {
"agentProfileVersion": 29,
"agentProfileVersion": 30,
"agentServerPackage": "pi-acp",
"agentServerVersion": "0.0.33",
"agentRuntimePackage": "@earendil-works/pi-coding-agent",
"agentRuntimeVersion": "1.0.0",
"commandDigest": "sha256:e80c6dc7bde12790371345ee5a12f0201d30941d26f1a780aec3d090545b2a06",
"commandDigest": "sha256:4517ab123dd89140ef462fc1905ac7a16a0dfcbaf21d67299a3c62de29c08a4d",
"requiresProviderPolicy": true
},
"cursor": {
@@ -32,12 +32,12 @@
"requiresProviderPolicy": true
},
"copilot": {
"agentProfileVersion": 65,
"agentProfileVersion": 70,
"agentServerPackage": "@github/copilot",
"agentServerVersion": "1.0.88",
"agentRuntimePackage": null,
"agentRuntimeVersion": null,
"commandDigest": "sha256:277894e850e07e6801cc24299c237513f58b41eb93c24f751d984fc4ddb04c47",
"commandDigest": "sha256:618c3b69bc98b9a7e3c1b3411700eab8b90fd9245ba35d0bc85484319247fdbc",
"requiresProviderPolicy": true
},
"claude": {
@@ -57,6 +57,110 @@ PRP v2 command vocabulary. Incoming wire commands still undergo the negotiated
protocol validation before execution; ordinary persisted v1 summaries remain
compatible.
## Adoption and retired launch artifacts
A live runner is adopted with its admitted artifact binding and authenticated
process identity. ACPX and OpenCode launch profiles are constructed only when a
new runner will actually be spawned. Adoption does not read the retired
controller's provider-pack executables, sidecar or proxy files. Fresh launches
still require the existing artifact digests and provider-pack authority.
The component regression starts the production runner and OpenCode proxy against
a scripted harness, removes the controller's launch files, then recovers the
same process and provider session and completes another turn. This exercises the
real transport; it does not qualify a hosted OpenCode or ACPX deployment.
## Source-bound profile history
Copilot profile v68 binds the merged hot-upgrade and current master transport.
Published v66 and the earlier hot-upgrade v67 identity remain byte-for-byte
historical records. The distinct development v66 identity is retained under
`copilot-profile-v66-hot-upgrade-candidate-historical-identity.json`; it does not
replace the published v66 fixture. Decoding historical versions does not grant
fresh-launch permission. Installation still requires the exact current profile,
package and digest. These records do not establish hosted harness qualification.
## External controller checkpoints
The optional controller persistence callback stores recovery control state, not
an output transcript. Text deltas do not trigger callbacks; usage updates are
coalesced into the next control checkpoint. Authentication, command intents and
outcomes, semantic inputs, lifecycle and result events retain commit barriers.
Priority-zero and unknown event kinds are never treated as disposable output.
Installing the callback makes it the controller's durable authority. The local
state file is replaced once with a fail-closed marker; ordinary control changes
no longer rewrite or fsync a second local journal. The controller can continue
if that directory disappears. A new controller must restore the committed
external checkpoint before opening the directory; the marker cannot recover
an execution or authorize a new one. This does not change the sandbox runner's
journal or controllers without an external persistence callback.
Fresh semantic inputs reserve their original live callback before awaiting the
checkpoint barrier. Recovery cannot steal first dispatch when no saved outcome
exists. The reservation is released if persistence or admission fails; business
execution still starts only after durable persistence succeeds.
Recovery and live semantic-tool dispatch share per-call ownership. Overlapping
reconciliation calls cannot invoke the same callback twice while its outcome is
pending. An already queued authoritative result is replayed without resolving
its business outcome again. Failed or unknown outcomes retain the existing
reconciliation and authority rules.
Pending completion inputs (`paperclip_finish` and `paperclip_block`) use the
driver's existing completion handler even when its caller supplies no recovery
lookup for ordinary tools. Codex transport waits for both authenticated startup
and handler installation; the external driver waits for session opening.
An ordinary tool with no outcome lookup remains held. Reconciliation does not
turn a missing lookup into permission to repeat that tool.
Transport readers use a committed view when external persistence is enabled.
Lifecycle and result events, command outcomes and warm-handoff authority become
visible only after the checkpoint callback succeeds. A failed write leaves the
previous view in place until a successful retry. Internal admission still sees
staged intents so overlapping recovery cannot duplicate them. Command results
flush their checkpoint even when no next command is waiting.
Streamed output continues without checkpoint writes. The view stops before a
pending control event to preserve event order, reuses immutable event records
and unchanged decoded commands, and caches the visible projection between
changes. It neither copies a second transcript nor adds an idle scan.
A rejected persistence callback closes the current connection and keeps the last
successful revision. A later flush or reconnect retries from a complete checkpoint,
including when the failed response hid a successful commit. Queued writes also
rebuild their delta against the last successful callback. Commit barriers remain
in force, and no retry timer runs when the controller is idle.
Supporting runners advertise `recoveryCursorVersion: 1` in `auth_hello`. The core
includes that capability and `recoveryAckedSourceSeq` in both authentication
proofs. The runner verifies the cursor equals its persisted acknowledgement.
After authentication and owner admission, a replacement core can reconcile that
cursor with its sparse checkpoint. It only skips already-acknowledged history;
new delivery remains contiguous. Unacknowledged events use the existing outbox.
A peer without this capability cannot attach to a sparse external checkpoint.
New runners can still authenticate to older controllers that use full journals.
There is no per-output external checkpoint timer or remote-controller cache
fsync. Local-only controllers retain their full file journal. Existing application
callbacks, the bounded runner outbox, and backpressure remain. Losing the
controller before acknowledged display output reaches its application log can
leave a display tail gap; this must not trigger provider or tool reexecution.
Prepared environments retain at most two opaque configuration IDs, each bound to
its exact execution identity across a warm handoff. They are captured with control
state, never output, and do not duplicate instructions or preparation requirements.
An execution driver can provide `authorizeRunnerAdoption(configurationId, identity)`
to recheck current authority before registering a surviving runner, without
preparing its resources again. The driver owns policy; Runner knows no project or
account rules. Missing bindings and rejected checks fail closed. Without this
optional hook, adoption retains the existing `prepareLaunch` recovery callback
and requires its controller-local preparation cache. Older checkpoints without
a configuration binding cannot use the cache-independent adoption path.
Supplying the hook for adoption without `prepareLaunch` is rejected at construction;
the transport must not silently skip a requested authority check. Drivers that do
not use prepared environments keep their existing adoption contract.
## Connection and authentication
The connection starts in this order:
@@ -424,11 +528,49 @@ A `revoke` envelope persists the `revoked` lifecycle. The runner flushes any
existing durable events and exits. It does not accept new work or delete
unacknowledged facts.
## Process identity
An external process owner can publish `startedAt` after launch dispatch returns.
`spawnRunner` preserves that live value through its restart wrapper. Once the
owner has a verified process identity, consumers use its birth time rather than
controller construction or attachment time. Replacing the controller must not
change the admitted process identity.
## Verification
The production durable control plane and live-session suites cover reconnect,
ACK replay, restart recovery, backpressure, lease expiry, drain, and revoke.
`src/control-plane/runner-recovery-fixture.test.ts` builds the test-only Cargo
example `recovery-fixture`. It links the unchanged production durable runner with
a scripted `CommandExecutor`, whose worker thread performs bounded hashing steps
and retains each resulting output until the normal executor ACK transfers it to
the runner outbox. There is one process, no HTTP counter service, and no production
provider-admission switch. Private fixture receipts report completed steps, queue
depth and process identity; they do not acknowledge or recover protocol events.
The short experiment starts one turn, removes the controller, then releases real
work. Eight retained events fill the fixture queue and pause work explicitly.
A replacement authenticates and drains through PRP. At step 32 the executor waits
for a controller command, survives another replacement, then completes 64 steps
without repeating the turn. The test runs with ordinary filesystem persistence
and with opaque external checkpoints restored onto empty controller storage.
An independent short-deadline case checks termination with recovery state intact.
These prove the runner mechanism, not Codex, Claude, Daytona or application
authorization. Server adoption and installed-artifact qualification have separate
owners and evidence.
Run the short experiment from this package with:
```sh
pnpm exec vitest run src/control-plane/runner-recovery-fixture.test.ts --maxWorkers=1
```
The fixture builds through Cargo during test setup. Production distribution builds
do not include the example. Existing control-plane tests separately inject failed
checkpoint persistence, stale/unauthenticated cursors and output floods, including
the zero-per-text-checkpoint-write gate.
Use `--json` for the complete trace or `--output <path>` to write it. The CLI
and browser show connection counts, safe lease ID and expiry, stable identities,
source and ACK cursors, outbox current/peak bytes, command delivery/effect
@@ -441,3 +583,5 @@ Regenerate the checked fault matrix and every exact per-fault trace with:
```sh
# Recorded evidence generation is deferred from this release.
```
Checkpoint decoding also exposes read-only recovery evidence: the admitted runner/session identity, authenticated connection count, and highest recorded source sequence. This projection grants no authority and adds no checkpoint writes. Qualification compares it across controller replacement; a replayed canonical event by itself does not prove that a replacement controller has authenticated a runner.
@@ -930,3 +930,31 @@ portable runtime isolation, private archive replay, and complete/partial/empty
campaign accounting. They do not establish current vendor prices, future
retention charges, live Braintrust attribution or invoice settlement. Keep
paid recurrence disabled until these independent gates are qualified.
### Token-spend qualification with separate operating reconciliation
A policy may explicitly set `overhead.accountingReadiness: "model_requests"`
when model-token qualification is the operator's priority. In that mode an
absent campaign capacity proof and absent observability cost proof do not block
admission. Any supplied proofs must still validate. Current allowance/pricing
windows, trace quotas, immutable source/request/runtime bindings, complete
selection, per-cell allocations, request limits, no retries and the shared
campaign admission cap remain required. Model reservation proofs remain required.
The final report retains the once-only operating allowance and labels operating
reconciliation pending. Missing, failed or canceled model evidence still holds
the full cell allocation and fails completion. This policy does not certify a
vendor operating-cost bound. Omit the field to retain full-lifecycle proof
requirements. Existing manifests and held amounts are not rewritten.
For this policy version, `operatingReconciliation` is always `pending`; the
immutable controller summary has no operating invoice settlement step. Mixed
histories retain that warning when any campaign uses the policy. A shared
`accountingReadiness` label appears only when every campaign uses it.
The trusted controller artifact preserves hidden dependency metadata as well as
code. Its inventory verifier requires every recorded byte, including package
`.npmignore` files. Only the fresh provider-credential-free controller build uses
this upload setting; private attempts still use the protected archive path.
@@ -85,7 +85,7 @@
{
"path": "schemas/provider-descriptor.schema.json",
"id": "https://paperclip.dev/schemas/prp/v1/provider-descriptor.schema.json",
"sha256": "862c0c13f23564c3530cefcf01579b0535a565b919cc3dcff7bc1bee9a668baf"
"sha256": "81d525335245dd4cb8a9bf41e3390dc60c6dd3e32474aad48b4f18a94d5f9711"
},
{
"path": "schemas/provider-event.schema.json",
@@ -168,6 +168,18 @@
"xhigh",
"max"
]
},
"grokEffort": {
"enum": [
"minimal",
"low",
"medium",
"high",
"xhigh"
]
},
"grokDisableWebSearch": {
"type": "boolean"
}
},
"allOf": [
@@ -518,6 +530,46 @@
}
}
}
},
{
"if": {
"required": [
"grokEffort"
]
},
"then": {
"properties": {
"provider": {
"const": "acpx"
},
"agent": {
"const": "grok"
}
},
"required": [
"agent"
]
}
},
{
"if": {
"required": [
"grokDisableWebSearch"
]
},
"then": {
"properties": {
"provider": {
"const": "acpx"
},
"agent": {
"const": "grok"
}
},
"required": [
"agent"
]
}
}
],
"$defs": {
@@ -0,0 +1,233 @@
use std::collections::VecDeque;
use std::fs;
use std::path::PathBuf;
use std::sync::{Arc, Condvar, Mutex};
use std::thread;
use std::time::Duration;
use paperclip_runner_core::durable::{
capture_bootstrap_ticket, run_durable_runner, Command, CommandExecution, CommandExecutor,
DurableRunnerConfig, DurableRunnerError, EventPriority, PolledEvent,
};
use serde::Deserialize;
use serde_json::json;
use sha2::{Digest, Sha256};
#[derive(Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
struct Script {
steps: usize,
wait_after: usize,
retention_limit: usize,
release_file: PathBuf,
receipt_file: PathBuf,
}
#[derive(Default)]
struct Work {
started: bool,
stopped: bool,
resumed: bool,
completed: usize,
acknowledged: usize,
peak_retained: usize,
events: VecDeque<PolledEvent>,
}
type Shared = Arc<(Mutex<Work>, Condvar)>;
fn receipt(script: &Script, work: &Work, phase: &str) {
let bytes = serde_json::to_vec(&json!({
"pid": std::process::id(), "phase": phase, "completedSteps": work.completed,
"retainedEvents": work.events.len(), "acknowledgedEvents": work.acknowledged,
"peakRetainedEvents": work.peak_retained,
}))
.unwrap();
let temporary = script.receipt_file.with_extension("pending");
fs::write(&temporary, bytes).unwrap();
fs::rename(temporary, &script.receipt_file).unwrap();
}
fn workload(script: Script, shared: Shared) {
let (lock, changed) = &*shared;
let mut work = lock.lock().unwrap();
receipt(&script, &work, "admitted");
loop {
if work.stopped {
receipt(&script, &work, "stopped");
return;
}
if !work.started || !script.release_file.exists() {
work = changed
.wait_timeout(work, Duration::from_millis(10))
.unwrap()
.0;
continue;
}
if work.completed == script.steps {
receipt(&script, &work, "completed");
work = changed.wait(work).unwrap();
continue;
}
if work.completed == script.wait_after && !work.resumed {
receipt(&script, &work, "awaiting_controller");
work = changed.wait(work).unwrap();
continue;
}
if work.events.len() == script.retention_limit {
receipt(&script, &work, "backpressured");
work = changed.wait(work).unwrap();
continue;
}
let step = work.completed + 1;
drop(work);
let mut digest = Sha256::digest(step.to_le_bytes());
for _ in 0..4096 {
digest = Sha256::digest(digest);
}
work = lock.lock().unwrap();
work.events.push_back(PolledEvent {
executor_event_id: format!("fixture-step-{step}"),
event_type: "item.delta".to_owned(),
priority: EventPriority::P2,
payload: json!({"text": format!("step={step} sha256={digest:x}\n")}),
});
work.completed = step;
work.peak_retained = work.peak_retained.max(work.events.len());
receipt(&script, &work, "computing");
}
}
struct ScriptedExecutor {
shared: Shared,
worker: Option<thread::JoinHandle<()>>,
}
impl ScriptedExecutor {
fn new(script: Script) -> Self {
let shared = Arc::new((Mutex::new(Work::default()), Condvar::new()));
let worker_shared = shared.clone();
Self {
shared,
worker: Some(thread::spawn(move || workload(script, worker_shared))),
}
}
}
impl CommandExecutor for ScriptedExecutor {
fn execute(&mut self, command: &Command) -> Result<CommandExecution, DurableRunnerError> {
let mut work = self.shared.0.lock().unwrap();
match command.command_type.as_str() {
"turn.start" => {
if work.started {
return Err(DurableRunnerError::invalid("fixture turn started twice"));
}
work.started = true;
}
"turn.steer" => work.resumed = true,
"session.snapshot" => {}
"turn.stop" | "session.close" | "runner.shutdown" | "runner.suspend" => {
work.stopped = true
}
_ => return Err(DurableRunnerError::invalid("unsupported fixture command")),
}
self.shared.1.notify_all();
Ok(CommandExecution::result(json!({
"completedSteps": work.completed, "retainedEvents": work.events.len(),
"acknowledgedEvents": work.acknowledged, "pid": std::process::id(),
})))
}
fn poll_events(&mut self) -> Result<Vec<PolledEvent>, DurableRunnerError> {
self.retained_events()
}
fn retained_events(&mut self) -> Result<Vec<PolledEvent>, DurableRunnerError> {
Ok(self
.shared
.0
.lock()
.unwrap()
.events
.iter()
.cloned()
.collect())
}
fn acknowledge_events(&mut self, count: usize) -> Result<(), DurableRunnerError> {
let mut work = self.shared.0.lock().unwrap();
if count > work.events.len() {
return Err(DurableRunnerError::invalid(
"fixture acknowledged an unowned event",
));
}
work.events.drain(..count);
work.acknowledged += count;
self.shared.1.notify_all();
Ok(())
}
fn shutdown(&mut self) -> Result<(), DurableRunnerError> {
self.shared.0.lock().unwrap().stopped = true;
self.shared.1.notify_all();
if let Some(worker) = self.worker.take() {
worker
.join()
.map_err(|_| DurableRunnerError::invalid("fixture worker failed"))?;
}
Ok(())
}
}
impl Drop for ScriptedExecutor {
fn drop(&mut self) {
let _ = self.shutdown();
}
}
fn main() -> Result<(), Box<dyn std::error::Error>> {
let args: Vec<String> = std::env::args().skip(1).collect();
let value = |key: &str| -> Result<String, DurableRunnerError> {
args.windows(2)
.find(|pair| pair[0] == key)
.map(|pair| pair[1].clone())
.ok_or_else(|| DurableRunnerError::invalid(format!("missing {key}")))
};
let script: Script = serde_json::from_slice(&fs::read(value("--fixture-script")?)?)?;
if script.steps == 0
|| script.steps > 4096
|| script.wait_after > script.steps
|| !(1..=256).contains(&script.retention_limit)
{
return Err("fixture workload exceeds its bounds".into());
}
let config = DurableRunnerConfig {
connect_url: value("--connect-url")?,
ca_bundle_path: None,
state_dir: value("--state-dir")?.into(),
runner_instance_id: value("--runner-id")?,
environment_lease_id: value("--environment-lease-id")?,
run_id: value("--run-id")?,
normalized_session_id: value("--session-id")?,
turn_id: value("--turn-id")?,
item_id: value("--item-id")?,
runner_version: value("--runner-version")?,
runner_digest: value("--runner-digest")?,
acpx_launch_profile: None,
opencode_launch_profile: None,
max_outbox_bytes: value("--max-outbox-bytes")?.parse()?,
p0_reserve_bytes: value("--p0-reserve-bytes")?.parse()?,
max_frame_bytes: 1024 * 1024,
reconnect_delay: Duration::from_millis(20),
reconnect_grace: Some(Duration::from_millis(
value("--reconnect-grace-ms")?.parse()?,
)),
max_runtime: Duration::ZERO,
};
run_durable_runner(
config,
capture_bootstrap_ticket()?.ok_or("missing ticket")?,
ScriptedExecutor::new(script),
)?;
Ok(())
}
@@ -145,6 +145,10 @@ struct AcpxProviderDescriptor {
pi_thinking_level: Option<PiThinkingLevel>,
#[serde(default, skip_serializing_if = "Option::is_none")]
claude_effort: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
grok_effort: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
grok_disable_web_search: Option<bool>,
permission_mode_pinned: bool,
#[serde(default)]
provider_policy: Option<AcpxProviderRuntimePolicy>,
@@ -273,6 +277,14 @@ impl AcpxProviderDescriptor {
self.agent != "claude"
|| !matches!(effort.as_str(), "low" | "medium" | "high" | "xhigh" | "max")
})
|| (self.grok_disable_web_search.is_some() && self.agent != "grok")
|| self.grok_effort.as_ref().is_some_and(|effort| {
self.agent != "grok"
|| !matches!(
effort.as_str(),
"minimal" | "low" | "medium" | "high" | "xhigh"
)
})
|| self.mode.as_ref().is_some_and(|mode| {
mode.trim().is_empty()
|| mode.chars().count() > 240
@@ -375,6 +387,8 @@ impl AcpxProviderDescriptor {
mode: self.mode.clone(),
pi_thinking_level: self.pi_thinking_level,
claude_effort: self.claude_effort.clone(),
grok_effort: self.grok_effort.clone(),
grok_disable_web_search: self.grok_disable_web_search.clone(),
permission_mode_pinned: self.permission_mode_pinned,
provider_policy: self.provider_policy.clone(),
system_instructions: self.instructions.clone(),
@@ -482,6 +496,12 @@ impl AcpxProviderDescriptor {
if let Some(effort) = &self.claude_effort {
descriptor["claudeEffort"] = json!(effort);
}
if let Some(effort) = &self.grok_effort {
descriptor["grokEffort"] = json!(effort);
}
if let Some(disabled) = self.grok_disable_web_search {
descriptor["grokDisableWebSearch"] = json!(disabled);
}
if let Some(level) = self.pi_thinking_level {
descriptor["piThinkingLevel"] = json!(level);
}
@@ -622,6 +642,8 @@ impl AcpxDurableState {
|| identity.mode != self.descriptor.mode
|| identity.pi_thinking_level != self.descriptor.pi_thinking_level
|| identity.claude_effort != self.descriptor.claude_effort
|| identity.grok_effort != self.descriptor.grok_effort
|| identity.grok_disable_web_search != self.descriptor.grok_disable_web_search
{
return Err(DurableRunnerError::invalid(
"ACPX durable identity no longer matches its qualified profile or provider mode",
@@ -1309,6 +1331,12 @@ impl AcpxCommandExecutor {
if let Some(effort) = &identity.claude_effort {
result["claudeEffort"] = json!(effort);
}
if let Some(effort) = &identity.grok_effort {
result["grokEffort"] = json!(effort);
}
if let Some(disabled) = identity.grok_disable_web_search {
result["grokDisableWebSearch"] = json!(disabled);
}
if let Some(level) = identity.pi_thinking_level {
// This identity is admitted only after the sidecar's effective-mode ACK.
result["piThinkingLevel"] = json!(level);
@@ -2783,6 +2811,8 @@ mod tests {
mode: descriptor.mode.clone(),
pi_thinking_level: descriptor.pi_thinking_level,
claude_effort: descriptor.claude_effort.clone(),
grok_effort: descriptor.grok_effort.clone(),
grok_disable_web_search: descriptor.grok_disable_web_search.clone(),
provider_lifetime_fence_candidates: [60_001, 60_002, 60_003],
};
let operations = Vec::new();
@@ -3060,6 +3090,8 @@ mod tests {
mode: None,
pi_thinking_level: None,
claude_effort: None,
grok_effort: None,
grok_disable_web_search: None,
provider_lifetime_fence_candidates: [60_001, 60_002, 60_003],
};
@@ -3214,7 +3246,7 @@ mod tests {
#[test]
fn pi_profile_matches_published_identity_and_rejects_prior_profiles() {
let published: Value = serde_json::from_str(include_str!(
"../../../../test-fixtures/pi-acp/profile-v29-identity.json"
"../../../../test-fixtures/pi-acp/profile-v30-identity.json"
))
.unwrap();
let mut value = descriptor("codex");
@@ -3231,6 +3263,7 @@ mod tests {
current.validate(&context()).unwrap();
for prior in [
include_str!("../../../../test-fixtures/pi-acp/profile-v29-identity.json"),
include_str!("../../../../test-fixtures/pi-acp/profile-v28-identity.json"),
include_str!("../../../../test-fixtures/pi-acp/profile-v24-identity.json"),
include_str!("../../../../test-fixtures/pi-acp/profile-v23-identity.json"),
@@ -3550,6 +3583,8 @@ mod tests {
mode: original_descriptor.mode.clone(),
pi_thinking_level: original_descriptor.pi_thinking_level,
claude_effort: original_descriptor.claude_effort.clone(),
grok_effort: original_descriptor.grok_effort.clone(),
grok_disable_web_search: original_descriptor.grok_disable_web_search.clone(),
provider_lifetime_fence_candidates: [60_001, 60_002, 60_003],
};
let operations = Vec::new();
@@ -3836,6 +3871,8 @@ mod tests {
mode: original_descriptor.mode.clone(),
pi_thinking_level: original_descriptor.pi_thinking_level,
claude_effort: original_descriptor.claude_effort.clone(),
grok_effort: original_descriptor.grok_effort.clone(),
grok_disable_web_search: original_descriptor.grok_disable_web_search.clone(),
provider_lifetime_fence_candidates: [60_001, 60_002, 60_003],
};
let operations = Vec::new();
@@ -4155,6 +4192,8 @@ mod tests {
mode: original_descriptor.mode.clone(),
pi_thinking_level: None,
claude_effort: None,
grok_effort: None,
grok_disable_web_search: None,
provider_lifetime_fence_candidates: [60_001, 60_002, 60_003],
};
let operations = Vec::new();
@@ -4404,6 +4443,8 @@ mod tests {
mode: descriptor.mode.clone(),
pi_thinking_level: descriptor.pi_thinking_level,
claude_effort: descriptor.claude_effort.clone(),
grok_effort: descriptor.grok_effort.clone(),
grok_disable_web_search: descriptor.grok_disable_web_search.clone(),
provider_lifetime_fence_candidates,
};
let operations = Vec::new();
@@ -4578,6 +4619,8 @@ mod tests {
mode: provider_descriptor.mode.clone(),
pi_thinking_level: provider_descriptor.pi_thinking_level,
claude_effort: provider_descriptor.claude_effort.clone(),
grok_effort: provider_descriptor.grok_effort.clone(),
grok_disable_web_search: provider_descriptor.grok_disable_web_search.clone(),
provider_lifetime_fence_candidates,
});
state.provider_exit_unconfirmed = true;
@@ -55,6 +55,10 @@ struct PersistedAcpxProviderSessionIdentity {
pi_thinking_level: Option<PiThinkingLevel>,
#[serde(default, skip_serializing_if = "Option::is_none")]
claude_effort: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
grok_effort: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
grok_disable_web_search: Option<bool>,
provider_lifetime_fence_candidates: [u16; 3],
}
@@ -80,6 +84,8 @@ impl PersistedAcpxProviderSessionIdentity {
mode: identity.mode,
pi_thinking_level: identity.pi_thinking_level,
claude_effort: identity.claude_effort.clone(),
grok_effort: identity.grok_effort.clone(),
grok_disable_web_search: identity.grok_disable_web_search.clone(),
provider_lifetime_fence_candidates: identity.provider_lifetime_fence_candidates,
})
}
@@ -99,6 +105,8 @@ impl PersistedAcpxProviderSessionIdentity {
mode: self.mode.clone(),
pi_thinking_level: self.pi_thinking_level,
claude_effort: self.claude_effort.clone(),
grok_effort: self.grok_effort.clone(),
grok_disable_web_search: self.grok_disable_web_search.clone(),
provider_lifetime_fence_candidates: self.provider_lifetime_fence_candidates,
}
}
@@ -122,6 +130,8 @@ impl AcpxSuspensionCheckpoint {
|| identity.mode != config.mode
|| identity.pi_thinking_level != config.pi_thinking_level
|| identity.claude_effort != config.claude_effort
|| identity.grok_effort != config.grok_effort
|| identity.grok_disable_web_search != config.grok_disable_web_search
|| config
.expected_identity
.as_ref()
@@ -74,6 +74,10 @@ pub struct AcpxProviderSessionIdentity {
pub pi_thinking_level: Option<PiThinkingLevel>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub claude_effort: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub grok_effort: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub grok_disable_web_search: Option<bool>,
pub provider_lifetime_fence_candidates: [u16; 3],
}
@@ -97,6 +101,8 @@ pub struct AcpxProviderSessionConfig {
pub mode: Option<String>,
pub pi_thinking_level: Option<PiThinkingLevel>,
pub claude_effort: Option<String>,
pub grok_effort: Option<String>,
pub grok_disable_web_search: Option<bool>,
pub permission_mode_pinned: bool,
pub provider_policy: Option<AcpxProviderRuntimePolicy>,
pub system_instructions: String,
@@ -120,6 +126,22 @@ impl AcpxProviderSessionConfig {
"Claude effort must be a supported Claude-only value",
));
}
if self.grok_disable_web_search.is_some() && self.agent != "grok" {
return Err(LocalRunnerError::invalid(
"Grok search policy requires Grok",
));
}
if self.grok_effort.as_ref().is_some_and(|effort| {
self.agent != "grok"
|| !matches!(
effort.as_str(),
"minimal" | "low" | "medium" | "high" | "xhigh"
)
}) {
return Err(LocalRunnerError::invalid(
"Grok effort must be a supported Grok-only value",
));
}
if (self.agent == "pi") != self.pi_thinking_level.is_some() {
return Err(LocalRunnerError::invalid(
"ACPX Pi thinking level must be explicit for Pi and absent for other agents",
@@ -200,6 +222,8 @@ impl AcpxProviderSessionConfig {
|| expected_identity.mode != self.mode
|| expected_identity.pi_thinking_level != self.pi_thinking_level
|| expected_identity.claude_effort != self.claude_effort
|| expected_identity.grok_effort != self.grok_effort
|| expected_identity.grok_disable_web_search != self.grok_disable_web_search
{
return Err(LocalRunnerError::invalid(
"ACPX expected identity conflicts with the requested session",
@@ -227,6 +251,16 @@ impl AcpxProviderSessionIdentity {
"ACPX identity contains invalid Claude effort",
));
}
if self.grok_effort.as_ref().is_some_and(|effort| {
!matches!(
effort.as_str(),
"minimal" | "low" | "medium" | "high" | "xhigh"
)
}) {
return Err(LocalRunnerError::invalid(
"ACPX identity contains invalid Grok effort",
));
}
for (value, label) in [
(&self.normalized_session_id, "normalized session"),
(&self.acpx_record_id, "record"),
@@ -1336,6 +1370,12 @@ fn session_open_params(config: &AcpxProviderSessionConfig, sidecar_tools: &[Valu
if let Some(effort) = &config.claude_effort {
params["claudeEffort"] = json!(effort);
}
if let Some(effort) = &config.grok_effort {
params["grokEffort"] = json!(effort);
}
if let Some(disabled) = config.grok_disable_web_search {
params["grokDisableWebSearch"] = json!(disabled);
}
if let Some(level) = config.pi_thinking_level {
params["piThinkingLevel"] = json!(level);
}
@@ -1454,6 +1494,8 @@ fn verify_open_response(
|| identity.mode != config.mode
|| identity.pi_thinking_level != config.pi_thinking_level
|| identity.claude_effort != config.claude_effort
|| identity.grok_effort != config.grok_effort
|| identity.grok_disable_web_search != config.grok_disable_web_search
|| config
.expected_identity
.as_ref()
@@ -1789,6 +1831,8 @@ mod mode_tests {
mode: Some("plan".to_owned()),
pi_thinking_level: None,
claude_effort: None,
grok_effort: None,
grok_disable_web_search: None,
permission_mode_pinned: true,
provider_policy: Some(AcpxProviderRuntimePolicy { read_only: false }),
system_instructions: String::new(),
@@ -1817,6 +1861,8 @@ mod mode_tests {
mode: Some("plan".to_owned()),
pi_thinking_level: None,
claude_effort: None,
grok_effort: None,
grok_disable_web_search: None,
provider_lifetime_fence_candidates: [60_001, 60_002, 60_003],
}
}
@@ -1976,6 +2022,39 @@ mod mode_tests {
cfg.claude_effort = None;
cfg.validate().unwrap();
}
#[test]
fn grok_search_policy_is_grok_only_and_preserves_false_in_the_sidecar() {
let mut cfg = config();
cfg.agent = "grok".to_owned();
for disabled in [true, false] {
cfg.grok_disable_web_search = Some(disabled);
cfg.validate().unwrap();
assert_eq!(
session_open_params(&cfg, &[])["grokDisableWebSearch"],
json!(disabled)
);
}
cfg.agent = "codex".to_owned();
assert!(cfg.validate().is_err());
}
#[test]
fn grok_effort_is_explicit_grok_only_and_forwarded_without_clamping() {
let mut cfg = config();
cfg.agent = "grok".to_owned();
for effort in ["minimal", "low", "medium", "high", "xhigh"] {
cfg.grok_effort = Some(effort.to_owned());
cfg.validate().unwrap();
assert_eq!(session_open_params(&cfg, &[])["grokEffort"], json!(effort));
}
cfg.grok_effort = Some("ultra".to_owned());
assert!(cfg.validate().is_err());
cfg.grok_effort = Some("high".to_owned());
cfg.agent = "codex".to_owned();
assert!(cfg.validate().is_err());
cfg.grok_effort = None;
cfg.validate().unwrap();
}
#[test]
fn pi_thinking_level_is_closed_explicit_and_pi_only() {
@@ -893,6 +893,23 @@ fn stderr_diagnostic_categories(value: &str) -> BTreeSet<&'static str> {
fn response_error_classification(error: &ResponseError) -> &'static str {
match error.code.as_str() {
"GROK_EFFORT_ACKNOWLEDGEMENT_MISSING" => return "grok_effort_acknowledgement_missing",
"GROK_EFFORT_MISMATCH" => return "grok_effort_mismatch",
"GROK_MODEL_MISMATCH" => return "grok_model_mismatch",
"GROK_SESSION_MISMATCH" => return "grok_session_mismatch",
"GROK_CONFIGURATION_CONFLICT" => return "grok_configuration_conflict",
"GROK_CONFIGURATION_DRIFT" => return "grok_configuration_drift",
"GROK_ADMISSION_PROTOCOL_VIOLATION" => return "grok_admission_protocol_violation",
"ACPX_ADMISSION_BINDING" => return "admission_binding",
"ACPX_ADMISSION_INSTALLATION" => return "admission_installation",
"ACPX_ADMISSION_SANDBOX" => return "admission_sandbox",
"ACPX_ADMISSION_CREDENTIAL" => return "admission_credential",
"ACPX_ADMISSION_CONFIGURATION" => return "admission_configuration",
"ACPX_ADMISSION_SKILLS" => return "admission_skills",
"ACPX_ADMISSION_COMMAND" => return "admission_command",
"ACPX_ADMISSION_TOOL_BRIDGE" => return "admission_tool_bridge",
"ACPX_ADMISSION_RUNTIME" => return "admission_runtime",
"ACPX_ADMISSION_IDENTITY" => return "admission_identity",
"ACPX_RUNTIME_PACKAGE_MISSING" => return "runtime_package_missing",
"ACP_MODEL_UNSUPPORTED" => return "requested_model_unsupported",
"AGENT_STARTUP_FAILED" => return "agent_startup_failed",
@@ -1120,6 +1137,20 @@ mod tests {
message: message.to_owned(),
retryable: false,
};
for (code, expected) in [
("ACPX_ADMISSION_SANDBOX", "admission_sandbox"),
("ACPX_ADMISSION_CREDENTIAL", "admission_credential"),
("ACPX_ADMISSION_RUNTIME", "admission_runtime"),
(
"GROK_EFFORT_ACKNOWLEDGEMENT_MISSING",
"grok_effort_acknowledgement_missing",
),
] {
assert_eq!(
response_error_classification(&error(code, "private-token-canary")),
expected
);
}
assert_eq!(
response_error_classification(&error(
"acpx_sidecar_command_failed",
@@ -262,7 +262,7 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
"id": id,
"ok": false,
"error": {
"code": "Q7Z9",
"code": request["params"]["errorCode"].as_str().unwrap_or("Q7Z9"),
"message": "violet-circuit-4821",
"retryable": false,
},
@@ -43,6 +43,7 @@ const OPENCODE_PROVIDER_ENVIRONMENT_KEYS: &[&str] = &[
"PAPERCLIP_NATIVE_MCP_TOKEN",
"PAPERCLIP_OPENCODE_PERMISSION_MODE",
"PAPERCLIP_OPENCODE_RUNTIME_DIR",
"PAPERCLIP_OPENCODE_PERFORMANCE_DIAGNOSTICS",
"PAPERCLIP_RUNNER_INSTANCE_ID",
"PAPERCLIP_RUN_ID",
"PAPERCLIP_NORMALIZED_SESSION_ID",
@@ -1014,6 +1014,7 @@ impl AuthenticatedTransport {
"protocolMin": PROTOCOL_MIN_VERSION,
"protocolMax": PROTOCOL_VERSION,
"warmTransitionVersion": 1,
"recoveryCursorVersion": 1,
"runnerInstanceId": state.runner_instance_id,
"environmentLeaseId": state.environment_lease_id,
"runId": state.run_id,
@@ -1200,6 +1201,10 @@ struct AuthChallenge {
warm_transition_version: Option<u64>,
#[serde(default)]
warm_transition_id: Option<String>,
#[serde(default)]
recovery_cursor_version: Option<u64>,
#[serde(default)]
recovery_acked_source_seq: Option<u64>,
}
fn validate_challenge(
@@ -1275,6 +1280,15 @@ fn validate_challenge(
"warm transition capability or receipt was not authenticated",
));
}
if (challenge.recovery_cursor_version.is_some()
|| challenge.recovery_acked_source_seq.is_some())
&& (challenge.recovery_cursor_version != Some(1)
|| challenge.recovery_acked_source_seq != Some(state.acked_source_seq))
{
return Err(DurableRunnerError::invalid(
"recovery cursor does not match the runner's durable acknowledgement",
));
}
match expected_lease {
Some(lease)
if challenge.credential_lease_id.as_deref() == Some(lease.lease_id.as_str())
@@ -1319,6 +1333,12 @@ fn challenge_signing_bytes(challenge: &AuthChallenge) -> Vec<u8> {
if let Some(id) = &challenge.warm_transition_id {
body["warmTransitionId"] = json!(id);
}
if let Some(version) = challenge.recovery_cursor_version {
body["recoveryCursorVersion"] = json!(version);
}
if let Some(cursor) = challenge.recovery_acked_source_seq {
body["recoveryAckedSourceSeq"] = json!(cursor);
}
canonical_json(&body).into_bytes()
}
@@ -1806,6 +1826,8 @@ mod tests {
server_proof: String::new(),
warm_transition_version: None,
warm_transition_id: None,
recovery_cursor_version: Some(1),
recovery_acked_source_seq: payload["resume"]["ackedSourceSeq"].as_u64(),
};
let signing = challenge_signing_bytes(&challenge);
challenge.server_proof = hex_encode(&hmac_domain(
@@ -1838,6 +1860,8 @@ mod tests {
"credentialLeaseId": &challenge.credential_lease_id,
"revocationEpoch": challenge.revocation_epoch,
"serverProof": &challenge.server_proof,
"recoveryCursorVersion": challenge.recovery_cursor_version,
"recoveryAckedSourceSeq": challenge.recovery_acked_source_seq,
},
}),
config.max_frame_bytes,
@@ -2505,6 +2529,8 @@ mod tests {
server_proof: String::new(),
warm_transition_version: None,
warm_transition_id: None,
recovery_cursor_version: None,
recovery_acked_source_seq: None,
};
let signing = challenge_signing_bytes(&challenge);
challenge.server_proof = hex_encode(&hmac_domain(
@@ -29,7 +29,7 @@ pub(crate) fn acpx_release_profile(agent: &str) -> Option<AcpxReleaseProfile> {
agent_runtime_package: Some("@earendil-works/pi-coding-agent"),
agent_runtime_version: Some("1.0.0"),
command_digest:
"sha256:e80c6dc7bde12790371345ee5a12f0201d30941d26f1a780aec3d090545b2a06",
"sha256:4517ab123dd89140ef462fc1905ac7a16a0dfcbaf21d67299a3c62de29c08a4d",
requires_provider_policy: true,
},
"cursor" => AcpxReleaseProfile {
@@ -47,7 +47,7 @@ pub(crate) fn acpx_release_profile(agent: &str) -> Option<AcpxReleaseProfile> {
agent_runtime_package: None,
agent_runtime_version: None,
command_digest:
"sha256:277894e850e07e6801cc24299c237513f58b41eb93c24f751d984fc4ddb04c47",
"sha256:618c3b69bc98b9a7e3c1b3411700eab8b90fd9245ba35d0bc85484319247fdbc",
requires_provider_policy: true,
},
"claude" => AcpxReleaseProfile {
@@ -671,6 +671,27 @@ fn measurement(value: &Value) -> Value {
/// Converts Codex app-server notifications into provider-neutral PRP events.
/// Provider-native envelopes are consumed here and never cross the PRP boundary.
fn provider_performance_spans(params: &Value) -> Vec<Value> {
let mut counts = std::collections::BTreeMap::<&str, usize>::new();
params.get("spans").and_then(Value::as_array).into_iter().flatten().take(128).filter_map(|span| {
let name = span.get("name")?.as_str()?;
if !matches!(name,
"provider.context.environment" | "provider.context.instructions" | "provider.skills.discovery"
| "provider.tools.resolve" | "provider.model.resolve" | "provider.model.stream"
| "provider.workspace.snapshot.track" | "provider.workspace.snapshot.patch" | "provider.workspace.snapshot.diff"
| "provider.workspace.git.discover" | "provider.workspace.git.default_branch"
| "provider.dependencies.install" | "provider.plugins.initialize") { return None; }
let start = span.get("startedAtMs")?.as_f64()?;
let end = span.get("endedAtMs")?.as_f64()?;
if !start.is_finite() || !end.is_finite() || start < 0.0 || end < start || end - start > 86_400_000.0 { return None; }
let count = counts.entry(name).or_default();
if *count >= 16 { return None; }
*count += 1;
Some(json!({"name":name,"startedAtMs":start,"endedAtMs":end,
"outcome":if span.get("outcome").and_then(Value::as_str)==Some("failed") {"failed"} else {"ok"}}))
}).collect()
}
pub fn normalize_codex_notification(method: &str, params: &Value) -> Vec<NormalizedProviderEvent> {
let mut events = Vec::new();
let push = |events: &mut Vec<NormalizedProviderEvent>,
@@ -685,6 +706,17 @@ pub fn normalize_codex_notification(method: &str, params: &Value) -> Vec<Normali
};
match method {
"paperclip/providerPerformance" => {
let spans = provider_performance_spans(params);
if !spans.is_empty() {
push(
&mut events,
"harness.diagnostic",
EventPriority::P1,
json!({"code":"provider_performance_spans","spans":spans}),
);
}
}
"thread/compacted" => push(
&mut events,
"context.compacted",
@@ -1464,6 +1496,37 @@ fn has_rfc_uri_scheme_prefix(value: &str) -> bool {
mod tests {
use super::*;
#[test]
fn performance_diagnostics_keep_only_bounded_content_free_measurements() {
let mut spans = vec![json!({"name":"private prompt","startedAtMs":1,"endedAtMs":2})];
for _ in 0..200 {
spans.push(
json!({"name":"provider.model.stream","startedAtMs":1000,"endedAtMs":1010,
"outcome":"failed","authorization":"private","attributes":{"prompt":"private"}}),
);
}
let events =
normalize_codex_notification("paperclip/providerPerformance", &json!({"spans":spans}));
assert_eq!(events.len(), 1);
assert_eq!(events[0].event_type, "harness.diagnostic");
assert_eq!(events[0].priority, EventPriority::P1);
let rows = events[0].payload["spans"].as_array().unwrap();
assert_eq!(rows.len(), 16);
assert_eq!(
rows[0],
json!({"name":"provider.model.stream","startedAtMs":1000.0,"endedAtMs":1010.0,"outcome":"failed"})
);
assert!(!events[0].payload.to_string().contains("private"));
assert!(normalize_codex_notification(
"paperclip/providerPerformance",
&json!({"spans":[
{"name":"provider.model.stream","startedAtMs":10,"endedAtMs":1},
{"name":"provider.model.stream","startedAtMs":0,"endedAtMs":86400001}
]})
)
.is_empty());
}
#[test]
fn unrelated_information_retains_only_bounded_run_log_diagnostics() {
let events = normalize_codex_notification(
@@ -53,6 +53,8 @@ fn config(directory: &std::path::Path) -> AcpxProviderSessionConfig {
mode: None,
pi_thinking_level: None,
claude_effort: None,
grok_effort: None,
grok_disable_web_search: None,
permission_mode_pinned: true,
provider_policy: None,
system_instructions: "Complete the supplied task.".to_owned(),
@@ -82,6 +84,8 @@ fn identity() -> AcpxProviderSessionIdentity {
mode: None,
pi_thinking_level: None,
claude_effort: None,
grok_effort: None,
grok_disable_web_search: None,
provider_lifetime_fence_candidates: [60_001, 60_002, 60_003],
}
}
@@ -52,6 +52,8 @@ fn config(mode: &str) -> AcpxProviderSessionConfig {
mode: None,
pi_thinking_level: None,
claude_effort: None,
grok_effort: None,
grok_disable_web_search: None,
permission_mode_pinned: true,
provider_policy: None,
system_instructions: "Complete the supplied task.".to_owned(),
@@ -48,6 +48,8 @@ fn config(mode: &str) -> AcpxProviderSessionConfig {
mode: None,
pi_thinking_level: None,
claude_effort: None,
grok_effort: None,
grok_disable_web_search: None,
permission_mode_pinned: true,
provider_policy: None,
system_instructions: "Complete the supplied task.".to_owned(),
@@ -72,6 +74,8 @@ fn expected_identity() -> AcpxProviderSessionIdentity {
mode: None,
pi_thinking_level: None,
claude_effort: None,
grok_effort: None,
grok_disable_web_search: None,
provider_lifetime_fence_candidates: [60_001, 60_002, 60_003],
}
}
@@ -28,6 +28,8 @@ fn config(mode: &str) -> AcpxProviderSessionConfig {
mode: None,
pi_thinking_level: None,
claude_effort: None,
grok_effort: None,
grok_disable_web_search: None,
permission_mode_pinned: true,
provider_policy: None,
system_instructions: "Complete the supplied task.".to_owned(),
@@ -51,6 +51,8 @@ fn config(mode: &str) -> AcpxProviderSessionConfig {
permission_mode: AcpxPermissionMode::ApproveReads,
mode: None,
claude_effort: None,
grok_effort: None,
grok_disable_web_search: None,
pi_thinking_level: if mode.starts_with("controls") {
Some(paperclip_runner_core::acpx_provider_session::PiThinkingLevel::Low)
} else {
@@ -219,6 +219,34 @@ fn keeps_valid_command_rejections_separate_from_protocol_failures() {
transport.shutdown().expect("fake sidecar should stop");
}
#[test]
fn preserves_closed_admission_categories_over_the_sidecar_pipe() {
let mut transport = transport("remote-error", Duration::from_secs(1));
for (code, category) in [
("ACPX_ADMISSION_INSTALLATION", "admission_installation"),
("ACPX_ADMISSION_CREDENTIAL", "admission_credential"),
(
"GROK_EFFORT_ACKNOWLEDGEMENT_MISSING",
"grok_effort_acknowledgement_missing",
),
("GROK_MODEL_MISMATCH", "grok_model_mismatch"),
("ACPX_ADMISSION_CREDENTIAL_SECRET", "unclassified"),
("GROK_MODEL_MISMATCH_SECRET", "unclassified"),
] {
let error = transport
.request(
GeneratedAcpxSidecarCommand::SessionOpen,
json!({"errorCode":code}),
)
.unwrap_err();
let message = error.to_string();
assert!(message.contains(&format!("classification={category}")));
assert!(!message.contains("violet-circuit-4821"));
assert!(!message.contains("SECRET"));
}
transport.shutdown().unwrap();
}
#[test]
fn redacts_sidecar_stderr_when_the_process_exits() {
let mut transport = transport("exit-secret", Duration::from_secs(1));
@@ -17,7 +17,7 @@ use sha2::{Digest, Sha256};
const CODEX_ACPX_DIGEST: &str =
"sha256:443491f4e814501a5dc47347e4510e84faeec9c9382e2d4e53bfe69386d5aabc";
const PI_ACPX_DIGEST: &str =
"sha256:e80c6dc7bde12790371345ee5a12f0201d30941d26f1a780aec3d090545b2a06";
"sha256:4517ab123dd89140ef462fc1905ac7a16a0dfcbaf21d67299a3c62de29c08a4d";
fn temporary_directory(label: &str) -> PathBuf {
let nonce = SystemTime::now()
@@ -1160,7 +1160,7 @@ fn acpx_transport_failure_retains_its_error_without_reopening_the_provider() {
assert_eq!(snapshot["status"], "closed");
assert_eq!(snapshot["activeProviderTurnId"], Value::Null);
assert!(executor.poll_events().unwrap().is_empty());
executor.shutdown().unwrap();
shutdown_recovered_acpx_fixture(&mut executor, &directory);
fs::remove_dir_all(directory).unwrap();
}
@@ -170,6 +170,9 @@ export async function summarizeControlledEvidence(bundles, options) {
function history(campaigns) {
const sum = field => usd(campaigns.reduce((total, campaign) => total + units(campaign[field]), 0n));
return { schema: 'paperclip.runner-eval.control-history/v1', costStatus: campaigns.some(c => c.costStatus === 'incomplete') ? 'incomplete' : 'provisional',
...(campaigns.length && campaigns.every(c => c.accountingReadiness === 'model_requests')
? { accountingReadiness: 'model_requests' } : {}),
...(campaigns.some(c => c.operatingReconciliation === 'pending') ? { operatingReconciliation: 'pending' } : {}),
knownUsd: sum('knownUsd'), heldUsd: sum('heldUsd'), committedUsd: sum('committedUsd'), operatingHeldUsd: sum('operatingHeldUsd'),
incompleteAttempts: campaigns.reduce((total, c) => total + c.incompleteAttempts, 0), overBudget: campaigns.some(c => c.overBudget), campaigns };
}
@@ -787,3 +787,42 @@ test('retained failure codes distinguish native startup, child exit and invalid
const artifact = await fixture(t, { missingRoute: true, requests: 0 });
assert.equal((await artifact.run()).report.failure, 'artifact_invalid');
});
test('model readiness rejects a missing terminal model receipt even when its reservation has a reviewed bound', async t => {
const f = await fixture(t, { requests: 1 });
f.manifest.overhead.accountingReadiness = 'model_requests';
for (const component of f.manifest.overhead.components) delete component.provisionalBound;
await f.save();
const result = await f.run(0, async () => Response.json({ error: 'synthetic refusal' }, { status: 429 }));
assert.equal(result.report.failure, 'request_accounting_unbounded');
const summary = await summarizeControlledCampaign(f.manifest);
assert.equal(summary.incompleteAttempts, 1);
assert.equal(summary.heldUsd, '0.014000000');
assert.equal(summary.operatingReconciliation, 'pending');
});
test('model readiness rejects zero model requests and keeps the full allocation', async t => {
const f = await fixture(t, { requests: 0 });
f.manifest.overhead.accountingReadiness = 'model_requests';
for (const component of f.manifest.overhead.components) delete component.provisionalBound;
await f.save();
const result = await f.run();
assert.equal(result.report.failure, 'request_accounting_unbounded');
assert.equal(result.report.accounting.requests.length, 0);
const summary = await summarizeControlledCampaign(f.manifest);
assert.equal(summary.incompleteAttempts, 1); assert.equal(summary.heldUsd, '0.014000000');
});
test('mixed-policy private histories keep the pending operating reconciliation warning', async t => {
const model = await fixture(t, { requests: 1 }), strict = await fixture(t, { requests: 1 });
model.manifest.campaignId = 'model-readiness-history';
model.manifest.overhead.accountingReadiness = 'model_requests';
for (const component of model.manifest.overhead.components) delete component.provisionalBound;
await model.save(); await model.run(); await strict.run();
const bundles = await Promise.all([model, strict].map(f => packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-0', environment: f.environment })));
const history = await summarizeControlledEvidence(bundles);
assert.equal(history.incompleteAttempts, 0); assert.equal(history.operatingHeldUsd, '0.008000000');
assert.equal(history.accountingReadiness, undefined); assert.equal(history.operatingReconciliation, 'pending');
});
@@ -23,7 +23,9 @@ export function prepareHostedManifest({ catalog, catalogSha256, prepared, policy
|| !Number.isSafeInteger(policy.maxCells) || policy.maxCells < 1 || policy.maxCells > 512
|| !Array.isArray(policy.allocations) || !policy.allocations.length) throw new Error('A bounded hosted policy with tracing is required');
if (!/^[a-f0-9]{64}$/.test(policy.controllerLockSha256 ?? '')) throw new Error('Invalid hosted controller lock binding');
costRuntime.reservationBound('operating_allowance', policy.capacityBound, hostedPolicyConfiguration(policy));
if (policy.overhead?.accountingReadiness !== 'model_requests' || policy.capacityBound !== undefined) {
costRuntime.reservationBound('operating_allowance', policy.capacityBound, hostedPolicyConfiguration(policy));
}
fields(policy.observability, ['projectId', 'capture', 'nativeLimits']);
const runId = environment.GITHUB_RUN_ID, paths = hostedPaths(runId);
if (prepared.schema !== 'paperclip.runner-eval.prepared-requests/v1' || prepared.githubRunId !== runId || prepared.githubRunAttempt !== 1
@@ -65,7 +67,7 @@ export function prepareHostedManifest({ catalog, catalogSha256, prepared, policy
models: policy.models, overhead: policy.overhead, observability: { ...policy.observability },
sourceRevision: prepared.sourceRevision, definitionsRevision: prepared.definitionsRevision, ...artifacts,
outputDirectory: paths.evidence, hosted: { repository: environment.GITHUB_REPOSITORY, runId, runAttempt: 1, workflowRevision: environment.GITHUB_SHA, controllerLockSha256: policy.controllerLockSha256, policySha256: sha256(canonical(policy)) }, attempts };
manifest.observability.provisionalBound = { ...policy.capacityBound, configurationSha256: sha256(canonical(observabilityBoundConfiguration(manifest))) };
if (policy.capacityBound !== undefined) manifest.observability.provisionalBound = { ...policy.capacityBound, configurationSha256: sha256(canonical(observabilityBoundConfiguration(manifest))) };
const validated = validateManifest(manifest, now);
assertHostedAdmission(manifest, attempts[0], { ...environment, PAPERCLIP_RUNNER_EVAL_EXECUTION_ID: attempts[0].executionId });
return validated;
@@ -119,3 +119,52 @@ test('hosted CLI reports safe admission reasons without echoing private paths or
assert.match(result.stderr, /A required hosted input file is missing/);
assert.doesNotMatch(result.stderr + result.stdout, /SENSITIVE_SENTINEL/);
});
test('model-readiness policy retains operating holds without requiring vendor cost proofs', async t => {
const c = await campaign(t);
c.input.policy.overhead.accountingReadiness = 'model_requests';
for (const component of c.input.policy.overhead.components) delete component.provisionalBound;
delete c.input.policy.capacityBound;
const result = c.build();
assert.equal(result.manifest.overhead.accountingReadiness, 'model_requests');
assert.equal(result.manifest.observability.provisionalBound, undefined);
Object.assign(c.f.manifest, result.manifest); await c.f.save();
const missing = await summarizeControlledManifest(result.manifest, []);
assert.equal(missing.incompleteAttempts, 2);
assert.equal(missing.operatingHeldUsd, '0.004000000');
Object.assign(c.f.environment, c.environment, { PAPERCLIP_RUNNER_EVAL_EXECUTION_ID: 'execution-0' });
assert.equal((await c.f.run(0, () => success())).code, 0);
const first = await packControlledAttempt({ manifest: result.manifest, attemptId: 'attempt-0', environment: c.f.environment });
Object.assign(c.f.environment, { PAPERCLIP_RUNNER_EVAL_EXECUTION_ID: 'execution-1' });
assert.equal((await c.f.run(1, () => success())).code, 0);
const second = await packControlledAttempt({ manifest: result.manifest, attemptId: 'attempt-1', environment: c.f.environment });
const complete = await summarizeControlledManifest(result.manifest, [first, second]);
assert.equal(complete.incompleteAttempts, 0);
assert.equal(complete.operatingHeldUsd, '0.004000000');
assert.equal(complete.accountingReadiness, 'model_requests');
assert.equal(complete.operatingReconciliation, 'pending');
delete c.input.policy.overhead.accountingReadiness;
assert.throws(() => c.build(), /cost bound|operating bound/);
});
test('model readiness still validates supplied operating and capacity proofs', async t => {
const c = await campaign(t), policy = c.input.policy;
policy.overhead.accountingReadiness = 'model_requests';
for (const component of policy.overhead.components) {
component.provisionalBound.configurationSha256 = sha256(canonical({ accountingReadiness: 'model_requests',
reference: policy.overhead.reference, runner: policy.overhead.runner, verifiedAt: policy.overhead.verifiedAt,
expiresAt: policy.overhead.expiresAt, component: { component: component.component, maxUsd: component.maxUsd, reference: component.reference } }));
}
policy.capacityBound.configurationSha256 = sha256(canonical(hostedPolicyConfiguration(policy)));
assert.equal(c.build().manifest.overhead.accountingReadiness, 'model_requests');
const mismatch = structuredClone(c.input); mismatch.policy.capacityBound.evidenceSha256 = '0'.repeat(64);
assert.throws(() => prepareHostedManifest(mismatch), /Observability capacity requires its operating bound evidence/);
const stale = structuredClone(c.input); stale.policy.capacityBound.configurationSha256 = '0'.repeat(64);
assert.throws(() => prepareHostedManifest(stale), /Reviewed cost bound configuration changed/);
const absent = structuredClone(c.input);
for (const component of absent.policy.overhead.components) delete component.provisionalBound;
absent.policy.capacityBound.configurationSha256 = sha256(canonical(hostedPolicyConfiguration(absent.policy)));
assert.throws(() => prepareHostedManifest(absent), /Observability capacity requires its operating bound evidence/);
});
Loaded 100 of 398 files, more files were not shown because too many files have changed in this diff. Show more