mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-04 19:28:46 +01:00
Merge pinned production for the post-custody chat pilot
Preserve frozen 244-scenario YAML intent and 191-capability coverage. Production baseline: 39e811c9c5.
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
commit
0195fcb016
398 files changed
+90564
-66016
No files matched your search
@@ -89,6 +89,7 @@ packages/grok-acp/bin
|
||||
|
||||
!packages/paperclip-runner/test-fixtures/pi-acp/profile-v28-identity.json
|
||||
!packages/paperclip-runner/test-fixtures/pi-acp/profile-v29-identity.json
|
||||
!packages/paperclip-runner/test-fixtures/pi-acp/profile-v30-identity.json
|
||||
|
||||
!packages/paperclip-runner/test/fixtures/copilot-profile-v23-identity.json
|
||||
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
import test from 'node:test';
|
||||
import { verifyControllerRuntime } from '../../packages/paperclip-runner/scripts/controlled-evals/verify-runtime.mjs';
|
||||
|
||||
test('the uploaded controller preserves every inventoried runtime file', {
|
||||
skip: !process.env.PAPERCLIP_CONTROLLER_TRANSPORT_DIRECTORY,
|
||||
}, async () => {
|
||||
await verifyControllerRuntime({
|
||||
directory: process.env.PAPERCLIP_CONTROLLER_TRANSPORT_DIRECTORY,
|
||||
sourceRevision: process.env.GITHUB_SHA,
|
||||
receiptSha256: process.env.PAPERCLIP_CONTROLLER_RECEIPT_SHA256,
|
||||
});
|
||||
});
|
||||
@@ -164,6 +164,16 @@ jobs:
|
||||
with:
|
||||
node-version: 24
|
||||
package-manager-cache: false
|
||||
- name: Apply reviewed qualification lock corrections
|
||||
if: inputs.release_run_id == '' && (inputs.build_eval_image || inputs.publish_eval_image)
|
||||
env:
|
||||
TARGET_SHA: ${{ needs.authorize_manual.outputs.target_sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "$TARGET_SHA" = "$GITHUB_SHA" ]; then
|
||||
node tests/runner-e2e/prepare-report-lock.mjs
|
||||
fi
|
||||
|
||||
- name: Resolve source dependencies without lifecycle scripts
|
||||
if: inputs.release_run_id == ''
|
||||
run: |
|
||||
|
||||
@@ -187,6 +187,18 @@ jobs:
|
||||
echo '::notice title=Paid runner routing::Using a standard GitHub-hosted runner'
|
||||
fi
|
||||
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
ref: ${{ github.sha }}
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: 24
|
||||
|
||||
- name: Preflight reviewed report dependencies before paid admission
|
||||
run: node tests/runner-e2e/prepare-report-lock.mjs
|
||||
|
||||
target_lock:
|
||||
name: Resolve target pnpm lockfile
|
||||
needs: authorize
|
||||
@@ -1273,6 +1285,9 @@ jobs:
|
||||
with:
|
||||
version: 9.15.4
|
||||
|
||||
- name: Apply reviewed reporting lock corrections
|
||||
run: node tests/runner-e2e/prepare-report-lock.mjs
|
||||
|
||||
- name: Install pinned reporting dependencies
|
||||
run: pnpm install --frozen-lockfile --ignore-scripts --store-dir "${{ runner.temp }}/eval-report-store"
|
||||
|
||||
@@ -1500,6 +1515,9 @@ jobs:
|
||||
|
||||
# Use a job-local store and disable lifecycle scripts throughout install.
|
||||
# OIDC is available to every step, even before the AWS credential exchange.
|
||||
- name: Apply reviewed publication lock corrections
|
||||
run: node tests/runner-e2e/prepare-report-lock.mjs
|
||||
|
||||
- name: Install pinned publication dependencies
|
||||
run: pnpm install --frozen-lockfile --ignore-scripts --store-dir "${{ runner.temp }}/eval-publisher-store"
|
||||
|
||||
|
||||
@@ -513,10 +513,25 @@ jobs:
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: runner-protocol-runtime-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
include-hidden-files: true
|
||||
path: ${{ runner.temp }}/runner-protocol-runtime/
|
||||
retention-days: 30
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Download controller artifact for a credential-free transport test
|
||||
if: needs.authorize.outputs.accounting_mode == 'controlled'
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
|
||||
with:
|
||||
artifact-ids: ${{ steps.upload.outputs.artifact-id }}
|
||||
path: ${{ runner.temp }}/runner-controller-roundtrip
|
||||
|
||||
- name: Verify the real uploaded controller inventory before private preparation
|
||||
if: needs.authorize.outputs.accounting_mode == 'controlled'
|
||||
env:
|
||||
PAPERCLIP_CONTROLLER_TRANSPORT_DIRECTORY: ${{ runner.temp }}/runner-controller-roundtrip
|
||||
PAPERCLIP_CONTROLLER_RECEIPT_SHA256: ${{ steps.build.outputs.runtime_receipt_sha256 }}
|
||||
run: node --test .github/tests/runner-controller-artifact-transport.test.mjs
|
||||
|
||||
prepare_controller:
|
||||
name: Prepare one shared cost and trace manifest
|
||||
needs: [authorize, catalog, build_runner, build_controller]
|
||||
|
||||
@@ -49,6 +49,18 @@ and leaves all existing cases passive. Source identities are not grants.
|
||||
|
||||
### Task query performance
|
||||
|
||||
Native session resume selects its newest progress barrier within the existing
|
||||
32-run task/session window in one SQL snapshot. Only the chosen checkpoint source
|
||||
returns its saved native input, checkpoint, tool fingerprint, wake envelope, and
|
||||
canonical task brief. Unused bootstrap candidates do not return full context or
|
||||
profiles. Wrong runtime, nonterminal status, process identity, provider events,
|
||||
invalid but non-null checkpoints, and future checkpoint barriers retain their
|
||||
existing selection rules. Stored run payloads and history reads stay complete.
|
||||
Run the synthetic PostgreSQL comparison with a pinned pre-change revision:
|
||||
`pnpm --filter @paperclipai/server exec tsx ../scripts/benchmark-native-session-resume-reads.ts --baseline-ref=<commit>`.
|
||||
An optional `PAPERCLIP_SESSION_RESUME_BENCHMARK_DATABASE_URL` must name a dedicated
|
||||
test database ending in `_test`; otherwise the harness starts disposable PostgreSQL.
|
||||
|
||||
Agent request authentication uses a compact run-identity read. It omits saved
|
||||
execution payloads, results, and log excerpts from the query response. The read
|
||||
still locks the task before the run and reconciles pending steering from durable
|
||||
|
||||
+81
-2
@@ -838,6 +838,41 @@ Set `SEARCH_EVAL_SCALE=1` to additionally measure a disposable 10,000-task,
|
||||
30,000-comment dataset. `SEARCH_EVAL_REPORT=/tmp/search-quality.json` saves
|
||||
per-query results and latency measurements; scale measurements are opt-in.
|
||||
|
||||
### Native skill preparation benchmark
|
||||
|
||||
Measure native skill preparation through the production service, real PostgreSQL,
|
||||
and an in-memory OpenTelemetry exporter:
|
||||
|
||||
```sh
|
||||
pnpm --filter @paperclipai/server exec tsx scripts/benchmark-native-skill-preparation.ts
|
||||
```
|
||||
|
||||
The default fixture has a 208-file repository of about 10.8 MB. The report includes
|
||||
cold and warm resolution, a fresh service instance, detailed phase timings, and
|
||||
regular-pool SQL statement counts (excluding the separate cache-lock connection).
|
||||
Every delivered file is checked outside the timed section. No provider is started.
|
||||
`--snapshot-file=/absolute/path/snapshot.json --entrypoint=skills/example/SKILL.md`
|
||||
uses a captured repository file inventory instead. `--network-delay-ms=25` adds
|
||||
25 ms in each direction through a local TCP relay; this is simulated transport
|
||||
latency, not a measurement of a hosted database.
|
||||
|
||||
`--actor=agent` adds a real agent, run, and responsible user, then records version
|
||||
pins through the same callback boundary as execution. `--staging-shape=true`
|
||||
seeds 14 projects, 10 sources, 8 bindings, and 156 other skills. These match one
|
||||
observed staging inventory's counts; their content is synthetic. Combine these
|
||||
options to include authorization and unselected library rows in the measurement.
|
||||
|
||||
By default, spans stay in memory. `--export-otlp=true` also exports them using
|
||||
the operator's `OTEL_EXPORTER_OTLP_ENDPOINT`, protocol, and headers. Keep header
|
||||
values out of command arguments and reports. The report records source hashes,
|
||||
the export mode, and trace IDs. Exported traces identify this local diagnostic
|
||||
in their service version; they are not evidence of staging performance.
|
||||
|
||||
If creating another embedded cluster is impractical, `--postgres-port=55448
|
||||
--postgres-user=paperclip` uses an existing **local** server. It creates and migrates
|
||||
a uniquely named disposable database and drops that database afterward; it does
|
||||
not reuse an application database.
|
||||
|
||||
### Repository workspace admission benchmark
|
||||
|
||||
Measure a fresh checkout with and without the repository object cache:
|
||||
@@ -1853,6 +1888,17 @@ TLS and protocol metadata. To use an isolated external database, set
|
||||
`PAPERCLIP_CONTINUATION_BENCHMARK_DATABASE_URL` to a disposable database ending
|
||||
in `_test`. The harness applies migrations there and adds synthetic fixtures.
|
||||
|
||||
Continuation history reads select completed tool receipts and the typed
|
||||
conversation/workspace recovery fields used to choose the prior terminal run.
|
||||
Every completed receipt keeps its full result. Receipt order, company/agent/task
|
||||
scope, and the complete retained history stay unchanged. Saved results remain
|
||||
available to detail, recovery, and execution paths.
|
||||
Use `pnpm --filter @paperclipai/server exec tsx ../scripts/benchmark-continuation-history.ts --baseline-ref=<pre-fix-commit>`
|
||||
to compare generated SQL, complete continuation envelopes, and DataRow bytes.
|
||||
The benchmark includes a large required-receipt control. Its external database
|
||||
option is `PAPERCLIP_CONTINUATION_HISTORY_BENCHMARK_DATABASE_URL`, with the same
|
||||
disposable `_test` database restriction.
|
||||
|
||||
Retry scheduling is in `server/src/services/heartbeat/retries.ts`. It owns bounded
|
||||
retry schedules, connection and workspace contention deferrals, shared-workspace
|
||||
holder checks, due retry promotion, and retry-now requests. `createHeartbeatRetries`
|
||||
@@ -1984,7 +2030,7 @@ agent conversion are outside this playbook.
|
||||
|
||||
## Paperclip Runner Adapter Conversion
|
||||
|
||||
Paperclip Runner offers native Codex, OpenCode, **ACPX Claude**, and **ACPX Pi**. New
|
||||
Paperclip Runner offers native Codex, OpenCode, **ACPX Claude**, **ACPX Grok Build**, and **ACPX Pi**. New
|
||||
Claude Code agents default to ACPX Claude on Apple Silicon, Intel Mac, Linux x64
|
||||
and Linux ARM64 with supported glibc. New OpenCode and Pi agents default to their native
|
||||
runtimes on Apple Silicon, Intel Mac, and Linux x64 with supported glibc;
|
||||
@@ -1995,11 +2041,15 @@ requires matching remote assets. The runner override is in Advanced. Ordinary Cl
|
||||
and provider routing remain authoritative. Existing agents keep their saved runner. Converting an existing Claude, Codex, OpenCode, or Pi agent selects its
|
||||
corresponding provider, preserves compatible models, credentials, workspace,
|
||||
and instructions, and resets execution sessions while retaining run history.
|
||||
Grok Build uses its qualified macOS ARM64 and Linux x64 targets. Its conversion
|
||||
preserves the selected model, account, and explicit reasoning effort through ACP.
|
||||
Custom commands, extra arguments, legacy permission modes, search flags, and
|
||||
nonzero turn limits require Legacy; conversion rejects them before saving.
|
||||
Other adapters require an explicit provider choice. Legacy ACPX Codex agent
|
||||
settings normalize to native Codex on configuration updates and before fresh
|
||||
runs; immutable run descriptors remain readable. The **Paperclip Runner**
|
||||
experimental setting applies to other native providers. Native Codex, OpenCode,
|
||||
ACPX Claude, and ACPX Pi do not require it. Company access checks still apply.
|
||||
ACPX Claude, ACPX Grok Build, and ACPX Pi do not require it. Company access checks still apply.
|
||||
|
||||
Native Codex preserves an explicit `adapterConfig.search: false` when converting
|
||||
or saving an agent. The disabled hosted-search policy is recorded in the native
|
||||
@@ -3332,6 +3382,35 @@ pnpm --filter @paperclipai/server exec tsx ../scripts/benchmark-workspace-prepar
|
||||
It compares actual preparation guards with synthetic history. Local timings are
|
||||
separate from Git materialization, environment acquisition, and provider startup.
|
||||
|
||||
### Sidebar badge run-query benchmark
|
||||
|
||||
The badge query keeps the newest 200 runs of nonterminated same-company agents.
|
||||
It selects each agent's first observed run before status or Mine filters. An
|
||||
observation ordinal preserves equal-time input order. Per-agent ranking orders
|
||||
by full creation time first, then that ordinal for exact ties. SQL returns only failure
|
||||
IDs, issue bindings, and creation times. Dismissals and fresh issue authorization
|
||||
still run in the service.
|
||||
|
||||
Run the synthetic PostgreSQL comparison with a pinned pre-change revision:
|
||||
|
||||
```sh
|
||||
PAPERCLIP_LOG_LEVEL=error SIDEBAR_REQUIRE_BYTE_REDUCTION=1 pnpm --filter @paperclipai/server exec tsx ../scripts/benchmark-sidebar-badges.ts --baseline-ref=bf3ef12f3a78d0b15a32ad57f7e16520fbf6cd5e
|
||||
```
|
||||
|
||||
The harness starts disposable embedded PostgreSQL by default. To use an existing
|
||||
isolated test database with current migrations, set
|
||||
`PAPERCLIP_SIDEBAR_TEST_DATABASE_URL` to a dedicated database whose name ends in
|
||||
`_test`. The harness removes only its own synthetic company fixtures. Do not use
|
||||
a development or hosted company database.
|
||||
|
||||
The report includes service-generated SQL, complete badge equality, candidate
|
||||
bindings, alternating warm samples, plans, and UTF-8 response-value bytes before
|
||||
Drizzle decoding. This byte count excludes protocol framing. The mixed fixture
|
||||
covers shared agents, superseding successes, timestamp ties, Mine users,
|
||||
dismissal equality, private and deleted issues, and more than 200 runs. The
|
||||
all-failed, all-succeeded, and empty controls expose neutral or slower paths.
|
||||
Timings describe local SQL work and do not establish hosted request latency.
|
||||
|
||||
### Budget accounting performance
|
||||
|
||||
Budget admission and settlement share the company accounting lock. They evaluate
|
||||
|
||||
+1
-1
@@ -49,7 +49,7 @@ start execution. This preference does not change the org chart or permissions.
|
||||
|
||||
### Agent Execution
|
||||
|
||||
Existing Claude and Codex agents can upgrade from the legacy runner on their
|
||||
Existing Claude, Codex, and Grok Build agents can upgrade from the legacy runner on their
|
||||
Harness / Runtime page when Paperclip Runner is the qualified default for their
|
||||
environment. One action saves pending edits and translates compatible settings
|
||||
while keeping the harness, selected model, account bindings, instructions, and
|
||||
|
||||
@@ -1356,7 +1356,7 @@ Invites tab does not hide this Cloud action.
|
||||
## 10.3 Agents
|
||||
|
||||
The Harness / Runtime configuration page offers **Upgrade runner** for existing
|
||||
legacy Claude and Codex agents when the target environment selects Paperclip
|
||||
legacy Claude, Codex, and Grok Build agents when the target environment selects Paperclip
|
||||
Runner by default. The action uses the normal authorized agent PATCH with
|
||||
`runner: "paperclip"`, includes pending configuration edits, and retains the
|
||||
model and compatible settings. A rejected conversion leaves saved configuration
|
||||
|
||||
@@ -2,14 +2,48 @@
|
||||
|
||||
Select **Grok Build** in the native runner provider selector. The stored contract is
|
||||
`adapterType: "paperclip_runner"` with `provider: "acpx"`, `acpxAgent: "grok"`,
|
||||
and `model: "grok-4.7"`. Existing `grok_local` agents keep their legacy adapter.
|
||||
and `model: "grok-4.7"`. Existing `grok_local` agents keep their legacy adapter
|
||||
until an explicit upgrade. On a qualified macOS ARM64 or Linux x64 computer,
|
||||
**Harness / Runtime → Upgrade runner** preserves the agent, model, account,
|
||||
instructions, and history. It starts fresh execution sessions. Advanced also
|
||||
offers an explicit Runner choice. Unsupported legacy settings are rejected
|
||||
before saving; select compatible settings or keep Legacy.
|
||||
New Grok runner agents default to **Full auto (approve all)**
|
||||
(`acpxPermissionMode: "approve-all"`) in setup and the configuration form.
|
||||
API configurations that omit the permission mode use the same default. No
|
||||
additional permission setting is needed for unattended execution. Explicitly
|
||||
saved restrictions remain unchanged.
|
||||
On Cloud, an operator must enable `enableNativeRunner` for the instance before
|
||||
the new-agent picker or direct setup page offers the native runner.
|
||||
Grok Build's qualified harness does not require the `enableNativeRunner` flag.
|
||||
|
||||
Explicit thinking effort uses `reasoningEffort` (`minimal`, `low`, `medium`,
|
||||
`high`, or `xhigh`). The immutable v10 execution input, provider descriptor,
|
||||
and recovery identity bind this value as `grokEffort`. The Grok provider boundary
|
||||
adds native `_meta.modelId` and `_meta.reasoningEffort` to session create, load,
|
||||
and resume requests before transport writes and observation. Grok 1.0.13 reports
|
||||
the selected effort in `_meta["x.ai/sessionConfig"].options`; it does not support
|
||||
the standard `session/set_config_option` effort API. The Runner checks that native
|
||||
acknowledgement and the effective model before a prompt. A lazy restored handle
|
||||
must connect through read-only native session inspection first. Reapplying the
|
||||
bound metadata on restore prevents Grok from resetting effort to its default.
|
||||
Missing configuration, a rejected value, or effort drift stops admission.
|
||||
Admission failures keep a closed Grok reason code through the sidecar boundary.
|
||||
Other startup failures retain their closed admission stage when no more specific
|
||||
known error code is available. These diagnostics contain no native payload,
|
||||
credential, arbitrary provider message, or workspace path. They do not change
|
||||
admission or retry policy.
|
||||
Omitting effort retains Grok's provider default. This setting remains outside
|
||||
the model-visible task prompt. Historical execution inputs retain their schemas.
|
||||
|
||||
The upgrade resolves an unset model or the legacy `grok-build` default to the
|
||||
native `grok-4.7` default. Task-level model and effort overrides use the Grok
|
||||
catalog and keep the account and tool policy unchanged.
|
||||
|
||||
Migration also preserves the effective `disableWebSearch` setting, whose legacy
|
||||
default is `true`. Native input v10 carries it as `grokDisableWebSearch` through
|
||||
startup and recovery. Grok's private `requirements.toml` enforces disabled search
|
||||
above compatible project settings. Explicit `false` retains enabled search;
|
||||
unset native configurations retain their provider default. See Grok's
|
||||
[configuration reference](https://github.com/xai-org/grok-build/blob/main/crates/codegen/xai-grok-pager/docs/user-guide/26-config-reference.md).
|
||||
|
||||
Grok Build speaks [ACP over stdio](https://github.com/xai-org/grok-build/blob/main/crates/codegen/xai-grok-pager/docs/user-guide/15-agent-mode.md).
|
||||
The runner owns `grok agent --no-leader stdio` through ACPX, including session
|
||||
|
||||
@@ -0,0 +1,231 @@
|
||||
# Approval queue seeding without retention polling
|
||||
|
||||
## Obligation and stack dependency
|
||||
|
||||
The attention maintenance scan does two different jobs: retention and automatic
|
||||
queue membership. Its call to `materializeSeededQueues` adds pending approvals to
|
||||
the PR queue when the first linked issue has a pull-request work product. Queue
|
||||
API readers use saved membership; they do not first build the attention feed.
|
||||
Removing approvals from maintenance therefore lost automatic queue population.
|
||||
A real PostgreSQL regression reproduced an empty queue with a pending linked
|
||||
approval and PR work product, without an attention reader.
|
||||
|
||||
This slice owns automatic PR-queue membership for approvals. It depends on the
|
||||
approval producer boundaries introduced in #17497 and is required with that
|
||||
retention migration before treating the combined behavior as complete. Join
|
||||
requests have no issue association or matching seed rule. Other attention-source
|
||||
seed discovery remains in the existing feed scan and must migrate with an
|
||||
explicit owner before those sources leave that scan.
|
||||
|
||||
## Source of truth and ownership
|
||||
|
||||
Existing approvals, issue approvals, issue work products, queues, and queue items
|
||||
are the only durable state. No new table, outbox, external API, or provider branch
|
||||
is introduced. `approvalQueueSeedingService` selects missing memberships for:
|
||||
|
||||
- A pending approval in an active company.
|
||||
- Its first linked issue, ordered by issue ID, in the same company and eligible
|
||||
under the attention feed's execution-issue visibility predicate.
|
||||
- A pull-request work product belonging to that issue and company.
|
||||
- A missing PR queue or an enabled seed rule without this approval's membership.
|
||||
|
||||
Disabled rules, nonmatching approvals, completed approvals, and existing seeded
|
||||
history are passive. A pending approval itself prevents its linked Slack/Speko
|
||||
conversation from projecting as idle, so this source does not depend on every
|
||||
other conversation-state producer. Hidden/harness/conversation-container issue
|
||||
changes retain their explicit signals.
|
||||
|
||||
The queue service owns seed matching, creation, membership uniqueness, and audit
|
||||
writes. The feed and worker use the same writer through a small normalized source
|
||||
contract. The candidate lookup and writer share the pull-request product SQL
|
||||
predicate, so the matching policy has one owner. A worker does not construct a presentation-layer AttentionItem. Queue
|
||||
reads keep their existing source authorization checks. A queue-rule update and
|
||||
seeding serialize on the existing queue row. Existing memberships are not pruned
|
||||
when an approval resolves. This preserves current behavior; pruning is a separate
|
||||
change, including the proposal in #13364.
|
||||
|
||||
## Producer audit
|
||||
|
||||
| Producer | Commit boundary and effect |
|
||||
| --- | --- |
|
||||
| Approval create, cancel, revision, resubmit, resolve | `writeApprovalChange` notifies retention and company-scoped queue seeding before writes in the owning transaction. Approve/reject/cancel retain their explicit tool-review notification; ordinary mutations do not wake those queues. |
|
||||
| Shared approval records, budget approvals, formal tool approvals | The same approval notification covers their existing transactions. |
|
||||
| Issue approval link, unlink, link-many, tool-created links | Existing approval transaction helper. Link removal can expose a different first issue. |
|
||||
| Issue visibility changes and issue deletion | Existing approval notification before the update or cascading delete. Routine failure cleanup retains this signal. |
|
||||
| Company activation, pause, archive, deletion and budget pause/resume | Existing company notification wakes both approval consumers. |
|
||||
| Work-product create, type/issue update, import batch, delete | The owning work-product transaction notifies seeding when a pull-request row is involved. |
|
||||
| PR queue creation, rule update and manual approval membership removal | The queue mutation transaction notifies seeding. An enabled rule can re-add a still-matching manually removed item, as the old scan did. |
|
||||
|
||||
Other direct work-product writes only create artifacts or update title, metadata,
|
||||
or trust. They cannot create a pull-request match. Supported writes use these
|
||||
in-process service boundaries. Independent SQL writers and multiple server
|
||||
replicas are not covered by local transaction signals.
|
||||
|
||||
Retention-only keep/archive/revive changes do not create queue projection debt.
|
||||
The old seed writer ran before retention filtering, so archived or kept approvals
|
||||
still follow the same seeding rule. User-specific dismissals do not filter the
|
||||
unscoped maintenance feed. No new deadline is inferred from those display states.
|
||||
|
||||
## Worker lifecycle
|
||||
|
||||
Startup reconstructs missing memberships across the instance. Each later committed
|
||||
producer carries its company ID and wakes only that company's reconciliation.
|
||||
Pending scopes coalesce in memory; one company's ordinary mutations cannot restart
|
||||
a global scan. Scope rotation prevents one failing company from starving another. Each sweep processes at most 100 approvals, with an ID cursor and a fixed upper ID per scan cycle. The cursor advances past
|
||||
failures and finishes that finite cycle before revisiting them; new arrivals cannot
|
||||
extend it indefinitely. Each row is rechecked under
|
||||
an approval lock before the queue writer runs. One bad row cannot permanently
|
||||
hide later candidates. Atomic existing membership uniqueness prevents duplicate
|
||||
rows and duplicate insertion audit events.
|
||||
|
||||
After a sweep, another lookup decides whether actual projection debt remains.
|
||||
Unprocessed pages and newer committed signals retain a one-second continuation.
|
||||
Failed scopes back off exponentially from one second to one minute. A successful
|
||||
nonempty batch resets consecutive failures; an empty cursor rollover while debt
|
||||
remains preserves outage backoff. New signals
|
||||
do not reset a failing scope's backoff; other companies can proceed independently.
|
||||
The next retry is registered with the shared scheduler, with no SQL during the
|
||||
wait. A failed scope remains tracked until its owed memberships are reconciled. The generation check follows the final awaited lookup, so a commit arriving
|
||||
during an empty read cannot be erased. Once reconciled, there is no deadline and
|
||||
no recurring worker SQL. The shared coordinator preserves requested work during
|
||||
drain. Passive pending checks inspect memory only; startup reconstructs from rows. Shutdown aborts between rows
|
||||
and waits for an admitted transaction; unprocessed rows remain recoverable.
|
||||
Registration follows the existing heartbeat-maintenance enablement switch.
|
||||
|
||||
This slice does not extend Cloud alarm coverage or loosen host sleep admission.
|
||||
It proves a worker's quiet/reactivation behavior, not deployed host or database
|
||||
suspension. The remaining attention maintenance interval is still present for
|
||||
other source owners.
|
||||
|
||||
## Verification
|
||||
|
||||
- Before the fix, the PostgreSQL regression found no PR queue after the normal
|
||||
maintenance path excluded approvals. After the fix, the queue API finds it
|
||||
without a full attention read.
|
||||
- Twenty-one real-database worker tests cover empty-to-active-to-quiet behavior,
|
||||
startup and completed history, actual approval/link/work-product producers,
|
||||
batch import, first-link selection, company isolation, hidden targets, queue
|
||||
rule enablement, company activation, resubmission, outer commit and rollback,
|
||||
first write during drain, database failure through drain, multiple pages,
|
||||
failed-page progress and retry, concurrent resolution, archived/kept history, shutdown/restart, and an admitted transaction at shutdown.
|
||||
- 115 existing tests pass across queue routes, attention, approval retention,
|
||||
approval service, work products, and startup wiring. The shared writer still
|
||||
passes existing plan/question/PR seed tests and authorization tests.
|
||||
- Workspace typechecks and the production build pass. Fresh PR checks are
|
||||
tracked in the PR and must pass before readiness is claimed.
|
||||
- The actual pending lookup on 40,000 completed approvals plus one pending PR
|
||||
approval uses the existing partial pending-retention index. The probe runs in
|
||||
a rolled-back transaction; execution took 0.094 ms on this local fixture. No new index or migration is needed for this case.
|
||||
|
||||
The first expanded test run caught two fixture errors: a hand-inserted queue
|
||||
omitted required creator attribution, and a failure test expected no idle hold
|
||||
while committed projection work was still unresolved. The corrected test waits
|
||||
for the active attempt to finish and checks passive SQL directly while preserving
|
||||
that required hold. Neither required weakening the worker's recovery contract.
|
||||
|
||||
Review found a queue/item lock inversion between seeded insertion and manual
|
||||
membership changes. A two-connection PostgreSQL regression reproduced `deadlock
|
||||
detected`. Manual add/remove now lock the queue before touching its items, matching
|
||||
the seed writer. The regression and the existing queue route suite pass (27 cases).
|
||||
Server typecheck and build pass after the repair. Fresh-head CI/review is required.
|
||||
|
||||
Security review found that normal producer wakes could restart a cross-company
|
||||
scan. The regression reproduced another company's queue being populated by an
|
||||
unrelated company write. Producers now include company hints, including batch
|
||||
imports and nested transactions. A separate controlled PostgreSQL test holds the
|
||||
final empty read while a real producer commits and verifies the retained wake.
|
||||
|
||||
After company scoping, all 169 selected cases pass across approval queue seeding,
|
||||
approval retention, queue routes, approval services, work products, and issue
|
||||
comment routes. Server typechecking passes. The preceding run suffered timeouts
|
||||
and a consequent cleanup foreign-key error; it is not counted as passing. The
|
||||
route suite separately exposed cold imports inside timed tests. Moving those
|
||||
imports to suite loading preserves assertions and prevents timed-out requests
|
||||
from leaking into later mock state. The watchdog fixture collision from CI uses
|
||||
monotonic identifiers instead; its 19 real-PostgreSQL cases pass.
|
||||
|
||||
The next CI run passed the security gate and the two originally failing shards.
|
||||
Its remaining server shard caught a historical cleanup test checking 33 provider
|
||||
inspections after only 32 had completed. The test advanced fake time before the
|
||||
previous asynchronous page had necessarily published its next deadline. It now
|
||||
waits for all 33 inspections, then retains the zero-further-work assertions. All
|
||||
14 environment cleanup cases pass on real PostgreSQL; production code is unchanged.
|
||||
|
||||
|
||||
Further review reproduced 120 failed transactions in two simulated minutes for a
|
||||
persistently failing projection. Per-scope backoff now bounds that test to at most
|
||||
eight attempts, even with new signals every second. Removing the fault recovers
|
||||
the membership. A real coordinator/producer case proves thirty seconds without
|
||||
SQL during backoff, another company's successful queue seeding during that wait,
|
||||
and eventual recovery without losing the original work. All 25 worker cases pass.
|
||||
The first consumer command omitted the external DB setting for queue routes and
|
||||
skipped those six cases; the corrected command executes and passes all six.
|
||||
The shared PR predicate is exercised by both those writer tests and worker tests.
|
||||
Server typecheck and build pass; fresh remote review/checks remain required.
|
||||
|
||||
The final startup, approval-retention, and attention consumer run passes all 93
|
||||
cases. With the 25 worker and six queue cases, 124 affected tests pass.
|
||||
|
||||
|
||||
## Current retention-parent integration
|
||||
|
||||
The seeding worker reuses `firstApprovalIssueId` from the approval-visibility
|
||||
owner rather than retaining its own first-link SQL. Company IDs are part of the
|
||||
approval notification contract because projection reconciliation is scoped.
|
||||
Generic approval writes notify retention and seeding only; the existing terminal
|
||||
approve/reject/cancel owners alone notify tool-review consumers. Current precise
|
||||
issue-deletion target notifications, unseen retention controls, finite retention
|
||||
cycles and the real source-row lock regressions are preserved.
|
||||
|
||||
All 522 cases in fifteen complete suites pass at `4e725e2055`, with server
|
||||
typecheck and production build. This includes seeding, queue routes, work products,
|
||||
issue comments, both retention workers, approvals, attention, retention controls,
|
||||
budgets, tool gateway, startup, scoped deadlines, environment cleanup and task
|
||||
watchdogs. Four new disposable fixtures were cloned from the already migrated
|
||||
parent fixture; this is not a fresh-empty-schema claim. The other suites used
|
||||
separate existing disposable databases. No database was shared with another
|
||||
concurrent local test run. The subsequent parent merge changes only two documents;
|
||||
all tested runtime and acceptance sources remain byte-identical.
|
||||
|
||||
The first integration typecheck caught two incorrect gateway company references.
|
||||
They now use the existing session company, and typecheck plus the complete gateway
|
||||
suite pass. Earlier deadlock, cross-company wake, backoff, skipped-fixture and
|
||||
teardown failures remain in the historical sections above. The parent's attention
|
||||
cleanup timeout and expiry-test CI assertion are recorded in its document; this
|
||||
larger current run passes without changing their assertions or timeouts. Fresh
|
||||
exact-head reviews and CI remain required before this dependent PR can merge.
|
||||
|
||||
|
||||
## Final master and review repairs
|
||||
|
||||
Join retention merged as `8406047b4e`. The normal master reconciliation first
|
||||
proved that landed tree equals the existing parent. Conflict resolution preserved
|
||||
the complete tested seeding tree; it also removed a duplicate helper introduced
|
||||
by automatic merge. The PR again differs from master in only its original 25
|
||||
seeding paths. Database schema and migration metadata are identical to master.
|
||||
|
||||
Three retry regressions failed before repair: ongoing arrivals skipped a recovered
|
||||
membership, a partial successful batch retained the old sixty-second failure
|
||||
penalty, and resetting that counter on an empty cursor rollover weakened outage
|
||||
backoff. The worker now fixes the upper ID per cycle and resets consecutive
|
||||
failures only on real successful progress or complete reconciliation. Exact
|
||||
one-second recovery and two-second consecutive-failure assertions pass; repeated
|
||||
signals still cannot reset a failing scope's delay. No new table or retry wrapper
|
||||
is introduced.
|
||||
|
||||
The external seeding, queue-route and watchdog fixtures now apply pending
|
||||
migrations before creating their client. Each of three empty-database regressions
|
||||
failed with missing tables before this setup repair. All 55 cases then passed on
|
||||
a second set of genuinely empty databases, with the suites applying their own
|
||||
migrations. The final additional rollover guard brings the three complete suites
|
||||
to 56 passing cases (28 seeding, 6 queue-route, 22 watchdog), using those now
|
||||
migrated fixtures. Server types/build pass at `7948136ef8`. The earlier 522-case
|
||||
integration run remains evidence for its stated revision and unchanged consumers;
|
||||
it is not relabeled as a current full-suite run.
|
||||
|
||||
The first expanded fresh run passed 54 cases and failed the new backoff test's
|
||||
cleanup expectation: after an entirely failed full page, a separate empty pass
|
||||
ends the cycle before the next pass revisits rows. The corrected fixture permits
|
||||
that explicit continuation and still checks complete recovery and the exact retry
|
||||
deadline. Earlier failures remain recorded. Fresh final-head CI and reviews are
|
||||
required before merge.
|
||||
@@ -0,0 +1,164 @@
|
||||
# Approval retention without idle polling
|
||||
|
||||
This slice removes approval retention from the recurring attention-feed sweep.
|
||||
It does not migrate the other attention source kinds or establish host sleep.
|
||||
|
||||
## Obligation and owner
|
||||
|
||||
A pending approval in an active company is eligible for automatic archiving
|
||||
90 days after its last approval activity, unless kept or already archived.
|
||||
This is a presentation/attention archive: it does not approve, reject, cancel,
|
||||
or delete the approval. The deadline and policy are unchanged.
|
||||
|
||||
Approval rows are the source of truth, including those never viewed in the
|
||||
attention UI. Scheduling only existing `decision_retention` rows would miss
|
||||
unseen approvals because the old feed reader materializes those rows lazily.
|
||||
`approval-retention.ts` discovers eligible approvals directly and registers
|
||||
the earliest actual deadline with the existing delivery coordinator.
|
||||
|
||||
The query retains the attention feed's first-linked-issue rule and consumes
|
||||
`executionIssueCondition` from the issue visibility owner. Hidden, harness and
|
||||
agent-conversation tasks remain excluded. A waiting external conversation with
|
||||
a linked pending approval is active under that same shared visibility policy;
|
||||
its chat state alone cannot hide this approval. Tests exercise the Slack case.
|
||||
|
||||
Each pass visits at most 100 candidates in activity/id order. The cursor keeps
|
||||
the database timestamp as text so sub-millisecond precision cannot repeat a
|
||||
failed page. A failed row does
|
||||
not prevent later pages from progressing. The failed durable record remains
|
||||
eligible for recovery. Each candidate is re-read and locked before archiving,
|
||||
so a resubmission committed while the worker waited cannot archive the new
|
||||
activity using an old selection. Existing retention version, keep and archive
|
||||
compare-and-set checks remain authoritative. This does not add cross-table
|
||||
serializability for concurrent link/visibility/company policy changes.
|
||||
|
||||
## Producer coverage
|
||||
|
||||
| Supported producer | Notification boundary |
|
||||
| --- | --- |
|
||||
| Approval create, cancel, revision, resubmit | SQL write transaction in approval records |
|
||||
| Approval resolution, including hire lifecycle | SQL write, retaining the caller's outer transaction |
|
||||
| Hard budget approval and budget resolution | Existing accounting transaction |
|
||||
| Formal tool approval and issue link | SQL write transaction; provider work is outside it |
|
||||
| Link, unlink and bulk link | Issue approval write transaction |
|
||||
| Issue visibility changes and deletion | Existing issue mutation transaction |
|
||||
| Routine creation rollback cleanup | Existing transaction before cascading issue deletion |
|
||||
| Company activation, pause, archive and deletion | Existing company/accounting transaction |
|
||||
| Keep, manual archive, revive and accepted archive manifest | Existing retention mutation transaction |
|
||||
| Retention materialization or source activity change | Signal only when inserting or changing actual state |
|
||||
|
||||
Approval comments do not change approval activity under the existing policy.
|
||||
Company portability imports do not write approvals; restored durable records
|
||||
are reconstructed at startup. Signals belong to the application DB owner and
|
||||
its dedicated child pools. Independent SQL writers or another active process
|
||||
need explicit integration; this change does not claim to observe them.
|
||||
|
||||
## Shared contracts and lifecycle
|
||||
|
||||
The main maintenance feed excludes the approval source before collecting it.
|
||||
Public attention reads keep their existing default behavior. Other attention
|
||||
sources remain in the aggregate sweep until separately migrated.
|
||||
|
||||
The retention service accepts the source fields it actually uses rather than
|
||||
requiring fabricated full UI attention items. Its existing versioned state,
|
||||
audit writes, authorization and archive-notification outbox are reused. No new
|
||||
queue table is introduced. Transaction-aware notifications retain intent until
|
||||
the outer transaction settles, including nested writes and rollback.
|
||||
|
||||
The deadline worker uses the shared scheduler's generation and drain behavior.
|
||||
Startup reconstructs pending work; idle drain retains requests without passive
|
||||
SQL; resume reconciles them. Failed work retains a 30-second recovery spacing.
|
||||
No eligible approval means no recurring deadline or query. Completed history,
|
||||
kept/archived approvals, and inactive companies are quiet until a supported
|
||||
mutation changes eligibility. Shutdown joins admitted work and stops between
|
||||
candidates. Notification delivery remains owned by the separate archive worker.
|
||||
|
||||
A pending-only `(updated_at, id)` index bounds deadline lookup independently of
|
||||
resolved approval history. The migration uses concurrent, idempotent creation
|
||||
and the existing migration runner's nontransactional execution list.
|
||||
|
||||
## Verification and limits
|
||||
|
||||
Real-Postgres lifecycle tests cover unseen sources, future deadlines, retained
|
||||
history, keep/unkeep, archive/revive, revision/resubmission, company activity,
|
||||
linked issue visibility/deletion/order, normal versus maintenance attention
|
||||
reads, nested commits/rollback, startup and first-write drain, failed database
|
||||
writes through drain, bounded progress past failure, shutdown, and a concurrent
|
||||
resubmission. The waiting-Slack case also checks durable notification creation.
|
||||
|
||||
Existing approval, attention, retention, budget, tool-gateway and startup suites
|
||||
are relevant consumers. Track final results and any failures in the PR and the
|
||||
[stack ledger](plans/2026-10-24-recurring-server-sleep-stack.md).
|
||||
|
||||
The Cloud sleep guards are unchanged. An in-process deadline cannot wake stopped
|
||||
compute, and this approval deadline has not been added to Cloud's durable wake
|
||||
contract. This proves only the named worker's quiet/reactivation behavior.
|
||||
|
||||
|
||||
## Initial review repair verification
|
||||
|
||||
The feed and retention worker now share `approval-visibility.ts` for the first
|
||||
linked issue and its execution visibility. The feed still applies actor-specific
|
||||
read access before including a linked approval. Both link orders are covered:
|
||||
hidden first/visible later stays excluded; visible first/hidden later stays
|
||||
eligible. The worker's current-state recheck remains under the approval row lock.
|
||||
|
||||
Manual keep, archive and revive materialize an unseen approval on demand inside
|
||||
the existing retention write transaction. They lock the company-scoped approval
|
||||
row in the same order as automatic archival, read its real activity timestamp,
|
||||
and use the existing versioned `syncItems` operation. Missing or unauthorized
|
||||
sources retain their existing rejection; other source kinds are unchanged. This
|
||||
adds no eager startup materialization or recurring scan.
|
||||
|
||||
Four real regressions failed before repair: unseen young keep/archive/revive
|
||||
returned 404, and the atomic materialization test could not reach the keep write.
|
||||
All pass after repair. A failing keep update rolls back the inserted retention
|
||||
row and emits no audit; the next successful keep persists exactly one audit.
|
||||
All 26 lifecycle cases pass, including the existing timestamp-precision failure
|
||||
rotation. The complete eight-suite consumer selection passes 311 cases at
|
||||
`063b61584d`: approval worker/service, attention, retention, budget, tool gateway,
|
||||
startup and scoped deadlines. Server typecheck and production build pass.
|
||||
|
||||
Normal parent reconciliation preserves every historical parent SQL file and all
|
||||
390 parent journal entries. The approval index is regenerated through the normal
|
||||
workflow as concurrent/idempotent `0392_small_frightful_four.sql`; five generated
|
||||
snapshots remain (0388–0392). Schema drift, safety/numbering and actual fresh
|
||||
migration apply/replay on five isolated PostgreSQL databases pass. Earlier
|
||||
transaction-stub, budget-import, query-probe and CI failures remain recorded in
|
||||
the stack ledger. Fresh current-head CI and both reviews are required.
|
||||
|
||||
|
||||
## Wake amplification repair
|
||||
|
||||
Ordinary creation, links, revision requests and resubmission signal retention
|
||||
only. Tool-review queues are notified by the actual terminal transition owners:
|
||||
approve/reject in approval resolution, and cancel in the approval service. These
|
||||
are the same terminal transitions that notified tool reviews before this PR.
|
||||
Cancellation must remain a producer because it settles parked tool actions as
|
||||
unavailable. No extra wrapper or scope argument is threaded through those calls.
|
||||
|
||||
The aggregate retention worker now coalesces sequential notifications for thirty
|
||||
seconds after a successful reconciliation. This is a single instance-wide read
|
||||
budget, stricter than granting each company an independent thirty-second budget,
|
||||
and matches the former default maintenance cadence. Known business deadlines
|
||||
still preempt the coalescing window. The window schedules no work on its own;
|
||||
without a notification or saved deadline the worker remains quiet. Newly changed
|
||||
eligibility can wait up to that window, as under the old default maintenance.
|
||||
|
||||
Three new real regressions failed before repair: approve and reject cases
|
||||
observed unwanted tool-review signals from ordinary mutations, and four
|
||||
sequential tenant writes caused eight global reads rather than one two-read
|
||||
reconciliation. Terminal approve/reject/cancel signals are now tested explicitly.
|
||||
With another company's pending approval present, sequential writes cause no extra
|
||||
SQL within the window and one reconciliation when the window ends.
|
||||
|
||||
The first coalescing fixture incorrectly expected the pending-work hold to clear
|
||||
before the requested reconciliation; it now lets the deferred wake remain owed
|
||||
and measures SQL directly. The first broad repair passed 313 cases and failed
|
||||
the existing parked-review cancellation case. Restoring the original cancel
|
||||
producer fixed that regression without weakening its outcome assertion. All
|
||||
315 cases in eight full suites now pass at `dd844f4e8c`, including 30 approval
|
||||
worker cases; server types/build pass. The earlier security check at `5991f257bb`
|
||||
was action-required and Greptile reported service error reference
|
||||
`8fb1835c-e564-476b-b802-d5be164ea915`. Neither is claimed as passing; the repaired
|
||||
head requires fresh security, CI and both bot reviews.
|
||||
@@ -0,0 +1,381 @@
|
||||
# Approved tool-action recovery
|
||||
|
||||
Approved tool actions recover at startup and after committed review changes.
|
||||
The aggregate maintenance tick no longer scans approved requests. The existing
|
||||
request and invocation rows remain authoritative; this change adds no queue table.
|
||||
|
||||
## Work and quiet states
|
||||
|
||||
The gateway owns the approved-work predicate. A signed agent action with
|
||||
`executionOnApprove: true` owes execution through the normal approval path.
|
||||
A legacy signed action without that flag remains deliberately inert. A Test-tab
|
||||
approval can owe execution even when no agent run exists. Recovery uses the saved
|
||||
origin and current authorization/target checks; the scheduler sees only whether
|
||||
another attempt is needed.
|
||||
|
||||
Startup discovers companies with saved approved requests. Committed review
|
||||
signals carry the affected company ID; subsequent discovery and recovery reads
|
||||
filter by that company. The shared scoped worker coalesces changes and dispatches
|
||||
one company per pass with a one-second minimum spacing. A company writing reviews
|
||||
does not scan or lock another company's approved backlog.
|
||||
|
||||
Recovery pages that company's approved requests in ID order using a partial index. It does not
|
||||
let a full batch of unavailable approvals starve later work. A failed read or
|
||||
unresolved executable approval retains a 30-second retry. New writes preserve
|
||||
that company's retry deadline, while other companies can run independently.
|
||||
A company pass still drains all its pages; this change does not cap provider
|
||||
calls within that pass or alter the provider's existing execution timeout. Once a pass leaves no
|
||||
such obligation, there is no recurring approved-recovery query. An empty system,
|
||||
configured idle tools, completed history, and deliberately inert approvals are
|
||||
quiet after reconciliation.
|
||||
|
||||
## Producer and transaction ownership
|
||||
|
||||
The existing tool-review notification boundary now signals both pending expiry
|
||||
and approved recovery. This covers human review transactions, gateway approval
|
||||
and accepted-interaction inference, administrative interaction resolution, review
|
||||
queue invalidation, and the gateway's approval claims and managed-argument
|
||||
invalidation. The notification is awaited before SQL in the existing transaction;
|
||||
caller-owned outer transactions retain commit/rollback ownership.
|
||||
|
||||
A pending-only change can cause one empty company-scoped discovery read. It does not
|
||||
create a periodic scan. No per-route recovery bookkeeping or database listener
|
||||
is added. Signals use the existing database owner's single-active-writer model.
|
||||
Independent SQL writers or another process need a separate notification contract.
|
||||
|
||||
## Single execution and outcome ownership
|
||||
|
||||
Agent actions retain their existing approved-to-executing claim. Test-tab
|
||||
foreground approval and recovery now share an equivalent claim before provider
|
||||
dispatch. The invocation then starts only if the request still holds that claim.
|
||||
Provider calls and target discovery remain outside SQL transactions. Current rate-policy admission and its receipt commit together in a short database transaction.
|
||||
|
||||
The Test-tab invocation outcome and terminal request status commit together.
|
||||
Claim timestamps fence late settlement. A later audit failure cannot overwrite
|
||||
an already committed outcome. Recovery repairs legacy approved Test-tab requests
|
||||
from a saved terminal invocation without decoding old arguments or calling the
|
||||
provider. A legacy executing invocation moves to the existing interrupted-work
|
||||
path; it is not dispatched again.
|
||||
|
||||
If the provider may have run but its result cannot be persisted, the executing
|
||||
claim remains authoritative. The existing stale-execution recovery quarantines
|
||||
unknown outcomes. This slice does not promise exactly-once external effects or
|
||||
replay an uncertain provider call.
|
||||
|
||||
## Lifecycle and remaining duties
|
||||
|
||||
The shared deadline worker coalesces notifications, preserves changes during a
|
||||
pass, and retains skipped work across idle drain/resume. Startup reconstructs
|
||||
saved approvals. Company deletion invalidates cached recovery and retries through
|
||||
the same transactional notification boundary. Shutdown stops its coordinator registration. With heartbeat
|
||||
scheduling disabled, startup keeps the existing one-off recovery behavior.
|
||||
|
||||
Pending expiry is owned by the preceding slice. Interrupted execution and
|
||||
interaction projection repair remain in the old sweep until their own migrations.
|
||||
The public combined sweep retains all duties for manual/recovery callers.
|
||||
No Cloud eligibility or external-wake proof changes here; worker quiet does not
|
||||
establish deployed database or host suspension.
|
||||
|
||||
## Verification
|
||||
|
||||
Real PostgreSQL service/startup tests cover producer commit and rollback,
|
||||
startup reconstruction, empty/configured/history quiet states, concrete retries,
|
||||
first-write drain/resume, legacy inert approvals, and multi-page recovery.
|
||||
Controlled Test-tab provider tests cover both foreground/recovery race winners,
|
||||
completed legacy outcomes, interrupted legacy calls, target changes, and a failed
|
||||
atomic outcome write followed by quarantine without another provider call.
|
||||
|
||||
The service/startup selection passes 123 tests. The existing gateway acceptance
|
||||
suite passes 92 tests. The strengthened paging and outcome-failure cases pass
|
||||
again after their final edits. Workspace typecheck and build pass. Full CI and
|
||||
review remain separate PR gates; no real external provider was invoked by these
|
||||
fixtures.
|
||||
|
||||
Review found that a one-shot success-write failure could fall into the provider
|
||||
error handler and save a false failure on its second attempt. The added sequence-
|
||||
based PostgreSQL fault reproduces that bug before the fix. Provider execution,
|
||||
pre-dispatch validation, and outcome persistence now have separate error
|
||||
boundaries. Both persistent and one-shot result-write failures retain the claim
|
||||
for quarantine without repeating the call. The complete 123-test selection and
|
||||
92 gateway acceptance cases pass after the repair, as do typecheck and build.
|
||||
|
||||
The existing tool-access service regression now expects a completed Test-tab
|
||||
request to be executed and verifies exactly one provider call before asking for
|
||||
a new approval. Its full 469-test suite passes. The initial local command skipped
|
||||
its database cases; the suite now honors its supplied disposable database through
|
||||
the standard test-database guard independently of embedded Postgres support.
|
||||
|
||||
Company-scoping regression: a real signed/reviewed action in company A previously
|
||||
caused an unfiltered approved-request SELECT over company B's inert legacy action.
|
||||
The captured-SQL test fails before this repair and passes with scoped reads. Added
|
||||
lifecycle cases check retry preservation across committed writes, independent
|
||||
company progress, and disarming after the real company-deletion producer. These
|
||||
checks supplement the earlier execution, transaction, and paging evidence above.
|
||||
|
||||
After company scoping, the complete gateway-service and startup selection passes
|
||||
149 tests against a disposable PostgreSQL database, and server typecheck passes.
|
||||
Stack integration, workspace checks, and fresh remote CI remain separate gates.
|
||||
|
||||
## Preparation failures retain an approved Test-tab action
|
||||
|
||||
A real PostgreSQL policy-read timeout reproduced a permanent-failure bug after
|
||||
claiming a Test-tab action: the provider had never been called, but both request
|
||||
and invocation were saved as failed, so approved recovery could not retry them.
|
||||
|
||||
The preparation boundary now distinguishes explicit validation/access rejections
|
||||
from unexpected dependency errors and transient HTTP errors. The existing
|
||||
`tool-approval-execution` owner releases only its exact claim: invocation and request
|
||||
are locked in the established order, their link/company/claim timestamp must
|
||||
match, and the invocation must still await approval with no recorded start.
|
||||
The request returns to `approved` with a transaction-aware notification. Signed
|
||||
arguments and the human approval remain unchanged; the error propagates so the
|
||||
existing recovery worker retains its retry obligation.
|
||||
|
||||
This boundary ends before `runTestToolInvocation`. Provider dispatch and outcome
|
||||
persistence keep their existing conservative no-replay behavior. A changed claim,
|
||||
started invocation or prior-dispatch evidence cannot be released by this path.
|
||||
If the requeue transaction itself fails, the error propagates and no terminal
|
||||
failure is fabricated; existing interrupted-execution reconciliation remains the
|
||||
fallback. This change does not infer replay safety from a failed requeue.
|
||||
|
||||
Before the fix, both foreground and recovery timeout regressions failed. They
|
||||
now observe the committed claim under a real PostgreSQL table lock, wait for
|
||||
`55P03`, verify unchanged approval and no provider call, then prove automatic
|
||||
recovery invokes the provider once and becomes quiet. Additional real-row cases
|
||||
cover changed ownership, mismatched invocation, a started call, prior-dispatch
|
||||
history, and outer-transaction rollback. A released claim cannot start execution.
|
||||
All 117 gateway-service cases pass on the owning slice, including existing target
|
||||
invalidation, provider-outcome persistence failures and unknown-outcome no-replay
|
||||
cases. Integrated consumer checks, workspace verification and remote PR gates
|
||||
must still be recorded after propagation.
|
||||
|
||||
|
||||
## Agent-origin preparation and dispatch ownership
|
||||
|
||||
Agent-origin approvals use the same pre-dispatch retry contract as Test-tab
|
||||
approvals. Previously, a policy or identity dependency failure after claiming an
|
||||
approved action permanently failed it; earlier issue/signing failures could
|
||||
leave the claim executing. Preparation now has one owning error boundary around
|
||||
issue, signature, identity, target, policy, managed-argument, and content checks.
|
||||
Validation and nontransient access rejections retain their terminal handling.
|
||||
Unexpected dependency failures and transient HTTP errors release only the exact,
|
||||
still-undispatched claim to the existing approved queue and publish its existing
|
||||
transactional work signal. No table, maintenance scan, or retry wrapper per
|
||||
producer is added.
|
||||
|
||||
`tool-approval-execution.ts` owns the shared claim token, release, and dispatch
|
||||
transition. Its Test-tab claim/outcome functions remain specific to that consumer.
|
||||
Both origins use the same invocation-then-request locks and verify invocation,
|
||||
company, request linkage, claim timestamp, awaiting-approval state, and absence
|
||||
of an earlier start before dispatch. An agent that loses its claim during
|
||||
preparation cannot call the provider or overwrite its successor. The Test-tab
|
||||
unapproved direct-call path retains its original behavior.
|
||||
|
||||
Only preparation is retryable here. Recording the invocation start, provider
|
||||
execution, and outcome persistence remain outside that catch. A provider error,
|
||||
unknown outcome, or failed durable release does not establish replay safety.
|
||||
Existing interruption handling remains the conservative fallback. Closed-issue
|
||||
expiry, signature/target invalidation, identity authority, and policy rejection
|
||||
remain enforced, including the final issue check before recording dispatch.
|
||||
|
||||
Real PostgreSQL lock-timeout cases cover foreground approval and recovery:
|
||||
observe the committed executing claim, block the policy read, then require the
|
||||
same signed request to remain approved with no invocation start and zero provider
|
||||
calls. Recovery after releasing the table lock executes once and becomes quiet.
|
||||
A separate contention case replaces the claim while preparation is blocked and
|
||||
proves the old owner cannot dispatch; the surviving approval later executes once.
|
||||
A provider-error case proves that an already-started call is not requeued as a
|
||||
preparation failure. The existing Test-tab timeout and ownership cases exercise
|
||||
the other consumer of the shared boundary.
|
||||
|
||||
|
||||
## New or edited rate rules during approval
|
||||
|
||||
A reviewed invocation already has durable `consumedRateAdmissions` in its policy
|
||||
explanation. The original read-only recheck retained unchanged rules correctly,
|
||||
but new or edited rules only inspected their counters. Two approved actions could
|
||||
therefore both execute against one available slot. The real recovery regression
|
||||
failed for both a newly added limit and an increased limit before this repair.
|
||||
|
||||
The policy service now owns `admitInvocation`: lock the company-scoped invocation,
|
||||
validate its exact actor/task/tool/argument binding, recheck current policy, spend
|
||||
only admissions not already recorded for that policy generation, and save the
|
||||
receipt atomically. The gateway supplies its execution claim; that claim is locked
|
||||
and checked by ID, invocation, company, executing status and timestamp. This also
|
||||
supports historical approved requests whose invocation still says pending, without
|
||||
accepting a stale claimant. Existing claim/start/settlement owners still control
|
||||
provider execution. No queue table, periodic query, or per-route wrapper is added.
|
||||
|
||||
Counter updates preserve the number of already spent slots when the limit changes.
|
||||
A rejection by a later policy or failure to save the receipt rolls back new charges.
|
||||
A retry of a committed admission retains the saved receipt, including concurrent
|
||||
rechecks. Read-only approval preflight and browser authority probes stay read-only;
|
||||
trust-rule usage is not counted again during execution revalidation. The policy
|
||||
service and its metric helper accept only the database capabilities they use, so
|
||||
transaction clients do not require a cast to a root connection.
|
||||
|
||||
The owning gateway and policy suites pass all 181 cases against disposable real
|
||||
PostgreSQL databases. Coverage includes two competing invocations, concurrent
|
||||
retries of one invocation, changed limits, failed receipt writes, rejection after
|
||||
an earlier rate charge, unchanged admissions, historical approvals, scope mismatch,
|
||||
claim loss, quiet recovery and unknown provider outcomes. Server typechecking
|
||||
passes after preparing current runner dependencies.
|
||||
|
||||
Earlier verification failures remain part of this evidence: the first full run
|
||||
was stopped because an old claim-loss fixture held the policy table while trying
|
||||
to revoke the newly locked invocation. It now pauses discovery before admission
|
||||
and retains the no-provider-dispatch assertion. A new receipt assertion was fixed
|
||||
to permit the intended additional receipt while preserving original fields. The
|
||||
next full run exposed an overly strict historical-approval check; exact execution-
|
||||
claim validation fixed it. Typechecking then exposed root-client-only signatures,
|
||||
which were narrowed to their actual database capabilities. The final complete
|
||||
suites and TypeScript check pass; stack integration and current-head CI remain
|
||||
separate gates. This proves worker and admission behavior, not deployed host sleep.
|
||||
|
||||
|
||||
## Managed-argument expiry ordering
|
||||
|
||||
Latest-head CI exposed a preparation-order regression in
|
||||
`tool-gateway.test.ts`: a valid historical Shopify approval whose managed UCP
|
||||
arguments were obsolete became `failed` instead of `expired`. The new consuming
|
||||
admission compared the current invocation identity before the existing managed
|
||||
argument expiry owner ran. It prevented dispatch, but bypassed the correct saved
|
||||
expiry outcome.
|
||||
|
||||
Approved execution now verifies target/signed/managed arguments and content
|
||||
before consuming admission. Exact claim fencing, fresh policy checks, atomic
|
||||
rate receipts, and the final issue-open check remain before dispatch. No check
|
||||
was removed and stale payloads are not permitted to execute.
|
||||
|
||||
The existing regression reproduced the CI failure on this owner in
|
||||
`/tmp/sleep-approved-managed-expiry-red.log`. All 92 gateway acceptance cases
|
||||
pass after the ordering change (`/tmp/sleep-approved-managed-expiry-full.log`),
|
||||
including the historical approval's expired request, failed invocation with
|
||||
`approvalState: expired`, and zero provider requests. Server typecheck passes
|
||||
in `/tmp/sleep-approved-managed-expiry-types.log`. Latest-head remote CI and
|
||||
combined-stack acceptance still need their own results.
|
||||
|
||||
## Delegated Test-tab admission after policy changes
|
||||
|
||||
A delegated Test-tab call has an originating agent but no gateway token. The
|
||||
named-gateway admission path therefore did not evaluate its caller policy. If a
|
||||
rate rule was added while approval was pending, approval charged only the target.
|
||||
A shared company/tool bucket then rejected the caller against that same charge;
|
||||
a caller-specific rule instead allowed execution without recording its charge.
|
||||
|
||||
The Test-tab owner now supplies its caller/target policy evaluation to the existing
|
||||
claim-transaction admission API. Both decisions use the claimed invocation's
|
||||
receipts. The target can share the caller's charge only when the rule and bucket
|
||||
match. Both receipts commit together. A denial rolls back new charges and receipts.
|
||||
Management reauthorization and credential identity checks remain with their owners,
|
||||
and provider execution stays outside the transaction. This adds no queue or timer.
|
||||
|
||||
Three actual HTTP/PostgreSQL cases cover a newly added shared rate rule, separate
|
||||
caller and target rules, and a target denial after caller admission. The unchanged
|
||||
implementation failed the first two cases: the shared rule rejected execution,
|
||||
and separate rules left only one charge. The denial case already passed. After
|
||||
the repair, all three pass, including duplicate approval, both saved receipts,
|
||||
one provider execution, and rejection of a subsequent authorized call.
|
||||
|
||||
The full connection-management suite passes all 78 cases against a fresh database.
|
||||
All 92 existing gateway tests and the server production typecheck also pass.
|
||||
Intermediate fixture failures remain accounted for: the new assertion initially
|
||||
used a nonexistent counter count field instead of remaining; the subsequent-call
|
||||
fixture first reused an ended run and then lacked responsible-user authority.
|
||||
The final fixture uses a new live run with the original responsible user. A reused
|
||||
full-suite database also retained a conflicting Slack bot identity; final acceptance
|
||||
uses a fresh isolated database. These are local checks, not current-head CI or a
|
||||
host-suspension qualification.
|
||||
|
||||
|
||||
## Historical argument identity compatibility repair
|
||||
|
||||
Earlier invocations stored the hash of their redacted argument preview. Modern
|
||||
admission correctly requires the complete request identity, including a keyed hash
|
||||
for sensitive arguments. The gateway already verified signed historical arguments,
|
||||
but admission rejected their saved preview hash even with no rate policy. That
|
||||
permanently failed an otherwise valid approval before the provider call.
|
||||
|
||||
Both agent-origin and Test-tab preparation now bind the verified complete identity
|
||||
for a historical invocation before current policy admission. Signature, reviewed
|
||||
target and managed-argument checks still run first. The approval execution owner
|
||||
performs the identity transition in an invocation-then-request lock transaction.
|
||||
It requires the exact current claim, company and invocation relationship, an
|
||||
undispatched awaiting-approval invocation, and the expected reviewed/preview hash.
|
||||
Unrelated saved hashes are not accepted. Modern matching identities do not add
|
||||
this transaction. Current policies and rate charges still apply afterward.
|
||||
|
||||
Four real-PostgreSQL regressions reproduced the defect: valid sensitive historical
|
||||
arguments, with and without a newly configured rate rule, through each of the two
|
||||
origins. Four controls keep invalid signatures and unrelated invocation hashes
|
||||
rejected without dispatch or identity changes. The provider receives the original
|
||||
verified arguments exactly once, rather than their redacted preview. A separate
|
||||
real-transaction case proves a replaced claim cannot change the saved identity.
|
||||
|
||||
The first draft fixtures used a nonexistent request-level tool name and then
|
||||
non-sensitive arguments whose preview and complete hashes were equal. Those
|
||||
fixture failures are not credited as product regressions. The corrected four
|
||||
positive cases fail before the repair and all eight parameter cases pass after.
|
||||
The complete gateway, policy and startup selection passes 230 cases with no skips;
|
||||
server types pass. The complete assembled-server connection-management suite also
|
||||
passes all 78 cases, including delegated approval and rate-policy behavior. Fresh
|
||||
remote CI and reviews remain required after publication.
|
||||
|
||||
|
||||
The repair was then integrated with the master-based parent, including #17918's
|
||||
tracing changes, at `81231824d773a79fc50c835eccf7d9b3977e79fc`. Server types and the
|
||||
production server build pass. The gateway, policy, startup and preparation-timing
|
||||
suites pass all 242 cases. The complete HTTP management suite passes all 78 cases
|
||||
on a fresh dedicated database, for 320 passing cases across the accepted runs.
|
||||
|
||||
The first combined integrated run passed 319 cases and failed the Slack setup
|
||||
case because its reused HTTP database still held the previous run's bot identity.
|
||||
The existing ownership guard correctly rejected the duplicate connection. A fresh
|
||||
dedicated fixture resolved that environment collision; no product behavior or
|
||||
assertion changed for the rerun. That failed run remains part of the evidence.
|
||||
|
||||
|
||||
## Gateway admission uses its transaction connection
|
||||
|
||||
Review of the master-targeted branch exposed a delegated gateway deadlock: the
|
||||
admission transaction held the sole pool connection while issuer/token authority
|
||||
reads used the root client. Foreground approval and approved recovery each failed
|
||||
the real one-connection regression; a revoked-issuer control already passed.
|
||||
The admission callback now receives its transaction and passes it through the
|
||||
existing issuer authority checks. Both connection authorization checks and the
|
||||
token read use that connection. Ordinary checks retain their root client, and
|
||||
current authority, claim, company and rate enforcement remain in place.
|
||||
The three constrained-pool cases pass, as do all 81 HTTP integration cases and
|
||||
190 gateway, policy, startup and deadline consumer cases. The initial consumer
|
||||
command selected an incorrect policy filename and omitted the gateway database
|
||||
override; it was interrupted and is not counted as passing evidence.
|
||||
|
||||
## Pending historical approvals retain their original admission
|
||||
|
||||
A second review finding concerned the normal foreground approval gate, before
|
||||
execution has claimed and rebound historical arguments. A real PostgreSQL
|
||||
regression with signed sensitive arguments and an already-consumed limit of one
|
||||
failed with `rate_limited`; the Test-tab and invalid-signature controls passed.
|
||||
The gateway now supplies the preview digest only after verifying the signed
|
||||
payload and matching either supported form of the review's canonical digest. Policy may match that digest
|
||||
only in a read-only preflight, with every other invocation-context field and
|
||||
stored rate-rule receipt still checked. The consuming admission continues to
|
||||
require the complete keyed identity, rebound under the exact execution claim.
|
||||
New or changed rate rules still apply. Historical recovery, invalid signatures,
|
||||
foreign invocation hashes, stale claims, and provider argument preservation stay
|
||||
covered. This adds no schema, recurring query or additional connection.
|
||||
|
||||
The preceding CI head also failed E2E shard 3/8. Its failure is retained in the
|
||||
validation record; local gateway checks do not erase it. Fresh-head CI and both
|
||||
required automated reviews remain merge gates.
|
||||
|
||||
The five full gateway/policy/startup/deadline suites passed 329 cases before the
|
||||
additional mixed historical identity control. The first combined attempt is
|
||||
retained: a Test-tab failure-result assertion incorrectly expected a thrown
|
||||
error, and reusing the earlier HTTP database left formal-approval foreign keys
|
||||
that prevented gateway acceptance setup. The assertion now checks the existing
|
||||
failed-result contract, with unchanged no-provider, hash and counter checks;
|
||||
the complete acceptance rerun used an independently migrated gateway database.
|
||||
Server production typechecking and build passed. The earlier E2E failure was a
|
||||
500 during the agent-chat project-plan fixture restoration, with a PostgreSQL
|
||||
deadlock in cancellation; it remains a fresh-head CI gate.
|
||||
@@ -0,0 +1,127 @@
|
||||
# Decision-archive notification delivery
|
||||
|
||||
Auto-archive already writes `decision_archive_notification_outbox` in the same
|
||||
transaction that archives its retention row. The old retention sweep called
|
||||
notification delivery on every maintenance tick, including an empty outbox.
|
||||
Delivery is now its own coordinator worker. Retention scanning remains a separate
|
||||
migration; this PR does not change what is archived or when.
|
||||
|
||||
## Ownership and producer audit
|
||||
|
||||
`decision-retention.ts` owns eligibility, current-version checks, source authority,
|
||||
origin resolution, and archive writes. Its sole outbox producer is `autoArchive`.
|
||||
It now awaits the delivery notification immediately before inserting the outbox
|
||||
row in the existing transaction. Caller-owned transactions and savepoints retain
|
||||
outer commit/rollback ownership through the shared database signal mechanism.
|
||||
Manual archive/revive and reviewed-manifest archive do not create these
|
||||
notifications today; this change does not add notifications to those actions.
|
||||
|
||||
`decision-archive-delivery.ts` owns claim recovery, selection, grouping, delivery,
|
||||
and settlement. `decisionRetentionService.deliverNotifications` remains a
|
||||
compatible delegate. Its notification callback and batch type retain their
|
||||
existing contract. The scheduler only sees the delivery operation and next
|
||||
work deadline. Claim and acknowledgement writes belong to this worker; they do
|
||||
not broadcast self-generated wakes. No table is added. A concurrent partial index supports the pending delivery
|
||||
order on the existing outbox.
|
||||
|
||||
The server registers the worker in enabled-heartbeat startup housekeeping and
|
||||
removes notification delivery from `runRetentionSweep`. Registration scans saved
|
||||
work before the retention pass can enqueue more. The disabled-heartbeat branch
|
||||
continues not to dispatch agent notifications. A service without a callback
|
||||
leaves outbox rows intact and does not attempt notification delivery.
|
||||
|
||||
## Work and deadlines
|
||||
|
||||
Pending outbox rows are due for delivery. Each pass claims up to 500 rows and
|
||||
groups them by company and origin agent, preserving company scope and the
|
||||
existing notification payload. Failed or remaining pending rows retain a
|
||||
30-second retry. Never-attempted rows sort first, then the oldest attempts, so
|
||||
one failing batch cannot repeatedly hide later rows.
|
||||
|
||||
A delivering row retains its existing five-minute claim deadline. Startup
|
||||
reconstructs that deadline from `lastAttemptAt`; a legacy missing timestamp is
|
||||
immediately recoverable. A worker waiting only for a future claim deadline does
|
||||
not repeatedly query until that deadline. Settlement and failure reset require
|
||||
the same claim timestamp, so a delayed old result cannot settle or release a
|
||||
replacement claim. Delivered counts include only rows actually settled.
|
||||
|
||||
An empty outbox or retained delivered history has no worker deadline and no
|
||||
recurring SQL. The coordinator coalesces writes and prevents overlapping passes.
|
||||
The shared deadline worker preserves first enqueue during drain, changes during
|
||||
delivery, and failed retries across drain/resume. Coordinator shutdown joins
|
||||
in-flight work. These guarantees do not replace the durable rows.
|
||||
|
||||
## Recovery and limits
|
||||
|
||||
A failed enqueue rolls back both the archive mutation and its outbox insertion.
|
||||
An uncertain transaction retains the shared intent until its outcome is known.
|
||||
A notification failure returns the same rows to pending. A process stopping
|
||||
while delivering leaves the claim for deadline recovery on restart.
|
||||
|
||||
Delivery remains at least once. A wake accepted before a lost acknowledgement
|
||||
can be retried. This change does not promise exactly-once agent wakeups or add
|
||||
new deduplication storage. The five-minute stale threshold is unchanged.
|
||||
|
||||
Supported producers share the process-local database signal owner. Independent
|
||||
SQL or another active server needs a separate wake contract. The existing Cloud
|
||||
sleep proof still blocks this outbox conservatively; no external-wake coverage
|
||||
or sleep guard is relaxed. Worker quiet does not prove actual database or host
|
||||
suspension. Retention scanning and decision continuations remain separate duties.
|
||||
|
||||
## Verification
|
||||
|
||||
The original migration selection passed 54 tests: 15 real-PostgreSQL retention/delivery cases
|
||||
and 39 startup cases. Twelve new cases cover configured idle, retained delivered
|
||||
history, real enqueue, restart, exact stale-claim timing, legacy missing timestamps,
|
||||
first enqueue during drain, failed-delivery retry through drain, outer commit and
|
||||
rollback, enqueue failure rollback, writes during delivery, more than 500 rows,
|
||||
and stale-result fencing. Quiet assertions observe no reads or writes across an
|
||||
hour of controlled time. Workspace typecheck and build pass.
|
||||
|
||||
The first 52-case selection passed. Adding fault/concurrency tests then exposed
|
||||
two fixture errors: duplicate globally unique issue identifiers across two seeded
|
||||
companies, and asserting quiet before rollback reconciliation had run. Seeded
|
||||
identifiers now use their company prefix; quiet checks advance the retained retry
|
||||
and await authoritative settlement. All 54 cases passed on rerun. No production
|
||||
hold, recovery deadline, or timeout was weakened. Full repository CI and fresh
|
||||
review are required before this PR is ready.
|
||||
|
||||
|
||||
## Review repairs: startup, claim results and backlog selection
|
||||
|
||||
Startup regressions now exercise `startServer` itself. Enabled startup must
|
||||
register `decisionArchiveDelivery` and invoke the delivery owner; calling the
|
||||
captured heartbeat interval still performs retention reads without another
|
||||
notification delivery. Calling the registered worker invokes delivery again.
|
||||
The disabled-heartbeat case explicitly preserves the existing no-notification
|
||||
contract. An initial expanded test incorrectly expected that disabled mode to
|
||||
register the worker and failed; inspection confirmed the documented disabled
|
||||
contract, so the test now asserts it rather than enabling new background work.
|
||||
|
||||
The real stale-claim regression first returned `notifiedAgents: 1` after the
|
||||
replacement claim prevented any settlement. The counter now increments only
|
||||
for a group that actually settles rows, while the existing timestamp fence and
|
||||
at-least-once delivery contract remain intact. Failure reset uses one timestamp
|
||||
for `lastAttemptAt` and `updatedAt`.
|
||||
|
||||
Generated migration `0387_boring_professor_monster.sql` creates a concurrent
|
||||
partial index on `(last_attempt_at NULLS FIRST, created_at, id)` for pending rows.
|
||||
The existing top-level migration executor handles concurrent creation and
|
||||
invalid-index recovery. No history migration or existing journal entry changes;
|
||||
the generated snapshots retain the five newest versions.
|
||||
|
||||
A real migration and query-plan check used a separate database with 50,000
|
||||
outbox rows, including 30,000 pending. The existing claim-selection query returned
|
||||
500 rows in both cases. Without the new index PostgreSQL sorted the pending
|
||||
backlog with a top-N heapsort and touched 944 shared buffers; with the index it
|
||||
used an index-only scan without sorting and touched 505 buffers. The baseline
|
||||
index removal was rolled back inside that dedicated database, and the migrated
|
||||
index was confirmed valid afterwards. These local measurements establish the
|
||||
access path, not production latency or cost savings.
|
||||
|
||||
All 58 cases across real PostgreSQL delivery/retention, startup and the shared
|
||||
deadline worker pass. Generated snapshot drift, migration numbering/safety,
|
||||
server types and build pass. The earlier stale-counter failure and invalid
|
||||
scheduler-disabled expectation are retained above. The existing 500-row batch
|
||||
limit and thirty-second retry for remaining pending work are unchanged. Fresh
|
||||
current-head CI, Greptile and Ponytail remain required before merge.
|
||||
@@ -0,0 +1,193 @@
|
||||
# Decision continuation delivery while idle
|
||||
|
||||
A terminal decision can owe an agent wake. Its existing row stores that debt:
|
||||
`continuationPolicy = wake_origin_agent`, `metadata.continuationPending = true`,
|
||||
and either expired/cancelled status or a decided status with succeeded/partial/
|
||||
failed execution. Open decisions and running effects are not ready to deliver.
|
||||
This migration adds no queue table and changes no wake payload.
|
||||
|
||||
## Ownership and producers
|
||||
|
||||
`decisions.ts` owns decision creation, authorization, signatures, effects, and
|
||||
state changes. Its ten update sites now use a narrow transaction-aware write
|
||||
helper before the SQL statement. These cover the running claim; ordinary effect
|
||||
completion; archive-proposal signature, manifest, and execution failures; archive
|
||||
success; TTL rejection on use; cancel; direct dismissal; and background expiry.
|
||||
The helper acquires intent in the existing outer transaction or its savepoint.
|
||||
It does not include effect execution, provider calls, activity publication, or
|
||||
agent wake callbacks in a new database transaction.
|
||||
|
||||
Creation and bundle creation insert open decisions, so they do not create ready
|
||||
continuation debt. Acknowledgement only clears debt and belongs to the delivery
|
||||
owner; it does not signal itself. The production decision services are created
|
||||
by server startup and routes with the application's DB handle. Independent SQL
|
||||
writers and additional active processes require a separate notification and
|
||||
ownership contract. This change does not make wake-producing service methods
|
||||
safe to call inside an uncommitted caller transaction; only their SQL mutation
|
||||
boundary has the existing outer-commit guarantee.
|
||||
|
||||
`decision-continuations.ts` owns the readiness predicate, delivery, and recovery.
|
||||
Each worker pass attempts at most 100 candidates. A saved in-memory cursor
|
||||
advances past failed candidates between passes and wraps after the last page.
|
||||
Remaining durable work retains another pass. This avoids starving later rows behind a failing first page.
|
||||
A partial ID index contains ready pending continuations only. The predicate uses
|
||||
fixed SQL literals so a prepared generic plan can still use that index.
|
||||
|
||||
The shared DB package exposes an opaque work-scope identity. Raw DB object
|
||||
identity is insufficient because dedicated and liveness child pools already
|
||||
share their owner's signals. The key is shared by those children and nested
|
||||
transactions; independent roots have different keys. No connection credentials,
|
||||
private transaction state, or decision policy are exposed. Existing signal
|
||||
consumers and transactions that do not register work keep their behavior.
|
||||
|
||||
## Foreground, recovery, and failure
|
||||
|
||||
Foreground delivery remains immediate. All service instances sharing the DB
|
||||
owner coalesce a simultaneous wake for the same decision into one promise.
|
||||
Delivery re-reads durable readiness after claiming that local flight, so a late
|
||||
foreground call cannot resend a continuation already acknowledged by recovery.
|
||||
The promise is removed on success or failure. Effect execution and its final
|
||||
state write also share one local flight per decision. This prevents concurrent
|
||||
archive callers from turning a successful archive into a stale-manifest failure. This is process-local coalescing,
|
||||
not a new durable claim or an exactly-once guarantee.
|
||||
|
||||
Effect completion requires the row to remain running. A competing completion
|
||||
cannot restore pending metadata after another caller has completed and delivered
|
||||
it. Effects retain their own existing durable claims and authorization checks.
|
||||
The delivery worker never executes effects.
|
||||
|
||||
A wake failure leaves pending metadata intact. A failed acknowledgement also
|
||||
leaves debt, so a later attempt can send the wake again. A crash after accepting
|
||||
the wake but before acknowledgement retains the existing at-least-once contract.
|
||||
An uncertain producer transaction remains held until the shared work-signal
|
||||
protocol establishes settlement; one empty scan cannot clear that uncertainty.
|
||||
|
||||
## Lifecycle and scope
|
||||
|
||||
Startup reconstructs pending delivery. Committed changes wake the worker through
|
||||
the existing coordinator. Failures or remaining ready debt retain a 30-second
|
||||
retry. Open/running decisions, no decisions, and delivered history have no
|
||||
recurring continuation scan or deadline. Changes during a pass retain another
|
||||
request. Drain/standby defer dispatch without consuming that request; shutdown
|
||||
waits for the current admitted delivery and stops before starting another.
|
||||
The shared deadline adapter forwards its existing AbortSignal to task execution;
|
||||
existing callbacks that do not consume the signal keep their behavior. A future process-local retry is not a Cloud alarm.
|
||||
|
||||
Enabled-heartbeat startup registers this worker before the combined decision
|
||||
sweep. The main tick skips that sweep's independent continuation scan. Its public
|
||||
API retains combined behavior by default. Expiry or effect recovery can still
|
||||
deliver their newly produced continuations immediately. Disabled-heartbeat
|
||||
startup keeps its existing behavior and does not dispatch this new worker.
|
||||
|
||||
Decision expiry, invalidated-target scanning, interrupted effect recovery, and
|
||||
retention remain separate migrations. Existing Cloud safety blockers are not
|
||||
relaxed. This establishes worker quiet under the named conditions, not deployed
|
||||
host/database suspension or measured fleet savings.
|
||||
|
||||
## Verification
|
||||
|
||||
The selected run passes 93 tests: 50 real-PostgreSQL decision cases, 39 startup
|
||||
cases, and four database work-signal unit cases. Fifteen new continuation cases
|
||||
cover real decide/cancel/dismiss/expiry producers, running-to-terminal completion,
|
||||
foreground contention, dedicated child pools, stale caller re-read, restart,
|
||||
failed acknowledgement, outer rollback/commit, first write during drain, failed
|
||||
retry through drain, changes during delivery, shutdown, and paging past a failed
|
||||
candidate. Quiet checks observe no SQL calls across an hour of controlled time.
|
||||
Nine pre-existing embedded-PostgreSQL signal tests are skipped locally because
|
||||
the embedded runtime is unavailable; CI must run that supported environment.
|
||||
Workspace typecheck and build pass. The final literal-predicate adjustment also
|
||||
passes all 49 decision tests before the additional archive-race regression.
|
||||
|
||||
The original continuation index applied and replayed without error. After parent integration it is generated migration `0388_ambitious_micromax.sql`; the fresh apply and replay also pass. A rolled-back synthetic
|
||||
40,000-row history with one pending continuation uses the partial index under
|
||||
`force_generic_plan`, reads two shared buffers, and executes in 0.015 ms on the
|
||||
local PostgreSQL server. This is query evidence, not a production latency claim.
|
||||
|
||||
The first test selection had 90 passes and one failure: the injected PostgreSQL
|
||||
acknowledgement error is wrapped by Drizzle, but the assertion expected its
|
||||
message at the top level. The assertion now checks the original cause. No
|
||||
production retry, hold, or timeout was weakened. A recursive unit-fixture type
|
||||
initially prevented migration generation; an explicit fixture type fixed it.
|
||||
Full CI and fresh review remain required before PR readiness.
|
||||
|
||||
|
||||
Review found two real regressions. Before repair, a controlled concurrent archive
|
||||
reported failed after archiving successfully, and shutdown sent a second wake
|
||||
after stop was requested. Both regressions failed before the fix. Shared local
|
||||
effect ownership fixes archive completion; bounded passes and abort propagation
|
||||
fix backlog shutdown. The 93-case selection passes after repair. The paging test
|
||||
also proves a failed first candidate cannot hide the 101st candidate across
|
||||
bounded passes. A result-type alias initially made the known decision ID optional;
|
||||
the alias now matches the existing outcome contract.
|
||||
|
||||
The existing tool-review and archive-delivery consumers also pass 124 real-DB
|
||||
tests after AbortSignal propagation. Together with the decision/startup/signal
|
||||
selection, 217 selected tests pass; the same nine older embedded-only cases
|
||||
remain locally skipped. Workspace typecheck passes after the outcome type fix.
|
||||
|
||||
|
||||
## Explicit archive-failure delivery
|
||||
|
||||
The three archive-proposal failure paths now explicitly set
|
||||
`continuationPending: true` for `wake_origin_agent`, rather than assuming the
|
||||
pre-read running row already carries that metadata. The existing running-state
|
||||
fence remains, so stale settlement cannot re-arm an acknowledged result.
|
||||
Invalid signatures, manifest mismatches and archive execution failures each
|
||||
failed a real-PostgreSQL recovery regression before repair. Each now saves the
|
||||
failed result, wakes once, acknowledges delivery and remains quiet on replay.
|
||||
|
||||
Normal parent integration preserves the projection interaction-link index and
|
||||
archive pending-order index. The unmerged continuation index was regenerated
|
||||
as 0388 from the parent's generated metadata; every parent journal entry is
|
||||
unchanged, five snapshots remain, the SQL uses the existing top-level concurrent
|
||||
executor, and snapshot/schema drift passes. No generated snapshot was hand-edited.
|
||||
|
||||
The final affected selection passes 111 cases: 53 decisions, 15 retention/archive,
|
||||
42 startup and one scoped-deadline case, with all four suites exercised. The
|
||||
first combined command used the wrong retention DB variable and skipped those
|
||||
15 cases; the corrected external-PostgreSQL retention run passes all 15. Server
|
||||
typecheck and build pass. The historical nine embedded-only work-signal skips
|
||||
above are not reclassified as executed by this selection. Fresh CI and reviews
|
||||
remain required.
|
||||
|
||||
|
||||
## Bounded notification-driven recovery
|
||||
|
||||
Security review identified that sequential decision writes could each start an
|
||||
instance-wide continuation scan. A real-producer regression reproduced six
|
||||
sweeps for six sequential cancellations. The worker now uses the existing
|
||||
`notificationCoalesceMs: 30_000` option. The first wake after a quiet window still
|
||||
reconciles promptly, and subsequent notifications in that window share its end
|
||||
deadline. Foreground continuation delivery is unchanged and immediate. Recovery
|
||||
can wait up to thirty seconds, matching the prior maintenance cadence. This
|
||||
bounds global scan frequency; it does not make queries company-scoped or add a
|
||||
cross-process notification contract.
|
||||
|
||||
The regression verifies one scan during repeated real create/cancel writes,
|
||||
retention and eventual delivery of a later pending row, and no recurring SQL
|
||||
after settlement. Its first green attempt wrongly expected zero active work
|
||||
while the coalesced request was pending. The existing intent hold correctly
|
||||
remains until reconciliation; the test now checks that hold's actual terminal
|
||||
boundary through the existing quiet assertion. No production hold was removed.
|
||||
All 293 integrated cases across decisions, archive, gateway/projection, startup
|
||||
and scoped-deadline suites pass without skips; server types/build pass. Fresh
|
||||
security review, Greptile, Ponytail and exact-head CI remain merge gates.
|
||||
|
||||
|
||||
The archive contention regression no longer waits a fixed 100 ms. A dedicated
|
||||
single-connection contender signals its decision read; two FIFO SQL round trips
|
||||
then establish that it reached shared execution while the first archive remains
|
||||
held. The test requires one contender decision read and one archive execution.
|
||||
Temporarily removing the effect-flight guard made this test fail (two reads);
|
||||
restoring the production guard passes. The mutation was restored before commit.
|
||||
This verifies actual cross-pool contention without a timing delay or a new
|
||||
production-only test hook.
|
||||
|
||||
|
||||
Archive #17475 is merged. Normal reconciliation with its exact squash
|
||||
`5785090ff1` preserves all 31 independent incoming master files, including
|
||||
Runner controller-authority recovery. All 97 decision/startup/deadline cases and
|
||||
server types/build pass after that integration. The Runner vendor rebuilt from
|
||||
its changed inputs. The preceding full five-suite selection passed 293 cases;
|
||||
the continuation owner's production behavior is unchanged by reconciliation.
|
||||
Fresh exact-head reviews and CI remain required on the master-target candidate.
|
||||
@@ -0,0 +1,255 @@
|
||||
# Idle decision-effect recovery
|
||||
|
||||
Interrupted decision effects now have a dedicated deadline worker. The main
|
||||
maintenance tick no longer scans running decisions. Target invalidation remains
|
||||
in that tick; TTL expiry and continuation delivery have their own workers.
|
||||
|
||||
## Obligation and activation
|
||||
|
||||
The existing decision row is durable truth. A row with `status = decided` and
|
||||
`executionStatus = running` owes effect recovery. Its next deadline is
|
||||
`updatedAt + PAPERCLIP_DECISIONS_RECOVERY_GRACE_MS`, default 60 seconds. Zero is
|
||||
supported; invalid or negative values retain the existing default. Reads do not
|
||||
move the timestamp. Open decisions and completed history create no recovery debt.
|
||||
|
||||
The existing `writeDecisionChange` transaction boundary signals effect recovery
|
||||
for running claims, effect/archive settlement, cancellation, dismissal and expiry.
|
||||
The generic `notifyDecisionChange` helper signals continuation and expiry only;
|
||||
open-decision creation does not owe effect recovery. Notifications are awaited
|
||||
before writes. Outer commit activates work; rollback does not publish a claim.
|
||||
The additional topic does not wrap provider work or agent wakes in SQL transactions
|
||||
and does not introduce a new queue table.
|
||||
|
||||
Committed notification bursts coalesce for up to one second after a successful
|
||||
pass. This bounds sequential tenant writes to at most one notification-driven
|
||||
global pass per window, rather than only coalescing overlapping scans. A saved
|
||||
recovery deadline can preempt that window. New obligations discovered by a signal
|
||||
can wait up to one second; foreground effects still execute immediately. The
|
||||
window alone never schedules work when there is no notification or saved debt.
|
||||
|
||||
The supported writer model is the application DB signal owner and its child pools.
|
||||
Independent SQL clients and multiple active writers need a separate notification
|
||||
contract. Startup reconstructs committed claims; it does not observe future writes
|
||||
from unsupported writers.
|
||||
|
||||
## Ownership and recovery
|
||||
|
||||
`decision-effect-recovery.ts` owns selection, grace, bounded paging, and earliest
|
||||
deadline lookup. `decisions.ts` owns execution, current membership, effect claims,
|
||||
audit, archive policy, and continuation delivery. The worker adapter only binds
|
||||
that domain contract to the shared coordinator. Startup registers it only when
|
||||
heartbeat scheduling is enabled. The public combined sweep retains its default
|
||||
recovery behavior for existing callers.
|
||||
|
||||
Each pass honors the existing configured batch size (default 100, finite values
|
||||
clamped to at least one integer). Its timestamp/ID cursor advances past failed
|
||||
candidates within a frozen final timestamp/ID. Newer due claims wait for the next
|
||||
rotation, so a continuous stream cannot postpone an earlier failure forever.
|
||||
Both cursor positions retain PostgreSQL precision. Genuine execution or settlement
|
||||
failures retain their retry obligation.
|
||||
|
||||
A missing saved decider or chosen option is a deterministic malformed outcome.
|
||||
Recovery rechecks it under a row lock, records an audited failed outcome (partial
|
||||
if a completed effect receipt exists), and preserves all effect receipts. It
|
||||
creates no authority and executes no effect for the malformed claim. The failure
|
||||
reason and continuation obligation are saved in the same transaction as the audit.
|
||||
An audit or settlement failure rolls back and leaves the claim pending. A claim
|
||||
repaired before the lock is acquired remains eligible for normal recovery.
|
||||
|
||||
The decision service re-reads current status before recovery. Foreground and
|
||||
recovery execution share the existing per-decision flight scoped to the DB owner.
|
||||
Individual effects keep their durable records and transaction advisory locks.
|
||||
Completed effects are not repeated after a failed final decision status write.
|
||||
Recovery derives authority from the saved decider's current active membership.
|
||||
It preserves the current authorization intersection and current target checks.
|
||||
|
||||
The scheduler has a 30-second minimum retry spacing for pending or failed work.
|
||||
A first claim or failed retry during drain retains its wake without passive SQL.
|
||||
Shutdown stops subscriptions and checks cancellation between candidates. An
|
||||
already admitted effect/continuation is awaited; later candidates remain durable
|
||||
for restart. No periodic pass remains after all claims settle.
|
||||
|
||||
## Existing guarantees and limits
|
||||
|
||||
Regular effects preserve their existing per-effect transaction boundaries.
|
||||
Continuation delivery retains its separate at-least-once wake/acknowledgement
|
||||
contract. This migration does not establish exactly-once external delivery.
|
||||
|
||||
Archive proposals have a separate versioned manifest contract. Recovery still
|
||||
verifies the saved signature/manifest and current authority. A crash after an
|
||||
archive commits but before the decision settles can encounter the changed
|
||||
manifest version on restart. The existing result is a failed decision with an
|
||||
archive error, while the archive remains committed once. The new worker preserves
|
||||
and tests that outcome; it does not claim to repair this preexisting atomicity gap.
|
||||
|
||||
Cloud sleep guards remain conservative. The internal next deadline is not a
|
||||
persisted external alarm and does not authorize host sleep with running decisions.
|
||||
This change establishes worker quiet, not actual instance/database suspension.
|
||||
|
||||
## Verification
|
||||
|
||||
The new real-PostgreSQL suite covers 17 cases: empty/open/completed-history quiet,
|
||||
real foreground claim and failed settlement, exact grace and zero override,
|
||||
restart from a claimed effect, active foreground/recovery overlap, outer commit
|
||||
and rollback, first claim and failed retry through drain, revoked authority,
|
||||
continuation failure, aggregate exclusion, fair paging past missing-decider and
|
||||
malformed claims, archive recovery before and after an archive commit, a claim
|
||||
created during delivery, and shutdown between candidates. Quiet assertions observe
|
||||
SQL methods through one hour of fake time after settling.
|
||||
|
||||
The combined selection passes 212 tests: the 17 new cases, 50 existing decision
|
||||
service cases, 14 TTL cases, 39 startup cases, and 92 agent-history route cases.
|
||||
The first 15-case recovery run also passed. Workspace typecheck/build and the
|
||||
final server typecheck pass. No failure was suppressed or timeout increased.
|
||||
|
||||
The current regenerated migration `0390_salty_killraven.sql` adds one partial index concurrently, with `IF NOT EXISTS`, through
|
||||
the migration owner's top-level statement path. Generation retains the newest
|
||||
five schema snapshots. Fresh application, replay, safety, and numbering checks
|
||||
pass. With 40,000 completed decisions and one running claim, both due selection
|
||||
and earliest-deadline lookup use the partial index and read two buffers (0.012 ms
|
||||
and 0.003 ms in the local probe). The probe transaction is rolled back.
|
||||
|
||||
Full PR CI and review must pass separately before this slice is ready.
|
||||
|
||||
A follow-up regression seeds a legacy invalid claim with a PostgreSQL-generated
|
||||
microsecond timestamp ahead of a valid claim. With a one-row page, the original
|
||||
Date cursor selected the invalid claim again and starved the valid effect. That
|
||||
regression failed before repair. Recovery now retains the timestamp as database
|
||||
text for cursor comparisons, leaving business grace deadlines unchanged. The
|
||||
unresolved claim remains durable debt and is revisited after paging wraps; the
|
||||
valid claim progresses without repeating its completed effect. The regression
|
||||
now holds a candidate unresolved through the dispatch callback, because known
|
||||
malformed claims receive a durable failed outcome as described above.
|
||||
All 18 effect-recovery lifecycle cases and server typechecking pass after repair.
|
||||
|
||||
### Foreground join test timing
|
||||
|
||||
CI exposed a timing assumption in the foreground/recovery overlap test: it
|
||||
expected exactly two lifetime recovery sweeps. A controlled PostgreSQL replay
|
||||
waited for the committed claim's deadline discovery before advancing time. It
|
||||
recorded two completed, zero-resumption scans before the deadline (startup and
|
||||
claim notification), followed by the legitimate deadline scan. The old assertion
|
||||
then failed deterministically with three calls. No recurring idle poll or duplicate
|
||||
effect was found in that sequence.
|
||||
|
||||
The test now waits for discovery, measures the deadline recovery relative to that
|
||||
point, and proves that its promise stays pending while foreground execution holds
|
||||
the effect lock. Advancing through several retry periods cannot start another
|
||||
sweep or effect. After release, recovery completes, one comment and one continuation
|
||||
wake exist, and the worker performs no SQL through the existing hour-long quiet
|
||||
check. Counting actual lock attempts also distinguishes joining from a second
|
||||
execution blocked on the same PostgreSQL advisory lock. A temporary mutation that
|
||||
removed flight joining made this stronger assertion fail; production code was
|
||||
restored before final verification. No runtime code, retry policy, timeout, or
|
||||
business deadline changed.
|
||||
|
||||
All 123 tests across the four complete effect-recovery, decision-service, expiry,
|
||||
and startup suites passed, followed by server typechecking. Evidence:
|
||||
`/tmp/sleep-effect-join-red.log`, `/tmp/sleep-effect-join-green.log`,
|
||||
`/tmp/sleep-effect-join-mutation.log`, `/tmp/sleep-effect-join-owning-tests.log`,
|
||||
and `/tmp/sleep-effect-join-root-types.log` with their status files. The initial
|
||||
failure and mutation failure are expected evidence, not passing runs. Remaining
|
||||
review findings and fresh stack CI remain separate acceptance requirements.
|
||||
|
||||
|
||||
### Recovery review repairs (October 26)
|
||||
|
||||
Two regressions failed before repair: an old failure was not revisited while newer
|
||||
due claims kept each page full, and creating an open decision caused an unnecessary
|
||||
effect-recovery scan. Frozen rotation boundaries now preserve both retry fairness
|
||||
and forward progress. Creation still signals its continuation/expiry consumers;
|
||||
the effect queue is signaled by the existing claim/settlement write boundary.
|
||||
|
||||
Four malformed-outcome cases also failed before repair. Recovery now records
|
||||
`effectRecoveryError` and a `decision.recovery_failed` activity atomically. Live
|
||||
activity publication follows commit. Existing completed effect receipts remain
|
||||
unchanged, and the normal continuation owner delivers the result. A real audit
|
||||
insert failure leaves both status and audit uncommitted, then later recovery
|
||||
settles successfully. A real PostgreSQL row-lock test repairs a claim while
|
||||
settlement waits; the locked recheck leaves that claim running, and its valid
|
||||
effect executes once on the next pass.
|
||||
|
||||
The shared execution flight now loads the current claim and derives recovery
|
||||
authority together. Recovery no longer performs a redundant full-row lookup
|
||||
before entering the flight. Foreground overlap, current membership checks,
|
||||
per-effect locks, continuation behavior, and the resumption count are preserved.
|
||||
Only the two known malformed states are terminalized; other failures retain their
|
||||
existing retry policy. No schema, queue, or external-wake contract is added.
|
||||
|
||||
All 24 recovery lifecycle cases and server typechecking pass. Controlled failures
|
||||
and final evidence are retained in `/tmp/sleep-effect-fairness-red.log`,
|
||||
`/tmp/sleep-effect-invalid-red.log`, `/tmp/sleep-effect-owner-tests.log`, and
|
||||
`/tmp/sleep-effect-owner-types.log`. Broader stack and remote acceptance remain
|
||||
separate requirements; these results do not prove host suspension.
|
||||
|
||||
The owning branch also passes all 129 cases across the four complete recovery,
|
||||
decision-service, expiry, and startup suites. Database-backed suites use separate
|
||||
migrated disposable databases. See `/tmp/sleep-effect-review-owner-acceptance.log`.
|
||||
|
||||
|
||||
### Bounded notification review repair
|
||||
|
||||
Ten sequential real cancellation transactions reproduced ten extra recovery
|
||||
scans. After enabling the existing shared notification-coalescing capability,
|
||||
the same transactions cause one follow-up scan and then return to SQL quiet.
|
||||
A separate real-database case preserves an already loaded deadline 500 milliseconds
|
||||
away. The initial deadline fixture observed the scheduler while its SQL pass was
|
||||
still active; waiting for that read removed the fixture race before the baseline
|
||||
was recorded. The cancellation amplification still failed before the production fix.
|
||||
|
||||
All 131 tests in the four complete recovery, decision-service, expiry and startup
|
||||
suites pass at the owning branch, as does server typechecking. The helper's optional
|
||||
coalescing capability is promoted from the later plugin scheduling slice into this
|
||||
PR; consumers that omit it keep their previous behavior. The integrated stack is
|
||||
verified separately before publication. The original full-suite and external-wake
|
||||
limitations remain.
|
||||
|
||||
|
||||
## Current parent integration
|
||||
|
||||
The expiry and continuation review repairs are integrated normally. The running
|
||||
claim index is regenerated as 0390 after parent 0388/0389, with concurrent creation
|
||||
registered in the existing executor. All 388 parent journal entries are unchanged;
|
||||
the five newest generated snapshots remain. Schema drift, numbering/safety and
|
||||
fresh migration apply/replay pass. No generated snapshot was hand-edited.
|
||||
|
||||
All 141 cases in the five complete effect-recovery, decision/continuation, expiry,
|
||||
startup and scoped-deadline suites pass. This includes all 26 recovery cases and
|
||||
the parent's deterministic cross-pool archive contention test. Server types and
|
||||
build pass. The first integrated run passed 140 cases and failed an old scheduler
|
||||
assertion: the coordinator now also holds an earlier continuation-coalescing
|
||||
deadline. The overlap test services that earlier registered wake, then still
|
||||
requires the original effect grace deadline, one active effect flight, two total
|
||||
advisory-lock attempts (test gate plus foreground), one committed effect and one
|
||||
continuation. No production timing or retry was changed to pass the test.
|
||||
|
||||
Existing one-second effect-notification coalescing and thirty-second failure
|
||||
retry remain. Continuation notification coalescing is owned separately by the
|
||||
parent. Extra upstream embedded-only tests attempted during parent integration
|
||||
could not initialize locally; fresh current-head CI must cover that environment.
|
||||
Full CI and both automated reviews remain merge gates.
|
||||
|
||||
|
||||
After normal integration of master-target expiry parent `f7c61e9912`, the
|
||||
effect-recovery owner, worker, decision execution and notification implementations
|
||||
are unchanged. All 69 current recovery/startup/deadline cases and server types/
|
||||
build pass with the updated Runner vendor. The earlier full 141-case selection
|
||||
remains evidence at its stated revision. Parent migration history is an exact
|
||||
prefix and five generated snapshots remain. Fresh head CI and both automated
|
||||
reviews are required after this parent integration.
|
||||
|
||||
|
||||
## Landed expiry integration and CI interruption
|
||||
|
||||
Expiry-only creation notification from `aa03d2c4c2` is integrated. All 143 cases
|
||||
in the complete recovery, decision, expiry, startup and scoped-deadline suites
|
||||
pass, including single/bundle creation without continuation signals. The normal
|
||||
merge of landed expiry `3421366903` also preserves all 21 independently merged
|
||||
master files. The same 143 cases pass after that reconciliation.
|
||||
|
||||
The previous candidate `f54a707d74` had both reviews at 5/5, but CI run
|
||||
40597921712 failed because server shard 6 was interrupted. Its test step was
|
||||
cancelled and orphan cleanup ran; no failing test assertion or suite summary was
|
||||
reported. The workflow eventually completed with that shard and its dependent
|
||||
verify gate failed. It was not retried or presented as passing. The necessary
|
||||
expiry parent repair creates a new candidate with fresh CI and reviews required.
|
||||
@@ -0,0 +1,177 @@
|
||||
# Decision TTL expiry while idle
|
||||
|
||||
Open decisions already contain an `expiresAt` deadline. The expiry worker now
|
||||
uses that timestamp instead of scanning TTL candidates on each maintenance tick.
|
||||
This slice owns time-based expiry only. Target invalidation, interrupted effect
|
||||
recovery, and retention remain separate duties.
|
||||
|
||||
## Producers and transaction ownership
|
||||
|
||||
Creation and bundle creation call `notifyDecisionCreated` before the decision
|
||||
insert in their existing transaction. It signals expiry only: an open row cannot
|
||||
owe continuation delivery yet. Existing decision updates share `notifyDecisionChange`, which now
|
||||
signals expiry as well as continuation delivery. Cancellation, dismissal, and
|
||||
execution claims therefore remove obsolete expiry deadlines. All production
|
||||
inserts and updates are in the decision owner; acknowledgement only changes
|
||||
terminal delivery metadata and cannot change an open expiry. No supported API
|
||||
currently edits an existing decision's expiry timestamp. A future such writer
|
||||
must use this same notification boundary.
|
||||
|
||||
The helper adds no recovery wrappers to routes. Outer transactions and nested
|
||||
savepoints retain the shared commit/rollback and uncertain-outcome protocol.
|
||||
Creation without a committed row cannot publish a business deadline. Startup
|
||||
reconstructs saved deadlines, including already-overdue work. Independent SQL
|
||||
writers and another active server process need their own wake contract.
|
||||
|
||||
`decision-expiry.ts` owns due selection, the next deadline, and expiry settlement.
|
||||
`decisions.ts` delegates expiry-on-use and the existing target-driven expiration
|
||||
transition to that owner. Target selection remains outside the TTL worker.
|
||||
Startup registers TTL work before the combined decision sweep. The periodic
|
||||
combined sweep opts out of TTL selection and cannot expire a row merely because
|
||||
its TTL elapsed. Its public default still includes TTL and preserves the
|
||||
configured combined-sweep batch limit.
|
||||
|
||||
## Deadline, settlement, and recovery
|
||||
|
||||
The next deadline is the earliest open `expiresAt`. A partial index on
|
||||
`(expires_at, id)` covers open decisions. The query selects one indexed row;
|
||||
retained terminal history does not enlarge the deadline lookup. Expiry is due
|
||||
when `expiresAt <= now`, including equality. The old combined path skipped the
|
||||
exact equality despite selecting that row; this migration fixes that boundary.
|
||||
|
||||
Each worker pass honors `PAPERCLIP_DECISIONS_SWEEP_BATCH_SIZE` (default 100),
|
||||
ordered by expiry and ID.
|
||||
Its in-memory cursor advances across failures and wraps after the last page.
|
||||
Remaining overdue work retains a one-second continuation through the saved
|
||||
earliest deadline after a successful pass.
|
||||
Failures retain the existing 30-second retry throttle. New earlier creation or
|
||||
resolution of the earliest decision causes deadline reconciliation. When only a
|
||||
future expiry remains, the worker does not query again until that deadline or a
|
||||
committed change. No open decisions means no recurring expiry work.
|
||||
|
||||
Settlement requires the row to remain open. TTL settlement also rechecks that
|
||||
the saved deadline is due. Metadata is merged into the current row. The existing
|
||||
expiry activity is persisted in the same transaction as the terminal transition.
|
||||
Publication and origin continuation follow commit. If the audit insert fails,
|
||||
the expiry and continuation obligation both roll back. A deadline-driven expiry
|
||||
records `ttl`; target-driven expiration retains its own reason. When both causes
|
||||
are possible, the first successful terminal transition determines the reason.
|
||||
Continuation failures belong to the continuation worker after the decision is
|
||||
expired. Neither owner replays decision effects.
|
||||
|
||||
Drain and standby defer dispatch while retaining the request. The worker checks
|
||||
the coordinator's AbortSignal between candidates. Shutdown finishes the admitted
|
||||
expiry and leaves untouched candidates for restart. It does not process the
|
||||
whole backlog before stopping. There is no new queue table or permanent DB
|
||||
listener, and no provider call is held inside a new transaction.
|
||||
|
||||
## Limits
|
||||
|
||||
The disabled-heartbeat branch keeps its existing behavior and does not register
|
||||
this worker. Expiry-on-use remains enforced independently of background work.
|
||||
The in-process deadline is not an external Cloud alarm. Existing conservative
|
||||
sleep blockers remain; this migration does not permit an instance to sleep with
|
||||
unsupported future obligations. Worker quiet is not proof of host or database
|
||||
suspension.
|
||||
|
||||
## Verification
|
||||
|
||||
The original combined selection passed 103 tests: 14 initial real-PostgreSQL expiry worker
|
||||
cases, 50 decision service/continuation cases, and 39 startup cases. It covered
|
||||
empty and terminal-history quiet, actual create and bundle producers, exact
|
||||
expiry, moving the next deadline earlier, cancellation, restart, outer rollback
|
||||
and commit, first creation during drain, failed retry through drain, failed wake
|
||||
delivery, separate aggregate/TTL ownership, 101 candidates with a failed first
|
||||
row, creation during delivery, and shutdown/restart with outstanding candidates.
|
||||
The parent archive race and bounded continuation shutdown regressions also pass.
|
||||
|
||||
The first selection had 97 passes and one fixture failure. The fault test cleared
|
||||
its error collector before an already-admitted failed query had finished. It now
|
||||
checks all collected causes after recovery settles. The next selection passed
|
||||
101 cases; after the parent review repairs and bounded expiry/shutdown coverage,
|
||||
all 103 pass. No production hold, retry, or timeout was weakened.
|
||||
|
||||
The original index applied and replayed successfully; after normal parent
|
||||
integration it is regenerated as `0389_brave_hulk.sql`. Fresh apply also passes. With 40,000 synthetic decisions
|
||||
and one open row, the next-deadline query uses the partial index, reads two shared
|
||||
buffers, and executes in 0.022 ms locally. The probe transaction rolls back. This
|
||||
is query evidence, not a production latency or savings claim. Workspace
|
||||
typecheck and build pass on the combined branch. Full PR CI and fresh review
|
||||
remain required before readiness.
|
||||
|
||||
A follow-up paging regression uses 101 overdue rows sharing a PostgreSQL
|
||||
microsecond timestamp, with the first 100 writes failing. The original Date-based
|
||||
cursor rounded the timestamp to milliseconds and repeatedly selected that failed
|
||||
page, starving the last valid row. The cursor now carries PostgreSQL timestamp
|
||||
text into its tuple comparison; business deadline calculations still use Date.
|
||||
The regression failed before the fix and verifies progress past the failed page,
|
||||
then recovery of all failed rows after the fault is removed.
|
||||
The repaired expiry suite passes all 15 cases, and server typechecking passes.
|
||||
|
||||
|
||||
## Current review repairs and verification
|
||||
|
||||
Three review findings failed real PostgreSQL regressions before repair: the
|
||||
standalone worker ignored the configured batch limit, a successful overdue
|
||||
backlog waited thirty seconds between batches, and a failed audit write left the
|
||||
row expired without durable activity. The worker now honors its existing batch
|
||||
configuration and continues successful overdue work after one second. The
|
||||
shared deadline adapter accepts an optional `continuationMs`; omitted values
|
||||
preserve every existing consumer's retry interval. Errors still retain the
|
||||
existing thirty-second retry. Expiry and durable activity commit atomically,
|
||||
then publish activity and deliver the origin continuation. Provider calls stay
|
||||
outside that transaction.
|
||||
|
||||
All 129 selected cases pass: 18 expiry, 53 decision/continuation, 15 archive, 42
|
||||
startup and one scoped-deadline case. The archive selection exercises an existing
|
||||
shared-deadline consumer with the new option absent. Server typecheck/build and
|
||||
generated snapshot/schema drift pass. Parent journal entries remain unchanged
|
||||
and five generated snapshots remain. No snapshot was hand-edited.
|
||||
|
||||
The first complete selection passed 113 cases and failed the future-startup
|
||||
assertion because it inspected the scheduler before an additional committed
|
||||
notification reconciliation finished. Coordinator `ready` awaits the first
|
||||
admitted pass, while a notification arriving during that pass can schedule
|
||||
another immediate pass. The assertion now waits for reconciliation to expose
|
||||
the exact saved future deadline, then still proves the decision does not expire
|
||||
early. The three production regressions and this fixture failure remain
|
||||
recorded; no error retry or business deadline was relaxed to pass the tests.
|
||||
Fresh current-head CI and reviews remain merge gates.
|
||||
|
||||
|
||||
After normal integration of continuation recovery's final coalescing and
|
||||
deterministic contention tests and pinned master `8161f2a813`, all 130 selected
|
||||
cases pass: 76 expiry/archive/startup/deadline cases plus all 54 decision cases.
|
||||
Server types/build pass. Existing notification coalescing remains thirty
|
||||
seconds for continuation recovery; expiry's successful backlog continuation is
|
||||
one second, and failure retry remains thirty seconds. Migration replay reports
|
||||
no pending migrations and the parent journal remains an exact prefix. Extra
|
||||
upstream embedded-only tests attempted during parent integration could not
|
||||
initialize locally; fresh CI must cover that environment, as recorded in the
|
||||
archive parent PR.
|
||||
|
||||
|
||||
Normal integration with master-target continuation parent `18f682863c` preserves
|
||||
expiry, deadline and producer implementations exactly. All 61 expiry/startup/
|
||||
scoped-deadline cases and server types/build pass with the rebuilt Runner vendor.
|
||||
The previous full 130-case selection remains evidence at its stated revision.
|
||||
No migration SQL changed, and five generated snapshots remain. Fresh head CI
|
||||
and reviews are required after this parent integration.
|
||||
|
||||
|
||||
## Creation notification review repair
|
||||
|
||||
Single and bundle creation previously notified continuation recovery even though
|
||||
the inserted decision was open. Two real producer regressions failed before
|
||||
repair because continuation received intent and settlement. Creation now uses the
|
||||
existing central notification file's expiry-only `notifyDecisionCreated` helper.
|
||||
Update and terminal transitions keep `notifyDecisionChange`, including the normal
|
||||
continuation signal. No optional flags, per-route bookkeeping or new table is
|
||||
needed. Both regressions verify the exact future expiry, no early wake, and a
|
||||
subsequent cancellation that still signals and delivers continuation normally.
|
||||
|
||||
All 117 current expiry/decision/startup/deadline cases pass, including 20 expiry
|
||||
cases. Server types/build pass. The preceding f7 candidate's identical-tree
|
||||
merge proof remains history; this subsequent creation-signal repair changes
|
||||
runtime behavior and has its own acceptance above. Fresh current-head CI and
|
||||
both reviews are required.
|
||||
@@ -0,0 +1,164 @@
|
||||
# Idle decision target invalidation
|
||||
|
||||
Open decisions no longer require a periodic target scan. The target worker
|
||||
reconciles committed issue changes and decision creation. Startup reconstructs
|
||||
open decisions in bounded pages. After reconciliation, valid open decisions and
|
||||
completed history cause no recurring SQL. TTL expiry, continuation delivery,
|
||||
interrupted effects, and retention remain separate responsibilities.
|
||||
|
||||
## Domain policy
|
||||
|
||||
Only strict effect references participate. The existing shared
|
||||
`decisionEffectTargetIssueIds` helper includes primary and secondary references.
|
||||
A missing or cancelled target expires a decision with `target_gone`. If all
|
||||
strict targets are done and at least one was not done in the proposal snapshot,
|
||||
it expires with `target_completed`. Lenient decisions and proposals intentionally
|
||||
acting on already-done targets remain open. A target that reopened before
|
||||
reconciliation is checked in its current state; the notification is not authority
|
||||
to expire it. The existing open-status compare-and-set arbitrates expiry against
|
||||
other decision transitions.
|
||||
|
||||
`decision-targets.ts` owns this policy and scoped reconciliation. It delegates
|
||||
expiry, activity, and continuation to the existing expiry owner. Startup
|
||||
registers the target worker when scheduling is enabled. The combined internal
|
||||
`sweepExpired` method is removed after the final duty is split. No new table or
|
||||
migration is added.
|
||||
|
||||
## Producer audit
|
||||
|
||||
| Producer | Intent before write | Scope |
|
||||
| --- | --- | --- |
|
||||
| Decision creation, including bundles and caller-owned transactions | Existing creation transaction, before insertion | Decision ID |
|
||||
| `issueService.update` entering/reasserting done or cancelled | Existing issue update transaction, before the patch | Issue ID |
|
||||
| Issue removal | Lock the issue and capture linked open decision IDs in the existing transaction, before deletion | Affected decision IDs; no signal when none |
|
||||
| Routine dispatch failure deleting its created issue | Same locked pre-delete lookup in the existing dispatch transaction | Affected decision IDs; usually no signal |
|
||||
|
||||
The existing target link table supports ordinary issue scopes. Before a target
|
||||
is deleted, the decision owner locks its issue row, reads distinct linked open
|
||||
decisions in the same company, and signals their IDs. The links may then cascade
|
||||
without losing notification scope. The row lock closes the race in which a new
|
||||
proposal could add its target reference after discovery but before deletion.
|
||||
This remains part of the caller's existing transaction; no provider operation
|
||||
is wrapped in a new transaction. Production hints require an issue or decision
|
||||
ID, so routine cleanup and repeated removals do not restart full startup paging.
|
||||
|
||||
Company deletion no longer signals target work. With no decision rows it cannot
|
||||
create a same-company invalidation obligation. Pre-existing restrictive origin
|
||||
run/issue foreign keys still prevent deleting some companies with decisions;
|
||||
that separate limitation is unchanged. This migration does not claim to implement
|
||||
complete decision-data deletion for those companies.
|
||||
|
||||
Other direct issue writers were inspected: checkout/adoption enter in-progress;
|
||||
conversation writers enter todo/in-progress/in-review; recovery enters blocked;
|
||||
release preserves existing terminal status; administrative release changes locks
|
||||
and assignment. Tree restoration excludes terminal snapshots. Route aliases and
|
||||
raw privacy SQL do not write terminal status. These paths do not create new
|
||||
invalidation obligations. They can restore a target before the worker runs; the
|
||||
worker reads its current status.
|
||||
|
||||
The initial decision feature (0197 / commit `9c1f8e7887`) already stored primary
|
||||
and secondary target links in the creation transaction. The later shared-helper
|
||||
extraction `ef33c1d9ed` preserved that behavior. There is no newly assumed legacy
|
||||
secondary-link backfill. Direct SQL writers or damaged/incomplete link data are
|
||||
outside the supported live writer contract; startup still examines all open rows.
|
||||
|
||||
## Scheduling and lifecycle
|
||||
|
||||
The owning service holds pending scopes and cursors, not a second durable queue.
|
||||
The database decisions, options, snapshots, and target rows remain truth. Each
|
||||
pass examines at most 100 rows/scopes, ordered by expiry and ID using the existing
|
||||
open-decision index. The cursor retains PostgreSQL timestamp text, including
|
||||
microseconds, rather than rounding through a JavaScript Date. An empty scope consumes one unit of the pass budget. It
|
||||
rotates unfinished scopes so a long startup scan does not starve new changes.
|
||||
|
||||
A settled notification replaces its scope with a fresh cursor. An admitted pass
|
||||
cannot delete that replacement when it completes. Failed rows receive individual
|
||||
retry scopes while later rows continue; query failures retain their scope.
|
||||
Retries use the coordinator's existing 30-second cadence only while reconciliation
|
||||
is unfinished. Unknown hints request a bounded full scan. Once all scopes settle,
|
||||
`hasPending` returns from memory and no future target scan is scheduled.
|
||||
|
||||
Intent registration is awaited before writes. Nested writes belong to the outer
|
||||
transaction. Rollback does not publish a change; ambiguous outcomes retain the
|
||||
existing XID reconciliation contract. Notifications cover the application's DB
|
||||
owner and child pools, not arbitrary SQL writers or independent active processes.
|
||||
|
||||
Drain preserves pending scopes without passive target SQL. Resume processes them.
|
||||
Shutdown checks cancellation between rows and awaits admitted expiry/delivery.
|
||||
Startup reconstructs anything left after process exit. Cloud's conservative
|
||||
sleep guards are unchanged; this establishes worker quiet, not permission to
|
||||
sleep with unsupported obligations or proof of host suspension.
|
||||
|
||||
## Current verification
|
||||
|
||||
All 248 cases in seven complete suites pass: 25 target, 80 routine, 54 decision,
|
||||
26 effect-recovery, 20 expiry, 42 startup and one scoped-deadline case. PostgreSQL
|
||||
suites use separate migrated external fixtures. Server typecheck and build pass.
|
||||
No new migration or table is added. The normal parent merge preserves landed
|
||||
expiry and all 21 independent master files. Server types and build use the
|
||||
updated Runner vendor (1,524 inputs and 1,563 outputs).
|
||||
|
||||
Four real regressions failed before repair: sub-millisecond paging, deletion
|
||||
behind a full valid startup page, unrelated deletion notifications and successful
|
||||
empty-company deletion notifications. They all pass. The new concurrency test
|
||||
observes the proposal's actual PostgreSQL lock wait while deletion is held.
|
||||
Temporarily removing the issue-row lock made the test fail because the proposal
|
||||
was accepted instead of waiting; restoring it passes. The mutation was restored
|
||||
before commit. No timeout or production hold was weakened.
|
||||
|
||||
The combined internal method had no production caller after this split. The
|
||||
server package exposes only its startup module; neither its exports nor the
|
||||
plugin SDK exposes the decision service. Its tests now call the dedicated TTL,
|
||||
target, effect and continuation workers, preserving their outcome assertions.
|
||||
The obsolete cursor, scan options and exported target-policy method are removed.
|
||||
HTTP and plugin SDK contracts are unchanged.
|
||||
|
||||
Fresh current-head CI and both automated reviews remain required. Historical
|
||||
fixture and CI failures below remain part of the verification record. Extra
|
||||
upstream embedded-only tests attempted during parent integration could not
|
||||
initialize locally; CI must cover that environment.
|
||||
|
||||
## Earlier verification
|
||||
|
||||
The initial 17-case real-PostgreSQL target suite passed. The combined selection
|
||||
passed 233 tests: those 17, 17 effect-recovery cases, 14 TTL cases, 50 decision
|
||||
service cases, 39 startup cases, 80 routine-service cases, and 16 routine-route
|
||||
cases. The final target run passed all 18 cases, including a change to the same scope
|
||||
during reconciliation. In total, 234 selected cases pass across these runs.
|
||||
|
||||
Coverage includes actual issue update/removal, decision creation after cancellation,
|
||||
secondary targets, all-target completion, intentional done targets, lenient history,
|
||||
outer commit/rollback, startup after deleted links, first change and failed expiry
|
||||
through drain, reopened targets, scope isolation, more than one page of valid
|
||||
open rows, failure fairness, creation during delivery, and shutdown. The original routine
|
||||
failure test observed intent and settlement around its cleanup transaction. The
|
||||
current test requires no target signal when that created issue has no linked
|
||||
decisions.
|
||||
Quiet assertions observe SQL through one hour of fake time after settlement.
|
||||
|
||||
Workspace typecheck/build passed before the final test-only addition. No runtime
|
||||
test failure was suppressed or timeout increased. Existing warnings from startup
|
||||
fixture signal listeners remain visible. The routine suite now supports the
|
||||
validated disposable external test database path, so local embedded PostgreSQL
|
||||
unavailability does not silently skip its 80 cases.
|
||||
|
||||
A local query probe with 40,000 completed decisions, one open decision, and 40,001
|
||||
target links used existing indexes: scoped lookup read five buffers (0.012 ms),
|
||||
full open-page lookup two (0.003 ms). The probe was rolled back. No new schema
|
||||
index is needed for this change. Full PR CI and fresh review remain required.
|
||||
|
||||
## Earlier review and CI repair
|
||||
|
||||
Review found that an absent deletion hint was dropped when the outer transaction
|
||||
also carried a scoped hint. Both delete-before-update and update-before-delete
|
||||
regressions reproduced the missing expiry after the target links cascaded away.
|
||||
That revision sent the explicit `all` hint alongside scoped hints; the current
|
||||
pre-delete decision-ID contract below supersedes the global hint.
|
||||
The repaired target suite passes 20 cases; the TTL suite passes 14 cases.
|
||||
|
||||
The initial CI run failed the TTL worker's drain assertion. It measured SQL
|
||||
before already-admitted sweeps had finished. The test now joins those specific
|
||||
sweep promises before measuring passive SQL; unresolved-intent holds remain
|
||||
intact. Waiting for the entire idle count to reach zero was tested and rejected:
|
||||
that count correctly includes the retained recovery holds. No production drain
|
||||
or retry behavior changed for this test repair.
|
||||
@@ -101,3 +101,48 @@ moves here. Other maintenance duties, external probes, and connected transports
|
||||
still affect idle traffic. No Cloud alarm coverage or host-sleep exemption changes.
|
||||
These tests prove the changed workers become quiet and recover; they do not prove
|
||||
deployed suspension or fleet savings.
|
||||
|
||||
## Custom-image setup expiry
|
||||
|
||||
Setup expiry uses a separate `custom-image-expiry` worker. The custom-image
|
||||
service owns the active-status predicate, the earliest saved `expiresAt`, and
|
||||
provider cancellation. Startup registers the worker in the existing enabled
|
||||
heartbeat housekeeping path; the maintenance tick no longer calls its sweep.
|
||||
|
||||
All supported setup producers live in `environment-custom-images.ts`: initial
|
||||
creation, provider refresh, cancellation/failure status, and promotion. They
|
||||
register intent in their owning transaction. A provider refresh can move the
|
||||
deadline earlier; a terminal transition removes it. Historical verification
|
||||
metadata does not change active expiry eligibility and does not need this signal.
|
||||
Environment deletion can only remove obligations; an already scheduled pass
|
||||
observes the removal. Independent SQL and other DB roots are outside the existing
|
||||
process-local notification contract.
|
||||
|
||||
No active dated session means no recurring reads or provider calls. Overdue
|
||||
batches and failures retain a bounded retry, while first writes and failed
|
||||
reconciliation survive drain. A missing expiry retains the existing behavior
|
||||
(no inferred expiration); sleep safety still treats unsupported live resources
|
||||
conservatively. Provider cancellation, failure state, and historical resource
|
||||
verification keep their existing semantics. No Cloud alarm coverage changes.
|
||||
|
||||
## Historical retry test observation
|
||||
|
||||
The historical verification worker intentionally does not delay server readiness.
|
||||
Its provider sweep finishes before the following database pending-state read
|
||||
registers a retry deadline. The regression test now waits for that future
|
||||
deadline before inspecting it; sweep completion alone is not scheduler settlement.
|
||||
The runtime worker, retry interval, and readiness contract are unchanged.
|
||||
|
||||
A controlled 150ms delay before the real pending-state query reproduced the
|
||||
current CI failure: the immediate assertion observed a null deadline. The
|
||||
corrected assertion passed with the same delay. The temporary delay was removed,
|
||||
and all 14 cleanup tests passed against an isolated PostgreSQL database. The
|
||||
existing external test-database hook is carried at the stack foundation so
|
||||
intermediate branches can use this same verification path.
|
||||
|
||||
An initial attempt at the foundation could not start embedded PostgreSQL because
|
||||
that earlier branch lacked the external fixture hook. A broader fixture wait for
|
||||
all idle holds to clear then failed three cases that legitimately retain pending
|
||||
cleanup obligations. That wait was removed. Only the historical deadline assertion
|
||||
changed; its requirement is now a deadline strictly in the future. The complete
|
||||
suite passes with the original readiness and idle-hold behavior preserved.
|
||||
@@ -0,0 +1,141 @@
|
||||
# Gateway credential expiry and idle behavior
|
||||
|
||||
Gateway sessions and named gateway tokens already persist their expiry times.
|
||||
`cleanupExpiredSessions` remains the owner of physical deletion. Authentication
|
||||
continues to check expiry, revocation, company scope, run ownership, and active
|
||||
run state on use. No table, authorization rule, or external alarm is added.
|
||||
|
||||
## Producer inventory
|
||||
|
||||
- `createSession` registers intent before inserting a run-bound session in a
|
||||
transaction. Its existing audit and returned credential remain unchanged.
|
||||
- `createNamedGatewayToken` registers intent before inserting an expiring token
|
||||
in a transaction. A token without an expiry owes no expiry cleanup and does
|
||||
not notify this worker.
|
||||
- Last-use updates do not change expiry. Token/session revocation, including
|
||||
heartbeat completion and connection removal, also leaves the saved expiry
|
||||
intact. Revoked rows retain the same physical deletion deadline.
|
||||
- Gateway/company deletion removes dependent rows and can leave one harmless
|
||||
stale deadline. It cannot create overdue work. No supported service changes
|
||||
a saved credential expiry after creation.
|
||||
|
||||
The database signal layer owns outer commit, rollback, savepoints, and uncertain
|
||||
outcomes. The worker supports producers sharing that signal owner. Independent
|
||||
SQL writers and another process need a separate live ingress contract. Startup
|
||||
reconstructs existing rows but does not provide continuous replica discovery.
|
||||
|
||||
## Worker contract
|
||||
|
||||
`nextCredentialExpiryAt` reads one earliest non-null expiry from each table using
|
||||
expiry-first indexes, then compares those at most two values. The migration adds
|
||||
the session expiry index; the token expiry index already exists.
|
||||
An empty credential store or retained non-expiring tokens returns no deadline.
|
||||
`registerGatewayCredentialExpiryWork` supplies cleanup and deadline selection to
|
||||
the shared `registerDeadlineWork` helper. Provider and authentication behavior
|
||||
stay in the gateway service; startup only registers the worker.
|
||||
|
||||
The worker starts where the old startup cleanup ran. With heartbeat scheduling
|
||||
enabled, committed changes and actual expiry times replace the recurring cleanup
|
||||
call. With heartbeat scheduling disabled, the existing startup-only cleanup is
|
||||
preserved. Tool action-review recovery is a separate obligation and is unchanged.
|
||||
|
||||
Token and session deletion are each bounded to 500 rows per pass. Session
|
||||
selection orders by expiry and rechecks expiry when deleting the selected IDs.
|
||||
A remaining overdue expiry schedules another pass after 30 seconds. Failed reconciliation also
|
||||
retries at that bound. No unfinished reconciliation or future expiry means no
|
||||
recurring SQL. A generation retains writes arriving during reconciliation.
|
||||
Startup restores deadlines; drain retains first writes without passive SQL;
|
||||
resume completes overdue cleanup. The coordinator owns shutdown and waits for
|
||||
in-flight work.
|
||||
|
||||
Sequential committed writes are coalesced for up to 30 seconds after a
|
||||
successful cleanup pass, matching the former heartbeat cleanup cadence. A
|
||||
previously loaded expiry can preempt that window. A newly reported credential
|
||||
may physically remain until that bounded follow-up; authentication still rejects
|
||||
it immediately at expiry. A successful pass with no future expiry and no new
|
||||
notifications creates no timer. This uses the shared optional notification
|
||||
coalescing capability, promoted from the later plugin/decision recovery slices;
|
||||
workers that omit it retain their existing behavior.
|
||||
|
||||
## Evidence and limits
|
||||
|
||||
The concurrent session index runs through the migration executor's existing
|
||||
per-file top-level path. Initial application and repeat application were tested.
|
||||
A PostgreSQL plan with 20,000 sessions and 20,000 tokens, including 19,000
|
||||
non-expiring tokens, used both expiry indexes and read one row from each.
|
||||
This establishes bounded lookup on that fixture, not production latency.
|
||||
|
||||
The new PostgreSQL suite exercises real session/token creation, configured idle
|
||||
state, non-expiring history, expiry ordering, revoked-token cleanup, outer commit
|
||||
and rollback, startup reconstruction, a backlog larger than one cleanup batch,
|
||||
and drain/resume. It tests expired authentication before deferred physical
|
||||
cleanup, through both session and named-gateway entry points. Existing gateway
|
||||
acceptance and server-startup tests cover the changed integration boundaries.
|
||||
|
||||
Use only disposable migrated databases for the test URL overrides. Their names
|
||||
must end in `_test`; the suites delete their fixtures. Ordinary CI uses embedded
|
||||
PostgreSQL. Test credentials are synthetic and no external provider is contacted
|
||||
by the expiry suite.
|
||||
|
||||
The migration proves worker behavior, not host suspension. Existing Cloud idle
|
||||
proof remains conservative. An in-process deadline cannot wake stopped compute;
|
||||
external wake coverage must be verified before changing sleep eligibility.
|
||||
|
||||
## Related unmerged token-lifetime work
|
||||
|
||||
At implementation time, PRs #17221 and #17274 change run-scoped token lifetime
|
||||
and cleanup eligibility. This slice preserves the current master policy and
|
||||
changes scheduling only. When combining those changes, the deadline query and
|
||||
producer signals must use the same token eligibility as cleanup: run-token idle
|
||||
markers must not schedule the operator-token expiry worker. Keep the run-liveness
|
||||
reaper with its own owner and retain both regression suites. A conflict-free
|
||||
text merge alone does not prove this integration.
|
||||
|
||||
## Notification amplification review repair (2026-10-27)
|
||||
|
||||
Ten sequential real session/token creation transactions reproduced ten extra
|
||||
instance-wide cleanup passes. With coalescing they cause one follow-up pass.
|
||||
A separate 501-session fixture reproduced the unbounded delete, then passed
|
||||
with 500 removed initially and the remaining overdue row removed at the saved
|
||||
30-second retry. A still-valid session survives both passes. The tests also
|
||||
preserve an already loaded expiry only 500 milliseconds away and verify that a
|
||||
new one-second session is rejected while physical cleanup waits for coalescing.
|
||||
|
||||
The initial full run exposed four outer-transaction tests whose fake-clock wait
|
||||
expected the intent hold to release before the new coalescing follow-up. Their
|
||||
settlement helper now advances the clock through that window while preserving
|
||||
the original commit/rollback assertions. The complete gateway expiry, gateway
|
||||
acceptance, custom-image deadline, and startup suites pass: 147 tests in four
|
||||
files. Server typechecking also passes on the owning branch. This is local
|
||||
behavioral evidence; combined-stack and fresh remote checks remain required.
|
||||
The full local test:run remains unverified because of embedded PostgreSQL
|
||||
startup/capacity limits; the two real gateway suites used separate migrated,
|
||||
task-owned disposable PostgreSQL databases.
|
||||
|
||||
A later combined run exposed a test-clock overshoot: repeated blind 30-second
|
||||
advances could expire the future session before its intermediate assertion.
|
||||
The settlement helper now lets real SQL settle and advances only to an actually
|
||||
armed follow-up within the coalescing window. All 12 credential expiry tests pass
|
||||
on the owning gateway branch with the existing exact-deadline and transaction
|
||||
assertions intact. This follow-up changes the test fixture only.
|
||||
|
||||
|
||||
## One-shot cleanup preserves complete backlog removal
|
||||
|
||||
Review found that bounding worker passes also truncated startup cleanup when the
|
||||
heartbeat scheduler was disabled. That path calls cleanup only once, so it could
|
||||
leave every expired token or session after the first 500 rows indefinitely.
|
||||
|
||||
`cleanupExpiredSessions` now drains the existing backlog in 500-row SQL batches
|
||||
by default, using one fixed expiry cutoff for the invocation. The deadline worker
|
||||
opts into `singleBatch: true` and retains its existing coalescing and next-page
|
||||
deadline. Future and nonexpiring credentials survive both paths. The return value
|
||||
still counts deleted sessions, as before. No periodic fallback was added.
|
||||
|
||||
Three real-PostgreSQL regressions reproduced the truncation with 1,001 expired
|
||||
sessions, tokens, or both, without registering a worker. They pass after the fix.
|
||||
All 55 expiry/startup cases pass, including the existing worker's 501-row bounded
|
||||
pass and commit/rollback tests. The original explicit-cleanup compatibility case
|
||||
and server typecheck also pass. Evidence is in `/tmp/sleep-master-gateway-manual-`
|
||||
logs: `red`, `green`, `compatibility`, and `types`. This fixes one-shot behavior;
|
||||
it does not establish provider suspension or complete the stack's CI acceptance.
|
||||
@@ -0,0 +1,121 @@
|
||||
# Idle Grok routine dispatch
|
||||
|
||||
The Grok dispatcher previously started a one-second timer whenever public MCP
|
||||
was configured. Each tick read experimental settings, even when Grok was
|
||||
otherwise disabled. Enabled ticks scanned prepared dispatches regardless of
|
||||
whether any work existed. The disabled baseline regression observed three
|
||||
settings queries in 3.2 seconds.
|
||||
|
||||
The dispatcher now registers with the existing delivery coordinator and shared
|
||||
scheduler. It performs one startup reconciliation. The worker becomes quiet
|
||||
when Grok is disabled or when no prepared dispatch remains. Quiet means no
|
||||
recurring worker SQL or provider calls, including with retained terminal history.
|
||||
This does not establish whole-instance or database suspension.
|
||||
|
||||
## Ownership and producers
|
||||
|
||||
`grokRoutineService.reserve` is the only supported creator of prepared dispatches.
|
||||
It signals intent before its insert in the existing transaction. Route verification,
|
||||
manual retry of a confirmed rejection, and native-broker assignment admission all
|
||||
use this service. The native broker passes its transaction, so the wake follows
|
||||
the outer commit. Savepoint release is not publication; rollback does not send.
|
||||
|
||||
`instanceSettingsService.updateExperimental` signals the same queue before its
|
||||
existing transactional update. This covers enablePublicMcp, enableNativeRunner,
|
||||
and enableGrokBot changes, including caller-owned outer transactions. It permits
|
||||
prepared work retained while disabled to resume when enabled. Managed settings
|
||||
and environment overlays are process configuration and are reconstructed on restart.
|
||||
Other settings writers update general/default-environment fields, not these flags.
|
||||
|
||||
All other dispatch writers move rows away from prepared: send claims, delivery
|
||||
settlement, acknowledgement, explicit recovery, assignment completion, and
|
||||
revocation. None reopens a row as prepared. They cannot introduce hidden new work.
|
||||
A previously scheduled backlog pass may observe cancellation and disarm.
|
||||
|
||||
Signals cover the current process's shared database owner and child pools.
|
||||
Independent SQL writers or another process need explicit ingress integration;
|
||||
this change does not claim cross-process notifications. Startup reconstructs
|
||||
committed prepared rows from the existing dispatch table. No queue table is added.
|
||||
|
||||
## Delivery and recovery
|
||||
|
||||
The Grok service continues to own current authority checks, binding locks,
|
||||
prepared-to-sending compare-and-set, secret access, provider delivery, and outcome
|
||||
settlement. The worker does not interpret provider payloads or relocate policy
|
||||
into app wiring. A foreground send and background recovery share the durable
|
||||
claim; only the winner contacts the provider.
|
||||
|
||||
Each pass processes at most ten prepared rows. The next lookup uses the same
|
||||
predicate and schedules the existing one-second cadence only while prepared work
|
||||
remains, or after a reconciliation failure. New commits wake the shared worker;
|
||||
its deadline adapter preserves requested generations through concurrent changes
|
||||
and drain. The partial created_at/id index keeps both selection and pending-work
|
||||
lookup independent of retained completed history. The migration builds it
|
||||
concurrently, is replay-safe, and uses the normal snapshot-retention workflow.
|
||||
|
||||
The existing send-error policy is preserved: if sending throws and the row is
|
||||
still prepared, the dispatcher cancels it. A row already claimed as sending is
|
||||
never reset or replayed. Accepted, sending, unknown, acknowledged, rejected, and
|
||||
cancelled rows do not become automatic retry work. Explicit operator retry still
|
||||
creates a new intent only for confirmed rejection. This migration does not claim
|
||||
exactly-once delivery or resolve uncertain provider outcomes automatically.
|
||||
|
||||
## Lifecycle and sleep boundary
|
||||
|
||||
The app registers only when public MCP is configured, after the shared coordinator
|
||||
exists. Registration does not delay readiness for a provider send. The coordinator
|
||||
tracks startup work and joins admitted sends on shutdown. Abort prevents the next
|
||||
row from starting; existing HTTP timeouts bound a current transport operation.
|
||||
Later prepared rows remain durable for restart.
|
||||
|
||||
Drain and warm standby retain pending requests without passive worker queries.
|
||||
Resume uses the same coordinator retry path; a first write or failed sweep cannot
|
||||
be forgotten merely because dispatch was paused. A disabled feature may retain
|
||||
prepared rows without scheduling a scan; a committed settings change reactivates
|
||||
it. Disabled feature state is not permission to discard durable work or relax
|
||||
Cloud sleep guards. Existing binding/assignment/hold blockers remain unchanged.
|
||||
|
||||
## Verification
|
||||
|
||||
- The baseline failed the disabled-feature quiet assertion with three SQL reads.
|
||||
- 21 real-PostgreSQL lifecycle cases cover disabled/configured idle, first real
|
||||
reservation, prepared startup recovery, all six non-prepared states, re-enable,
|
||||
outer commit/rollback, drain/startup, failed scan through drain, concurrent
|
||||
enqueue, two-page draining, foreground claim overlap, expired unsent work,
|
||||
settings transaction rollback/commit, and shutdown joining a send.
|
||||
- The existing 45 Grok qualification cases and 39 startup cases pass together
|
||||
with the new suite: 105 cases total.
|
||||
- The four affected settings suites pass 154 cases (259 selected cases total).
|
||||
- Workspace typecheck, production build, migration checks and replay pass. A
|
||||
generic-plan probe with 40,000 completed dispatches and one prepared row uses
|
||||
the partial index for both queries (six shared buffers including the routine
|
||||
join; 0.032ms batch and 0.025ms next-work execution in this local probe).
|
||||
- Remote PR review and CI must pass before the slice is ready.
|
||||
|
||||
The tests use the existing real OAuth/agent/routine fixture and a controlled
|
||||
provider response. A validated disposable external PostgreSQL option permits
|
||||
local execution where embedded PostgreSQL is unavailable; CI retains embedded
|
||||
PostgreSQL by default. Provider-host suspension is outside this local proof.
|
||||
|
||||
|
||||
## Current parent reconciliation
|
||||
|
||||
The normal merge of target parent `85ce6545bf` preserves its complete tree outside
|
||||
this duty's explicit files. Grok dispatch, reservation, settings and app wiring
|
||||
are unchanged from the preceding candidate. The index is regenerated as
|
||||
`0391_third_roulette.sql`, with concurrent/idempotent execution. All 389 parent
|
||||
journal entries are unchanged; five generated snapshots remain (0387–0391).
|
||||
Schema drift, migration safety/numbering and fresh apply/replay on two isolated
|
||||
PostgreSQL databases pass.
|
||||
|
||||
All 273 cases in nine complete suites pass at `6d81746fe2`: dispatch lifecycle,
|
||||
Grok qualification, server startup, scoped deadlines and all five instance-settings
|
||||
suites. Server typecheck and production build pass with the current Runner vendor.
|
||||
The earlier 259-case and populated-plan measurements above remain historical
|
||||
evidence; fresh exact-head reviews and CI are required.
|
||||
|
||||
An earlier browser CI run failed during agent-chat fixture restoration with a
|
||||
lock timeout/HTTP 500; the subsequent case observed the experiment left enabled.
|
||||
Those failures were not dispatcher assertions and are retained in the evidence.
|
||||
The later pre-reconciliation `a74305e2b4` checks were terminal and green. Neither
|
||||
that older result nor local acceptance substitutes for this candidate's CI.
|
||||
@@ -0,0 +1,116 @@
|
||||
# Interrupted tool-action recovery
|
||||
|
||||
An executing tool-action request has an existing ten-minute stale deadline from
|
||||
its `updatedAt` claim timestamp. Recovery now registers the earliest actual
|
||||
claim deadline with the shared scheduler. It no longer scans executing requests
|
||||
on every aggregate maintenance tick.
|
||||
|
||||
## Ownership and producers
|
||||
|
||||
The gateway owns execution claims, provider calls, and result settlement. The
|
||||
new interruption service owns only the deadline query and stale-row settlement.
|
||||
The worker uses the existing deadline registration and exposes no provider
|
||||
protocol or signed argument representation to the scheduler.
|
||||
|
||||
The review write helper signals interruption alongside approved recovery and
|
||||
pending expiry. All three agent claim paths and the Test-tab claim use that
|
||||
helper. Test-tab outcomes already commit both rows inside it. Agent pre-dispatch
|
||||
and execution failure settlement uses the same notification transaction, and
|
||||
successful or failed direct execution request updates now notify before writing.
|
||||
Administrative review transitions continue to use the same existing boundary.
|
||||
|
||||
Unrelated pending review changes can cause a bounded empty reconciliation. They
|
||||
do not create a periodic deadline. Process-local signals cover the existing
|
||||
single-active-writer database owner; independent writers require a wake contract.
|
||||
|
||||
## Settlement and quiet states
|
||||
|
||||
The query uses a partial executing-request index and reads one earliest row.
|
||||
There is no deadline after all executing claims settle. Retained terminal
|
||||
requests and configured-but-idle tools produce no recurring worker queries.
|
||||
|
||||
At the deadline, settlement locks invocation then request, matching the existing
|
||||
execution-failure lock order. It rechecks the claim status and timestamp after
|
||||
locking. A changed or completed claim cannot be settled from an earlier scan.
|
||||
Unknown outcomes atomically mark both rows failed with
|
||||
`tool_execution_outcome_unknown`. Recovery never calls the external provider.
|
||||
|
||||
A saved terminal invocation is authoritative. Recovery repairs its request to
|
||||
executed or failed without replacing the saved result, failure reason, or
|
||||
completion time with an unknown-outcome failure. This covers a crash between
|
||||
legacy invocation completion and request settlement. Interaction projection
|
||||
runs after the transaction through the existing gateway owner. A failed
|
||||
projection still relies on the separate projection-repair duty.
|
||||
|
||||
A database failure rolls back both outcome rows and retains a 30-second retry.
|
||||
Startup reconstructs saved deadlines. Drain pauses dispatch without forgetting
|
||||
the first claim or failed retry; resume processes overdue claims. Completion
|
||||
notifications cancel unnecessary future scans. The scheduler-disabled path keeps
|
||||
one startup pass. The public combined review sweep retains its existing duties.
|
||||
|
||||
## Verification and limits
|
||||
|
||||
Sixteen new real-PostgreSQL cases cover real foreground execution activation
|
||||
and completion, configured idle and terminal history, restart/deadline timing,
|
||||
six saved terminal invocation states, first-write drain/resume, outer commit and
|
||||
rollback, transactional failure/retry, concurrent claim change, and multi-page
|
||||
settlement. The service/startup selection passes 136 tests on the combined parent-review
|
||||
head. Workspace typecheck/build and migration safety checks pass on that head.
|
||||
Migration application and reapplication passed. The combined run also exercised
|
||||
469 tool-access tests: 468 passed and an unchanged connection-archive case timed
|
||||
out under concurrent build load. That case passes in isolation. The separate full tool-access suite passes all
|
||||
469 tests and gateway acceptance passes all 92 (697 across the final selections).
|
||||
Fresh full CI/review remains required.
|
||||
|
||||
The first typecheck caught optional approval IDs losing narrowing in transaction
|
||||
callbacks; capturing them before the callback fixes it. Two new fault/transaction
|
||||
cases initially waited for idle before advancing their next fake-clock retry.
|
||||
They now advance time while waiting for the authoritative outcome, and the full
|
||||
selection passes. These are fixture corrections, not permission to clear an
|
||||
unresolved work hold.
|
||||
|
||||
The ten-minute policy is unchanged. Active provider execution, unknown external
|
||||
outcomes, and other worker obligations still prevent unsafe sleep. No Cloud
|
||||
eligibility or external-alarm coverage changes here. Worker quiet is not evidence
|
||||
of whole-instance or database suspension.
|
||||
|
||||
## Sequential-write cost correction (October 27)
|
||||
|
||||
Review found that sequential tenant writes could each request a global
|
||||
interruption pass, even when no execution was stale. A real-SQL regression
|
||||
created ten tool review requests through the gateway and allowed each preceding
|
||||
pass to finish. It measured 11 stale-claim scans including startup. Concurrent
|
||||
notification coalescing alone did not cover this sequence.
|
||||
|
||||
The worker now uses the existing deadline helper's 30-second notification
|
||||
coalescing window, matching the former default maintenance cadence. The same
|
||||
regression measures one startup scan and one follow-up scan for those ten
|
||||
requests, then no worker SQL through an additional idle hour. A loaded stale
|
||||
claim deadline only 500 milliseconds away still preempts the window and
|
||||
quarantines the unknown outcome without invoking the provider. Foreground
|
||||
execution and result settlement remain immediate. Newly discovered work can
|
||||
wait up to this window; the ten-minute stale-claim policy is unchanged.
|
||||
|
||||
This is the bounded-frequency alternative raised in review. Discovery is still
|
||||
instance-wide, and real stale backlogs still drain through the existing pages.
|
||||
It does not claim company-isolated scans or constant cost for a real backlog.
|
||||
No notification and no saved execution deadline means no periodic scan. No new
|
||||
table, index, or scheduler is added. Startup, drain, retry and shutdown retain
|
||||
the existing owners.
|
||||
|
||||
The first full verification passed 175 cases and failed three fake-clock waits:
|
||||
settlement writes leave a notification pending for its coalesced follow-up, so
|
||||
waiting for the idle hold without advancing that armed deadline cannot finish.
|
||||
An initial shared clock adjustment then advanced an unrelated five-second
|
||||
expiry test too far (177 passed, one failed). The adjustment is now confined to
|
||||
the interruption tests. It lets real SQL finish and advances only to an actually
|
||||
armed short follow-up. The imminent-deadline test checks quarantine before that
|
||||
30-second window and separately waits for eventual quiet. Product assertions
|
||||
were retained.
|
||||
|
||||
The final complete gateway-service and startup-wiring suites pass all 178 cases
|
||||
without skips, and server typechecking passes, on owner source based on
|
||||
`e255d8eff3363cf0bb63541bdc7ef055d4321f83`. The attempted third test selector
|
||||
`deadline-work.test.ts` did not exist at this owner; no separate helper-suite
|
||||
pass is claimed. The shared helper is exercised by the full gateway suites.
|
||||
Fresh combined-source checks and remote review remain required.
|
||||
@@ -0,0 +1,183 @@
|
||||
# Join request retention while idle
|
||||
|
||||
Pending join requests already have durable activity timestamps. Retention now
|
||||
uses their next 90-day archive deadline instead of including them in the main
|
||||
maintenance tick's attention-feed scan. The regular attention UI still includes
|
||||
join requests. This changes automatic retention discovery, not access approval.
|
||||
|
||||
## Owner and eligibility
|
||||
|
||||
`join-request-retention.ts` owns candidate eligibility, its locked recheck and
|
||||
deadline. `decision-retention-sweep.ts` shares bounded paging, per-row retention
|
||||
transactions and failure handling with the approval-retention owner. The shared
|
||||
sweep consumes normalized candidates; each owner retains its table/query policy.
|
||||
A candidate is a pending_approval request belonging to an active company, with
|
||||
an invite whose company matches the request. Kept and already archived items do
|
||||
not schedule work. Invite expiration or revocation does not remove the existing
|
||||
attention item and therefore does not disable its retention. No provider policy
|
||||
is interpreted by the worker or shared scheduler.
|
||||
|
||||
The next deadline is the earliest eligible updatedAt plus 90 days, rounded up
|
||||
in SQL to the scheduler's millisecond resolution. It must not wake a fraction of
|
||||
a millisecond early and then wait thirty seconds for recovery. The partial
|
||||
pending index orders updated_at and id. Resolved request history is excluded by
|
||||
the index predicate. Each pass attempts at most 100 due rows. The private cursor
|
||||
preserves database timestamp precision so a failed page cannot starve later work.
|
||||
Errors keep the shared 30-second retry; successful full pages continue until
|
||||
work is exhausted. No eligible rows means no scheduled SQL or network scan.
|
||||
|
||||
Each candidate is re-read under its request row lock before retention changes.
|
||||
A writer that wins that lock first is rechecked before archival. If archival
|
||||
commits first, a later activity write leaves the item archived until explicit
|
||||
revival; updating activity does not clear existing retention choices. The tests
|
||||
cover the writer-first race and the archive-first committed outcome. Supported
|
||||
invite replay expires before the 90-day cutoff and Slack access replay deduplicates;
|
||||
a future producer that needs to reappear after archival must request revival.
|
||||
The shared retention owner still controls keep, archive/revive, version checks,
|
||||
activity logging, and notification creation. Archiving does not approve or reject a join.
|
||||
Join requests have no origin agent, so their archival creates no agent delivery.
|
||||
Company and invite joins preserve existing visibility policy; this does not add
|
||||
cross-table serializability beyond the existing retention contract.
|
||||
|
||||
## Producer audit
|
||||
|
||||
| Producer | Notification boundary |
|
||||
| --- | --- |
|
||||
| Invite acceptance creates a pending request | Existing acceptance transaction, before the request insert |
|
||||
| Supported gateway invite replay changes activity | Small writeJoinRequestChange transaction around the existing SQL update |
|
||||
| Automatic human approval after acceptance | Same helper around the status update |
|
||||
| Explicit approval and rejection | Same helper around the status update |
|
||||
| Slack self-service access request | Existing platform transaction, before the request insert |
|
||||
| Company pause, resume, archive, or deletion | Company-level retention notification inside the existing transaction |
|
||||
| Budget-driven company pause or resume | Same company notification in the accounting transaction |
|
||||
| Keep/unkeep, manual archive/revive, accepted archive proposal | Existing typed retention notification now routes join_request to its worker |
|
||||
|
||||
Provider calls, membership grants, and unrelated route behavior stay outside the
|
||||
small SQL mutation helper. Nested callers retain the existing outer transaction
|
||||
settlement and uncertain-outcome protocol. The access router itself requires a
|
||||
root database connection; it is not wrapped in a caller transaction.
|
||||
|
||||
API-key consumption only permits an already-approved request. It cannot create
|
||||
pending retention work and does not acquire a new wrapper or signal. Invite
|
||||
acceptance that reuses a human request does not update that request. Revocation,
|
||||
expiry, and acceptedAt changes do not change the feed predicate. Invite/company
|
||||
identity is immutable in supported routes; company deletion is the invite delete
|
||||
owner. A future producer that changes these facts must join the same contract.
|
||||
Process-local signals do not cover independent SQL writers or another instance.
|
||||
|
||||
## Lifecycle and limits
|
||||
|
||||
Startup reconstructs pending deadlines. The worker is registered only when the
|
||||
existing heartbeat maintenance switch is enabled. Writes during drain retain a
|
||||
wake without passive SQL. Failed work retains its retry across drain. Shutdown
|
||||
joins an admitted archive and stops before subsequent candidates; restart picks
|
||||
up the durable remainder. There is no new queue table or permanent DB listener.
|
||||
The in-process deadline is not a Cloud alarm. Existing unsupported-future-work
|
||||
sleep blockers remain in place.
|
||||
|
||||
## Earlier verification
|
||||
|
||||
Seventeen new real-PostgreSQL cases pass. They cover actual invite acceptance,
|
||||
rejection, human approval, gateway replay, and Slack access requests; empty and
|
||||
resolved-history quiet; saved deadlines; startup reconstruction; keep and manual
|
||||
archival controls; company pause/resume; unchanged expired/revoked invite policy;
|
||||
normal versus maintenance feeds; nested commit/rollback; first writes and failed
|
||||
work during drain; concurrent source updates; 101-row microsecond paging with a
|
||||
full failed page; and shutdown/restart.
|
||||
|
||||
Another 139 consumer cases pass: 71 invitation/startup cases, 21 approval worker
|
||||
cases, and 47 shared retention/attention/budget cases. This is 156 selected tests,
|
||||
not a full local suite claim. The first consumer command used a nonexistent
|
||||
retention test filename and ran only the approval suite; the corrected selection
|
||||
ran the actual retention suite against its dedicated external database.
|
||||
|
||||
Initial test failures exposed fixture assumptions: the access router cannot run
|
||||
on a transaction handle; unresolved recovery cannot be expected to become idle;
|
||||
Slack links require actual endpoint/principal ownership rows; and shutdown tests
|
||||
must establish admission and deterministic candidate order. Those fixtures were
|
||||
corrected without weakening assertions or production checks. The first replay
|
||||
fixture lacked required gateway credentials. The final fixture supplies test-only
|
||||
credentials. A reused transaction spy retained admission calls during the quiet
|
||||
assertion; restoring it before the quiet measurement fixed the measurement.
|
||||
|
||||
Migration 0387 applies, the full migration set replays, and the concurrent index
|
||||
statement replays idempotently. Migration safety and numbering checks pass; the
|
||||
five newest schema snapshots are retained. The actual next-deadline query with
|
||||
40,000 resolved requests and one pending request uses the pending index and
|
||||
executes in 0.017 ms locally. The probe rolls back its data. This is query-plan
|
||||
evidence, not production latency or measured savings. Workspace typechecking and the production build pass. Fresh PR CI/review remain
|
||||
required before readiness.
|
||||
|
||||
|
||||
## Current review repair and verification
|
||||
|
||||
The duplicate approval/join archive loop is now one `createDecisionRetentionSweep`
|
||||
implementation. Each owner supplies its candidate ordering boundary, page, locked current-state
|
||||
lookup and next deadline. This keeps eligibility, company/invite joins and
|
||||
source-row locks with their owner while sharing cursor/error/retention behavior.
|
||||
The existing 100-row budget, PostgreSQL text cursor and failure rotation remain.
|
||||
Both workers coalesce notification-driven global reads under a thirty-second
|
||||
aggregate budget; saved deadlines preempt it and no idle scan is scheduled.
|
||||
|
||||
Manual keep/archive/revive can materialize an unseen join request in the existing
|
||||
retention write transaction, as approvals already can. The company-scoped source
|
||||
is locked, its real activity is read, and existing versioned state is synchronized.
|
||||
Authorization, archive choices and other source kinds retain their contracts.
|
||||
No route bookkeeping, queue table, permanent listener or Cloud exemption is added.
|
||||
|
||||
Five real regressions failed before repair: three unseen-request controls returned
|
||||
404 and both owners rounded a saved microsecond deadline down. All pass. The first
|
||||
two-owner run passed 50 cases but failed two old producer expectations that used
|
||||
a truncated JavaScript Date. They now compute the expected scheduler millisecond
|
||||
independently from integer database microseconds; exact assertions remain.
|
||||
|
||||
The first missing-lock mutation exposed a weak test: its timing-only writer gate
|
||||
released before proving the sweep had reached the lock, so it still passed without
|
||||
`FOR UPDATE`. The strengthened test observes the actual PostgreSQL lock wait before
|
||||
releasing the writer. It passes with the production lock and fails with zero waiters
|
||||
when that lock is removed. The production source was restored in `finally` before
|
||||
commit. A separate archive-first committed-order case preserves the existing
|
||||
archived-stays-archived behavior and verifies explicit revival.
|
||||
|
||||
All 340 cases in nine complete suites pass at `405c7dcdef`, including all 31
|
||||
approval and 22 join worker cases. Server typecheck and production build pass.
|
||||
Normal integration of parent `0b0e844b77` changes only four independently merged
|
||||
Runner E2E accounting files and the ledger; runtime and acceptance sources remain
|
||||
byte-identical. Existing results retain their tested revision.
|
||||
|
||||
Generated concurrent/idempotent migration `0393_open_grandmaster.sql` adds only the
|
||||
pending-request index. All 391 parent journal entries and historical SQL files are
|
||||
unchanged; five generated snapshots remain (0389–0393). Migration safety/numbering,
|
||||
schema drift and actual apply/replay on six isolated PostgreSQL fixtures pass.
|
||||
Earlier setup, timing, mutation and CI failures remain recorded here and in the
|
||||
stack ledger. Fresh final-head CI, security and both 5/5 reviews remain required.
|
||||
|
||||
|
||||
## Retry progress under continuing arrivals
|
||||
|
||||
Each scan cycle saves the final due `(updated_at, id)` tuple, at full database
|
||||
precision. Its pages stop at that boundary. Later due rows cannot keep extending
|
||||
the cycle and postpone failed rows forever. The cursor still advances past an
|
||||
individual failure, so one bad request—or an entire failed page—does not block
|
||||
later healthy rows. Once that finite cycle ends, the next cycle revisits failures.
|
||||
There is no new durable queue or per-row retry registry. Restart reconstructs
|
||||
eligibility from existing rows.
|
||||
|
||||
Two real PostgreSQL regressions failed before this repair: continued newer overdue
|
||||
arrivals left a recovered approval and join request behind the cursor. Both now
|
||||
archive in the next cycle. All 55 worker cases pass, including the existing
|
||||
100-failure-page progress and microsecond-cursor cases. The nine-suite run passed
|
||||
341 cases but failed the attention suite's cleanup hook after its 16,400-run case:
|
||||
`delete from heartbeat_runs` exceeded thirty seconds. After vacuum/analyze of only
|
||||
the disposable attention fixture, all 31 attention cases pass without code or
|
||||
timeout changes. The other eight suites passed 311 cases in the original run;
|
||||
342 distinct cases pass across those complete-suite executions. This demonstrates
|
||||
recovery, not a proven explanation of the cleanup slowdown. Server typecheck and
|
||||
production build pass at `df1815fbba`; the subsequent master integration has an
|
||||
identical complete source tree.
|
||||
|
||||
Earlier CI at `df1d4d0` also failed the unchanged decision-expiry restart test:
|
||||
its earliest wake was a thirty-second retry rather than the saved expiry. The
|
||||
complete unchanged expiry suite passes all twenty cases locally. Its CI timing
|
||||
cause remains unproven; it is not labeled infrastructure or silently discarded.
|
||||
Fresh final-head CI and reviews must pass before merging.
|
||||
@@ -95,3 +95,20 @@ cover the surrounding login contracts.
|
||||
The fake-clock database fixture warms its connections and disables pool idle/lifetime
|
||||
expiry so time jumps test the worker rather than racing socket connection timers.
|
||||
Database suspension and reconnect behavior require separate qualification.
|
||||
|
||||
## CI fixture settlement across the migration stack
|
||||
|
||||
The setup-token consume fixture can finish its initial fake-clock jump while
|
||||
cleanup is still performing database work. A write during that pass can request
|
||||
another generation whose coalesced deadline lies beyond the jump. Waiting for
|
||||
zero tracked work without advancing to that registered deadline then times out,
|
||||
even though cleanup has a valid wake and can finish.
|
||||
|
||||
The existing fixture repair from `59f8c8b3e5` is now carried at the earliest stack
|
||||
branch so every intermediate PR receives it. The helper advances to registered
|
||||
deadlines only while work remains tracked, preserving its existing five-second
|
||||
observation limit and all production behavior. All 16 login-cleanup cases passed
|
||||
on this branch against a dedicated PostgreSQL database; the prior later-branch
|
||||
verification and failed CI history remain documented in the GitHub review-check
|
||||
migration audit. No runtime change or weakened quiet assertion is included.
|
||||
Evidence: `/tmp/sleep-login-settlement-owning-tests.log` and its zero status file.
|
||||
@@ -0,0 +1,90 @@
|
||||
# Secret-proposal expiry and idle behavior
|
||||
|
||||
Secret proposals already store `status`, `expiresAt`, and encrypted pending
|
||||
values. This worker uses those records; it adds no queue table or migration.
|
||||
|
||||
## Ownership and activation
|
||||
|
||||
`secret-proposals.ts` still owns quotas, authority checks, approval, expiry,
|
||||
ciphertext scrubbing, dependent binding rejection, audit records, and interaction
|
||||
reflection. `expiryDeadlines(companyId?)` returns one saved deadline per pending
|
||||
company. Normal write notifications reload only the affected company. Startup
|
||||
reconstruction loads all companies once. The manual `nextExpiryAt()` query remains
|
||||
available to existing callers.
|
||||
Terminal history does not create a deadline. Approval still rejects an expired
|
||||
proposal synchronously, even when maintenance is paused.
|
||||
|
||||
The supported producers register `secretProposalExpiry` intent before their
|
||||
writes in the owning transaction:
|
||||
|
||||
- `createWithinQuota`: both secret and binding creation, after quota admission.
|
||||
- `markApproved`: ordinary approval and prerequisite approval during a cascade.
|
||||
- `transition`: rejection, withdrawal, expiry, and dependent binding rejection.
|
||||
- Interaction resolution: rejection, withdrawal/cancellation, expiry, and failed
|
||||
acceptance execution that resolves a linked pending binding.
|
||||
- Company deletion: a company hint removes its cached deadline after deletion commits.
|
||||
|
||||
Each producer includes its company ID. The remaining proposal write only attaches
|
||||
an interaction ID and does not change expiry eligibility. Independent SQL writers
|
||||
or another DB root are not supported live producers of these process-local signals;
|
||||
startup reconstructs their persisted rows, but it is not a live replica contract.
|
||||
|
||||
The shared database signal boundary defers notification until the outer transaction
|
||||
settles. Nested savepoints, rollback, and uncertain commit outcomes retain the
|
||||
existing protocol. Callers do not pass recovery counters or reconnect the database.
|
||||
|
||||
## Scheduling and lifecycle
|
||||
|
||||
`registerSecretProposalExpiryWork` runs once at startup while heartbeat scheduling
|
||||
is enabled. It replaces the secret-proposal sweep in the aggregate maintenance
|
||||
tick. Disabled heartbeat scheduling keeps its existing behavior.
|
||||
|
||||
The worker reloads the affected company's saved minimum expiry after a committed
|
||||
change. It does not sweep future work. At a due deadline it sweeps at most 100
|
||||
expired proposals for that company, then reads that company's next deadline.
|
||||
Companies rotate fairly between bounded passes. Retained history creates no work.
|
||||
No pending proposal, unresolved write, or failed reconciliation means no recurring
|
||||
query. Unscoped legacy hints still reconstruct all companies.
|
||||
|
||||
`scoped-deadline-work.ts` owns scope invalidation, cached deadlines, fair dispatch,
|
||||
and failure backoff. It owns no SQL or proposal policy. Failed loads and failed
|
||||
company sweeps back off independently from 30 seconds to five minutes. A new
|
||||
write reloads that company without erasing its failed sweep's backoff. Other
|
||||
companies keep their own deadlines, including already-due work. The coordinator's
|
||||
optional `nextRunAtOnError` contract permits immediate eligible work only after
|
||||
the worker has recorded per-item backoffs. Failed transaction reconciliation and
|
||||
failed pending-state inspection retain the coordinator's normal retry.
|
||||
|
||||
Secret expiry opts out of the shared helper's default one-second pass spacing so
|
||||
newly learned short deadlines are not clamped. Other scoped consumers retain that
|
||||
default. Custom-image expiry continues to use `deadline-work.ts` unchanged. No
|
||||
shared scheduler or startup wiring interprets secret-proposal policy.
|
||||
|
||||
A request stays pending until reconciliation succeeds. Changes arriving during a
|
||||
pass, first writes during drain, and failures followed by drain retain a request.
|
||||
Passive drain performs no proposal SQL. Resume reconciles retained requests;
|
||||
restart reconstructs deadlines. Coordinator shutdown unsubscribes and waits for
|
||||
active work. Deferred physical cleanup does not extend approval authorization.
|
||||
|
||||
## Verification and limits
|
||||
|
||||
The PostgreSQL route suite exercises actual service producers, nested outer commit
|
||||
and rollback, quiet empty and terminal states, quiet future deadlines, automatic
|
||||
ciphertext scrubbing and dependent-card resolution, startup reconstruction,
|
||||
withdrawal, interaction rejection/failed execution, failure retry, and drain/resume.
|
||||
It also verifies that expired approval fails while cleanup is paused. The existing
|
||||
coordinator suite verifies fallback retry and uncertainty boundaries. New cases
|
||||
record actual SQL to prove normal writes and due expiry stay company-scoped; they
|
||||
also cover five-second deadlines, earlier replacement deadlines, company deletion,
|
||||
and another company making progress while one sweep fails. A deterministic
|
||||
shared-worker test covers a 250ms replacement deadline.
|
||||
|
||||
The worker tests use a controlled clock and a real PostgreSQL database. Existing
|
||||
route tests cover quotas, isolation, authorization, concurrency, and lifecycle
|
||||
semantics. The test-only database URL must refer to a disposable migrated database:
|
||||
the suite deletes its fixture tables. Ordinary CI uses its isolated embedded DB.
|
||||
|
||||
This is worker quiet, not proof of host or database suspension. No Cloud alarm or
|
||||
sleep-eligibility exemption is added. Existing sleep-safety checks remain in force;
|
||||
a local expiry timer cannot wake a stopped host. Tool review expiry and gateway
|
||||
session cleanup are separate migrations.
|
||||
@@ -0,0 +1,103 @@
|
||||
# Terminal workspace scan and retry deadline precision
|
||||
|
||||
Terminal workspace maintenance already uses the shared delivery coordinator and
|
||||
retains deadlines only for outstanding work. Its bounded scan must progress past
|
||||
ineligible rows so that later eligible work can finish and the worker can park.
|
||||
|
||||
The source query orders candidates by `(updated_at, id)`. PostgreSQL timestamps
|
||||
can contain microseconds. The former cursor used a JavaScript Date, which loses
|
||||
that precision. A skipped row could then compare greater than its own rounded
|
||||
cursor. Repeated full pages kept recovery active and prevented later cleanup.
|
||||
|
||||
The query now carries the database timestamp as text alongside each workspace.
|
||||
Only the private scan cursor uses that value. Workspace data, business deadlines,
|
||||
and the frozen rotation boundary keep their existing types and behavior. The
|
||||
cursor comparison uses the same timestamp/ID tuple as the ordering. The source
|
||||
service continues to own cleanup eligibility, authorization, concurrency fences,
|
||||
page limits, abort handling, failed-page recovery, and rotation under churn.
|
||||
No scheduler API, queue, migration, producer signal, or external alarm changes.
|
||||
|
||||
## Verification
|
||||
|
||||
The existing full-page starvation regression now gives its skipped rows actual
|
||||
PostgreSQL microsecond timestamps. Before the repair, the eligible workspace was
|
||||
never archived. After the repair, the full workspace service and terminal worker
|
||||
selection passes all 87 tests, including bounded rotation under continuous churn,
|
||||
changed rows behind the cursor, failed-page recovery, quiet workers, and shutdown.
|
||||
The service suite accepts the repository's validated disposable external
|
||||
PostgreSQL URL. Setup applies pending migrations before opening the test client,
|
||||
then clears fixture tables before the first test as well as after each test. This
|
||||
supports a fresh database and reuse after an interrupted test run. The database
|
||||
connection closes during teardown. Server typechecking passes.
|
||||
|
||||
This fixes recovery progress. It does not establish provider suspension or change
|
||||
which active or unresolved resources prevent instance sleep.
|
||||
|
||||
Workspace-wide typechecking and the production build also pass. Full sharded CI
|
||||
and fresh review remain required before this PR is ready.
|
||||
|
||||
Review follow-up reproduced the missing-schema failure against a newly created,
|
||||
unmigrated disposable database. The setup repair migrates that same database
|
||||
inside the suite; the full workspace-service selection then runs without manual
|
||||
schema preparation. The production cursor implementation is unchanged.
|
||||
|
||||
## Preserve one retry deadline through early inspection
|
||||
|
||||
Nested-clone migration and isolated-source rehoming kept a retry guard and a
|
||||
separate advertised scheduler deadline. Each value called the clock separately.
|
||||
When the clock advanced between those calls, an early inspection copied the
|
||||
earlier guard back into the advertised deadline. CI observed a one-millisecond
|
||||
backward change in an otherwise unchanged retry.
|
||||
|
||||
Each path now computes one retry timestamp and stores that value in all of its
|
||||
retry and scheduling state. The cadence, cleanup eligibility, quiet conditions,
|
||||
rotation, and wake contract do not change. No new timer, queue, or table is added.
|
||||
|
||||
The existing nested-clone regression and the isolated-rehome dry-run case now
|
||||
use a clock that advances one millisecond per read. Both failed before the fix
|
||||
with the same one-millisecond drift. Both exact-deadline assertions remain. The
|
||||
complete migration suites passed all 49 cases afterward, using real PostgreSQL
|
||||
and Git, including preserved files/history and interrupted migration recovery.
|
||||
The complete workspace-service and terminal-worker suites passed another 87
|
||||
cases; server typechecking passed. The two focused pre-fix failures used a name
|
||||
filter intentionally; the reported final 136 cases are four complete suites
|
||||
without skips. Validated external test database hooks keep each migration suite
|
||||
on its own disposable database and retain the existing embedded fixture fallback.
|
||||
|
||||
These are local correctness and scheduling results, not deployed host-sleep or
|
||||
external-wake proof. Remote acceptance must be checked at the published head.
|
||||
|
||||
## External fixture migrations (October 26 review repair)
|
||||
|
||||
Review found that the two migration suites opened external test databases without
|
||||
applying migrations. The previous verification used prepared databases, which
|
||||
hid this setup gap. Both selected retry cases failed against new empty disposable
|
||||
databases with missing `companies` and restore-point tables. Setup now applies
|
||||
pending migrations before creating its test client, matching the workspace
|
||||
service fixture. The embedded path is unchanged.
|
||||
|
||||
All 49 cases in the two complete suites then passed against those same databases,
|
||||
including Git/filesystem migration, recovery, and exact retry-deadline checks.
|
||||
The fixture itself prepared the databases. This repair changes no production
|
||||
recovery code. Evidence: `/tmp/sleep-workspace-fresh-red.log` and
|
||||
`/tmp/sleep-workspace-fresh-green.log`.
|
||||
|
||||
|
||||
## Current parent verification
|
||||
|
||||
Approval-retention parent `3efda33812` is integrated with normal merges. All
|
||||
workspace precision, retry and fixture implementations remain unchanged from the
|
||||
preceding candidate. All 136 cases in the complete workspace-service, terminal
|
||||
worker, isolated-rehome and nested-clone suites pass at `9d846c84e7`; server types
|
||||
and production build pass with the current Runner vendor.
|
||||
|
||||
The preceding `f0f0de1f6f` run also passed all 136 cases with three newly created,
|
||||
unmigrated external databases: the suites themselves applied the current
|
||||
migrations. The `9d846c84e7` run reused those clean fixtures after the parent's
|
||||
approval wake repair. This retains actual proof of fixture self-preparation
|
||||
without mislabeling the reused run as a fresh-database test. No new migration,
|
||||
queue, wake contract or workspace policy change is added by this PR.
|
||||
|
||||
Earlier microsecond starvation, one-millisecond retry drift and missing-schema
|
||||
failures remain documented above. Fresh exact-head CI and both 5/5 reviews remain
|
||||
required in stack order; the parent is not claimed merged.
|
||||
@@ -0,0 +1,155 @@
|
||||
# Pending tool-review expiry and idle behavior
|
||||
|
||||
Pending tool action requests already save their review deadline in `expiresAt`.
|
||||
This worker replaces the recurring pending-expiry scan with committed changes,
|
||||
startup reconstruction, and that saved deadline. It does not add a queue table
|
||||
or change the one-hour review policy.
|
||||
|
||||
## Ownership and producer inventory
|
||||
|
||||
The tool gateway owns request signing and authorization. `toolReviewExpiryService`
|
||||
owns pending-expiry selection and settlement. `registerToolReviewExpiryWork`
|
||||
supplies company-scoped operations to the existing shared scoped deadline worker. Startup wires
|
||||
that contract without interpreting tool or provider behavior.
|
||||
|
||||
The supported writers are:
|
||||
|
||||
- `tool-access-policy.ts` inserts an unsigned pending request without an expiry.
|
||||
It creates no expiry obligation yet. The existing signing grace remains intact.
|
||||
- The normal agent gateway signing path and the board test-call path assign the
|
||||
saved deadline. Each signals in its SQL transaction before that update.
|
||||
- Gateway signing failures, competing-call invalidation, invalid signatures,
|
||||
and legacy pending-to-approved/expired transitions signal before removing a
|
||||
pending obligation.
|
||||
- `commitToolActionReview` signals inside its existing human-resolution
|
||||
transaction. The issue/interaction locks, human authority checks, and delivery
|
||||
enqueue remain owned there.
|
||||
- Administrative interaction resolution signals in the caller-owned transaction
|
||||
before resolving linked requests. Review-queue invalidation signals before
|
||||
invalidating unreadable or unsigned requests.
|
||||
- Company deletion signals its company before removing saved review obligations.
|
||||
- The expiry worker signals before its own settlement. A compare-and-set against
|
||||
pending status and the observed update time protects concurrent resolution.
|
||||
|
||||
Every notification requires the company ID. Normal commits reload only that
|
||||
company's saved minimum expiry; due cleanup selects only that company. Startup
|
||||
reconstructs all saved company deadlines once. A company with only unsigned
|
||||
null-expiry requests has no deadline.
|
||||
|
||||
Only the SQL mutation is wrapped. Signing, provider requests, telemetry, and
|
||||
interaction publication are not moved inside an enclosing recovery transaction.
|
||||
The database signal layer handles nested savepoints, outer commit/rollback, and
|
||||
uncertain transaction outcomes. No recovery object is passed through callers.
|
||||
Independent SQL writers and other database owners/replicas still require a live
|
||||
notification contract. Startup reconstructs saved rows but is not ongoing
|
||||
cross-process discovery.
|
||||
|
||||
## State and recovery
|
||||
|
||||
Expiry updates the authoritative request and invocation in one transaction. If
|
||||
invocation settlement fails, the request remains pending and the worker retries.
|
||||
After commit, the gateway reflects the result in the linked interaction. The
|
||||
existing terminal projection-repair duty remains in `sweepActionReviews` and
|
||||
repairs a crash or failure between authoritative settlement and projection.
|
||||
That duty is a separate migration; this worker must not claim it has completed.
|
||||
|
||||
The public/manual `sweepActionReviews()` still includes expiry by default. Server
|
||||
startup and the recurring legacy review sweep pass `includeExpiry: false` when
|
||||
expiry has its own startup pass or worker. Approved-action recovery, interrupted
|
||||
execution quarantine, and terminal projection repair retain their behavior.
|
||||
An unknown external outcome is never replayed by this expiry worker.
|
||||
|
||||
The next deadline comes from pending rows only. Startup uses the existing
|
||||
partial expiry index to reconstruct company deadlines. A company notification
|
||||
uses an ordered, single-row lookup backed by the partial `(company_id,
|
||||
expires_at)` index, rather than aggregating that company's pending backlog.
|
||||
Null-expiry unsigned requests and retained resolved history do not keep this
|
||||
worker active. The concurrent index migration uses the existing top-level
|
||||
executor and is safe to apply again after interruption.
|
||||
|
||||
The shared coordinator coalesces changes and preserves a write arriving during
|
||||
reconciliation. Each due pass expires at most 100 rows for one company; companies
|
||||
rotate between passes until the backlog drains. Failed scopes retry independently
|
||||
with exponential backoff from 30 seconds to five minutes. Other companies retain
|
||||
their own deadlines. New writes do not erase a failed sweep's backoff.
|
||||
Once reconciliation completes with no deadline, it has no recurring SQL. Drain
|
||||
retains first writes without passive database scans; resume processes overdue
|
||||
reviews. Startup reconstructs persisted deadlines. Shutdown stops subscription
|
||||
and timer ownership and joins active work.
|
||||
|
||||
Approval still rejects an expired request synchronously before physical cleanup.
|
||||
With heartbeat scheduling disabled, startup-only expiry remains as before.
|
||||
|
||||
## Verification scope
|
||||
|
||||
Real PostgreSQL tests record SQL to prove signing, human decisions, and due expiry
|
||||
remain company-scoped. They also verify five-second and earlier replacement
|
||||
deadlines, company deletion, and bounded multi-pass draining. Existing cases cover
|
||||
normal gateway signing and human decisions,
|
||||
configured idle and terminal history, outer commit/rollback, startup/restart,
|
||||
drain/resume, concurrent producer/read, unsigned rows, multi-page expiry, and
|
||||
injected invocation-write failure with atomic rollback and retry. Existing
|
||||
service tests cover approval/execution, interrupted outcomes, and projection
|
||||
repair through the compatible manual entry point. Startup tests cover wiring.
|
||||
The external test database override requires a disposable database ending in
|
||||
`_test`; the suite deletes fixtures and stops the worker before database cleanup.
|
||||
|
||||
This is one worker migration, not proof that the instance can sleep. Other tool
|
||||
review duties remain periodic. Existing Cloud sleep-safety gates and external
|
||||
alarm coverage are unchanged. An in-process deadline cannot wake a stopped host.
|
||||
|
||||
## Scoped lookup review repair (2026-10-27)
|
||||
|
||||
The original scoped aggregate examined 12,004 rows to return one deadline in a
|
||||
real PostgreSQL regression. The repaired lookup returns the earliest non-null
|
||||
pending expiry for the requested company with `ORDER BY expires_at LIMIT 1`.
|
||||
The grouped startup inventory remains unchanged. Existing company signals,
|
||||
authority checks, expiry settlement, paging, backoff, and sleep gates remain
|
||||
owned by their existing services.
|
||||
|
||||
The additive company-expiry index is migration `0383_tool_review_company_expiry`.
|
||||
It builds the index concurrently and idempotently through the existing top-level
|
||||
migration executor. Initially reserving number 0402 passed per-branch checks but
|
||||
caused CI to reject descendants whose migrations preceded their target branch.
|
||||
A local 0380 correction fixed that ordering, but master had meanwhile added its
|
||||
own 0378 and 0379 migrations. The integrated stack preserves master's history,
|
||||
renumbers its unmerged migrations after 0379 in PR dependency order, and preserves
|
||||
every migration's SQL bytes. Exact filename references and journal tags/indices
|
||||
follow the new names. Stack journal timestamps remain ordered after master's
|
||||
latest entry. Each generated snapshot includes both master and that PR's schema.
|
||||
|
||||
Each PR must pass the exact target-branch migration-order gate, in addition to
|
||||
the existing numbering, safety, five-snapshot retention, and schema-drift checks.
|
||||
Local branch checks alone did not establish the stack's append-only ordering.
|
||||
Fresh application, upgrades from the previous numbered task databases, and
|
||||
replay also require verification. These unmerged stack migrations have not
|
||||
changed the SQL of migrations already on master.
|
||||
|
||||
Two failures shaped the repair. The first migration attempt omitted its explicit
|
||||
top-level registration and PostgreSQL rejected the concurrent build inside a
|
||||
transaction; adding the registration fixed application and replay. An initial
|
||||
three-column index also lost to the global expiry index in the planner, which
|
||||
filtered 12,001 foreign-company rows before returning a result. The final index
|
||||
omits the unused request ID. With 12,000 additional pending rows in each of two
|
||||
companies, including earlier foreign deadlines, all three measured reloads
|
||||
examine one row. The assertion counts filtered rows as well as returned rows.
|
||||
This is evidence for the recorded fixture and query plan, not a production
|
||||
latency guarantee or a claim that PostgreSQL can never choose another plan.
|
||||
|
||||
The complete gateway service, gateway acceptance, credential expiry, startup,
|
||||
custom-image deadline, and schema snapshot suites pass: 223 tests across six
|
||||
files. The run also exposed a credential test-clock overshoot; its correction
|
||||
belongs to the earlier gateway PR and waits for real SQL before advancing to
|
||||
an actually armed follow-up. Commit/rollback and exact expiry assertions remain.
|
||||
Server typechecking passes. Combined-stack migration upgrades, full build, and
|
||||
fresh remote checks remain required. Full local test:run is still unverified
|
||||
because of embedded PostgreSQL startup/capacity limits; real gateway suites used
|
||||
isolated, migrated test databases in the task-owned PostgreSQL instance.
|
||||
|
||||
|
||||
A later review found an unbound `query` name in the SQL-capture assertion.
|
||||
The due-row predicate short-circuited before that expression in the passing
|
||||
fixture, and server typechecking excludes test sources. The assertion now
|
||||
independently requires both the due expiry scan and the non-null deadline lookup
|
||||
through each captured `read.query`. The earlier passing run did not detect this
|
||||
name error; acceptance must include the service suite with both assertions.
|
||||
@@ -0,0 +1,229 @@
|
||||
# Tool-review interaction repair
|
||||
|
||||
The tool-action request and invocation are authoritative. The interaction card
|
||||
is a projection of their outcome. Previously the aggregate maintenance tick
|
||||
joined these tables repeatedly, even when every card already matched.
|
||||
|
||||
The projection worker reconstructs affected companies at startup and reconciles
|
||||
after committed changes. Producer hints carry the company ID. Subsequent
|
||||
candidate discovery and paginated repair queries filter by that company; review
|
||||
writes cannot drive a scan of another company's terminal history. The shared
|
||||
scoped worker dispatches one company per pass with one-second minimum spacing.
|
||||
It retains a 30-second retry only for a mismatch or failed reconciliation.
|
||||
Configured idle tools, pending reviews without terminal outcomes, and matching
|
||||
terminal history have no recurring projection SQL or provider calls. This is
|
||||
worker quiet, not whole-instance suspension.
|
||||
|
||||
## Ownership and trigger contract
|
||||
|
||||
`tool-review-projection.ts` owns candidate discovery and pagination. The gateway
|
||||
retains card interpretation, current-state checks, locks, activity publication,
|
||||
and the existing continuation callback. The scheduler consumes company discovery,
|
||||
`sweep(companyId)`, and the returned retry obligation. No provider payload enters shared scheduling.
|
||||
No table is added. A concurrent company/interaction lookup index supports the
|
||||
existing link checks and linked-request updates.
|
||||
|
||||
The existing review notification helper now also signals projection repair.
|
||||
Notifications are awaited before writes and inherit the outer transaction's
|
||||
commit, rollback, and uncertainty tracking. Unrelated review changes can cause
|
||||
one empty company-scoped reconciliation; they do not create a recurring deadline.
|
||||
Interaction-only writes first check for an actual linked tool request using the
|
||||
company and affected card IDs. Generic confirmation kinds alone do not signal
|
||||
any tool-review queue. This also preserves recovery for legacy linked cards
|
||||
without current tool payload flags. These reads happen in the mutation owner,
|
||||
not a background poll. Sibling selection shares the update predicate and runs
|
||||
under the existing issue lock before publishing the notification.
|
||||
|
||||
| Producer | Notification ownership |
|
||||
| --- | --- |
|
||||
| Gateway signing, approval inference, execution claims, provider outcomes, and invalidation | Existing `writeToolReviewChange`; the closed-issue expiry path now uses it too |
|
||||
| Human approval/rejection | `commitToolActionReview` signals in its existing transaction |
|
||||
| Pending expiry, approved recovery, interrupted claims, Test-tab settlement | Existing review-write boundary |
|
||||
| Review-queue invalidation | Existing review-write boundary |
|
||||
| Interaction withdrawal, skip, terminal-issue expiry | Linked-request resolution signals before the mutation |
|
||||
| Linked tool-card accept/reject, deleted-addressee cancellation | Projection notification before replacing card state in the owning transaction; unlinked confirmations do not notify |
|
||||
| Sibling supersession, startup duplicate cleanup, new/historical comment supersession | Projection notification before card replacement, including cases where the request is already terminal |
|
||||
| Comment-based confirmation resolution | Calls the interaction owner inside the same outer transaction; its later comment-ID update shares that signal |
|
||||
|
||||
The audited other interaction writers do not create this mismatch: native bridge
|
||||
and status-committer summary/status-only updates preserve the projection; native
|
||||
review retirement owns separate completion-review cards; connection and secret
|
||||
flows own their separate card types. Document-staleness and plan-resume paths
|
||||
use document targets, while tool cards use `tool-action:` custom targets.
|
||||
Reflection consent uses its instruction/profile/skill target keys, not tool-action
|
||||
keys. A future writer that replaces tool-card result state must use the same
|
||||
transaction notification boundary. A future writer that changes authoritative
|
||||
tool outcomes must signal the review boundary.
|
||||
|
||||
## Repair, concurrency, and lifecycle
|
||||
|
||||
The mismatch predicate preserves the previous policy: terminal executed, failed,
|
||||
expired, or cancelled requests with a linked interaction whose displayed tool
|
||||
status differs. Cancelled requests display `expired`. Invocation error and result
|
||||
summaries come from saved state. Repair never repeats a provider action.
|
||||
|
||||
Candidate queries page in request-ID order, 100 rows at a time. A card that cannot
|
||||
yet be projected does not starve later pages. Per-row failures do not prevent
|
||||
attempting the remaining candidates; failures keep the retry. A final mismatch
|
||||
check retains retries for no-op repairs rather than silently declaring them done.
|
||||
Failed companies retain their own backoff across notifications while other
|
||||
companies progress. An unresolved no-op projection retains a 30-second deadline;
|
||||
a new relevant company change can make it actionable sooner. A company pass
|
||||
still drains all its pages, preserving progress past an unprojectable card.
|
||||
The joins inspect that company's history on activation. This change removes repeated
|
||||
idle scans; it does not introduce an indexed projection-debt table or claim that
|
||||
an activated reconciliation has constant cost independent of history size.
|
||||
|
||||
The gateway locks the interaction and request, then checks the current request
|
||||
status. If another repair already wrote the same status and outcome fields,
|
||||
it avoids a second write and duplicate activity. Different saved details still
|
||||
use the existing update path. Missing projection state remains repairable.
|
||||
|
||||
The shared deadline worker retains first writes during drain, failed retries,
|
||||
and changes arriving during reconciliation. Shutdown joins coordinator work.
|
||||
Startup reconstructs debt from existing rows. With heartbeat scheduling disabled,
|
||||
startup still performs one pass, preserving the existing disabled-mode contract.
|
||||
The public/manual combined review sweep remains available; server maintenance
|
||||
no longer calls it periodically.
|
||||
|
||||
## Continuation and sleep limits
|
||||
|
||||
Card state is not proof that continuation delivery finished. Existing
|
||||
`tool_action_deliveries` receipts and their coordinator remain responsible for
|
||||
that work. Activity publication still occurs after the card transaction; this
|
||||
slice does not add exactly-once activity delivery or a new outbox.
|
||||
|
||||
Signals remain process-local to the supported database owner. Independent SQL
|
||||
writers and another active server require a separate notification/reconciliation
|
||||
contract. Cloud eligibility still conservatively blocks retained tool-action
|
||||
requests. No external wake or sleep-safety guard is relaxed here.
|
||||
|
||||
## Verification
|
||||
|
||||
Twelve new real-PostgreSQL cases exercise normal approval and single execution,
|
||||
quiet configured and terminal-history states, four persisted outcome states,
|
||||
outer commit/rollback, first write during drain, failure retry through drain,
|
||||
foreground/worker repair contention, multi-page progress past an unprojectable
|
||||
card, and an administrative card overwrite that reactivates repair. Quiet checks
|
||||
observe absence of database reads across an hour of controlled time.
|
||||
|
||||
The first focused run caught a missing-projection null access in the new
|
||||
idempotency check; optional access fixes it. The pagination fixture also attempted
|
||||
to change a referenced request ID; it now orders the additional candidates without
|
||||
mutating that key. The focused rerun passed all eleven then-existing tests.
|
||||
After adding the concurrent repair case, all 148 service/startup tests and 92
|
||||
gateway acceptance tests passed. Workspace typechecking and build passed. All 469 tool-access and 106 interaction
|
||||
service tests also pass (815 tests across the final selections). The first
|
||||
interaction run passed 105 cases but timed out in one cleanup hook during a
|
||||
53-minute local run; the isolated full rerun passed in nine seconds. No production
|
||||
change or timeout increase was made to force that result. Full repository tests
|
||||
will run in CI; the local selections do not claim a complete repository test run.
|
||||
Fresh PR CI/review remains required.
|
||||
|
||||
Company-scoping follow-up reproduced two regressions: captured recovery SQL lacked
|
||||
a company predicate, and ordinary confirmation creation/replacement/cancellation
|
||||
published three unnecessary tool-projection wakes. Both regressions now pass
|
||||
through real producers. Additional coverage keeps failed-company retry timing
|
||||
independent, proves ordinary mutations still complete without recovery scans,
|
||||
and exercises administrative repair of legacy cards identified by their saved
|
||||
link rather than payload flags.
|
||||
|
||||
Company-scoping follow-up verification: all 180 gateway and startup cases and all 152 interaction and secret-proposal route cases passed against dedicated PostgreSQL test databases. Server typechecking passed. These are local checks; stack CI and deployed sleep qualification remain separate.
|
||||
|
||||
|
||||
## Indexed interaction links
|
||||
|
||||
Review found that the new link check had no supporting interaction index. An
|
||||
ordinary unlinked confirmation could scan all tool-action history for its company.
|
||||
The generated `0386_steady_epoch.sql` migration adds the non-unique
|
||||
`(company_id, interaction_id)` index concurrently, and the existing top-level
|
||||
migration executor handles its creation and interrupted-build recovery. The
|
||||
schema and generated snapshot describe the same index; older SQL and journal
|
||||
entries are preserved and snapshot pruning retains the five newest snapshots.
|
||||
|
||||
A real PostgreSQL check applied the migration to a fresh dedicated test database
|
||||
and seeded 30,000 tool-action rows. The ordinary missing-link query returned no
|
||||
rows in both treatments: without the index it performed a sequential scan,
|
||||
filtered 30,000 rows and touched 429 shared buffers; with the index it used the
|
||||
new index and touched two buffers. These are local planner measurements, not a
|
||||
fleet latency or cost claim. The baseline index removal was rolled back in that
|
||||
dedicated database, and the index remained valid afterwards. An initial probe
|
||||
failed while binding a JSON fixture parameter before any comparison; correcting
|
||||
the parameter encoding made the complete probe pass. That failed fixture is not
|
||||
counted as a product regression.
|
||||
|
||||
Migration numbering/safety checks, generated snapshot drift, server types and
|
||||
server build pass. All 214 cases in the full gateway/projection/startup/scoped-
|
||||
deadline selection pass against a separate freshly migrated database, preserving
|
||||
notification quiet, company isolation, transaction settlement, and interruption
|
||||
behavior. Latest-head CI and automated reviews remain required merge gates.
|
||||
|
||||
## Bounded projection and lifecycle interruption
|
||||
|
||||
A review found that one company could drain an unlimited backlog in a single
|
||||
worker pass. Five real PostgreSQL regressions reproduced 501 rows being consumed
|
||||
instead of the 500-row budget, missing abort/drain handling, and the absence of
|
||||
continuation past a full page of unprojectable or failed cards.
|
||||
|
||||
The worker now gives its existing service a 500-row budget, the coordinator's
|
||||
abort signal, and its admission predicate. The service checks admission between
|
||||
rows and reads at most 100 candidates per page. It retains the last attempted
|
||||
request ID for each unfinished company, including after a failed pass, so a
|
||||
poisoned early page cannot hide later healthy cards. A truncated pass schedules
|
||||
a one-second continuation; the existing shared per-company fairness and failure
|
||||
backoff remain in force. A completed pass that still has an unprojectable card
|
||||
backs off from thirty seconds to five minutes. Fresh committed company changes
|
||||
reset that delay and wake the affected scope promptly. These are real unresolved projections,
|
||||
not an unconditional empty-history scan.
|
||||
|
||||
Reconstruction clears continuation state for companies that no longer have
|
||||
mismatches. A sixth real regression reproduced an abandoned cursor skipping the
|
||||
first new earlier card after an empty reconstruction; clearing that cursor makes
|
||||
the first new pass repair it immediately. The maps contain no durable truth:
|
||||
restart reconstructs from the existing rows, and completed rows stop matching.
|
||||
There is no cursor table, independent timer or per-producer recovery wrapper.
|
||||
|
||||
Legacy manual/global and scheduler-disabled one-shot repair calls retain their
|
||||
complete-sweep behavior unless they request a budget. The normal coordinator
|
||||
worker always supplies the bounded options. Cancellation is observed between
|
||||
rows and pages; an already issued database query or reflection finishes before
|
||||
the service stops. A sweep never replays a provider tool action.
|
||||
|
||||
All 220 cases across the full gateway/projection/interruption, startup and scoped
|
||||
deadline suites pass, with no skipped cases. The five initial regressions and
|
||||
sixth cursor regression failed before their fixes; server types and build pass.
|
||||
The earlier 219-case acceptance remains evidence for the preceding revision.
|
||||
Fresh integrated-head CI and both automated reviews remain merge gates.
|
||||
|
||||
|
||||
After normal reconciliation with master `f6bc653f7f`, all 320 integrated cases
|
||||
pass, including the 81 authenticated gateway HTTP cases and the upstream narrowed
|
||||
identity-read tests. Server types and build pass. Thirty-five independent master
|
||||
performance files are preserved exactly; the gateway's narrowed identity read
|
||||
is preserved alongside the projection extraction. The pre-integration 220-case
|
||||
selection above remains evidence for its stated revision.
|
||||
|
||||
|
||||
## Review efficiency follow-up
|
||||
|
||||
A company-scoped hint now enters the bounded owner directly rather than first
|
||||
repeating company discovery. Startup/global reconstruction still discovers
|
||||
companies from existing rows. An empty sweep with no saved cursor returns after
|
||||
one candidate query; a resumed cursor or attempted reflection still needs the
|
||||
final query to detect unresolved earlier cards. The reflection callback returns
|
||||
no settlement result, so an attempted reflection cannot establish that a card
|
||||
was repaired.
|
||||
|
||||
Returned unresolved work now backs off through 30, 60, 120, 240 and 300 seconds,
|
||||
matching the shared error-backoff cap. Truncated backlog continuation remains
|
||||
one second. Settlement removes the retry; a new committed hint resets backoff.
|
||||
No pending row is discarded, and no independent timer or table is added.
|
||||
|
||||
The fixed-cadence regression failed before repair and now verifies the full
|
||||
backoff cap, producer reactivation and quiet after settlement. An initial scan
|
||||
assertion incorrectly assumed that a complete approval emitted only one wake;
|
||||
its separate legitimate writes caused two scans even after optimization. The
|
||||
corrected case isolates one committed hint and verifies one candidate scan with
|
||||
matching history, followed by an empty direct sweep with one query. All 222
|
||||
cases in the full gateway, startup and scoped-deadline suites pass without skips;
|
||||
server typecheck and build pass. Fresh-head CI and both reviews are still required.
|
||||
@@ -35,6 +35,13 @@ maintains as a dependency; once you install the packages below, they become
|
||||
normal dependencies of **your own** project, and your own Dependabot updates
|
||||
them.
|
||||
|
||||
An initialized SDK does not confirm ingestion. Verify a fresh trace from the
|
||||
deployed revision at the configured destination. A rejected export can leave
|
||||
the application healthy. For Honeycomb, configure the ingest key in the
|
||||
`x-honeycomb-team` entry of `OTEL_EXPORTER_OTLP_HEADERS`; see
|
||||
[Honeycomb authentication](https://docs.honeycomb.io/api/query-annotations/authorization).
|
||||
Keep credential values out of logs and diagnostic reports.
|
||||
|
||||
## Budget admission, accounting, and enforcement
|
||||
|
||||
The operator-gated `paperclip.budgets` tracer records
|
||||
@@ -313,6 +320,9 @@ attempt or owning phase:
|
||||
| `runtime_selection.*` | Completion contracts, session/recovery reads, profiles, interaction materialization, backend discovery, bootstrap, run/coordinator writes, transaction entry/settlement, and workspace synchronization. |
|
||||
| `runtime_context.instructions`, `.skills`, `.mcp` | The three concurrent runtime-context branches. Their durations overlap. |
|
||||
| `runtime_context.asset_*` | Source reads, manifest hashing, cache lookup, cold materialization, and exact file verification on cold and warm paths. |
|
||||
| `skills.inventory`, `.wait`, `.bundled`, `.releases`, `.folders`, `.local_sources` | Company skill reconciliation, waiting for an existing company refresh, and its bundled imports, release verification, folder, and local-source work beneath runtime skill preparation. Inventory children overlap their parent. |
|
||||
| `skills.cache.acquire`, `skills.rows`, `skills.version.read`, `skills.repository.read`, `skills.version.materialize`, `skills.version.verify` | Dedicated connection/transaction startup plus advisory-lock acquisition, authorized skill rows, frozen version and repository reads, and exact runtime materialization/validation. Lock acquisition ends before the protected work; version verification is inside materialization. Only bounded file counts are added. |
|
||||
| `skills.consume` | The caller's delivery and durable version-pinning callback while the cache lock remains held. |
|
||||
| `workspace_sync.prepare_host_current`, `.prepare_durable_seed`, `.prepare_adopt_remote` | Runtime preparation under the selected inbound mode, including snapshot and staging work. A warm mismatch may produce a second preparation; it is real repeated work. |
|
||||
| `workspace_sync.source_root`, `.repositories`, `.ignore_rules`, `.snapshot_git`, `.snapshot_baseline` | Physical source capture, managed repositories, ignore rules, Git metadata and baseline content snapshot. |
|
||||
| `workspace_sync.pack`, `.git_clone`, `.git_archive`, `.overlay_copy`, `.overlay_archive`, `.deleted_manifest` | Temporary shallow Git clone, its archive callback, selected working-tree overlay copy/archive, and the deleted-file manifest. The archive is inside the clone's lifetime; do not sum parent and child. |
|
||||
@@ -334,6 +344,41 @@ attempt or owning phase:
|
||||
| `response.resolve`, `.authorize`, `.add_comment`, `.transaction*`, `.row_lock`, `.write`, `.publish` | Response selection, existing authorization, comment transaction including commit or rollback, row wait, insert, and external publication enqueue. |
|
||||
| `execution.realtime` | Synchronous live-event and plugin publication dispatch. It does not measure browser receipt or paint. |
|
||||
|
||||
For bounded native OpenCode diagnosis, set
|
||||
`PAPERCLIP_OPENCODE_PERFORMANCE_DIAGNOSTICS=1` as an agent/environment variable.
|
||||
The existing configured-environment projection carries it into the native Runner.
|
||||
It defaults off. The Runner creates an authenticated loopback OTLP sink and enables
|
||||
the pinned OpenCode runtime's internal tracing. The sink discards logs, resource
|
||||
attributes, span attributes, events, and unknown span names. It retains only closed
|
||||
phase names, wall timestamps, and success/failure, with a 1 MiB request limit,
|
||||
128 retained measurements, and 16 measurements per phase. No operator OTLP endpoint
|
||||
or ingestion credential is passed to OpenCode.
|
||||
|
||||
For fresh remote diagnostic runs, the selected pack must contain the current
|
||||
build's OpenCode proxy. An otherwise compatible image pack can predate the
|
||||
collector. Such a pack cannot satisfy the diagnostic request. The existing
|
||||
verified pack preparation path selects or stages the current proxy; it can add
|
||||
startup time or fail admission when matching runtime assets are unavailable.
|
||||
Ordinary runs retain compatible image-pack reuse. Recovery retains the pack
|
||||
identity already admitted for that session.
|
||||
|
||||
Available phases cover context/instruction loading, skill discovery, tool and model
|
||||
resolution, model streaming (`ai.streamText`, including stream consumption), Git
|
||||
discovery/default branch, snapshots, dependency installation, and plugin startup.
|
||||
These intervals overlap; do not sum them as elapsed time. At each terminal turn,
|
||||
already-received measurements travel in one `harness.diagnostic` event through the
|
||||
existing durable PRP channel. Late exports may be absent; collection never waits
|
||||
for a provider flush or delays terminal publication to complete the diagnostic.
|
||||
Late spans from a prior completed turn are discarded at the next boundary.
|
||||
|
||||
The server validates the closed fields again and records them under a foldable
|
||||
`provider.runtime` group inside `native.session.execute`. The normal operator
|
||||
endpoint gate controls Honeycomb export. The single diagnostic batch stays in the instance run log. Its OTel projection
|
||||
does not write duplicate phase rows or await additional database work before
|
||||
terminal consumption. Without an endpoint, no provider child spans are exported. Raw OpenCode OTLP must
|
||||
never be sent directly to the operator endpoint: it can contain prompts and
|
||||
authorization attributes before the loopback sink removes them.
|
||||
|
||||
Detail phases emit at most 16 spans of each name per attempt. Larger receipt and
|
||||
asset inventories continue under the containing group without unbounded per-row
|
||||
or per-file spans. Adding detail spans does not itself change work, query order,
|
||||
@@ -1954,6 +1999,12 @@ It carries no skill contents, identifiers, locations, or credentials. It uses th
|
||||
existing run performance events and operator-configured OpenTelemetry endpoint;
|
||||
no first-party Telemetry event is added.
|
||||
|
||||
The `skills.*` detail spans above distinguish inventory reconciliation and lock
|
||||
acquisition from repository transfer and file validation. A repository-backed
|
||||
skill's authored `SKILL.md` is not its complete runtime payload: the generated
|
||||
discovery entrypoint includes the pinned repository under `.paperclip-repository`.
|
||||
Measure the delivered bundle's file count and bytes when reproducing its cost.
|
||||
|
||||
Runtime preparation refreshes the company inventory once per listing. Local and
|
||||
catalog directories remain direct sources, so edits are visible on the next
|
||||
preparation. Explicit version selections still use their stored snapshots.
|
||||
|
||||
@@ -634,10 +634,11 @@ Longer polling intervals alone do not complete an item.
|
||||
[idle-login-cleanup.md](../idle-login-cleanup.md) for producer coverage,
|
||||
verification and remaining provider-orphan boundaries.
|
||||
|
||||
- [ ] **JOB-19 — Custom-image setup expiry.** Current: **30s** active-session
|
||||
expiry scans. Change: register `expiresAt` and cleanup attempts. **Complete
|
||||
when:** no sessions means no SQL and expired provider resources are released.
|
||||
Source: [environment-custom-images.ts](../../server/src/services/environment-custom-images.ts).
|
||||
- [x] **JOB-19 — Custom-image setup expiry.** Uses committed setup-session
|
||||
changes and the earliest existing active-session expiry. Startup reconstructs
|
||||
deadlines; empty and terminal history disarm. Failed reconciliation and first
|
||||
writes retain their wake through drain. Provider teardown and historical cleanup
|
||||
remain with the existing services. See [idle-environment-cleanup.md](../idle-environment-cleanup.md#custom-image-setup-expiry).
|
||||
|
||||
- [ ] **JOB-20 — Tool connection health.** Current: **30s** DB inventory;
|
||||
provider probes are conditional. Change: check on use/configuration change,
|
||||
@@ -646,8 +647,12 @@ Longer polling intervals alone do not complete an item.
|
||||
visible within the agreed freshness contract. Source:
|
||||
[index.ts](../../server/src/index.ts), `tools.sweepConnectionHealth`.
|
||||
|
||||
- [ ] **JOB-21 — Secret proposals and expired action tokens.** Current: **30s**
|
||||
expiry/cleanup queries. Change: enforce expiry synchronously on use and
|
||||
- [ ] **JOB-21 — Secret proposals and expired action tokens (partially migrated).**
|
||||
Secret-proposal expiry now uses transaction signals and saved expiry deadlines;
|
||||
see [the lifecycle contract](../idle-secret-proposal-expiry.md). Gateway token
|
||||
and session expiry now also uses saved deadlines and producer signals; see
|
||||
[its lifecycle contract](../idle-gateway-credential-expiry.md). Tool review
|
||||
recovery still uses **30s** queries. Change: enforce expiry synchronously on use and
|
||||
register required scrub/cleanup deadlines. **Complete when:** idle cleanup
|
||||
does not extend authorization or secret-retention periods. Sources:
|
||||
[secret-proposals.ts](../../server/src/services/secret-proposals.ts),
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,53 @@
|
||||
# Retained workspace recovery: fill a free slot across the cursor
|
||||
|
||||
This is a shared CI repair carried at the sleep-stack root. It fixes the existing
|
||||
retained-workspace owner; it does not add another background duty or migrate the
|
||||
remaining run/task recovery inventory. No PR is merged by this repair.
|
||||
|
||||
## Failure and ownership
|
||||
|
||||
The retained-workspace worker permits two physical exports. A held export stays
|
||||
in the durable unfinished candidates, even though this process already owns it.
|
||||
Discovery paginates by recovery-action ID and wraps only when its query returns
|
||||
no rows. A page containing only a held export was nonempty, but its only row was
|
||||
skipped in memory. Newly committed work before the cursor therefore waited for
|
||||
another 15-second recovery pass despite a free slot.
|
||||
|
||||
The CI failure at PR #17446, run 40423925338 / job 121925042581, leaves the third
|
||||
export `scheduled` at the existing 20-second assertion. The older #17450 failure
|
||||
has the same observable result. A deterministic real-PostgreSQL/filesystem
|
||||
regression reproduces it by starting the highest action ID, holding its provider
|
||||
resume, completing a middle action, then requesting the lowest action. The
|
||||
before-fix test failed at that same third-export assertion.
|
||||
|
||||
## Change and preserved boundaries
|
||||
|
||||
The existing candidate query accepts the locally in-flight run IDs and excludes
|
||||
them before pagination on both unfinished and passive discovery paths. The
|
||||
coordinator reads this set after its existing two-slot guard, so the exclusion
|
||||
contains at most one run for an admitted discovery pass. An empty page can now
|
||||
wrap in the same pass and find new work before the cursor. The in-memory duplicate
|
||||
guard remains to handle changes while a query is running.
|
||||
|
||||
The retained-workspace module owns candidate selection. Its callers, SQL source
|
||||
eligibility, physical ownership lock, retry policy, concurrency limit, completion
|
||||
callbacks, and shutdown joins retain their existing contracts. Other-process
|
||||
locks are still handled by the existing bounded cursor rotation. No new timer,
|
||||
queue, table, provider branch, or retry authority is added. Held work remains
|
||||
tracked as active and is not treated as settled or safe to sleep through.
|
||||
|
||||
## Verification
|
||||
|
||||
At the integrated pre-repair tip, the deterministic test failed with `scheduled`
|
||||
instead of `complete`. After the repair, that regression and the existing
|
||||
other-worker physical-lock rotation test both pass with real files and database
|
||||
state. The free-slot test keeps the original 20-second completion bound, verifies
|
||||
preserved files, no extra agent run, at most two concurrent resume operations,
|
||||
and shutdown waiting for the held export before its final completion.
|
||||
|
||||
At the stack root, all 56 retained-workspace recovery cases pass on a dedicated
|
||||
migrated PostgreSQL test database. The full suite also exercises retries,
|
||||
exhaustion, stop verification, cleanup authorization, source bindings and saved
|
||||
file recovery. Integrated workspace typecheck/build and fresh remote checks must
|
||||
be recorded in the PR after parent propagation. This is not a full repository or
|
||||
deployed-sleep claim.
|
||||
@@ -0,0 +1,127 @@
|
||||
# Connection-grants batch verification
|
||||
|
||||
Date: 2026-10-27. Implements the
|
||||
[selected design](2026-10-27-connection-grants-capability-batch.md).
|
||||
|
||||
## Change and authority boundary
|
||||
|
||||
`connectionManagementService.describeAgentInstallCapabilities` prepares a direct
|
||||
saved connection once, checks its retained installation targets once, and checks
|
||||
each additional candidate in order. It uses the same private target evaluator as
|
||||
ordinary authorization. The grants GET replaces only its editable-agent loop.
|
||||
|
||||
The optimized shape is board actors, non-AI saved connections, canonical retained
|
||||
and candidate IDs, and no contextual or proposed-resource selectors. Delegated
|
||||
agents and
|
||||
complex resources finish the first already prepared evaluation and then use the
|
||||
ordinary evaluator. Preparation is not restarted. Configure/use capabilities,
|
||||
grant identity filtering, audience disclosure, writes, and human continuations
|
||||
retain their existing authority paths.
|
||||
|
||||
Reuse lasts one descriptive call. It is not an atomic snapshot: a retained
|
||||
permission or target removed mid-call can leave a stale displayed choice. The
|
||||
next description and actual authorization must read fresh authority. No context,
|
||||
mutation authority, or reusable cache escapes the batch.
|
||||
|
||||
## Reproduction and measurements
|
||||
|
||||
Pinned baseline: `f6bc653f7f745adaafa26393852fb2f5a99256a4`.
|
||||
Treatment service SHA-256:
|
||||
`f5fca127e8a669d343b61455187e102dc6705d87389544ff4edfa5a066185656`.
|
||||
The benchmark loads the actual baseline service from Git, verifies that its other
|
||||
runtime dependencies and lockfile match, and compares it with the new production
|
||||
batch. Every positional capability, reason, and operation must match.
|
||||
|
||||
```sh
|
||||
node --import ./server/node_modules/tsx/dist/esm/index.mjs \
|
||||
scripts/benchmark-connection-install-capabilities.ts \
|
||||
--baseline-ref=f6bc653f7f745adaafa26393852fb2f5a99256a4
|
||||
```
|
||||
|
||||
The fixture is a Cloud board member who is not the creator, with a target-scoped
|
||||
grant covering the synthetic roster. Company installs retain the whole roster;
|
||||
partial installs retain one quarter, with at least one retained agent. Each arm
|
||||
has two warmups and five measured samples in alternating order. Counts below are
|
||||
per invocation. Full positional equality passed in every iteration. Individual
|
||||
SQL shapes are unchanged; the benchmark saves statement counts, parameters, rows,
|
||||
and treatment `EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON)` outside timed samples.
|
||||
|
||||
| Agents / installation | Old median ms (range) | Batch median ms (range) | SQL calls old → batch | Target lookups old → batch |
|
||||
| --- | ---: | ---: | ---: | ---: |
|
||||
| 1 / company | 5.13 (4.53–7.45) | 4.82 (4.38–5.08) | 8 → 8 | 1 → 1 |
|
||||
| 1 / partial | 5.16 (4.01–5.54) | 5.26 (4.83–6.13) | 7 → 7 | 1 → 1 |
|
||||
| 8 / company | 231.79 (221.63–238.96) | 31.11 (24.79–35.86) | 344 → 43 | 64 → 8 |
|
||||
| 8 / partial | 128.84 (114.42–160.09) | 43.07 (39.36–45.69) | 126 → 42 | 22 → 8 |
|
||||
| 32 / company | 2,804.04 (2,516.50–4,928.42) | 77.19 (75.56–132.76) | 5,216 → 163 | 1,024 → 32 |
|
||||
| 32 / partial | 697.51 (676.38–732.57) | 73.92 (73.21–77.78) | 1,464 → 162 | 280 → 32 |
|
||||
| 100 / company | 24,753.29 (23,849.21–25,019.72) | 246.39 (234.94–278.72) | 50,300 → 503 | 10,000 → 100 |
|
||||
| 100 / partial | 6,338.58 (6,239.68–6,526.36) | 236.37 (228.40–288.54) | 13,075 → 502 | 2,575 → 100 |
|
||||
|
||||
At 100 agents, returned Drizzle rows fell from 60,200 to 602 for company installs
|
||||
and from 15,475 to 526 for partial installs. The one-agent controls are neutral,
|
||||
including a small overlapping timing regression for the partial case. The table
|
||||
retains every sample range; do not extrapolate a latency curve from these medians.
|
||||
|
||||
The host could not start native PostgreSQL because SysV semaphore allocation
|
||||
failed, including an unsandboxed low-resource initdb attempt. Local proof used
|
||||
the existing test constructor seam with a scratch Docker provider: PostgreSQL
|
||||
18.6 Debian/aarch64 Linux, Node 26.4.0, loopback-only networking, tmpfs storage,
|
||||
and no simulated database delay. Each provider instance removed only its own
|
||||
container. Existing database creation, migrations, clients, service code and
|
||||
cleanup remained in use. The normal embedded package targets Darwin PostgreSQL
|
||||
18.1; minor version, platform, networking, and compute differ. Defaults included
|
||||
128 MiB shared buffers, 4 MiB work memory, and JIT enabled.
|
||||
|
||||
## Correctness evidence and limits
|
||||
|
||||
- Focused real-policy regressions cover scoped grants without installs, retained
|
||||
missing/denied targets, mixed and duplicate candidates, 403 versus reached
|
||||
409/500 faults, earlier denial masking an unreachable fault, forced mutation
|
||||
mode, fresh calls and reauthorization, actor/company isolation,
|
||||
deliberate within-call staleness, and ordinary fallback preparation counts.
|
||||
- Real PostgreSQL differential coverage includes company and partial installs,
|
||||
empty/duplicate candidates, foreign targets, uppercase UUID comparisons,
|
||||
profile/proposed-provider fallback, and an injected SQL division-by-zero fault
|
||||
that must escape only when reached.
|
||||
- The assembled grants GET compares complete old-loop and batch responses across
|
||||
creator, manager, member, viewer, inactive, and administrator fixtures. It
|
||||
verifies personal identity filtering, audience disclosure, ordered editable
|
||||
IDs, and an actual allowed install PUT followed by revocation, fresh denied
|
||||
capabilities, and a denied PUT with saved installs unchanged.
|
||||
- Independent review compared the pinned and extracted ordinary evaluator in
|
||||
18 context/query/lock-order cases and 18 complete batch cases. Additional
|
||||
checks covered delegated fallback, recursive profiles/gateways/AI pools, run
|
||||
capture reuse, a succeeded-run continuation, and fresh revocation. This
|
||||
supplemental proof uses synthetic database transport, not real lock contention.
|
||||
|
||||
Initial test failures were incorrect fixture expectations: inactive access hides
|
||||
the resource with 404, database roster order is not insertion order, and the
|
||||
existing final install mutation gate requires an unscoped agent-configuration
|
||||
grant. These were corrected in tests. Complete old/new GET equality passed before
|
||||
these extra assertions. No product behavior was changed to satisfy them.
|
||||
|
||||
Final guard review also found a real defect before merge: saved target IDs are
|
||||
text, and an empty saved target makes the legacy check depend on the current
|
||||
candidate. An executable regression returned `[false, true]` in the original
|
||||
service and `[false, false]` in the first batch. Real PostgreSQL reproduced it.
|
||||
The guard now requires canonical, nonempty retained IDs as well as candidates;
|
||||
unusual saved selectors keep ordinary evaluation. The fixed path matches both
|
||||
the complete capabilities and a later reached infrastructure fault. No stored
|
||||
target is dropped or normalized, and legacy selector semantics stay unchanged.
|
||||
|
||||
The bound is retained-plus-candidate target evaluations, not total CPU, SQL rows,
|
||||
or endpoint latency. Scoped grant arrays and company hierarchy reads can still
|
||||
grow inside each policy decision. Recursive and delegated fallback retain their
|
||||
existing costs. The benchmark measures the descriptive service component;
|
||||
hosted request attribution and production latency improvement remain unverified.
|
||||
|
||||
Local coverage contains 33 batch regressions and 83 connection-management
|
||||
integration cases, using disposable Docker PostgreSQL.
|
||||
The required workspace typecheck and build passed using pinned pnpm 9.15.4.
|
||||
The standard full Vitest workflow was attempted, then stopped after native
|
||||
database fixtures failed to initialize. A separately isolated failing native
|
||||
runner case reproduced the same PostgreSQL startup failure. This is not a full
|
||||
local test pass; the critical connection suites use the disposable Docker
|
||||
provider described above, and the complete CI suite remains a separate gate.
|
||||
Final test/CI outcomes are recorded in [PR #17945](https://github.com/paperclipai/paperclip/pull/17945); this document does not
|
||||
substitute for a final-head check or deployed acceptance.
|
||||
@@ -0,0 +1,220 @@
|
||||
# Connection-grants capability fan-out: design and safety proof
|
||||
|
||||
Date: 2026-10-27. This records the selected design before implementation.
|
||||
See [implementation verification](2026-10-27-connection-grants-batch-verification.md)
|
||||
for measured results and completed acceptance checks.
|
||||
|
||||
## Recommendation
|
||||
|
||||
Add one narrow, service-owned batch operation for describing which agents may
|
||||
receive a connection installation. Factor the common installed-agent checks out
|
||||
of the per-candidate loop. Keep authorization policy in its existing owner and
|
||||
keep actual mutations on the existing fresh authorization path.
|
||||
|
||||
Start with board actors describing a direct, saved, non-AI connection. Preserve
|
||||
the ordinary evaluator for delegated agents and complex resources until their
|
||||
additional obligations have executable parity evidence. This is a bounded first
|
||||
fix, not a general authorization cache or a rewrite of connection permissions.
|
||||
|
||||
The endpoint currently asks, for each candidate agent, whether the actor may
|
||||
change the connection's installations. Each answer checks **all retained
|
||||
installed agents**, followed by the candidate if it is not already installed.
|
||||
A company installation expands to the company roster. For 100 candidates with
|
||||
100 retained targets, this means 10,000 target evaluations before the other route
|
||||
work. The proposed eligible path needs 100 retained-target evaluations, with
|
||||
extra evaluations only for candidates outside that set.
|
||||
|
||||
Those retained checks are real policy obligations. The fix removes their repeated
|
||||
execution; it must not remove their authority requirements.
|
||||
|
||||
## What the evidence establishes
|
||||
|
||||
The prior observation window contains two successful grants GETs lasting
|
||||
56.506 and 66.388 seconds. There is no matching request trace attributing their
|
||||
duration to this loop. See [the measurement record](2026-10-27-next-bottlenecks.md).
|
||||
The source and local probes establish amplification, not the fraction of hosted
|
||||
latency it causes or the eventual improvement.
|
||||
|
||||
The production path is:
|
||||
|
||||
- `server/src/routes/tool-access.ts`, `describeConnectionCapabilities`: selects
|
||||
the company roster and calls `connections:profiles` once per agent, with
|
||||
mutation authority and `readOnly: false`.
|
||||
- `server/src/services/connection-management.ts`, `authorize`: resolves the
|
||||
saved connection, expands its retained installations, then checks the ordered
|
||||
target set. `capability` converts only errors with status 403 into denials;
|
||||
other faults escape.
|
||||
- `server/src/services/authorization.ts`: owns membership and scoped permission
|
||||
decisions. Some subtree grants can read the whole company hierarchy for each
|
||||
target, so linear target evaluations do not guarantee linear rows read.
|
||||
|
||||
Three independent designs were compared: structural factoring, general lazy
|
||||
request caching, and a narrow guarded batch. A separate judge selected the narrow
|
||||
batch. The final design incorporates the structural candidate's preparation
|
||||
continuation and differential test approach, plus the cache candidate's careful
|
||||
first-visit ordering and separate work counters.
|
||||
|
||||
| Local experiment | Observed result | Limit |
|
||||
| --- | --- | --- |
|
||||
| Existing exported service, all-installed fixtures | 4 agents caused 16 target reads; 8 caused 64 | Synthetic database transport, not PostgreSQL |
|
||||
| Handwritten factored terminal phase compared with the current service | Six static output/fault cases matched; target reads fell 16 to 4 and 64 to 8 | Treatment omits preparation and is not the proposed production implementation |
|
||||
| General read-reuse experiment, 32 agents | Physical transport reads fell 3,168 to 36, but 1,024 target invocations remained | Demonstrates why caching alone can retain quadratic work |
|
||||
| Existing policy with a grant scoped to one agent and no installations | Named candidate allowed; targetless request denied | Rules out an extra targetless authorization preflight |
|
||||
| Existing service with an unavailable retained target | Normal capability denied; `mutation: false` shortcut allowed | Rules out bypassing retained installation checks |
|
||||
| Existing continuation after membership removal | Fresh `reauthorize` denied | Does not prove a future batch's freshness |
|
||||
|
||||
All three local proof scripts were rerun successfully by the primary agent and
|
||||
the independent judge. They execute unchanged exported services over synthetic
|
||||
Drizzle transport. They are level-4 evidence for the exercised current-code
|
||||
facts, not proof of the unimplemented batch, assembled HTTP equivalence,
|
||||
PostgreSQL performance, or delegated-actor parity.
|
||||
|
||||
## Owning API and eligibility
|
||||
|
||||
The new operation belongs in `connectionManagementService`, beside `capability`.
|
||||
Its only initial consumer replaces the `editableAgentIds` loop. Use a fixed
|
||||
`connections:profiles` action and mutation-mode semantics rather than exposing
|
||||
general action or read-only knobs. Accept the original actor, base connection
|
||||
resource, ordered candidate IDs, and existing read context. Return positional
|
||||
capability descriptions; the route derives the existing editable-ID list.
|
||||
|
||||
Do not return an authorization context, audit authority, reusable session, cache
|
||||
handle, or continuation. Do not change the wire schema. Keep manager checks,
|
||||
configure/use capabilities, membership gates, grant identity filtering, and
|
||||
audience disclosure on their existing paths.
|
||||
|
||||
The initial optimized shape is deliberately explicit:
|
||||
|
||||
- Board actor; one direct saved connection whose actual saved purpose is not AI.
|
||||
- No non-null profile, gateway, routine, project, or issue selector.
|
||||
- No own `requestedProvider` or `requestedInstallCompanyIds` property, and no
|
||||
`selfServiceAgentInstalls: true` request.
|
||||
- Validated, nonempty roster IDs in their original spelling and input order.
|
||||
Retained target IDs must also be canonical and nonempty. Unusual identifiers
|
||||
use the ordinary path rather than normalization: a saved empty target is a
|
||||
candidate-dependent selector in the legacy evaluator.
|
||||
|
||||
Use the saved connection's provider, purpose, and creator derivation, including
|
||||
its ordinary personal-grant handling. Caller labels cannot establish eligibility.
|
||||
Unsupported shapes retain the existing per-candidate evaluator. In particular,
|
||||
AI pools, profile bindings and selectors, current/proposed gateway resources,
|
||||
and delegated actor contexts keep all recursive checks.
|
||||
|
||||
## Evaluation algorithm
|
||||
|
||||
Extract private preparation, target evaluation, and completion primitives from
|
||||
the existing evaluator. Both the ordinary and batch paths must consume these
|
||||
same primitives; do not copy policy into the route or implement a second policy
|
||||
engine. Ordinary authorization always starts with fresh private state.
|
||||
|
||||
1. An empty candidate list returns an empty result without authorization, as
|
||||
the current loop does.
|
||||
2. Begin preparation for the first actual candidate in the current order.
|
||||
Discover eligibility during the normal saved-connection lookup. The private
|
||||
preparation frame is consumed exactly once. If ineligible, finish that same
|
||||
evaluation through the ordinary path, then evaluate the remaining candidates
|
||||
normally. Do not restart preparation to fall back.
|
||||
3. For eligible inputs, form the same ordered set of retained installation
|
||||
targets, including the existing company-install expansion. Evaluate each
|
||||
retained target once, using the original check order: connection policy,
|
||||
applicable proposed-provider/resource trust checks, target existence, and
|
||||
principal-specific target authority. The shared primitive preserves these
|
||||
obligations even where the initial eligibility guard excludes some branches.
|
||||
4. A retained-target 403 denies every candidate with the same current denial.
|
||||
A reached non-403 fault aborts the operation. Do not inspect later targets to
|
||||
discover a fault the original evaluator would not reach.
|
||||
5. After the retained prefix succeeds, visit candidates in input order. A
|
||||
candidate already in the retained set uses the final retained decision,
|
||||
matching the current ordered-set behavior and success reason. An additional
|
||||
candidate uses the shared target primitive. Its 403 denies that candidate;
|
||||
a reached non-403 fault aborts. Return one result per input position.
|
||||
6. With no retained installations, the candidate is the policy target. Do not
|
||||
add a targetless preflight: a grant scoped to that candidate may correctly
|
||||
allow it while denying an unspecified target.
|
||||
|
||||
For N candidate positions and S distinct retained targets, the target phase is
|
||||
bounded by S + N evaluations, and often fewer when candidates are retained.
|
||||
This is not a promise of S + N SQL statements. Authorization may still read
|
||||
membership, grants, and hierarchy for each distinct target; complex fallback
|
||||
keeps its present cost. Measure those separately before widening the change.
|
||||
|
||||
Avoid eager prefetch and `Promise.all` here. A denied retained target currently
|
||||
prevents later target lookups. Loading those targets first changes which faults
|
||||
are reached. Parallelizing the old loop also retains its total work.
|
||||
|
||||
## Two safety facts and the blast radius
|
||||
|
||||
**1. A displayed permission must retain every existing installation obligation
|
||||
and its own target scope.** The batch cannot turn a denied or absent retained
|
||||
target into editable siblings, substitute a targetless decision, or reuse an
|
||||
allowed candidate's decision for a different candidate. Preserve positional
|
||||
results, reason, operation, ordered deduplication, and the 403/fatal-fault boundary.
|
||||
Existing-code counterexamples prove these shortcuts are unsafe. Actual old/new
|
||||
service and route comparisons remain required to prove the proposed change.
|
||||
|
||||
**2. A description must never become reusable mutation authority.** Batch state
|
||||
exists only inside one call and cannot cross actors, companies, connections,
|
||||
requests, operations, or human continuations. Actual writes and `reauthorize`
|
||||
start fresh. The batch returns descriptions only; identity disclosure does not
|
||||
consume them as authority.
|
||||
|
||||
There is a deliberate consistency change: reused facts and retained decisions
|
||||
are observed once during the description rather than reread for every candidate.
|
||||
This is neither a database snapshot nor an atomic revocation boundary. A revoke
|
||||
or target deletion during the call can leave a briefly stale displayed choice;
|
||||
a second GET and an actual mutation must observe fresh authority. Sequential
|
||||
first visits preserve static-fact failure order, not every possible failure or
|
||||
revocation between removed duplicate reads. Document and test this honestly.
|
||||
|
||||
Delegated agents remain on the ordinary path initially. `captureRunIdentity`
|
||||
locks and can reconcile pending steering identity; it is not a pure read. A
|
||||
future extension must prove run ownership, work mode, API-key status, responsible
|
||||
human and external-human ceilings, and pending-identity behavior. Preserve the
|
||||
original actor reference because existing read-context reuse checks identity by
|
||||
reference. Do not memoize capture or attach batch state to that context.
|
||||
|
||||
The intended change touches the connection-management owner, its route consumer,
|
||||
and focused tests. Existing connection policy is the authority. There is no
|
||||
schema migration, provider protocol change, background worker, persistent cache,
|
||||
UI contract change, or blanket creator/manager allow rule in this design.
|
||||
|
||||
## Implementation and acceptance gates
|
||||
|
||||
1. **Shared-helper refactor:** preserve ordinary evaluation before enabling the
|
||||
batch. Prove preparation runs exactly once for the first candidate on both
|
||||
eligible and fallback paths. Keep its continuation private and consumed.
|
||||
2. **Actual service differential:** compare the new production batch against the
|
||||
old full evaluator with a disposable PostgreSQL fixture. Compare every
|
||||
positional `allowed`, `reason`, and `operation`, not just editable counts.
|
||||
Cover empty/duplicate candidates; no, partial, and company installations;
|
||||
scoped grants with no installations; missing/foreign retained and candidate
|
||||
targets; local board and Cloud creator, owner, administrator, operator,
|
||||
ordinary member, viewer, and inactive membership.
|
||||
3. **Failures and fallback:** retained denial before an unreachable fault;
|
||||
candidate denial followed by a reached later fault; preparation/target 500
|
||||
and relevant reconciliation 409; only 403 converted. Exercise every excluded
|
||||
shape, including empty AI pools, explicit null selectors, proposed providers,
|
||||
company-install changes, profile selectors, and linked gateway obligations.
|
||||
4. **Full grants GET:** compare complete assembled responses, including order,
|
||||
editable IDs, configure/use flags, personal identity filtering, manager
|
||||
visibility, audience members, and route access gates. Exercise the existing
|
||||
connection Permissions and agent installation UI consumers.
|
||||
5. **Freshness and isolation:** controlled mid-batch revoke/delete; fresh second
|
||||
GET; GET then revoke then real mutation; delayed human/OAuth continuation;
|
||||
concurrent actors, companies, and connections. Agent fast-path enablement
|
||||
additionally requires real run/key/steering and lock-contention fixtures.
|
||||
6. **Performance:** at roster sizes 1, 8, 32, and 100 or larger, count preparation,
|
||||
target evaluations, policy decisions, SQL calls, returned rows, and wall time
|
||||
separately. Test subtree grants and unrelated-request responsiveness. Measure
|
||||
assembled route phases and database/pool delay before attributing savings.
|
||||
|
||||
After implementation, run the relevant regression suite and the repository's
|
||||
required typecheck, tests, and build before PR hand-off. Require passing final-head
|
||||
CI and review before hosted rollout. Repeat acceptance on the exact serving
|
||||
revision, verify both response behavior and latency with enough samples, and
|
||||
retain a way to restore the ordinary evaluator if parity fails.
|
||||
|
||||
This design pass changed no application behavior and performed no hosted mutation.
|
||||
It establishes a preferred implementation and executable current-code safety
|
||||
constraints. Complete batch parity and production latency improvement remain
|
||||
unproved until the implementation and acceptance gates above pass.
|
||||
@@ -0,0 +1,254 @@
|
||||
# Next bottleneck batch
|
||||
|
||||
## Scope and observation
|
||||
|
||||
Find the next bounded batch after the three merged slow-query fixes. This is a
|
||||
diagnosis pass: rank fresh costs, identify actual callers and existing work,
|
||||
and define the smallest performance and correctness experiments. No application
|
||||
code, schema, hosted configuration or deployment changes are part of this pass.
|
||||
|
||||
The reported workspace's Cloud, Neon database/runtime role, and Railway
|
||||
deployment were revalidated from the live registry. The staging app served
|
||||
`8548173fc9efdc0fa07644fb92dcc0b6851a95e5` throughout the query window. Upstream
|
||||
`master` was refreshed to `50b932a142216331b8d476042b454bc123739c40`. Private
|
||||
resource mapping, normalized SQL and raw logs remain outside Git.
|
||||
|
||||
After that stable measurement window, the final serving check at 18:49:08 UTC
|
||||
observed revision `6e781e0ab20cdaeb6570cebdb5f14dbb8557ba04`.
|
||||
The grants route/policy/authorization paths, wake-notice owner and skills
|
||||
route/service are byte-identical between that source and the measured source.
|
||||
The counts below describe the earlier window, not a post-rollout measurement.
|
||||
Re-baseline the assembled application before accepting a later performance fix.
|
||||
|
||||
The fresh query window is 2026-10-27 **18:32:51.270–18:38:07.464 UTC**, or
|
||||
316.194 seconds. Both snapshots read every statement for the exact application
|
||||
database OID and runtime-role OID through the existing monitoring view, using
|
||||
bounded read-only transactions. There is no top-100 or query-family filter.
|
||||
|
||||
- Statistics reset stayed at 14:21:16.446 UTC; eviction count stayed zero.
|
||||
- There were 4,573 starting and 4,590 ending entries, with no missing entries or
|
||||
invalid matched counters. Entry-level epochs were checked.
|
||||
- The 1,147 matched entries with positive call deltas account for 121,299 calls
|
||||
and 596.418 seconds of summed SQL execution time. They are all top-level
|
||||
entries, so this total does not add nested statement timings.
|
||||
- Seventeen new entries contribute 25 calls and 75.941 ms as separate lower
|
||||
bounds. They are excluded from the matched ranking.
|
||||
- I/O timing is disabled. Zero timing fields do not establish zero I/O latency.
|
||||
Point-in-time wait samples cannot establish sustained pressure or its absence.
|
||||
- The probe comparator passed eight controlled cases, including global/entry
|
||||
resets, decreasing counters, missing/new entries, evictions and scope changes.
|
||||
An initial diagnostic assertion about a pooler's default read-only setting
|
||||
was rejected before capture; the successful probe checks the actual transaction.
|
||||
|
||||
The matched HTTP read contains 997 completed log records from the same deployment
|
||||
and exact time window, below its 2,000-record cap. One WebSocket connection is
|
||||
excluded from ordinary request latency. The remaining 996 include 603 HTTP 304
|
||||
responses and one HTTP 500 from issue summaries; this is an observed traffic
|
||||
sample, not a clean acceptance run. Duration units follow
|
||||
[Railway's log contract](https://docs.railway.com/observability/logs).
|
||||
|
||||
SQL execution time can overlap across requests and background workers. Its sum
|
||||
is neither elapsed time nor endpoint latency. Honeycomb queries using the
|
||||
verified host, forwarded host and stack fields found no matching request spans
|
||||
in the checked dataset/window; the SQL share of these HTTP requests is unproved.
|
||||
|
||||
## Fresh rankings
|
||||
|
||||
| SQL caller | Calls | Mean SQL time | Total SQL time | Returned/affected rows | Disposition |
|
||||
| --- | ---: | ---: | ---: | ---: | --- |
|
||||
| Chat lifecycle milestone candidates | 167 | 1,497.65 ms | 250.108 s | 0 | Exact caller overlaps open draft #17606 |
|
||||
| Terminal workspace last activity | 245 | 591.64 ms | 144.951 s | 245 | No qualified open fix for these reads found |
|
||||
| Inbound wake-notice candidates | 647 | 48.99 ms | 31.694 s | 0 | Exact duty overlaps open draft #17647 |
|
||||
| GitHub stale-review repair UPDATE | 12 | 1,658.12 ms | 19.897 s | 0 | Exact duty overlaps open draft #17664 |
|
||||
| GitHub review source-run fallback | 82 | 230.25 ms | 18.880 s | 1 | Distinct query in GitHub maintenance owner |
|
||||
| GitHub stopped-response recovery | 12 | 1,340.53 ms | 16.086 s | 120 | Exact duty overlaps open draft #17661 |
|
||||
|
||||
The first two families still account for **66.24%** of matched SQL time after
|
||||
the earlier query improvements. Their existing optimized SQL shapes are active.
|
||||
The historical pre-fix fingerprints had no new calls in this window.
|
||||
|
||||
Guarded membership upserts used **41.660 ms across 84 calls**. The advisory-lock
|
||||
statement with a large historical maximum used **0.950 ms across 174 calls**.
|
||||
Neither historical cumulative total belongs on the new bottleneck shortlist.
|
||||
|
||||
| Ordinary HTTP route | Requests | Mean | Sample p95 | Maximum | Relevant result |
|
||||
| --- | ---: | ---: | ---: | ---: | --- |
|
||||
| GET connection grants | 2 | 61.447 s | 66.388 s | 66.388 s | Both 200; minimum 56.506 s; very small sample |
|
||||
| GET company skills | 20 | 2.733 s | 6.015 s | 13.945 s | All 304 |
|
||||
| GET company issues | 135 | 1.079 s | 2.957 s | 9.671 s | 112 returned 304 |
|
||||
| GET sidebar badges | 197 | 482 ms | 1.440 s | 5.851 s | All 304 |
|
||||
| GET run events | 203 | 346 ms | 1.063 s | 4.273 s | 137 returned 304 |
|
||||
|
||||
Percentiles use nearest rank within the observed sample. Upstream request
|
||||
duration is almost identical to total duration for these routes. That locates
|
||||
the observed delay behind the edge, but does not distinguish application CPU,
|
||||
database round trips, pool wait or other dependencies.
|
||||
|
||||
## Recommended next work
|
||||
|
||||
### 1. Connection-grants capability fan-out
|
||||
|
||||
This is the largest newly observed individual request delay. The route is
|
||||
`server/src/routes/tool-access.ts:1754`. Listing grants itself uses batched
|
||||
database reads. The route also describes connection capabilities by inspecting
|
||||
every company agent. Each inspection asks the connection-management policy for
|
||||
a capability; its default mutation-authority calculation expands existing
|
||||
company-wide installations to their agents and checks each target serially.
|
||||
This creates quadratic authorization fan-out in the reproduced service path. No provider HTTP call
|
||||
was found in this listing path. The relevant route and policy paths are
|
||||
unchanged between the measured source, later serving source and refreshed
|
||||
upstream master. The per-agent helper is `tool-access.ts:665–692`; the repeated
|
||||
install expansion and target checks are `connection-management.ts:143–158` and
|
||||
`:225–249`. It deliberately checks retained installed targets as well as the
|
||||
candidate target.
|
||||
|
||||
The fresh SQL window contains thousands of individually fast membership,
|
||||
project, issue and run-authority reads. Their counts alone do not attribute them
|
||||
to these two requests. A scratch local reproduction called the real exported
|
||||
`connectionManagementService.capability` → `authorize` → authorization service
|
||||
with a synthetic database transport and an implicit local board actor. Four
|
||||
agents caused 16 real target probes / 28 transport queries; eight agents caused
|
||||
64 probes / 88 queries. The outer iteration follows the private route helper's
|
||||
source; the helper itself was not exported or executed by the harness. This
|
||||
proves the policy amplification through real code (blast-radius evidence level
|
||||
4), not its share of the observed hosted request latency or a faster treatment.
|
||||
The Cloud connection-creator case also reproduced 16 → 64 target probes, with
|
||||
60 → 216 transport queries and 32 → 128 membership reads. A missing installed
|
||||
target negative case denied all capabilities. These are synthetic transport
|
||||
counts through real policy code, not real PostgreSQL or HTTP measurements.
|
||||
Next measure the assembled route with representative installed scopes and
|
||||
controlled database round-trip latency. The fix should make capability description share equivalent
|
||||
request-local facts while retaining each target's authorization decision.
|
||||
|
||||
**Safety fact to prove:** any reused authority fact belongs to the same actor,
|
||||
company, connection and request; a denied or absent target cannot inherit an
|
||||
allowed sibling's capability. Cover board, user and agent callers, personal and
|
||||
company installations, project/resource restrictions, removed grants, missing
|
||||
agents and mixed allowed/denied targets. Preserve current membership/revocation
|
||||
reads between requests. Do not make a one-minute route appear fast by skipping
|
||||
its checks. Do not bypass installed-target checks by passing `mutation:false` or
|
||||
`readOnly:true`; a missing installed target currently denies every candidate.
|
||||
|
||||
### 2. Qualify existing inbound wake-notice work
|
||||
|
||||
The measured query is owned by
|
||||
`server/src/services/chat-channels/inbound/wakeup-notices.ts`, in
|
||||
`createInboundWakeNotices.enqueueInboundWakeupPublications`. This selects
|
||||
historical notice eligibility; it is not the execution-admission gate. It ran
|
||||
647 times at approximately 2.05 calls per second and returned no rows. Its
|
||||
31.694 seconds of SQL time included 12,973,945 shared-buffer hits and no
|
||||
shared-block reads or temporary blocks in this window. The deployed and current
|
||||
upstream query paths are identical.
|
||||
|
||||
Direct overlap is confirmed in
|
||||
[#17647](https://github.com/paperclipai/paperclip/pull/17647), draft head
|
||||
`ca8f2d17c64fc7cecff4526817d1ac6e1d479ff4`. It changes this exact selector and
|
||||
removes its normal publication-maintenance call. Its coordinator reconstructs
|
||||
durable scopes at startup, retains cursors, and consumes committed-change
|
||||
notifications. Explicit/manual publication drains still call the selector, so
|
||||
eliminating all 647 observed calls is not established. The PR is stacked on
|
||||
`codex/sleep-cancel-contention` rather than master. Advance and qualify this
|
||||
existing work, its dependencies and remaining callers before creating a
|
||||
competing implementation. This pass did not run its tests or qualify its CI.
|
||||
|
||||
For remaining query work, capture the actual SQL and parameters from this service and
|
||||
reproduce long retained action/publication/comment history with no pending
|
||||
notice, followed by a single real eligible notice. Compare plans and all output
|
||||
columns before deciding whether to narrow the query or wake it from concrete
|
||||
committed obligations. A quiet path must still reconstruct unresolved work at
|
||||
startup and recover missed notifications.
|
||||
|
||||
**Safety fact to prove:** a narrower candidate path still finds every notice
|
||||
authorized by the original action/publication relationship, with unchanged
|
||||
company scope and deduplication. Cover legacy/native lineage, malformed or
|
||||
missing context, delayed commits, resolved/expired actions, failed publication,
|
||||
duplicate delivery and drain/resume. Removing the recurring call requires proof
|
||||
for every producer and restart/recovery path, not only an empty-queue test.
|
||||
|
||||
### 3. Skills-list reconciliation and repeated resolution
|
||||
|
||||
`server/src/routes/company-skills.ts:467` calls the service list at
|
||||
`server/src/services/company-skills.ts:3269`. Every request first awaits
|
||||
`ensureSkillInventoryCurrent` at line 3235: bundled imports, bundled-release
|
||||
verification, folder reconciliation and local-source inspection. The shared
|
||||
promise coalesces concurrent work but is removed when it finishes, so a later
|
||||
request repeats the refresh. A GET can therefore perform filesystem and
|
||||
database reconciliation before listing any skills.
|
||||
|
||||
The list then resolves every agent's desired skills again for each displayed
|
||||
skill (`company-skills.ts:3351`). Reference resolution performs searches over
|
||||
the skill list at line 2152. This is a concrete repeated-work lead, but its share
|
||||
of the measured 2.733-second HTTP mean remains unproved. These service/route
|
||||
files are identical in the deployed source, local integration and current
|
||||
upstream master.
|
||||
|
||||
First profile the real service with representative skill and agent counts,
|
||||
separating inventory refresh, database reads, desired-skill resolution and
|
||||
serialization. Compute each agent's desired-key set once per request as the
|
||||
first small candidate, comparing complete results. Treat any longer-lived
|
||||
inventory cache as a separate change requiring freshness proof.
|
||||
|
||||
**Safety fact to prove:** each agent contributes at most once to each selected
|
||||
skill, with identical exact-ID/key/slug resolution and ambiguous/missing alias
|
||||
handling. Preserve visibility, folder/subtree/search filtering, sort order,
|
||||
last-editor results and changed local-source discovery. A cached authorized
|
||||
inventory cannot be reused for another principal's visibility.
|
||||
|
||||
## Existing work and deferred candidates
|
||||
|
||||
- [#17606](https://github.com/paperclipai/paperclip/pull/17606) owns the exact chat
|
||||
lifecycle milestone path. Its draft branch moves lifecycle work to committed
|
||||
changes and reconstructs pending work. Audit its producer/recovery proof and
|
||||
drive its checks before starting a competing polling rewrite. Its draft
|
||||
status and source design are not proof that it is safe or merge-ready.
|
||||
- [#17664](https://github.com/paperclipai/paperclip/pull/17664) and
|
||||
[#17661](https://github.com/paperclipai/paperclip/pull/17661) overlap the two
|
||||
measured GitHub maintenance duties. The source-run fallback is a distinct
|
||||
delivery/link/run lookup in `chat-github-checks.ts:181–202`, used when the
|
||||
review lacks a resolvable run. It still merits a local plan/equality experiment,
|
||||
coordinated with those owner changes. A previous scalar delivery lookup fix
|
||||
in #17669 is already deployed and must not be proposed again.
|
||||
- Workspace activity is called at
|
||||
`server/src/services/execution-workspaces.ts:3576` through the shared terminal
|
||||
workspace worker, already migrated by #16282. Its query is at
|
||||
`server/src/services/git-workspace-expiration.ts:153–160`.
|
||||
[#17505](https://github.com/paperclipai/paperclip/pull/17505) adjusts cursor and
|
||||
retry deadlines but leaves this query/call intact. A second pass could batch
|
||||
repeated activity reads; fresh destructive eligibility, live leases and
|
||||
concurrent reopen checks must remain authoritative. No qualified open fix
|
||||
reducing these activity reads was found in the scoped check.
|
||||
- [#17896](https://github.com/paperclipai/paperclip/pull/17896),
|
||||
[#17894](https://github.com/paperclipai/paperclip/pull/17894), and
|
||||
[#17893](https://github.com/paperclipai/paperclip/pull/17893) are already merged
|
||||
after the measured deployment. They narrow explicit-continuation run reads,
|
||||
avoid duplicate GitHub webhook body transfer and narrow review-wait JSON.
|
||||
The final 18:49:08 serving source contains all three. Evaluate their impact in
|
||||
a new post-rollout window before duplicating those changes.
|
||||
- [#17907](https://github.com/paperclipai/paperclip/pull/17907) keeps the skill
|
||||
cache lock alive during slow preparation. Its documented scope does not
|
||||
explain why preparation is slow or remove the skills-list work above.
|
||||
|
||||
The prior Slack continuation's 353.059-second delay is confirmed by an exact
|
||||
deployed-version `heartbeat.queue` span without children. Executor startup begins
|
||||
after that wait. A periodic recovery chain stayed in flight for at least
|
||||
240.086 seconds and suppressed eight ticks, but an earlier recovery pass reached
|
||||
its post-resume stage while the continuation was still queued. Source cleanup
|
||||
had completed before the answer arrived. This narrows the failure to queue /
|
||||
admission or recovery scheduling; it does not establish the blocking gate, a
|
||||
lost wake, or SQL as the cause. Instrument gate decisions and recovery stages
|
||||
before choosing a repair. No task was replayed or re-woken for diagnosis.
|
||||
|
||||
## Finish conditions for the next implementation batch
|
||||
|
||||
Take at most the three priorities above, advancing existing work where it
|
||||
overlaps. Pin the deployed baseline, prove
|
||||
the suspected cost with the real caller, define the performance criterion, then
|
||||
compare complete behavior and adversarial authority/recovery cases. Keep
|
||||
independent changes in independent PRs with green final-head checks. No migration
|
||||
or boot script is justified by this discovery pass. Repeat hosted acceptance and
|
||||
a fresh comparable counter/request window after an authorized rollout.
|
||||
|
||||
The private evidence set retains all raw inputs, comparator validation, exact
|
||||
source inventories, request summaries and query snapshots. This document is the
|
||||
sanitized discovery handoff. Correctness of proposed optimizations is unproved
|
||||
until their real-code comparisons and regressions run.
|
||||
@@ -0,0 +1,35 @@
|
||||
# Run identity reads
|
||||
|
||||
`captureRunIdentity` locks the task before the run. It validates company, agent,
|
||||
and active status. Pending steering still requires its durable acknowledgement.
|
||||
The accepted requester is read for each operation. No scope adds a cache or skips
|
||||
these checks.
|
||||
|
||||
Callers choose a typed read scope:
|
||||
|
||||
| Scope | Retained run data | Consumers |
|
||||
| --- | --- | --- |
|
||||
| `identity` | Run and accepted requester identifiers, status | Gateway, native connection tools, connection intents, project skill setup |
|
||||
| `task_authorization` | Identity, task bindings, wake request, conversation generation, skill-test marker | Projects, workspace repositories, Slack, voice, connection management |
|
||||
| `token_broker` | Identity, invocation source, task/project/routine and requester attribution | Connection token broker |
|
||||
| `project_execution` | Task authorization plus runtime mode and the complete saved native execution input | Muse project admission |
|
||||
|
||||
Narrow return types omit unselected columns. JSON projections preserve the types
|
||||
of retained values. The default full read remains available for compatibility and
|
||||
explicit inspection. Existing GitHub-credential and request-authentication scopes
|
||||
keep their current contracts. Execution preparation and recovery retain their
|
||||
separate complete-input reads.
|
||||
|
||||
The synthetic real-PostgreSQL benchmark captures the actual locked query and
|
||||
compares every retained column, context and identity result. It keeps the same
|
||||
five-query reconciliation transaction, checks the unchanged stored run, records
|
||||
plans and alternating timings, and measures raw PostgreSQL DataRow bytes:
|
||||
|
||||
```sh
|
||||
pnpm --filter @paperclipai/server exec tsx ../scripts/benchmark-run-identity-projections.ts --baseline-ref=623f1dc289a14a699ebbe3f09b94179d38fc2ae2
|
||||
```
|
||||
|
||||
The default uses disposable embedded PostgreSQL. `PAPERCLIP_TEST_DATABASE_URL`
|
||||
can select a disposable local database whose name ends in `_test`. DataRow bytes
|
||||
exclude response metadata, TLS and TCP. Byte reduction does not establish a
|
||||
production latency or billing change.
|
||||
@@ -139,9 +139,11 @@ environment, and consent checks as other edits. It adds no agent permission or
|
||||
credential fallback. Pool and externally managed agents use their existing
|
||||
configuration surfaces. Board browser verification does not qualify agent access.
|
||||
|
||||
The existing-agent upgrade notice uses this configuration PATCH. It does not
|
||||
The existing-agent upgrade notice for Claude, Codex, and Grok Build uses this configuration PATCH. It does not
|
||||
grant configuration, account, or environment access. The server resolves the
|
||||
saved settings under normal company/configuration authorization. Obsolete Claude
|
||||
saved settings under normal company/configuration authorization. Grok migration preserves
|
||||
the effective legacy search setting. Task model/effort overrides retain the
|
||||
agent's account binding and tool policy without granting extra account access. Obsolete Claude
|
||||
tuning fields are removed with `configurationWarnings`; security incompatibilities
|
||||
remain errors. Unqualified sandbox upgrades first use the agent-scoped
|
||||
`POST /api/agents/{id}/qualify-runner` under the same `agent_config:update`
|
||||
|
||||
@@ -189,6 +189,16 @@ conversational resolution with a genuine later same-task user message supplies
|
||||
acceptance; an agent-only resolution does not. A plan approval must name the
|
||||
current same-task revision. Newer pending/rejected proposals stop new creation.
|
||||
Already admitted task replay and ordinary delegation retain their existing rules.
|
||||
|
||||
Native `paperclip_finish` does not grant completion authority from an opening
|
||||
answer. The onboarding parent requires a trusted saved human decision on its
|
||||
latest proposal. Acceptance permits completion under the existing policy;
|
||||
rejection permits acknowledgment and closure without authorizing new work.
|
||||
The native status commit checks this decision again under its existing task/run
|
||||
locks. This adds no new agent endpoint or cross-company authority. The
|
||||
[native completion controls](../../server/src/services/native-runtime/paperclip-control-plane-port.test.ts)
|
||||
check persisted outcomes on disposable PostgreSQL; they are not live Cloud
|
||||
credential or onboarding qualification.
|
||||
Source: [creation admission](../../server/src/services/onboarding-first-task-admission.ts).
|
||||
Controls: [native/REST PostgreSQL regressions](../../server/src/__tests__/onboarding-first-task-admission.test.ts).
|
||||
These controls do not qualify a live Cloud provider journey.
|
||||
|
||||
@@ -99,6 +99,9 @@ For an agent-owned plugin environment, verify preparation before connecting,
|
||||
same-agent probes, account preservation after an environment change, failed
|
||||
selection-save retry, and switching back while the computer stops. Confirm the
|
||||
setup hold still blocks ordinary work and API key creation before finishing.
|
||||
Start through the name/adapter dialog with a managed-only or forced Kubernetes
|
||||
policy and a draft that inherits its environment. Once preparation completes,
|
||||
confirm the auth choices, Advanced gateway, and saved subscription remain usable.
|
||||
The [setup lifecycle tests](../../server/src/modules/agent-lifecycle/__tests__/setup.test.ts)
|
||||
exercise durable transitions with synthetic plugin resources; they do not prove
|
||||
live VM provisioning.
|
||||
@@ -167,12 +170,16 @@ Selecting another runner must not silently discard security settings or change a
|
||||
recorded execution. Keep runner choice in Advanced and verify the next eligible
|
||||
run separately from historical transcripts.
|
||||
|
||||
Existing legacy Claude and Codex agents on qualified default environments, or
|
||||
Existing legacy Claude, Codex, and Grok Build agents on qualified default environments, or
|
||||
probe-capable Daytona snapshots / immutable sandbox images, show **Upgrade runner** above their configuration. Verify a company owner with `agents:configure` and no instance-admin role can qualify the saved computer and upgrade. A caller without configuration permission must be denied before a probe starts. Exercise that action with
|
||||
pending edits and an environment-variable draft. An unsaved change to a different
|
||||
effective computer must disable the action and explain that the computer change
|
||||
must be saved first; restoring the saved target must re-enable it. Reopen the page and compare the
|
||||
saved harness, model, account bindings, instructions, and run policy. Check a
|
||||
Grok upgrade with an unset or `grok-build` model: it must use `grok-4.7`.
|
||||
Compare effective search policy before and after, including the implicit disabled
|
||||
default and explicit enabled search. Verify task model and effort changes use the
|
||||
Grok catalog and reach execution while retaining the account and search policy. Check a
|
||||
saved `maxTurnsPerRun: 1000`: upgrade must warn and remove it. Test an initially
|
||||
unqualified Daytona Paperclip Computer: click must probe, record qualification,
|
||||
refresh availability, and only then save. A failed probe or changed selection must
|
||||
|
||||
@@ -142,6 +142,15 @@ admit new work. [Admission controls](../../server/src/__tests__/onboarding-first
|
||||
exercise native and REST calls against disposable PostgreSQL; live model and
|
||||
Cloud onboarding remain separate evidence.
|
||||
|
||||
Native completion also requires a saved human proposal decision. An opening
|
||||
answer alone must leave the first task in review. Verify that premature native
|
||||
completion is refused, including a result already saved before finalization.
|
||||
Accepted proposals and acknowledged human rejections can close the first task;
|
||||
rejection still does not authorize new tasks. The
|
||||
[native completion controls](../../server/src/services/native-runtime/paperclip-control-plane-port.test.ts)
|
||||
exercise these boundaries against disposable PostgreSQL. These controls do not
|
||||
identify a historical Cloud failure or establish live onboarding success.
|
||||
|
||||
For accepted file-producing work, verify exactly one self-assigned child with a
|
||||
self-contained description. The onboarding run must yield through the existing
|
||||
dependency/block disposition so the child's own execution can publish its file.
|
||||
|
||||
@@ -74,6 +74,16 @@ mutations. Qualify a useful model task separately from saving the account.
|
||||
|
||||
### `session`
|
||||
|
||||
For a repository project that inherits storage-plugin defaults, run a task and
|
||||
send two follow-ups. Start with unset task workspace settings, then repeat with
|
||||
an explicit isolated mode or strategy. Binding must save `reuse_existing` and
|
||||
the resolved mode without changing the effective policy,
|
||||
session fingerprint, and workspace identity. An actual project-policy change
|
||||
must still reset the session and preserve the stored workspace owner. The
|
||||
[session admission regression](../../server/src/__tests__/heartbeat-session-preparation.test.ts)
|
||||
checks this with PostgreSQL and synthetic provider execution; it is not live
|
||||
warm-process or latency evidence.
|
||||
|
||||
For `runs-adapters/session`, complete a task in a disposable warm sandbox and
|
||||
retain its saved workspace file. Let the warm session close normally and verify
|
||||
that the retained lease has a complete, lease-bound harness backup before
|
||||
|
||||
@@ -44,6 +44,7 @@ describe("Paperclip Runner permission defaults", () => {
|
||||
["codex_local", "codexPermissionMode", "never"],
|
||||
["opencode_local", "opencodePermissionMode", "allow"],
|
||||
["pi_local", "acpxPermissionMode", "approve-all"],
|
||||
["grok_local", "acpxPermissionMode", "approve-all"],
|
||||
])("uses full auto when converting %s to the new runner", (adapter, key, value) => {
|
||||
expect(paperclipRunnerTransitionConfig(adapter, undefined)).toMatchObject({ [key]: value });
|
||||
});
|
||||
@@ -53,6 +54,15 @@ describe("Paperclip Runner permission defaults", () => {
|
||||
expect(paperclipRunnerTransitionConfig("pi_local", "openrouter/provider/model")).toMatchObject({ provider: "acpx", acpxAgent: "pi", model: "openrouter/provider/model" });
|
||||
});
|
||||
|
||||
it("keeps Grok's harness and model when converting its adapter", () => {
|
||||
expect(paperclipRunnerTransitionConfig("grok_local", "grok-4.7"))
|
||||
.toMatchObject({ provider: "acpx", acpxAgent: "grok", model: "grok-4.7" });
|
||||
});
|
||||
|
||||
it.each([undefined, "", "grok-build"])("resolves Grok's legacy default %s to the native model", model => {
|
||||
expect(paperclipRunnerTransitionConfig("grok_local", model)).toMatchObject({ model: "grok-4.7" });
|
||||
});
|
||||
|
||||
it("recognizes only exact provider identifiers", () => {
|
||||
expect(isPaperclipRunnerProvider("codex")).toBe(true);
|
||||
expect(isPaperclipRunnerProvider("opencode")).toBe(true);
|
||||
|
||||
@@ -205,7 +205,7 @@ export function paperclipRunnerTransitionConfig(
|
||||
providerOverride?: unknown,
|
||||
): Record<string, unknown> {
|
||||
const previousProvider =
|
||||
(previousAdapterType === "claude_local" || previousAdapterType === "pi_local")
|
||||
(["claude_local", "pi_local", "grok_local"].includes(previousAdapterType))
|
||||
? "acpx"
|
||||
: previousAdapterType === "opencode_local"
|
||||
? "opencode"
|
||||
@@ -218,13 +218,15 @@ export function paperclipRunnerTransitionConfig(
|
||||
: previousProvider;
|
||||
return {
|
||||
provider,
|
||||
model: provider === "acpx" && previousAdapterType === "pi_local"
|
||||
model: provider === "acpx" && previousAdapterType === "grok_local"
|
||||
? typeof previousModel === "string" && previousModel.trim() && previousModel.trim() !== "grok-build" ? previousModel.trim() : "grok-4.7"
|
||||
: provider === "acpx" && previousAdapterType === "pi_local"
|
||||
? typeof previousModel === "string" ? previousModel.trim() : ""
|
||||
: resolvePaperclipRunnerModel(
|
||||
provider,
|
||||
provider === previousProvider ? previousModel : undefined,
|
||||
),
|
||||
...(provider === "acpx" ? { acpxAgent: previousAdapterType === "pi_local" ? "pi" : "claude" } : {}),
|
||||
...(provider === "acpx" ? { acpxAgent: previousAdapterType === "grok_local" ? "grok" : previousAdapterType === "pi_local" ? "pi" : "claude" } : {}),
|
||||
[PAPERCLIP_RUNNER_PERMISSION_CAPABILITIES[provider].configKey]:
|
||||
PAPERCLIP_RUNNER_PERMISSION_CAPABILITIES[provider].defaultMode,
|
||||
lifecycleMode: "per_turn",
|
||||
|
||||
@@ -501,6 +501,21 @@ const MIGRATIONS_WITH_TOP_LEVEL_STATEMENTS = new Set([
|
||||
"0332_little_night_nurse.sql",
|
||||
"0345_luxuriant_carmella_unuscione.sql",
|
||||
"0369_late_logan.sql",
|
||||
"0381_mean_scream.sql",
|
||||
"0382_narrow_the_executioner.sql",
|
||||
"0385_open_madame_web.sql",
|
||||
"0386_steady_epoch.sql",
|
||||
"0387_boring_professor_monster.sql",
|
||||
"0388_ambitious_micromax.sql",
|
||||
"0389_brave_hulk.sql",
|
||||
"0390_salty_killraven.sql",
|
||||
"0391_third_roulette.sql",
|
||||
"0392_small_frightful_four.sql",
|
||||
"0393_open_grandmaster.sql",
|
||||
|
||||
|
||||
"0384_funny_adam_warlock.sql",
|
||||
"0383_tool_review_company_expiry.sql",
|
||||
]);
|
||||
|
||||
async function applyPendingMigrationsManually(
|
||||
|
||||
@@ -45,4 +45,4 @@ export { issueRelations } from "./schema/issue_relations.js";
|
||||
export { issueReferenceMentions } from "./schema/issue_reference_mentions.js";
|
||||
export * from "./schema/index.js";
|
||||
|
||||
export { signalDatabaseWork, subscribeDatabaseWork, databaseWorkPending, reconcileDatabaseWork } from "./work-signals.js";
|
||||
export { databaseWorkScopeKey, signalDatabaseWork, subscribeDatabaseWork, databaseWorkPending, reconcileDatabaseWork } from "./work-signals.js";
|
||||
@@ -0,0 +1 @@
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS "tool_gateway_sessions_expiry_idx" ON "tool_gateway_sessions" USING btree ("expires_at");
|
||||
@@ -0,0 +1 @@
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS "tool_action_requests_pending_expiry_idx" ON "tool_action_requests" USING btree ("expires_at","id") WHERE "tool_action_requests"."status" = 'pending';
|
||||
@@ -0,0 +1 @@
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS "tool_action_requests_company_pending_expiry_idx" ON "tool_action_requests" USING btree ("company_id","expires_at") WHERE "tool_action_requests"."status" = 'pending';
|
||||
@@ -0,0 +1 @@
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS "tool_action_requests_approved_idx" ON "tool_action_requests" USING btree ("id") WHERE "tool_action_requests"."status" = 'approved';
|
||||
@@ -0,0 +1 @@
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS "tool_action_requests_executing_deadline_idx" ON "tool_action_requests" USING btree ("updated_at","id") WHERE "tool_action_requests"."status" = 'executing';
|
||||
@@ -0,0 +1 @@
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS "tool_action_requests_company_interaction_idx" ON "tool_action_requests" USING btree ("company_id","interaction_id");
|
||||
@@ -0,0 +1 @@
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS "decision_archive_notification_outbox_delivery_idx" ON "decision_archive_notification_outbox" USING btree ("last_attempt_at" NULLS FIRST,"created_at","id") WHERE "decision_archive_notification_outbox"."status" = 'pending';
|
||||
@@ -0,0 +1 @@
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS "decisions_pending_continuation_idx" ON "decisions" USING btree ("id") WHERE "decisions"."continuation_policy" = 'wake_origin_agent' AND "decisions"."metadata" ->> 'continuationPending' = 'true' AND ("decisions"."status" IN ('expired', 'cancelled') OR ("decisions"."status" = 'decided' AND "decisions"."execution_status" IN ('succeeded', 'partial', 'failed')));
|
||||
@@ -0,0 +1 @@
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS "decisions_open_expiry_idx" ON "decisions" USING btree ("expires_at","id") WHERE "decisions"."status" = 'open';
|
||||
@@ -0,0 +1 @@
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS "decisions_running_recovery_idx" ON "decisions" USING btree ("updated_at","id") WHERE "decisions"."status" = 'decided' AND "decisions"."execution_status" = 'running';
|
||||
@@ -0,0 +1 @@
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS "grok_dispatch_prepared_idx" ON "grok_bot_dispatches" USING btree ("created_at","id") WHERE "grok_bot_dispatches"."status" = 'prepared';
|
||||
@@ -0,0 +1 @@
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS "approvals_pending_retention_idx" ON "approvals" USING btree ("updated_at","id") WHERE "approvals"."status" = 'pending';
|
||||
@@ -0,0 +1 @@
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS "join_requests_pending_retention_idx" ON "join_requests" USING btree ("updated_at","id") WHERE "join_requests"."status" = 'pending_approval';
|
||||
-64014
File diff suppressed because it is too large.
Load diff
+186
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"id": "bfda87fa-662b-47bb-87c0-dabc814aebd7",
|
||||
"prevId": "2a9439e3-4a2b-4a20-ab2b-2aee3db8cb03",
|
||||
"id": "394e85a6-7fc2-4ab1-80a3-33bb4c7003aa",
|
||||
"prevId": "4f21a636-735f-4dce-9ab8-0583f3bc7b79",
|
||||
"version": "7",
|
||||
"dialect": "postgresql",
|
||||
"tables": {
|
||||
@@ -22110,6 +22110,34 @@
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"decision_archive_notification_outbox_delivery_idx": {
|
||||
"name": "decision_archive_notification_outbox_delivery_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "last_attempt_at",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "first"
|
||||
},
|
||||
{
|
||||
"expression": "created_at",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"decision_archive_notification_outbox\".\"status\" = 'pending'",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
}
|
||||
},
|
||||
"foreignKeys": {
|
||||
@@ -24030,6 +24058,44 @@
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"decisions_open_expiry_idx": {
|
||||
"name": "decisions_open_expiry_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "expires_at",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"decisions\".\"status\" = 'open'",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"decisions_pending_continuation_idx": {
|
||||
"name": "decisions_pending_continuation_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"decisions\".\"continuation_policy\" = 'wake_origin_agent' AND \"decisions\".\"metadata\" ->> 'continuationPending' = 'true' AND (\"decisions\".\"status\" IN ('expired', 'cancelled') OR (\"decisions\".\"status\" = 'decided' AND \"decisions\".\"execution_status\" IN ('succeeded', 'partial', 'failed')))",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"decisions_bundle_idx": {
|
||||
"name": "decisions_bundle_idx",
|
||||
"columns": [
|
||||
@@ -52812,6 +52878,88 @@
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"tool_action_requests_pending_expiry_idx": {
|
||||
"name": "tool_action_requests_pending_expiry_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "expires_at",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"tool_action_requests\".\"status\" = 'pending'",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"tool_action_requests_company_pending_expiry_idx": {
|
||||
"name": "tool_action_requests_company_pending_expiry_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "company_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "expires_at",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"tool_action_requests\".\"status\" = 'pending'",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"tool_action_requests_executing_deadline_idx": {
|
||||
"name": "tool_action_requests_executing_deadline_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "updated_at",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"tool_action_requests\".\"status\" = 'executing'",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"tool_action_requests_approved_idx": {
|
||||
"name": "tool_action_requests_approved_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"tool_action_requests\".\"status\" = 'approved'",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"tool_action_requests_invocation_idx": {
|
||||
"name": "tool_action_requests_invocation_idx",
|
||||
"columns": [
|
||||
@@ -52827,6 +52975,27 @@
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"tool_action_requests_company_interaction_idx": {
|
||||
"name": "tool_action_requests_company_interaction_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "company_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "interaction_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"tool_action_requests_issue_idx": {
|
||||
"name": "tool_action_requests_issue_idx",
|
||||
"columns": [
|
||||
@@ -55222,6 +55391,21 @@
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"tool_gateway_sessions_expiry_idx": {
|
||||
"name": "tool_gateway_sessions_expiry_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "expires_at",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"tool_gateway_sessions_run_idx": {
|
||||
"name": "tool_gateway_sessions_run_idx",
|
||||
"columns": [
|
||||
+214
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"id": "2a9439e3-4a2b-4a20-ab2b-2aee3db8cb03",
|
||||
"prevId": "a0fcd9b8-f125-4985-8795-f567466b9aef",
|
||||
"id": "f38ffbf3-c0ce-435e-aad8-9e5afa6b9417",
|
||||
"prevId": "394e85a6-7fc2-4ab1-80a3-33bb4c7003aa",
|
||||
"version": "7",
|
||||
"dialect": "postgresql",
|
||||
"tables": {
|
||||
@@ -22110,6 +22110,34 @@
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"decision_archive_notification_outbox_delivery_idx": {
|
||||
"name": "decision_archive_notification_outbox_delivery_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "last_attempt_at",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "first"
|
||||
},
|
||||
{
|
||||
"expression": "created_at",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"decision_archive_notification_outbox\".\"status\" = 'pending'",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
}
|
||||
},
|
||||
"foreignKeys": {
|
||||
@@ -24030,6 +24058,66 @@
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"decisions_running_recovery_idx": {
|
||||
"name": "decisions_running_recovery_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "updated_at",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"decisions\".\"status\" = 'decided' AND \"decisions\".\"execution_status\" = 'running'",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"decisions_open_expiry_idx": {
|
||||
"name": "decisions_open_expiry_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "expires_at",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"decisions\".\"status\" = 'open'",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"decisions_pending_continuation_idx": {
|
||||
"name": "decisions_pending_continuation_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"decisions\".\"continuation_policy\" = 'wake_origin_agent' AND \"decisions\".\"metadata\" ->> 'continuationPending' = 'true' AND (\"decisions\".\"status\" IN ('expired', 'cancelled') OR (\"decisions\".\"status\" = 'decided' AND \"decisions\".\"execution_status\" IN ('succeeded', 'partial', 'failed')))",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"decisions_bundle_idx": {
|
||||
"name": "decisions_bundle_idx",
|
||||
"columns": [
|
||||
@@ -52812,6 +52900,88 @@
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"tool_action_requests_pending_expiry_idx": {
|
||||
"name": "tool_action_requests_pending_expiry_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "expires_at",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"tool_action_requests\".\"status\" = 'pending'",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"tool_action_requests_company_pending_expiry_idx": {
|
||||
"name": "tool_action_requests_company_pending_expiry_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "company_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "expires_at",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"tool_action_requests\".\"status\" = 'pending'",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"tool_action_requests_executing_deadline_idx": {
|
||||
"name": "tool_action_requests_executing_deadline_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "updated_at",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"tool_action_requests\".\"status\" = 'executing'",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"tool_action_requests_approved_idx": {
|
||||
"name": "tool_action_requests_approved_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"tool_action_requests\".\"status\" = 'approved'",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"tool_action_requests_invocation_idx": {
|
||||
"name": "tool_action_requests_invocation_idx",
|
||||
"columns": [
|
||||
@@ -52827,6 +52997,27 @@
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"tool_action_requests_company_interaction_idx": {
|
||||
"name": "tool_action_requests_company_interaction_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "company_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "interaction_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"tool_action_requests_issue_idx": {
|
||||
"name": "tool_action_requests_issue_idx",
|
||||
"columns": [
|
||||
@@ -55222,6 +55413,21 @@
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"tool_gateway_sessions_expiry_idx": {
|
||||
"name": "tool_gateway_sessions_expiry_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "expires_at",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"tool_gateway_sessions_run_idx": {
|
||||
"name": "tool_gateway_sessions_run_idx",
|
||||
"columns": [
|
||||
@@ -55967,6 +56173,12 @@
|
||||
"notNull": true,
|
||||
"default": "''"
|
||||
},
|
||||
"responsible_user_id": {
|
||||
"name": "responsible_user_id",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"owner_note": {
|
||||
"name": "owner_note",
|
||||
"type": "text",
|
||||
+261
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"id": "7832df38-fa1b-4687-bd06-eccbc29bd28c",
|
||||
"prevId": "3e17203b-ec95-41e9-8363-685982450199",
|
||||
"id": "e080fc1b-af8e-4012-8ce5-3b5065b65143",
|
||||
"prevId": "f38ffbf3-c0ce-435e-aad8-9e5afa6b9417",
|
||||
"version": "7",
|
||||
"dialect": "postgresql",
|
||||
"tables": {
|
||||
@@ -22110,6 +22110,34 @@
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"decision_archive_notification_outbox_delivery_idx": {
|
||||
"name": "decision_archive_notification_outbox_delivery_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "last_attempt_at",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "first"
|
||||
},
|
||||
{
|
||||
"expression": "created_at",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"decision_archive_notification_outbox\".\"status\" = 'pending'",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
}
|
||||
},
|
||||
"foreignKeys": {
|
||||
@@ -24030,6 +24058,66 @@
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"decisions_running_recovery_idx": {
|
||||
"name": "decisions_running_recovery_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "updated_at",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"decisions\".\"status\" = 'decided' AND \"decisions\".\"execution_status\" = 'running'",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"decisions_open_expiry_idx": {
|
||||
"name": "decisions_open_expiry_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "expires_at",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"decisions\".\"status\" = 'open'",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"decisions_pending_continuation_idx": {
|
||||
"name": "decisions_pending_continuation_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"decisions\".\"continuation_policy\" = 'wake_origin_agent' AND \"decisions\".\"metadata\" ->> 'continuationPending' = 'true' AND (\"decisions\".\"status\" IN ('expired', 'cancelled') OR (\"decisions\".\"status\" = 'decided' AND \"decisions\".\"execution_status\" IN ('succeeded', 'partial', 'failed')))",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"decisions_bundle_idx": {
|
||||
"name": "decisions_bundle_idx",
|
||||
"columns": [
|
||||
@@ -32019,6 +32107,28 @@
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"grok_dispatch_prepared_idx": {
|
||||
"name": "grok_dispatch_prepared_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "created_at",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"grok_bot_dispatches\".\"status\" = 'prepared'",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"grok_dispatch_company_status_idx": {
|
||||
"name": "grok_dispatch_company_status_idx",
|
||||
"columns": [
|
||||
@@ -47782,6 +47892,18 @@
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"git_authorization_grant_id": {
|
||||
"name": "git_authorization_grant_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"git_authorized_by_user_id": {
|
||||
"name": "git_authorized_by_user_id",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"pause_reason": {
|
||||
"name": "pause_reason",
|
||||
"type": "text",
|
||||
@@ -47920,6 +48042,19 @@
|
||||
],
|
||||
"onDelete": "no action",
|
||||
"onUpdate": "no action"
|
||||
},
|
||||
"projects_git_authorization_grant_id_connection_grants_id_fk": {
|
||||
"name": "projects_git_authorization_grant_id_connection_grants_id_fk",
|
||||
"tableFrom": "projects",
|
||||
"tableTo": "connection_grants",
|
||||
"columnsFrom": [
|
||||
"git_authorization_grant_id"
|
||||
],
|
||||
"columnsTo": [
|
||||
"id"
|
||||
],
|
||||
"onDelete": "set null",
|
||||
"onUpdate": "no action"
|
||||
}
|
||||
},
|
||||
"compositePrimaryKeys": {},
|
||||
@@ -52787,6 +52922,88 @@
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"tool_action_requests_pending_expiry_idx": {
|
||||
"name": "tool_action_requests_pending_expiry_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "expires_at",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"tool_action_requests\".\"status\" = 'pending'",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"tool_action_requests_company_pending_expiry_idx": {
|
||||
"name": "tool_action_requests_company_pending_expiry_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "company_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "expires_at",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"tool_action_requests\".\"status\" = 'pending'",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"tool_action_requests_executing_deadline_idx": {
|
||||
"name": "tool_action_requests_executing_deadline_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "updated_at",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"tool_action_requests\".\"status\" = 'executing'",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"tool_action_requests_approved_idx": {
|
||||
"name": "tool_action_requests_approved_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"tool_action_requests\".\"status\" = 'approved'",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"tool_action_requests_invocation_idx": {
|
||||
"name": "tool_action_requests_invocation_idx",
|
||||
"columns": [
|
||||
@@ -52802,6 +53019,27 @@
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"tool_action_requests_company_interaction_idx": {
|
||||
"name": "tool_action_requests_company_interaction_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "company_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "interaction_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"tool_action_requests_issue_idx": {
|
||||
"name": "tool_action_requests_issue_idx",
|
||||
"columns": [
|
||||
@@ -55197,6 +55435,21 @@
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"tool_gateway_sessions_expiry_idx": {
|
||||
"name": "tool_gateway_sessions_expiry_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "expires_at",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"tool_gateway_sessions_run_idx": {
|
||||
"name": "tool_gateway_sessions_run_idx",
|
||||
"columns": [
|
||||
@@ -55942,6 +56195,12 @@
|
||||
"notNull": true,
|
||||
"default": "''"
|
||||
},
|
||||
"responsible_user_id": {
|
||||
"name": "responsible_user_id",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"owner_note": {
|
||||
"name": "owner_note",
|
||||
"type": "text",
|
||||
+283
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"id": "a0fcd9b8-f125-4985-8795-f567466b9aef",
|
||||
"prevId": "7832df38-fa1b-4687-bd06-eccbc29bd28c",
|
||||
"id": "6974dd25-b8e9-4143-b4b7-ee5a5c3ff54a",
|
||||
"prevId": "e080fc1b-af8e-4012-8ce5-3b5065b65143",
|
||||
"version": "7",
|
||||
"dialect": "postgresql",
|
||||
"tables": {
|
||||
@@ -5721,6 +5721,28 @@
|
||||
}
|
||||
},
|
||||
"indexes": {
|
||||
"approvals_pending_retention_idx": {
|
||||
"name": "approvals_pending_retention_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "updated_at",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"approvals\".\"status\" = 'pending'",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"approvals_company_status_type_idx": {
|
||||
"name": "approvals_company_status_type_idx",
|
||||
"columns": [
|
||||
@@ -22110,6 +22132,34 @@
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"decision_archive_notification_outbox_delivery_idx": {
|
||||
"name": "decision_archive_notification_outbox_delivery_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "last_attempt_at",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "first"
|
||||
},
|
||||
{
|
||||
"expression": "created_at",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"decision_archive_notification_outbox\".\"status\" = 'pending'",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
}
|
||||
},
|
||||
"foreignKeys": {
|
||||
@@ -24030,6 +24080,66 @@
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"decisions_running_recovery_idx": {
|
||||
"name": "decisions_running_recovery_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "updated_at",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"decisions\".\"status\" = 'decided' AND \"decisions\".\"execution_status\" = 'running'",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"decisions_open_expiry_idx": {
|
||||
"name": "decisions_open_expiry_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "expires_at",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"decisions\".\"status\" = 'open'",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"decisions_pending_continuation_idx": {
|
||||
"name": "decisions_pending_continuation_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"decisions\".\"continuation_policy\" = 'wake_origin_agent' AND \"decisions\".\"metadata\" ->> 'continuationPending' = 'true' AND (\"decisions\".\"status\" IN ('expired', 'cancelled') OR (\"decisions\".\"status\" = 'decided' AND \"decisions\".\"execution_status\" IN ('succeeded', 'partial', 'failed')))",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"decisions_bundle_idx": {
|
||||
"name": "decisions_bundle_idx",
|
||||
"columns": [
|
||||
@@ -32019,6 +32129,28 @@
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"grok_dispatch_prepared_idx": {
|
||||
"name": "grok_dispatch_prepared_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "created_at",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"grok_bot_dispatches\".\"status\" = 'prepared'",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"grok_dispatch_company_status_idx": {
|
||||
"name": "grok_dispatch_company_status_idx",
|
||||
"columns": [
|
||||
@@ -47782,6 +47914,18 @@
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"git_authorization_grant_id": {
|
||||
"name": "git_authorization_grant_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"git_authorized_by_user_id": {
|
||||
"name": "git_authorized_by_user_id",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"pause_reason": {
|
||||
"name": "pause_reason",
|
||||
"type": "text",
|
||||
@@ -47920,6 +48064,19 @@
|
||||
],
|
||||
"onDelete": "no action",
|
||||
"onUpdate": "no action"
|
||||
},
|
||||
"projects_git_authorization_grant_id_connection_grants_id_fk": {
|
||||
"name": "projects_git_authorization_grant_id_connection_grants_id_fk",
|
||||
"tableFrom": "projects",
|
||||
"tableTo": "connection_grants",
|
||||
"columnsFrom": [
|
||||
"git_authorization_grant_id"
|
||||
],
|
||||
"columnsTo": [
|
||||
"id"
|
||||
],
|
||||
"onDelete": "set null",
|
||||
"onUpdate": "no action"
|
||||
}
|
||||
},
|
||||
"compositePrimaryKeys": {},
|
||||
@@ -52787,6 +52944,88 @@
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"tool_action_requests_pending_expiry_idx": {
|
||||
"name": "tool_action_requests_pending_expiry_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "expires_at",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"tool_action_requests\".\"status\" = 'pending'",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"tool_action_requests_company_pending_expiry_idx": {
|
||||
"name": "tool_action_requests_company_pending_expiry_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "company_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "expires_at",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"tool_action_requests\".\"status\" = 'pending'",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"tool_action_requests_executing_deadline_idx": {
|
||||
"name": "tool_action_requests_executing_deadline_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "updated_at",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"tool_action_requests\".\"status\" = 'executing'",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"tool_action_requests_approved_idx": {
|
||||
"name": "tool_action_requests_approved_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"tool_action_requests\".\"status\" = 'approved'",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"tool_action_requests_invocation_idx": {
|
||||
"name": "tool_action_requests_invocation_idx",
|
||||
"columns": [
|
||||
@@ -52802,6 +53041,27 @@
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"tool_action_requests_company_interaction_idx": {
|
||||
"name": "tool_action_requests_company_interaction_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "company_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "interaction_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"tool_action_requests_issue_idx": {
|
||||
"name": "tool_action_requests_issue_idx",
|
||||
"columns": [
|
||||
@@ -55197,6 +55457,21 @@
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"tool_gateway_sessions_expiry_idx": {
|
||||
"name": "tool_gateway_sessions_expiry_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "expires_at",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"tool_gateway_sessions_run_idx": {
|
||||
"name": "tool_gateway_sessions_run_idx",
|
||||
"columns": [
|
||||
@@ -55942,6 +56217,12 @@
|
||||
"notNull": true,
|
||||
"default": "''"
|
||||
},
|
||||
"responsible_user_id": {
|
||||
"name": "responsible_user_id",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"owner_note": {
|
||||
"name": "owner_note",
|
||||
"type": "text",
|
||||
+64317
File diff suppressed because it is too large.
Load diff
+91
@@ -2654,6 +2654,97 @@
|
||||
"when": 1793060791724,
|
||||
"tag": "0380_broken_aqueduct",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 381,
|
||||
"version": "7",
|
||||
"when": 1793060791725,
|
||||
"tag": "0381_mean_scream",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 382,
|
||||
"version": "7",
|
||||
"when": 1793060791726,
|
||||
"tag": "0382_narrow_the_executioner",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 383,
|
||||
"version": "7",
|
||||
"when": 1793060791727,
|
||||
"tag": "0383_tool_review_company_expiry",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 384,
|
||||
"version": "7",
|
||||
"when": 1793060791728,
|
||||
"tag": "0384_funny_adam_warlock",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 385,
|
||||
"version": "7",
|
||||
"when": 1793060791729,
|
||||
"tag": "0385_open_madame_web",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 386,
|
||||
"version": "7",
|
||||
"when": 1793135146893,
|
||||
"tag": "0386_steady_epoch",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 387,
|
||||
"version": "7",
|
||||
"when": 1793136115698,
|
||||
"tag": "0387_boring_professor_monster",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 388,
|
||||
"version": "7",
|
||||
"when": 1793137983484,
|
||||
"tag": "0388_ambitious_micromax",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 389,
|
||||
"version": "7",
|
||||
"when": 1793138457589,
|
||||
"tag": "0389_brave_hulk",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 390,
|
||||
"version": "7",
|
||||
"when": 1793139659776,
|
||||
"tag": "0390_salty_killraven",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 391,
|
||||
"version": "7",
|
||||
"when": 1793142971791,
|
||||
"tag": "0391_third_roulette",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 392,
|
||||
"version": "7",
|
||||
"when": 1793143372644,
|
||||
"tag": "0392_small_frightful_four",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 393,
|
||||
"version": "7",
|
||||
"when": 1793145214502,
|
||||
"tag": "0393_open_grandmaster",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
import { sql } from "drizzle-orm";
|
||||
import { pgTable, uuid, text, timestamp, jsonb, index } from "drizzle-orm/pg-core";
|
||||
import { companies } from "./companies.js";
|
||||
import { agents } from "./agents.js";
|
||||
@@ -19,6 +20,7 @@ export const approvals = pgTable(
|
||||
updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(),
|
||||
},
|
||||
(table) => ({
|
||||
pendingRetentionIdx: index("approvals_pending_retention_idx").on(table.updatedAt, table.id).where(sql`${table.status} = 'pending'`),
|
||||
companyStatusTypeIdx: index("approvals_company_status_type_idx").on(
|
||||
table.companyId,
|
||||
table.status,
|
||||
|
||||
@@ -248,6 +248,9 @@ export const decisionArchiveNotificationOutbox = pgTable(
|
||||
table.originAgentId,
|
||||
),
|
||||
pendingIdx: index("decision_archive_notification_outbox_pending_idx").on(table.status, table.createdAt),
|
||||
pendingDeliveryIdx: index("decision_archive_notification_outbox_delivery_idx")
|
||||
.on(table.lastAttemptAt.asc().nullsFirst(), table.createdAt, table.id)
|
||||
.where(sql`${table.status} = 'pending'`),
|
||||
statusCheck: check(
|
||||
"decision_archive_notification_outbox_status_check",
|
||||
sql`${table.status} IN ('pending', 'delivering', 'delivered')`,
|
||||
|
||||
@@ -69,6 +69,12 @@ export const decisions = pgTable(
|
||||
table.status,
|
||||
table.expiresAt,
|
||||
),
|
||||
runningRecoveryIdx: index("decisions_running_recovery_idx").on(table.updatedAt, table.id)
|
||||
.where(sql`${table.status} = 'decided' AND ${table.executionStatus} = 'running'`),
|
||||
openExpiryIdx: index("decisions_open_expiry_idx").on(table.expiresAt, table.id)
|
||||
.where(sql`${table.status} = 'open'`),
|
||||
pendingContinuationIdx: index("decisions_pending_continuation_idx").on(table.id)
|
||||
.where(sql`${table.continuationPolicy} = 'wake_origin_agent' AND ${table.metadata} ->> 'continuationPending' = 'true' AND (${table.status} IN ('expired', 'cancelled') OR (${table.status} = 'decided' AND ${table.executionStatus} IN ('succeeded', 'partial', 'failed')))`),
|
||||
bundleIdx: index("decisions_bundle_idx").on(table.bundleId),
|
||||
originIssueIdx: index("decisions_origin_issue_idx").on(table.originIssueId),
|
||||
companyIdempotencyUq: uniqueIndex("decisions_company_idempotency_uq")
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { sql } from "drizzle-orm";
|
||||
import { index, integer, jsonb, pgTable, text, timestamp, uniqueIndex, uuid } from "drizzle-orm/pg-core";
|
||||
import { companies } from "./companies.js";
|
||||
import { agents } from "./agents.js";
|
||||
@@ -49,6 +50,7 @@ export const grokBotDispatches = pgTable("grok_bot_dispatches", {
|
||||
updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(),
|
||||
}, t => [
|
||||
uniqueIndex("grok_dispatch_event_uq").on(t.bindingId, t.bindingGeneration, t.routineRevision, t.sourceEventId),
|
||||
index("grok_dispatch_prepared_idx").on(t.createdAt, t.id).where(sql`${t.status} = 'prepared'`),
|
||||
index("grok_dispatch_company_status_idx").on(t.companyId, t.status),
|
||||
index("grok_dispatch_assignment_kind_idx").on(t.assignmentId, t.kind),
|
||||
]);
|
||||
@@ -31,6 +31,8 @@ export const joinRequests = pgTable(
|
||||
updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(),
|
||||
},
|
||||
(table) => ({
|
||||
pendingRetentionIdx: index("join_requests_pending_retention_idx").on(table.updatedAt, table.id)
|
||||
.where(sql`${table.status} = 'pending_approval'`),
|
||||
inviteUniqueIdx: uniqueIndex("join_requests_invite_unique_idx").on(table.inviteId),
|
||||
companyStatusTypeCreatedIdx: index("join_requests_company_status_type_created_idx").on(
|
||||
table.companyId,
|
||||
|
||||
@@ -714,6 +714,7 @@ export const toolGatewaySessions = pgTable(
|
||||
uniqueIndex("tool_gateway_sessions_token_hash_uq").on(table.tokenHash),
|
||||
index("tool_gateway_sessions_company_agent_idx").on(table.companyId, table.agentId),
|
||||
index("tool_gateway_sessions_company_expires_idx").on(table.companyId, table.expiresAt),
|
||||
index("tool_gateway_sessions_expiry_idx").on(table.expiresAt),
|
||||
index("tool_gateway_sessions_run_idx").on(table.companyId, table.runId),
|
||||
index("tool_gateway_sessions_issue_idx").on(table.companyId, table.issueId),
|
||||
index("tool_gateway_sessions_gateway_idx").on(table.companyId, table.gatewayId),
|
||||
@@ -831,7 +832,14 @@ export const toolActionRequests = pgTable(
|
||||
},
|
||||
(table) => [
|
||||
index("tool_action_requests_company_status_idx").on(table.companyId, table.status),
|
||||
index("tool_action_requests_pending_expiry_idx").on(table.expiresAt, table.id)
|
||||
.where(sql`${table.status} = 'pending'`),
|
||||
index("tool_action_requests_company_pending_expiry_idx").on(table.companyId, table.expiresAt)
|
||||
.where(sql`${table.status} = 'pending'`),
|
||||
index("tool_action_requests_executing_deadline_idx").on(table.updatedAt, table.id).where(sql`${table.status} = 'executing'`),
|
||||
index("tool_action_requests_approved_idx").on(table.id).where(sql`${table.status} = 'approved'`),
|
||||
index("tool_action_requests_invocation_idx").on(table.invocationId),
|
||||
index("tool_action_requests_company_interaction_idx").on(table.companyId, table.interactionId),
|
||||
index("tool_action_requests_issue_idx").on(table.companyId, table.issueId),
|
||||
],
|
||||
);
|
||||
|
||||
@@ -3,7 +3,7 @@ import { drizzle } from "drizzle-orm/postgres-js";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { createDb, withDedicatedDbConnection } from "./client.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./test-embedded-postgres.js";
|
||||
import { databaseWorkPending, installDatabaseWorkSignals, reconcileDatabaseWork, signalDatabaseWork, subscribeDatabaseWork } from "./work-signals.js";
|
||||
import { databaseWorkScopeKey, databaseWorkPending, installDatabaseWorkSignals, reconcileDatabaseWork, signalDatabaseWork, subscribeDatabaseWork } from "./work-signals.js";
|
||||
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
(support.supported ? describe : describe.skip)("database work signals", () => {
|
||||
@@ -187,3 +187,23 @@ describe("work signal query boundaries", () => {
|
||||
await expect(signalDatabaseWork(connection, "bad")).rejects.toThrow("inside a transaction");
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
describe("database work scope identity", () => {
|
||||
type TestConnection = { execute: () => Promise<unknown[]>; transaction: (callback: (tx: TestConnection) => unknown) => Promise<unknown> };
|
||||
function connection(): TestConnection {
|
||||
return { execute: vi.fn(async () => []), transaction: async callback => callback(connection()) };
|
||||
}
|
||||
it("shares owner identity through child pools and nested transactions, but isolates independent owners", async () => {
|
||||
const first = installDatabaseWorkSignals(connection());
|
||||
const second = installDatabaseWorkSignals(connection());
|
||||
const child = installDatabaseWorkSignals(connection(), first);
|
||||
const key = databaseWorkScopeKey(first);
|
||||
expect(databaseWorkScopeKey(second)).not.toBe(key);
|
||||
expect(databaseWorkScopeKey(child)).toBe(key);
|
||||
await child.transaction(async tx => {
|
||||
expect(databaseWorkScopeKey(tx)).toBe(key);
|
||||
await tx.transaction(async nested => { expect(databaseWorkScopeKey(nested)).toBe(key); });
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -5,7 +5,7 @@ import { sql } from "drizzle-orm";
|
||||
// an XID, so unrelated transactions keep their existing SQL behavior.
|
||||
type Listener = (event: "intent" | "settled", hints?: ReadonlySet<string>) => void;
|
||||
type Transaction = { topics: Set<string>; hints: Map<string, Set<string>>; xid?: Promise<string>; uncertain: boolean };
|
||||
type Scope = { listeners: Map<string, Set<Listener>>; transactions: Set<Transaction>; root: Connection };
|
||||
type Scope = { key: object; listeners: Map<string, Set<Listener>>; transactions: Set<Transaction>; root: Connection };
|
||||
type Context = { scope: Scope; transaction?: Transaction };
|
||||
type Connection = { transaction: (...args: any[]) => Promise<any>; execute: (...args: any[]) => Promise<any> };
|
||||
const contexts = new WeakMap<object, Context>();
|
||||
@@ -19,7 +19,7 @@ function publish(scope: Scope, topic: string, event: "intent" | "settled" = "set
|
||||
|
||||
/** Installed once by createDb; dedicated child pools share their owner's scope. */
|
||||
export function installDatabaseWorkSignals<T extends Connection>(db: T, owner?: object): T {
|
||||
const scope = (owner && contexts.get(owner)?.scope) || { listeners: new Map(), transactions: new Set(), root: db };
|
||||
const scope = (owner && contexts.get(owner)?.scope) || { key: {}, listeners: new Map(), transactions: new Set(), root: db };
|
||||
install(db, { scope });
|
||||
return db;
|
||||
}
|
||||
@@ -112,3 +112,7 @@ export function subscribeDatabaseWork(owner: object, topic: string, listener: Li
|
||||
}
|
||||
return () => { listeners.delete(listener); };
|
||||
}
|
||||
|
||||
export function databaseWorkScopeKey(connection: object): object {
|
||||
return contexts.get(connection)?.scope.key ?? connection;
|
||||
}
|
||||
@@ -388,6 +388,13 @@ support disabling reasoning; other runner providers reject an explicit selection
|
||||
before starting work. Native sessions expose support through the
|
||||
`perTurnReasoning` capability.
|
||||
|
||||
For the native `openrouter/` route, the parent-held provider proxy retains the
|
||||
upstream OpenRouter key. The child config keeps the selected `openrouter` provider
|
||||
and exact model, using a random loopback token restricted to that model's chat
|
||||
completions endpoint. The proxy and token expire when the session closes. The
|
||||
local token remains a credential and is redacted from diagnostics; this does not
|
||||
claim isolation for every child credential or persisted value.
|
||||
|
||||
OpenCode's proxy sends agent instructions once per turn through the system prompt. The
|
||||
task envelope retains task-specific constraints and the completion contract,
|
||||
without repeating the system instructions. This applies to all users of the
|
||||
|
||||
@@ -14,12 +14,12 @@
|
||||
"requiresProviderPolicy": false
|
||||
},
|
||||
"pi": {
|
||||
"agentProfileVersion": 29,
|
||||
"agentProfileVersion": 30,
|
||||
"agentServerPackage": "pi-acp",
|
||||
"agentServerVersion": "0.0.33",
|
||||
"agentRuntimePackage": "@earendil-works/pi-coding-agent",
|
||||
"agentRuntimeVersion": "1.0.0",
|
||||
"commandDigest": "sha256:e80c6dc7bde12790371345ee5a12f0201d30941d26f1a780aec3d090545b2a06",
|
||||
"commandDigest": "sha256:4517ab123dd89140ef462fc1905ac7a16a0dfcbaf21d67299a3c62de29c08a4d",
|
||||
"requiresProviderPolicy": true
|
||||
},
|
||||
"cursor": {
|
||||
@@ -32,12 +32,12 @@
|
||||
"requiresProviderPolicy": true
|
||||
},
|
||||
"copilot": {
|
||||
"agentProfileVersion": 65,
|
||||
"agentProfileVersion": 70,
|
||||
"agentServerPackage": "@github/copilot",
|
||||
"agentServerVersion": "1.0.88",
|
||||
"agentRuntimePackage": null,
|
||||
"agentRuntimeVersion": null,
|
||||
"commandDigest": "sha256:277894e850e07e6801cc24299c237513f58b41eb93c24f751d984fc4ddb04c47",
|
||||
"commandDigest": "sha256:618c3b69bc98b9a7e3c1b3411700eab8b90fd9245ba35d0bc85484319247fdbc",
|
||||
"requiresProviderPolicy": true
|
||||
},
|
||||
"claude": {
|
||||
|
||||
@@ -57,6 +57,110 @@ PRP v2 command vocabulary. Incoming wire commands still undergo the negotiated
|
||||
protocol validation before execution; ordinary persisted v1 summaries remain
|
||||
compatible.
|
||||
|
||||
## Adoption and retired launch artifacts
|
||||
|
||||
A live runner is adopted with its admitted artifact binding and authenticated
|
||||
process identity. ACPX and OpenCode launch profiles are constructed only when a
|
||||
new runner will actually be spawned. Adoption does not read the retired
|
||||
controller's provider-pack executables, sidecar or proxy files. Fresh launches
|
||||
still require the existing artifact digests and provider-pack authority.
|
||||
|
||||
The component regression starts the production runner and OpenCode proxy against
|
||||
a scripted harness, removes the controller's launch files, then recovers the
|
||||
same process and provider session and completes another turn. This exercises the
|
||||
real transport; it does not qualify a hosted OpenCode or ACPX deployment.
|
||||
|
||||
## Source-bound profile history
|
||||
|
||||
Copilot profile v68 binds the merged hot-upgrade and current master transport.
|
||||
Published v66 and the earlier hot-upgrade v67 identity remain byte-for-byte
|
||||
historical records. The distinct development v66 identity is retained under
|
||||
`copilot-profile-v66-hot-upgrade-candidate-historical-identity.json`; it does not
|
||||
replace the published v66 fixture. Decoding historical versions does not grant
|
||||
fresh-launch permission. Installation still requires the exact current profile,
|
||||
package and digest. These records do not establish hosted harness qualification.
|
||||
|
||||
## External controller checkpoints
|
||||
|
||||
The optional controller persistence callback stores recovery control state, not
|
||||
an output transcript. Text deltas do not trigger callbacks; usage updates are
|
||||
coalesced into the next control checkpoint. Authentication, command intents and
|
||||
outcomes, semantic inputs, lifecycle and result events retain commit barriers.
|
||||
Priority-zero and unknown event kinds are never treated as disposable output.
|
||||
|
||||
Installing the callback makes it the controller's durable authority. The local
|
||||
state file is replaced once with a fail-closed marker; ordinary control changes
|
||||
no longer rewrite or fsync a second local journal. The controller can continue
|
||||
if that directory disappears. A new controller must restore the committed
|
||||
external checkpoint before opening the directory; the marker cannot recover
|
||||
an execution or authorize a new one. This does not change the sandbox runner's
|
||||
journal or controllers without an external persistence callback.
|
||||
|
||||
Fresh semantic inputs reserve their original live callback before awaiting the
|
||||
checkpoint barrier. Recovery cannot steal first dispatch when no saved outcome
|
||||
exists. The reservation is released if persistence or admission fails; business
|
||||
execution still starts only after durable persistence succeeds.
|
||||
|
||||
Recovery and live semantic-tool dispatch share per-call ownership. Overlapping
|
||||
reconciliation calls cannot invoke the same callback twice while its outcome is
|
||||
pending. An already queued authoritative result is replayed without resolving
|
||||
its business outcome again. Failed or unknown outcomes retain the existing
|
||||
reconciliation and authority rules.
|
||||
|
||||
Pending completion inputs (`paperclip_finish` and `paperclip_block`) use the
|
||||
driver's existing completion handler even when its caller supplies no recovery
|
||||
lookup for ordinary tools. Codex transport waits for both authenticated startup
|
||||
and handler installation; the external driver waits for session opening.
|
||||
An ordinary tool with no outcome lookup remains held. Reconciliation does not
|
||||
turn a missing lookup into permission to repeat that tool.
|
||||
|
||||
Transport readers use a committed view when external persistence is enabled.
|
||||
Lifecycle and result events, command outcomes and warm-handoff authority become
|
||||
visible only after the checkpoint callback succeeds. A failed write leaves the
|
||||
previous view in place until a successful retry. Internal admission still sees
|
||||
staged intents so overlapping recovery cannot duplicate them. Command results
|
||||
flush their checkpoint even when no next command is waiting.
|
||||
|
||||
Streamed output continues without checkpoint writes. The view stops before a
|
||||
pending control event to preserve event order, reuses immutable event records
|
||||
and unchanged decoded commands, and caches the visible projection between
|
||||
changes. It neither copies a second transcript nor adds an idle scan.
|
||||
|
||||
A rejected persistence callback closes the current connection and keeps the last
|
||||
successful revision. A later flush or reconnect retries from a complete checkpoint,
|
||||
including when the failed response hid a successful commit. Queued writes also
|
||||
rebuild their delta against the last successful callback. Commit barriers remain
|
||||
in force, and no retry timer runs when the controller is idle.
|
||||
|
||||
Supporting runners advertise `recoveryCursorVersion: 1` in `auth_hello`. The core
|
||||
includes that capability and `recoveryAckedSourceSeq` in both authentication
|
||||
proofs. The runner verifies the cursor equals its persisted acknowledgement.
|
||||
After authentication and owner admission, a replacement core can reconcile that
|
||||
cursor with its sparse checkpoint. It only skips already-acknowledged history;
|
||||
new delivery remains contiguous. Unacknowledged events use the existing outbox.
|
||||
A peer without this capability cannot attach to a sparse external checkpoint.
|
||||
New runners can still authenticate to older controllers that use full journals.
|
||||
|
||||
There is no per-output external checkpoint timer or remote-controller cache
|
||||
fsync. Local-only controllers retain their full file journal. Existing application
|
||||
callbacks, the bounded runner outbox, and backpressure remain. Losing the
|
||||
controller before acknowledged display output reaches its application log can
|
||||
leave a display tail gap; this must not trigger provider or tool reexecution.
|
||||
|
||||
Prepared environments retain at most two opaque configuration IDs, each bound to
|
||||
its exact execution identity across a warm handoff. They are captured with control
|
||||
state, never output, and do not duplicate instructions or preparation requirements.
|
||||
An execution driver can provide `authorizeRunnerAdoption(configurationId, identity)`
|
||||
to recheck current authority before registering a surviving runner, without
|
||||
preparing its resources again. The driver owns policy; Runner knows no project or
|
||||
account rules. Missing bindings and rejected checks fail closed. Without this
|
||||
optional hook, adoption retains the existing `prepareLaunch` recovery callback
|
||||
and requires its controller-local preparation cache. Older checkpoints without
|
||||
a configuration binding cannot use the cache-independent adoption path.
|
||||
Supplying the hook for adoption without `prepareLaunch` is rejected at construction;
|
||||
the transport must not silently skip a requested authority check. Drivers that do
|
||||
not use prepared environments keep their existing adoption contract.
|
||||
|
||||
## Connection and authentication
|
||||
|
||||
The connection starts in this order:
|
||||
@@ -424,11 +528,49 @@ A `revoke` envelope persists the `revoked` lifecycle. The runner flushes any
|
||||
existing durable events and exits. It does not accept new work or delete
|
||||
unacknowledged facts.
|
||||
|
||||
## Process identity
|
||||
|
||||
An external process owner can publish `startedAt` after launch dispatch returns.
|
||||
`spawnRunner` preserves that live value through its restart wrapper. Once the
|
||||
owner has a verified process identity, consumers use its birth time rather than
|
||||
controller construction or attachment time. Replacing the controller must not
|
||||
change the admitted process identity.
|
||||
|
||||
## Verification
|
||||
|
||||
The production durable control plane and live-session suites cover reconnect,
|
||||
ACK replay, restart recovery, backpressure, lease expiry, drain, and revoke.
|
||||
|
||||
`src/control-plane/runner-recovery-fixture.test.ts` builds the test-only Cargo
|
||||
example `recovery-fixture`. It links the unchanged production durable runner with
|
||||
a scripted `CommandExecutor`, whose worker thread performs bounded hashing steps
|
||||
and retains each resulting output until the normal executor ACK transfers it to
|
||||
the runner outbox. There is one process, no HTTP counter service, and no production
|
||||
provider-admission switch. Private fixture receipts report completed steps, queue
|
||||
depth and process identity; they do not acknowledge or recover protocol events.
|
||||
|
||||
The short experiment starts one turn, removes the controller, then releases real
|
||||
work. Eight retained events fill the fixture queue and pause work explicitly.
|
||||
A replacement authenticates and drains through PRP. At step 32 the executor waits
|
||||
for a controller command, survives another replacement, then completes 64 steps
|
||||
without repeating the turn. The test runs with ordinary filesystem persistence
|
||||
and with opaque external checkpoints restored onto empty controller storage.
|
||||
An independent short-deadline case checks termination with recovery state intact.
|
||||
These prove the runner mechanism, not Codex, Claude, Daytona or application
|
||||
authorization. Server adoption and installed-artifact qualification have separate
|
||||
owners and evidence.
|
||||
|
||||
Run the short experiment from this package with:
|
||||
|
||||
```sh
|
||||
pnpm exec vitest run src/control-plane/runner-recovery-fixture.test.ts --maxWorkers=1
|
||||
```
|
||||
|
||||
The fixture builds through Cargo during test setup. Production distribution builds
|
||||
do not include the example. Existing control-plane tests separately inject failed
|
||||
checkpoint persistence, stale/unauthenticated cursors and output floods, including
|
||||
the zero-per-text-checkpoint-write gate.
|
||||
|
||||
Use `--json` for the complete trace or `--output <path>` to write it. The CLI
|
||||
and browser show connection counts, safe lease ID and expiry, stable identities,
|
||||
source and ACK cursors, outbox current/peak bytes, command delivery/effect
|
||||
@@ -441,3 +583,5 @@ Regenerate the checked fault matrix and every exact per-fault trace with:
|
||||
```sh
|
||||
# Recorded evidence generation is deferred from this release.
|
||||
```
|
||||
|
||||
Checkpoint decoding also exposes read-only recovery evidence: the admitted runner/session identity, authenticated connection count, and highest recorded source sequence. This projection grants no authority and adds no checkpoint writes. Qualification compares it across controller replacement; a replayed canonical event by itself does not prove that a replacement controller has authenticated a runner.
|
||||
@@ -930,3 +930,31 @@ portable runtime isolation, private archive replay, and complete/partial/empty
|
||||
campaign accounting. They do not establish current vendor prices, future
|
||||
retention charges, live Braintrust attribution or invoice settlement. Keep
|
||||
paid recurrence disabled until these independent gates are qualified.
|
||||
|
||||
|
||||
### Token-spend qualification with separate operating reconciliation
|
||||
|
||||
A policy may explicitly set `overhead.accountingReadiness: "model_requests"`
|
||||
when model-token qualification is the operator's priority. In that mode an
|
||||
absent campaign capacity proof and absent observability cost proof do not block
|
||||
admission. Any supplied proofs must still validate. Current allowance/pricing
|
||||
windows, trace quotas, immutable source/request/runtime bindings, complete
|
||||
selection, per-cell allocations, request limits, no retries and the shared
|
||||
campaign admission cap remain required. Model reservation proofs remain required.
|
||||
|
||||
The final report retains the once-only operating allowance and labels operating
|
||||
reconciliation pending. Missing, failed or canceled model evidence still holds
|
||||
the full cell allocation and fails completion. This policy does not certify a
|
||||
vendor operating-cost bound. Omit the field to retain full-lifecycle proof
|
||||
requirements. Existing manifests and held amounts are not rewritten.
|
||||
|
||||
For this policy version, `operatingReconciliation` is always `pending`; the
|
||||
immutable controller summary has no operating invoice settlement step. Mixed
|
||||
histories retain that warning when any campaign uses the policy. A shared
|
||||
`accountingReadiness` label appears only when every campaign uses it.
|
||||
|
||||
|
||||
The trusted controller artifact preserves hidden dependency metadata as well as
|
||||
code. Its inventory verifier requires every recorded byte, including package
|
||||
`.npmignore` files. Only the fresh provider-credential-free controller build uses
|
||||
this upload setting; private attempts still use the protected archive path.
|
||||
@@ -85,7 +85,7 @@
|
||||
{
|
||||
"path": "schemas/provider-descriptor.schema.json",
|
||||
"id": "https://paperclip.dev/schemas/prp/v1/provider-descriptor.schema.json",
|
||||
"sha256": "862c0c13f23564c3530cefcf01579b0535a565b919cc3dcff7bc1bee9a668baf"
|
||||
"sha256": "81d525335245dd4cb8a9bf41e3390dc60c6dd3e32474aad48b4f18a94d5f9711"
|
||||
},
|
||||
{
|
||||
"path": "schemas/provider-event.schema.json",
|
||||
|
||||
@@ -168,6 +168,18 @@
|
||||
"xhigh",
|
||||
"max"
|
||||
]
|
||||
},
|
||||
"grokEffort": {
|
||||
"enum": [
|
||||
"minimal",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh"
|
||||
]
|
||||
},
|
||||
"grokDisableWebSearch": {
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"allOf": [
|
||||
@@ -518,6 +530,46 @@
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"if": {
|
||||
"required": [
|
||||
"grokEffort"
|
||||
]
|
||||
},
|
||||
"then": {
|
||||
"properties": {
|
||||
"provider": {
|
||||
"const": "acpx"
|
||||
},
|
||||
"agent": {
|
||||
"const": "grok"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"agent"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"if": {
|
||||
"required": [
|
||||
"grokDisableWebSearch"
|
||||
]
|
||||
},
|
||||
"then": {
|
||||
"properties": {
|
||||
"provider": {
|
||||
"const": "acpx"
|
||||
},
|
||||
"agent": {
|
||||
"const": "grok"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"agent"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"$defs": {
|
||||
|
||||
@@ -0,0 +1,233 @@
|
||||
use std::collections::VecDeque;
|
||||
use std::fs;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::{Arc, Condvar, Mutex};
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
|
||||
use paperclip_runner_core::durable::{
|
||||
capture_bootstrap_ticket, run_durable_runner, Command, CommandExecution, CommandExecutor,
|
||||
DurableRunnerConfig, DurableRunnerError, EventPriority, PolledEvent,
|
||||
};
|
||||
use serde::Deserialize;
|
||||
use serde_json::json;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
struct Script {
|
||||
steps: usize,
|
||||
wait_after: usize,
|
||||
retention_limit: usize,
|
||||
release_file: PathBuf,
|
||||
receipt_file: PathBuf,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Work {
|
||||
started: bool,
|
||||
stopped: bool,
|
||||
resumed: bool,
|
||||
completed: usize,
|
||||
acknowledged: usize,
|
||||
peak_retained: usize,
|
||||
events: VecDeque<PolledEvent>,
|
||||
}
|
||||
|
||||
type Shared = Arc<(Mutex<Work>, Condvar)>;
|
||||
|
||||
fn receipt(script: &Script, work: &Work, phase: &str) {
|
||||
let bytes = serde_json::to_vec(&json!({
|
||||
"pid": std::process::id(), "phase": phase, "completedSteps": work.completed,
|
||||
"retainedEvents": work.events.len(), "acknowledgedEvents": work.acknowledged,
|
||||
"peakRetainedEvents": work.peak_retained,
|
||||
}))
|
||||
.unwrap();
|
||||
let temporary = script.receipt_file.with_extension("pending");
|
||||
fs::write(&temporary, bytes).unwrap();
|
||||
fs::rename(temporary, &script.receipt_file).unwrap();
|
||||
}
|
||||
|
||||
fn workload(script: Script, shared: Shared) {
|
||||
let (lock, changed) = &*shared;
|
||||
let mut work = lock.lock().unwrap();
|
||||
receipt(&script, &work, "admitted");
|
||||
loop {
|
||||
if work.stopped {
|
||||
receipt(&script, &work, "stopped");
|
||||
return;
|
||||
}
|
||||
if !work.started || !script.release_file.exists() {
|
||||
work = changed
|
||||
.wait_timeout(work, Duration::from_millis(10))
|
||||
.unwrap()
|
||||
.0;
|
||||
continue;
|
||||
}
|
||||
if work.completed == script.steps {
|
||||
receipt(&script, &work, "completed");
|
||||
work = changed.wait(work).unwrap();
|
||||
continue;
|
||||
}
|
||||
if work.completed == script.wait_after && !work.resumed {
|
||||
receipt(&script, &work, "awaiting_controller");
|
||||
work = changed.wait(work).unwrap();
|
||||
continue;
|
||||
}
|
||||
if work.events.len() == script.retention_limit {
|
||||
receipt(&script, &work, "backpressured");
|
||||
work = changed.wait(work).unwrap();
|
||||
continue;
|
||||
}
|
||||
let step = work.completed + 1;
|
||||
drop(work);
|
||||
let mut digest = Sha256::digest(step.to_le_bytes());
|
||||
for _ in 0..4096 {
|
||||
digest = Sha256::digest(digest);
|
||||
}
|
||||
work = lock.lock().unwrap();
|
||||
work.events.push_back(PolledEvent {
|
||||
executor_event_id: format!("fixture-step-{step}"),
|
||||
event_type: "item.delta".to_owned(),
|
||||
priority: EventPriority::P2,
|
||||
payload: json!({"text": format!("step={step} sha256={digest:x}\n")}),
|
||||
});
|
||||
work.completed = step;
|
||||
work.peak_retained = work.peak_retained.max(work.events.len());
|
||||
receipt(&script, &work, "computing");
|
||||
}
|
||||
}
|
||||
|
||||
struct ScriptedExecutor {
|
||||
shared: Shared,
|
||||
worker: Option<thread::JoinHandle<()>>,
|
||||
}
|
||||
|
||||
impl ScriptedExecutor {
|
||||
fn new(script: Script) -> Self {
|
||||
let shared = Arc::new((Mutex::new(Work::default()), Condvar::new()));
|
||||
let worker_shared = shared.clone();
|
||||
Self {
|
||||
shared,
|
||||
worker: Some(thread::spawn(move || workload(script, worker_shared))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl CommandExecutor for ScriptedExecutor {
|
||||
fn execute(&mut self, command: &Command) -> Result<CommandExecution, DurableRunnerError> {
|
||||
let mut work = self.shared.0.lock().unwrap();
|
||||
match command.command_type.as_str() {
|
||||
"turn.start" => {
|
||||
if work.started {
|
||||
return Err(DurableRunnerError::invalid("fixture turn started twice"));
|
||||
}
|
||||
work.started = true;
|
||||
}
|
||||
"turn.steer" => work.resumed = true,
|
||||
"session.snapshot" => {}
|
||||
"turn.stop" | "session.close" | "runner.shutdown" | "runner.suspend" => {
|
||||
work.stopped = true
|
||||
}
|
||||
_ => return Err(DurableRunnerError::invalid("unsupported fixture command")),
|
||||
}
|
||||
self.shared.1.notify_all();
|
||||
Ok(CommandExecution::result(json!({
|
||||
"completedSteps": work.completed, "retainedEvents": work.events.len(),
|
||||
"acknowledgedEvents": work.acknowledged, "pid": std::process::id(),
|
||||
})))
|
||||
}
|
||||
|
||||
fn poll_events(&mut self) -> Result<Vec<PolledEvent>, DurableRunnerError> {
|
||||
self.retained_events()
|
||||
}
|
||||
|
||||
fn retained_events(&mut self) -> Result<Vec<PolledEvent>, DurableRunnerError> {
|
||||
Ok(self
|
||||
.shared
|
||||
.0
|
||||
.lock()
|
||||
.unwrap()
|
||||
.events
|
||||
.iter()
|
||||
.cloned()
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn acknowledge_events(&mut self, count: usize) -> Result<(), DurableRunnerError> {
|
||||
let mut work = self.shared.0.lock().unwrap();
|
||||
if count > work.events.len() {
|
||||
return Err(DurableRunnerError::invalid(
|
||||
"fixture acknowledged an unowned event",
|
||||
));
|
||||
}
|
||||
work.events.drain(..count);
|
||||
work.acknowledged += count;
|
||||
self.shared.1.notify_all();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn shutdown(&mut self) -> Result<(), DurableRunnerError> {
|
||||
self.shared.0.lock().unwrap().stopped = true;
|
||||
self.shared.1.notify_all();
|
||||
if let Some(worker) = self.worker.take() {
|
||||
worker
|
||||
.join()
|
||||
.map_err(|_| DurableRunnerError::invalid("fixture worker failed"))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ScriptedExecutor {
|
||||
fn drop(&mut self) {
|
||||
let _ = self.shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let args: Vec<String> = std::env::args().skip(1).collect();
|
||||
let value = |key: &str| -> Result<String, DurableRunnerError> {
|
||||
args.windows(2)
|
||||
.find(|pair| pair[0] == key)
|
||||
.map(|pair| pair[1].clone())
|
||||
.ok_or_else(|| DurableRunnerError::invalid(format!("missing {key}")))
|
||||
};
|
||||
let script: Script = serde_json::from_slice(&fs::read(value("--fixture-script")?)?)?;
|
||||
if script.steps == 0
|
||||
|| script.steps > 4096
|
||||
|| script.wait_after > script.steps
|
||||
|| !(1..=256).contains(&script.retention_limit)
|
||||
{
|
||||
return Err("fixture workload exceeds its bounds".into());
|
||||
}
|
||||
let config = DurableRunnerConfig {
|
||||
connect_url: value("--connect-url")?,
|
||||
ca_bundle_path: None,
|
||||
state_dir: value("--state-dir")?.into(),
|
||||
runner_instance_id: value("--runner-id")?,
|
||||
environment_lease_id: value("--environment-lease-id")?,
|
||||
run_id: value("--run-id")?,
|
||||
normalized_session_id: value("--session-id")?,
|
||||
turn_id: value("--turn-id")?,
|
||||
item_id: value("--item-id")?,
|
||||
runner_version: value("--runner-version")?,
|
||||
runner_digest: value("--runner-digest")?,
|
||||
acpx_launch_profile: None,
|
||||
opencode_launch_profile: None,
|
||||
max_outbox_bytes: value("--max-outbox-bytes")?.parse()?,
|
||||
p0_reserve_bytes: value("--p0-reserve-bytes")?.parse()?,
|
||||
max_frame_bytes: 1024 * 1024,
|
||||
reconnect_delay: Duration::from_millis(20),
|
||||
reconnect_grace: Some(Duration::from_millis(
|
||||
value("--reconnect-grace-ms")?.parse()?,
|
||||
)),
|
||||
max_runtime: Duration::ZERO,
|
||||
};
|
||||
run_durable_runner(
|
||||
config,
|
||||
capture_bootstrap_ticket()?.ok_or("missing ticket")?,
|
||||
ScriptedExecutor::new(script),
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -145,6 +145,10 @@ struct AcpxProviderDescriptor {
|
||||
pi_thinking_level: Option<PiThinkingLevel>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
claude_effort: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
grok_effort: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
grok_disable_web_search: Option<bool>,
|
||||
permission_mode_pinned: bool,
|
||||
#[serde(default)]
|
||||
provider_policy: Option<AcpxProviderRuntimePolicy>,
|
||||
@@ -273,6 +277,14 @@ impl AcpxProviderDescriptor {
|
||||
self.agent != "claude"
|
||||
|| !matches!(effort.as_str(), "low" | "medium" | "high" | "xhigh" | "max")
|
||||
})
|
||||
|| (self.grok_disable_web_search.is_some() && self.agent != "grok")
|
||||
|| self.grok_effort.as_ref().is_some_and(|effort| {
|
||||
self.agent != "grok"
|
||||
|| !matches!(
|
||||
effort.as_str(),
|
||||
"minimal" | "low" | "medium" | "high" | "xhigh"
|
||||
)
|
||||
})
|
||||
|| self.mode.as_ref().is_some_and(|mode| {
|
||||
mode.trim().is_empty()
|
||||
|| mode.chars().count() > 240
|
||||
@@ -375,6 +387,8 @@ impl AcpxProviderDescriptor {
|
||||
mode: self.mode.clone(),
|
||||
pi_thinking_level: self.pi_thinking_level,
|
||||
claude_effort: self.claude_effort.clone(),
|
||||
grok_effort: self.grok_effort.clone(),
|
||||
grok_disable_web_search: self.grok_disable_web_search.clone(),
|
||||
permission_mode_pinned: self.permission_mode_pinned,
|
||||
provider_policy: self.provider_policy.clone(),
|
||||
system_instructions: self.instructions.clone(),
|
||||
@@ -482,6 +496,12 @@ impl AcpxProviderDescriptor {
|
||||
if let Some(effort) = &self.claude_effort {
|
||||
descriptor["claudeEffort"] = json!(effort);
|
||||
}
|
||||
if let Some(effort) = &self.grok_effort {
|
||||
descriptor["grokEffort"] = json!(effort);
|
||||
}
|
||||
if let Some(disabled) = self.grok_disable_web_search {
|
||||
descriptor["grokDisableWebSearch"] = json!(disabled);
|
||||
}
|
||||
if let Some(level) = self.pi_thinking_level {
|
||||
descriptor["piThinkingLevel"] = json!(level);
|
||||
}
|
||||
@@ -622,6 +642,8 @@ impl AcpxDurableState {
|
||||
|| identity.mode != self.descriptor.mode
|
||||
|| identity.pi_thinking_level != self.descriptor.pi_thinking_level
|
||||
|| identity.claude_effort != self.descriptor.claude_effort
|
||||
|| identity.grok_effort != self.descriptor.grok_effort
|
||||
|| identity.grok_disable_web_search != self.descriptor.grok_disable_web_search
|
||||
{
|
||||
return Err(DurableRunnerError::invalid(
|
||||
"ACPX durable identity no longer matches its qualified profile or provider mode",
|
||||
@@ -1309,6 +1331,12 @@ impl AcpxCommandExecutor {
|
||||
if let Some(effort) = &identity.claude_effort {
|
||||
result["claudeEffort"] = json!(effort);
|
||||
}
|
||||
if let Some(effort) = &identity.grok_effort {
|
||||
result["grokEffort"] = json!(effort);
|
||||
}
|
||||
if let Some(disabled) = identity.grok_disable_web_search {
|
||||
result["grokDisableWebSearch"] = json!(disabled);
|
||||
}
|
||||
if let Some(level) = identity.pi_thinking_level {
|
||||
// This identity is admitted only after the sidecar's effective-mode ACK.
|
||||
result["piThinkingLevel"] = json!(level);
|
||||
@@ -2783,6 +2811,8 @@ mod tests {
|
||||
mode: descriptor.mode.clone(),
|
||||
pi_thinking_level: descriptor.pi_thinking_level,
|
||||
claude_effort: descriptor.claude_effort.clone(),
|
||||
grok_effort: descriptor.grok_effort.clone(),
|
||||
grok_disable_web_search: descriptor.grok_disable_web_search.clone(),
|
||||
provider_lifetime_fence_candidates: [60_001, 60_002, 60_003],
|
||||
};
|
||||
let operations = Vec::new();
|
||||
@@ -3060,6 +3090,8 @@ mod tests {
|
||||
mode: None,
|
||||
pi_thinking_level: None,
|
||||
claude_effort: None,
|
||||
grok_effort: None,
|
||||
grok_disable_web_search: None,
|
||||
provider_lifetime_fence_candidates: [60_001, 60_002, 60_003],
|
||||
};
|
||||
|
||||
@@ -3214,7 +3246,7 @@ mod tests {
|
||||
#[test]
|
||||
fn pi_profile_matches_published_identity_and_rejects_prior_profiles() {
|
||||
let published: Value = serde_json::from_str(include_str!(
|
||||
"../../../../test-fixtures/pi-acp/profile-v29-identity.json"
|
||||
"../../../../test-fixtures/pi-acp/profile-v30-identity.json"
|
||||
))
|
||||
.unwrap();
|
||||
let mut value = descriptor("codex");
|
||||
@@ -3231,6 +3263,7 @@ mod tests {
|
||||
current.validate(&context()).unwrap();
|
||||
|
||||
for prior in [
|
||||
include_str!("../../../../test-fixtures/pi-acp/profile-v29-identity.json"),
|
||||
include_str!("../../../../test-fixtures/pi-acp/profile-v28-identity.json"),
|
||||
include_str!("../../../../test-fixtures/pi-acp/profile-v24-identity.json"),
|
||||
include_str!("../../../../test-fixtures/pi-acp/profile-v23-identity.json"),
|
||||
@@ -3550,6 +3583,8 @@ mod tests {
|
||||
mode: original_descriptor.mode.clone(),
|
||||
pi_thinking_level: original_descriptor.pi_thinking_level,
|
||||
claude_effort: original_descriptor.claude_effort.clone(),
|
||||
grok_effort: original_descriptor.grok_effort.clone(),
|
||||
grok_disable_web_search: original_descriptor.grok_disable_web_search.clone(),
|
||||
provider_lifetime_fence_candidates: [60_001, 60_002, 60_003],
|
||||
};
|
||||
let operations = Vec::new();
|
||||
@@ -3836,6 +3871,8 @@ mod tests {
|
||||
mode: original_descriptor.mode.clone(),
|
||||
pi_thinking_level: original_descriptor.pi_thinking_level,
|
||||
claude_effort: original_descriptor.claude_effort.clone(),
|
||||
grok_effort: original_descriptor.grok_effort.clone(),
|
||||
grok_disable_web_search: original_descriptor.grok_disable_web_search.clone(),
|
||||
provider_lifetime_fence_candidates: [60_001, 60_002, 60_003],
|
||||
};
|
||||
let operations = Vec::new();
|
||||
@@ -4155,6 +4192,8 @@ mod tests {
|
||||
mode: original_descriptor.mode.clone(),
|
||||
pi_thinking_level: None,
|
||||
claude_effort: None,
|
||||
grok_effort: None,
|
||||
grok_disable_web_search: None,
|
||||
provider_lifetime_fence_candidates: [60_001, 60_002, 60_003],
|
||||
};
|
||||
let operations = Vec::new();
|
||||
@@ -4404,6 +4443,8 @@ mod tests {
|
||||
mode: descriptor.mode.clone(),
|
||||
pi_thinking_level: descriptor.pi_thinking_level,
|
||||
claude_effort: descriptor.claude_effort.clone(),
|
||||
grok_effort: descriptor.grok_effort.clone(),
|
||||
grok_disable_web_search: descriptor.grok_disable_web_search.clone(),
|
||||
provider_lifetime_fence_candidates,
|
||||
};
|
||||
let operations = Vec::new();
|
||||
@@ -4578,6 +4619,8 @@ mod tests {
|
||||
mode: provider_descriptor.mode.clone(),
|
||||
pi_thinking_level: provider_descriptor.pi_thinking_level,
|
||||
claude_effort: provider_descriptor.claude_effort.clone(),
|
||||
grok_effort: provider_descriptor.grok_effort.clone(),
|
||||
grok_disable_web_search: provider_descriptor.grok_disable_web_search.clone(),
|
||||
provider_lifetime_fence_candidates,
|
||||
});
|
||||
state.provider_exit_unconfirmed = true;
|
||||
|
||||
@@ -55,6 +55,10 @@ struct PersistedAcpxProviderSessionIdentity {
|
||||
pi_thinking_level: Option<PiThinkingLevel>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
claude_effort: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
grok_effort: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
grok_disable_web_search: Option<bool>,
|
||||
provider_lifetime_fence_candidates: [u16; 3],
|
||||
}
|
||||
|
||||
@@ -80,6 +84,8 @@ impl PersistedAcpxProviderSessionIdentity {
|
||||
mode: identity.mode,
|
||||
pi_thinking_level: identity.pi_thinking_level,
|
||||
claude_effort: identity.claude_effort.clone(),
|
||||
grok_effort: identity.grok_effort.clone(),
|
||||
grok_disable_web_search: identity.grok_disable_web_search.clone(),
|
||||
provider_lifetime_fence_candidates: identity.provider_lifetime_fence_candidates,
|
||||
})
|
||||
}
|
||||
@@ -99,6 +105,8 @@ impl PersistedAcpxProviderSessionIdentity {
|
||||
mode: self.mode.clone(),
|
||||
pi_thinking_level: self.pi_thinking_level,
|
||||
claude_effort: self.claude_effort.clone(),
|
||||
grok_effort: self.grok_effort.clone(),
|
||||
grok_disable_web_search: self.grok_disable_web_search.clone(),
|
||||
provider_lifetime_fence_candidates: self.provider_lifetime_fence_candidates,
|
||||
}
|
||||
}
|
||||
@@ -122,6 +130,8 @@ impl AcpxSuspensionCheckpoint {
|
||||
|| identity.mode != config.mode
|
||||
|| identity.pi_thinking_level != config.pi_thinking_level
|
||||
|| identity.claude_effort != config.claude_effort
|
||||
|| identity.grok_effort != config.grok_effort
|
||||
|| identity.grok_disable_web_search != config.grok_disable_web_search
|
||||
|| config
|
||||
.expected_identity
|
||||
.as_ref()
|
||||
|
||||
@@ -74,6 +74,10 @@ pub struct AcpxProviderSessionIdentity {
|
||||
pub pi_thinking_level: Option<PiThinkingLevel>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub claude_effort: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub grok_effort: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub grok_disable_web_search: Option<bool>,
|
||||
pub provider_lifetime_fence_candidates: [u16; 3],
|
||||
}
|
||||
|
||||
@@ -97,6 +101,8 @@ pub struct AcpxProviderSessionConfig {
|
||||
pub mode: Option<String>,
|
||||
pub pi_thinking_level: Option<PiThinkingLevel>,
|
||||
pub claude_effort: Option<String>,
|
||||
pub grok_effort: Option<String>,
|
||||
pub grok_disable_web_search: Option<bool>,
|
||||
pub permission_mode_pinned: bool,
|
||||
pub provider_policy: Option<AcpxProviderRuntimePolicy>,
|
||||
pub system_instructions: String,
|
||||
@@ -120,6 +126,22 @@ impl AcpxProviderSessionConfig {
|
||||
"Claude effort must be a supported Claude-only value",
|
||||
));
|
||||
}
|
||||
if self.grok_disable_web_search.is_some() && self.agent != "grok" {
|
||||
return Err(LocalRunnerError::invalid(
|
||||
"Grok search policy requires Grok",
|
||||
));
|
||||
}
|
||||
if self.grok_effort.as_ref().is_some_and(|effort| {
|
||||
self.agent != "grok"
|
||||
|| !matches!(
|
||||
effort.as_str(),
|
||||
"minimal" | "low" | "medium" | "high" | "xhigh"
|
||||
)
|
||||
}) {
|
||||
return Err(LocalRunnerError::invalid(
|
||||
"Grok effort must be a supported Grok-only value",
|
||||
));
|
||||
}
|
||||
if (self.agent == "pi") != self.pi_thinking_level.is_some() {
|
||||
return Err(LocalRunnerError::invalid(
|
||||
"ACPX Pi thinking level must be explicit for Pi and absent for other agents",
|
||||
@@ -200,6 +222,8 @@ impl AcpxProviderSessionConfig {
|
||||
|| expected_identity.mode != self.mode
|
||||
|| expected_identity.pi_thinking_level != self.pi_thinking_level
|
||||
|| expected_identity.claude_effort != self.claude_effort
|
||||
|| expected_identity.grok_effort != self.grok_effort
|
||||
|| expected_identity.grok_disable_web_search != self.grok_disable_web_search
|
||||
{
|
||||
return Err(LocalRunnerError::invalid(
|
||||
"ACPX expected identity conflicts with the requested session",
|
||||
@@ -227,6 +251,16 @@ impl AcpxProviderSessionIdentity {
|
||||
"ACPX identity contains invalid Claude effort",
|
||||
));
|
||||
}
|
||||
if self.grok_effort.as_ref().is_some_and(|effort| {
|
||||
!matches!(
|
||||
effort.as_str(),
|
||||
"minimal" | "low" | "medium" | "high" | "xhigh"
|
||||
)
|
||||
}) {
|
||||
return Err(LocalRunnerError::invalid(
|
||||
"ACPX identity contains invalid Grok effort",
|
||||
));
|
||||
}
|
||||
for (value, label) in [
|
||||
(&self.normalized_session_id, "normalized session"),
|
||||
(&self.acpx_record_id, "record"),
|
||||
@@ -1336,6 +1370,12 @@ fn session_open_params(config: &AcpxProviderSessionConfig, sidecar_tools: &[Valu
|
||||
if let Some(effort) = &config.claude_effort {
|
||||
params["claudeEffort"] = json!(effort);
|
||||
}
|
||||
if let Some(effort) = &config.grok_effort {
|
||||
params["grokEffort"] = json!(effort);
|
||||
}
|
||||
if let Some(disabled) = config.grok_disable_web_search {
|
||||
params["grokDisableWebSearch"] = json!(disabled);
|
||||
}
|
||||
if let Some(level) = config.pi_thinking_level {
|
||||
params["piThinkingLevel"] = json!(level);
|
||||
}
|
||||
@@ -1454,6 +1494,8 @@ fn verify_open_response(
|
||||
|| identity.mode != config.mode
|
||||
|| identity.pi_thinking_level != config.pi_thinking_level
|
||||
|| identity.claude_effort != config.claude_effort
|
||||
|| identity.grok_effort != config.grok_effort
|
||||
|| identity.grok_disable_web_search != config.grok_disable_web_search
|
||||
|| config
|
||||
.expected_identity
|
||||
.as_ref()
|
||||
@@ -1789,6 +1831,8 @@ mod mode_tests {
|
||||
mode: Some("plan".to_owned()),
|
||||
pi_thinking_level: None,
|
||||
claude_effort: None,
|
||||
grok_effort: None,
|
||||
grok_disable_web_search: None,
|
||||
permission_mode_pinned: true,
|
||||
provider_policy: Some(AcpxProviderRuntimePolicy { read_only: false }),
|
||||
system_instructions: String::new(),
|
||||
@@ -1817,6 +1861,8 @@ mod mode_tests {
|
||||
mode: Some("plan".to_owned()),
|
||||
pi_thinking_level: None,
|
||||
claude_effort: None,
|
||||
grok_effort: None,
|
||||
grok_disable_web_search: None,
|
||||
provider_lifetime_fence_candidates: [60_001, 60_002, 60_003],
|
||||
}
|
||||
}
|
||||
@@ -1976,6 +2022,39 @@ mod mode_tests {
|
||||
cfg.claude_effort = None;
|
||||
cfg.validate().unwrap();
|
||||
}
|
||||
#[test]
|
||||
fn grok_search_policy_is_grok_only_and_preserves_false_in_the_sidecar() {
|
||||
let mut cfg = config();
|
||||
cfg.agent = "grok".to_owned();
|
||||
for disabled in [true, false] {
|
||||
cfg.grok_disable_web_search = Some(disabled);
|
||||
cfg.validate().unwrap();
|
||||
assert_eq!(
|
||||
session_open_params(&cfg, &[])["grokDisableWebSearch"],
|
||||
json!(disabled)
|
||||
);
|
||||
}
|
||||
cfg.agent = "codex".to_owned();
|
||||
assert!(cfg.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn grok_effort_is_explicit_grok_only_and_forwarded_without_clamping() {
|
||||
let mut cfg = config();
|
||||
cfg.agent = "grok".to_owned();
|
||||
for effort in ["minimal", "low", "medium", "high", "xhigh"] {
|
||||
cfg.grok_effort = Some(effort.to_owned());
|
||||
cfg.validate().unwrap();
|
||||
assert_eq!(session_open_params(&cfg, &[])["grokEffort"], json!(effort));
|
||||
}
|
||||
cfg.grok_effort = Some("ultra".to_owned());
|
||||
assert!(cfg.validate().is_err());
|
||||
cfg.grok_effort = Some("high".to_owned());
|
||||
cfg.agent = "codex".to_owned();
|
||||
assert!(cfg.validate().is_err());
|
||||
cfg.grok_effort = None;
|
||||
cfg.validate().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pi_thinking_level_is_closed_explicit_and_pi_only() {
|
||||
|
||||
@@ -893,6 +893,23 @@ fn stderr_diagnostic_categories(value: &str) -> BTreeSet<&'static str> {
|
||||
|
||||
fn response_error_classification(error: &ResponseError) -> &'static str {
|
||||
match error.code.as_str() {
|
||||
"GROK_EFFORT_ACKNOWLEDGEMENT_MISSING" => return "grok_effort_acknowledgement_missing",
|
||||
"GROK_EFFORT_MISMATCH" => return "grok_effort_mismatch",
|
||||
"GROK_MODEL_MISMATCH" => return "grok_model_mismatch",
|
||||
"GROK_SESSION_MISMATCH" => return "grok_session_mismatch",
|
||||
"GROK_CONFIGURATION_CONFLICT" => return "grok_configuration_conflict",
|
||||
"GROK_CONFIGURATION_DRIFT" => return "grok_configuration_drift",
|
||||
"GROK_ADMISSION_PROTOCOL_VIOLATION" => return "grok_admission_protocol_violation",
|
||||
"ACPX_ADMISSION_BINDING" => return "admission_binding",
|
||||
"ACPX_ADMISSION_INSTALLATION" => return "admission_installation",
|
||||
"ACPX_ADMISSION_SANDBOX" => return "admission_sandbox",
|
||||
"ACPX_ADMISSION_CREDENTIAL" => return "admission_credential",
|
||||
"ACPX_ADMISSION_CONFIGURATION" => return "admission_configuration",
|
||||
"ACPX_ADMISSION_SKILLS" => return "admission_skills",
|
||||
"ACPX_ADMISSION_COMMAND" => return "admission_command",
|
||||
"ACPX_ADMISSION_TOOL_BRIDGE" => return "admission_tool_bridge",
|
||||
"ACPX_ADMISSION_RUNTIME" => return "admission_runtime",
|
||||
"ACPX_ADMISSION_IDENTITY" => return "admission_identity",
|
||||
"ACPX_RUNTIME_PACKAGE_MISSING" => return "runtime_package_missing",
|
||||
"ACP_MODEL_UNSUPPORTED" => return "requested_model_unsupported",
|
||||
"AGENT_STARTUP_FAILED" => return "agent_startup_failed",
|
||||
@@ -1120,6 +1137,20 @@ mod tests {
|
||||
message: message.to_owned(),
|
||||
retryable: false,
|
||||
};
|
||||
for (code, expected) in [
|
||||
("ACPX_ADMISSION_SANDBOX", "admission_sandbox"),
|
||||
("ACPX_ADMISSION_CREDENTIAL", "admission_credential"),
|
||||
("ACPX_ADMISSION_RUNTIME", "admission_runtime"),
|
||||
(
|
||||
"GROK_EFFORT_ACKNOWLEDGEMENT_MISSING",
|
||||
"grok_effort_acknowledgement_missing",
|
||||
),
|
||||
] {
|
||||
assert_eq!(
|
||||
response_error_classification(&error(code, "private-token-canary")),
|
||||
expected
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
response_error_classification(&error(
|
||||
"acpx_sidecar_command_failed",
|
||||
|
||||
@@ -262,7 +262,7 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
|
||||
"id": id,
|
||||
"ok": false,
|
||||
"error": {
|
||||
"code": "Q7Z9",
|
||||
"code": request["params"]["errorCode"].as_str().unwrap_or("Q7Z9"),
|
||||
"message": "violet-circuit-4821",
|
||||
"retryable": false,
|
||||
},
|
||||
|
||||
@@ -43,6 +43,7 @@ const OPENCODE_PROVIDER_ENVIRONMENT_KEYS: &[&str] = &[
|
||||
"PAPERCLIP_NATIVE_MCP_TOKEN",
|
||||
"PAPERCLIP_OPENCODE_PERMISSION_MODE",
|
||||
"PAPERCLIP_OPENCODE_RUNTIME_DIR",
|
||||
"PAPERCLIP_OPENCODE_PERFORMANCE_DIAGNOSTICS",
|
||||
"PAPERCLIP_RUNNER_INSTANCE_ID",
|
||||
"PAPERCLIP_RUN_ID",
|
||||
"PAPERCLIP_NORMALIZED_SESSION_ID",
|
||||
|
||||
@@ -1014,6 +1014,7 @@ impl AuthenticatedTransport {
|
||||
"protocolMin": PROTOCOL_MIN_VERSION,
|
||||
"protocolMax": PROTOCOL_VERSION,
|
||||
"warmTransitionVersion": 1,
|
||||
"recoveryCursorVersion": 1,
|
||||
"runnerInstanceId": state.runner_instance_id,
|
||||
"environmentLeaseId": state.environment_lease_id,
|
||||
"runId": state.run_id,
|
||||
@@ -1200,6 +1201,10 @@ struct AuthChallenge {
|
||||
warm_transition_version: Option<u64>,
|
||||
#[serde(default)]
|
||||
warm_transition_id: Option<String>,
|
||||
#[serde(default)]
|
||||
recovery_cursor_version: Option<u64>,
|
||||
#[serde(default)]
|
||||
recovery_acked_source_seq: Option<u64>,
|
||||
}
|
||||
|
||||
fn validate_challenge(
|
||||
@@ -1275,6 +1280,15 @@ fn validate_challenge(
|
||||
"warm transition capability or receipt was not authenticated",
|
||||
));
|
||||
}
|
||||
if (challenge.recovery_cursor_version.is_some()
|
||||
|| challenge.recovery_acked_source_seq.is_some())
|
||||
&& (challenge.recovery_cursor_version != Some(1)
|
||||
|| challenge.recovery_acked_source_seq != Some(state.acked_source_seq))
|
||||
{
|
||||
return Err(DurableRunnerError::invalid(
|
||||
"recovery cursor does not match the runner's durable acknowledgement",
|
||||
));
|
||||
}
|
||||
match expected_lease {
|
||||
Some(lease)
|
||||
if challenge.credential_lease_id.as_deref() == Some(lease.lease_id.as_str())
|
||||
@@ -1319,6 +1333,12 @@ fn challenge_signing_bytes(challenge: &AuthChallenge) -> Vec<u8> {
|
||||
if let Some(id) = &challenge.warm_transition_id {
|
||||
body["warmTransitionId"] = json!(id);
|
||||
}
|
||||
if let Some(version) = challenge.recovery_cursor_version {
|
||||
body["recoveryCursorVersion"] = json!(version);
|
||||
}
|
||||
if let Some(cursor) = challenge.recovery_acked_source_seq {
|
||||
body["recoveryAckedSourceSeq"] = json!(cursor);
|
||||
}
|
||||
canonical_json(&body).into_bytes()
|
||||
}
|
||||
|
||||
@@ -1806,6 +1826,8 @@ mod tests {
|
||||
server_proof: String::new(),
|
||||
warm_transition_version: None,
|
||||
warm_transition_id: None,
|
||||
recovery_cursor_version: Some(1),
|
||||
recovery_acked_source_seq: payload["resume"]["ackedSourceSeq"].as_u64(),
|
||||
};
|
||||
let signing = challenge_signing_bytes(&challenge);
|
||||
challenge.server_proof = hex_encode(&hmac_domain(
|
||||
@@ -1838,6 +1860,8 @@ mod tests {
|
||||
"credentialLeaseId": &challenge.credential_lease_id,
|
||||
"revocationEpoch": challenge.revocation_epoch,
|
||||
"serverProof": &challenge.server_proof,
|
||||
"recoveryCursorVersion": challenge.recovery_cursor_version,
|
||||
"recoveryAckedSourceSeq": challenge.recovery_acked_source_seq,
|
||||
},
|
||||
}),
|
||||
config.max_frame_bytes,
|
||||
@@ -2505,6 +2529,8 @@ mod tests {
|
||||
server_proof: String::new(),
|
||||
warm_transition_version: None,
|
||||
warm_transition_id: None,
|
||||
recovery_cursor_version: None,
|
||||
recovery_acked_source_seq: None,
|
||||
};
|
||||
let signing = challenge_signing_bytes(&challenge);
|
||||
challenge.server_proof = hex_encode(&hmac_domain(
|
||||
|
||||
@@ -29,7 +29,7 @@ pub(crate) fn acpx_release_profile(agent: &str) -> Option<AcpxReleaseProfile> {
|
||||
agent_runtime_package: Some("@earendil-works/pi-coding-agent"),
|
||||
agent_runtime_version: Some("1.0.0"),
|
||||
command_digest:
|
||||
"sha256:e80c6dc7bde12790371345ee5a12f0201d30941d26f1a780aec3d090545b2a06",
|
||||
"sha256:4517ab123dd89140ef462fc1905ac7a16a0dfcbaf21d67299a3c62de29c08a4d",
|
||||
requires_provider_policy: true,
|
||||
},
|
||||
"cursor" => AcpxReleaseProfile {
|
||||
@@ -47,7 +47,7 @@ pub(crate) fn acpx_release_profile(agent: &str) -> Option<AcpxReleaseProfile> {
|
||||
agent_runtime_package: None,
|
||||
agent_runtime_version: None,
|
||||
command_digest:
|
||||
"sha256:277894e850e07e6801cc24299c237513f58b41eb93c24f751d984fc4ddb04c47",
|
||||
"sha256:618c3b69bc98b9a7e3c1b3411700eab8b90fd9245ba35d0bc85484319247fdbc",
|
||||
requires_provider_policy: true,
|
||||
},
|
||||
"claude" => AcpxReleaseProfile {
|
||||
|
||||
@@ -671,6 +671,27 @@ fn measurement(value: &Value) -> Value {
|
||||
|
||||
/// Converts Codex app-server notifications into provider-neutral PRP events.
|
||||
/// Provider-native envelopes are consumed here and never cross the PRP boundary.
|
||||
fn provider_performance_spans(params: &Value) -> Vec<Value> {
|
||||
let mut counts = std::collections::BTreeMap::<&str, usize>::new();
|
||||
params.get("spans").and_then(Value::as_array).into_iter().flatten().take(128).filter_map(|span| {
|
||||
let name = span.get("name")?.as_str()?;
|
||||
if !matches!(name,
|
||||
"provider.context.environment" | "provider.context.instructions" | "provider.skills.discovery"
|
||||
| "provider.tools.resolve" | "provider.model.resolve" | "provider.model.stream"
|
||||
| "provider.workspace.snapshot.track" | "provider.workspace.snapshot.patch" | "provider.workspace.snapshot.diff"
|
||||
| "provider.workspace.git.discover" | "provider.workspace.git.default_branch"
|
||||
| "provider.dependencies.install" | "provider.plugins.initialize") { return None; }
|
||||
let start = span.get("startedAtMs")?.as_f64()?;
|
||||
let end = span.get("endedAtMs")?.as_f64()?;
|
||||
if !start.is_finite() || !end.is_finite() || start < 0.0 || end < start || end - start > 86_400_000.0 { return None; }
|
||||
let count = counts.entry(name).or_default();
|
||||
if *count >= 16 { return None; }
|
||||
*count += 1;
|
||||
Some(json!({"name":name,"startedAtMs":start,"endedAtMs":end,
|
||||
"outcome":if span.get("outcome").and_then(Value::as_str)==Some("failed") {"failed"} else {"ok"}}))
|
||||
}).collect()
|
||||
}
|
||||
|
||||
pub fn normalize_codex_notification(method: &str, params: &Value) -> Vec<NormalizedProviderEvent> {
|
||||
let mut events = Vec::new();
|
||||
let push = |events: &mut Vec<NormalizedProviderEvent>,
|
||||
@@ -685,6 +706,17 @@ pub fn normalize_codex_notification(method: &str, params: &Value) -> Vec<Normali
|
||||
};
|
||||
|
||||
match method {
|
||||
"paperclip/providerPerformance" => {
|
||||
let spans = provider_performance_spans(params);
|
||||
if !spans.is_empty() {
|
||||
push(
|
||||
&mut events,
|
||||
"harness.diagnostic",
|
||||
EventPriority::P1,
|
||||
json!({"code":"provider_performance_spans","spans":spans}),
|
||||
);
|
||||
}
|
||||
}
|
||||
"thread/compacted" => push(
|
||||
&mut events,
|
||||
"context.compacted",
|
||||
@@ -1464,6 +1496,37 @@ fn has_rfc_uri_scheme_prefix(value: &str) -> bool {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn performance_diagnostics_keep_only_bounded_content_free_measurements() {
|
||||
let mut spans = vec![json!({"name":"private prompt","startedAtMs":1,"endedAtMs":2})];
|
||||
for _ in 0..200 {
|
||||
spans.push(
|
||||
json!({"name":"provider.model.stream","startedAtMs":1000,"endedAtMs":1010,
|
||||
"outcome":"failed","authorization":"private","attributes":{"prompt":"private"}}),
|
||||
);
|
||||
}
|
||||
let events =
|
||||
normalize_codex_notification("paperclip/providerPerformance", &json!({"spans":spans}));
|
||||
assert_eq!(events.len(), 1);
|
||||
assert_eq!(events[0].event_type, "harness.diagnostic");
|
||||
assert_eq!(events[0].priority, EventPriority::P1);
|
||||
let rows = events[0].payload["spans"].as_array().unwrap();
|
||||
assert_eq!(rows.len(), 16);
|
||||
assert_eq!(
|
||||
rows[0],
|
||||
json!({"name":"provider.model.stream","startedAtMs":1000.0,"endedAtMs":1010.0,"outcome":"failed"})
|
||||
);
|
||||
assert!(!events[0].payload.to_string().contains("private"));
|
||||
assert!(normalize_codex_notification(
|
||||
"paperclip/providerPerformance",
|
||||
&json!({"spans":[
|
||||
{"name":"provider.model.stream","startedAtMs":10,"endedAtMs":1},
|
||||
{"name":"provider.model.stream","startedAtMs":0,"endedAtMs":86400001}
|
||||
]})
|
||||
)
|
||||
.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unrelated_information_retains_only_bounded_run_log_diagnostics() {
|
||||
let events = normalize_codex_notification(
|
||||
|
||||
@@ -53,6 +53,8 @@ fn config(directory: &std::path::Path) -> AcpxProviderSessionConfig {
|
||||
mode: None,
|
||||
pi_thinking_level: None,
|
||||
claude_effort: None,
|
||||
grok_effort: None,
|
||||
grok_disable_web_search: None,
|
||||
permission_mode_pinned: true,
|
||||
provider_policy: None,
|
||||
system_instructions: "Complete the supplied task.".to_owned(),
|
||||
@@ -82,6 +84,8 @@ fn identity() -> AcpxProviderSessionIdentity {
|
||||
mode: None,
|
||||
pi_thinking_level: None,
|
||||
claude_effort: None,
|
||||
grok_effort: None,
|
||||
grok_disable_web_search: None,
|
||||
provider_lifetime_fence_candidates: [60_001, 60_002, 60_003],
|
||||
}
|
||||
}
|
||||
|
||||
@@ -52,6 +52,8 @@ fn config(mode: &str) -> AcpxProviderSessionConfig {
|
||||
mode: None,
|
||||
pi_thinking_level: None,
|
||||
claude_effort: None,
|
||||
grok_effort: None,
|
||||
grok_disable_web_search: None,
|
||||
permission_mode_pinned: true,
|
||||
provider_policy: None,
|
||||
system_instructions: "Complete the supplied task.".to_owned(),
|
||||
|
||||
@@ -48,6 +48,8 @@ fn config(mode: &str) -> AcpxProviderSessionConfig {
|
||||
mode: None,
|
||||
pi_thinking_level: None,
|
||||
claude_effort: None,
|
||||
grok_effort: None,
|
||||
grok_disable_web_search: None,
|
||||
permission_mode_pinned: true,
|
||||
provider_policy: None,
|
||||
system_instructions: "Complete the supplied task.".to_owned(),
|
||||
@@ -72,6 +74,8 @@ fn expected_identity() -> AcpxProviderSessionIdentity {
|
||||
mode: None,
|
||||
pi_thinking_level: None,
|
||||
claude_effort: None,
|
||||
grok_effort: None,
|
||||
grok_disable_web_search: None,
|
||||
provider_lifetime_fence_candidates: [60_001, 60_002, 60_003],
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,6 +28,8 @@ fn config(mode: &str) -> AcpxProviderSessionConfig {
|
||||
mode: None,
|
||||
pi_thinking_level: None,
|
||||
claude_effort: None,
|
||||
grok_effort: None,
|
||||
grok_disable_web_search: None,
|
||||
permission_mode_pinned: true,
|
||||
provider_policy: None,
|
||||
system_instructions: "Complete the supplied task.".to_owned(),
|
||||
|
||||
@@ -51,6 +51,8 @@ fn config(mode: &str) -> AcpxProviderSessionConfig {
|
||||
permission_mode: AcpxPermissionMode::ApproveReads,
|
||||
mode: None,
|
||||
claude_effort: None,
|
||||
grok_effort: None,
|
||||
grok_disable_web_search: None,
|
||||
pi_thinking_level: if mode.starts_with("controls") {
|
||||
Some(paperclip_runner_core::acpx_provider_session::PiThinkingLevel::Low)
|
||||
} else {
|
||||
|
||||
@@ -219,6 +219,34 @@ fn keeps_valid_command_rejections_separate_from_protocol_failures() {
|
||||
transport.shutdown().expect("fake sidecar should stop");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preserves_closed_admission_categories_over_the_sidecar_pipe() {
|
||||
let mut transport = transport("remote-error", Duration::from_secs(1));
|
||||
for (code, category) in [
|
||||
("ACPX_ADMISSION_INSTALLATION", "admission_installation"),
|
||||
("ACPX_ADMISSION_CREDENTIAL", "admission_credential"),
|
||||
(
|
||||
"GROK_EFFORT_ACKNOWLEDGEMENT_MISSING",
|
||||
"grok_effort_acknowledgement_missing",
|
||||
),
|
||||
("GROK_MODEL_MISMATCH", "grok_model_mismatch"),
|
||||
("ACPX_ADMISSION_CREDENTIAL_SECRET", "unclassified"),
|
||||
("GROK_MODEL_MISMATCH_SECRET", "unclassified"),
|
||||
] {
|
||||
let error = transport
|
||||
.request(
|
||||
GeneratedAcpxSidecarCommand::SessionOpen,
|
||||
json!({"errorCode":code}),
|
||||
)
|
||||
.unwrap_err();
|
||||
let message = error.to_string();
|
||||
assert!(message.contains(&format!("classification={category}")));
|
||||
assert!(!message.contains("violet-circuit-4821"));
|
||||
assert!(!message.contains("SECRET"));
|
||||
}
|
||||
transport.shutdown().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redacts_sidecar_stderr_when_the_process_exits() {
|
||||
let mut transport = transport("exit-secret", Duration::from_secs(1));
|
||||
|
||||
+2
-2
@@ -17,7 +17,7 @@ use sha2::{Digest, Sha256};
|
||||
const CODEX_ACPX_DIGEST: &str =
|
||||
"sha256:443491f4e814501a5dc47347e4510e84faeec9c9382e2d4e53bfe69386d5aabc";
|
||||
const PI_ACPX_DIGEST: &str =
|
||||
"sha256:e80c6dc7bde12790371345ee5a12f0201d30941d26f1a780aec3d090545b2a06";
|
||||
"sha256:4517ab123dd89140ef462fc1905ac7a16a0dfcbaf21d67299a3c62de29c08a4d";
|
||||
|
||||
fn temporary_directory(label: &str) -> PathBuf {
|
||||
let nonce = SystemTime::now()
|
||||
@@ -1160,7 +1160,7 @@ fn acpx_transport_failure_retains_its_error_without_reopening_the_provider() {
|
||||
assert_eq!(snapshot["status"], "closed");
|
||||
assert_eq!(snapshot["activeProviderTurnId"], Value::Null);
|
||||
assert!(executor.poll_events().unwrap().is_empty());
|
||||
executor.shutdown().unwrap();
|
||||
shutdown_recovered_acpx_fixture(&mut executor, &directory);
|
||||
fs::remove_dir_all(directory).unwrap();
|
||||
}
|
||||
|
||||
|
||||
@@ -170,6 +170,9 @@ export async function summarizeControlledEvidence(bundles, options) {
|
||||
function history(campaigns) {
|
||||
const sum = field => usd(campaigns.reduce((total, campaign) => total + units(campaign[field]), 0n));
|
||||
return { schema: 'paperclip.runner-eval.control-history/v1', costStatus: campaigns.some(c => c.costStatus === 'incomplete') ? 'incomplete' : 'provisional',
|
||||
...(campaigns.length && campaigns.every(c => c.accountingReadiness === 'model_requests')
|
||||
? { accountingReadiness: 'model_requests' } : {}),
|
||||
...(campaigns.some(c => c.operatingReconciliation === 'pending') ? { operatingReconciliation: 'pending' } : {}),
|
||||
knownUsd: sum('knownUsd'), heldUsd: sum('heldUsd'), committedUsd: sum('committedUsd'), operatingHeldUsd: sum('operatingHeldUsd'),
|
||||
incompleteAttempts: campaigns.reduce((total, c) => total + c.incompleteAttempts, 0), overBudget: campaigns.some(c => c.overBudget), campaigns };
|
||||
}
|
||||
|
||||
@@ -787,3 +787,42 @@ test('retained failure codes distinguish native startup, child exit and invalid
|
||||
const artifact = await fixture(t, { missingRoute: true, requests: 0 });
|
||||
assert.equal((await artifact.run()).report.failure, 'artifact_invalid');
|
||||
});
|
||||
|
||||
|
||||
test('model readiness rejects a missing terminal model receipt even when its reservation has a reviewed bound', async t => {
|
||||
const f = await fixture(t, { requests: 1 });
|
||||
f.manifest.overhead.accountingReadiness = 'model_requests';
|
||||
for (const component of f.manifest.overhead.components) delete component.provisionalBound;
|
||||
await f.save();
|
||||
const result = await f.run(0, async () => Response.json({ error: 'synthetic refusal' }, { status: 429 }));
|
||||
assert.equal(result.report.failure, 'request_accounting_unbounded');
|
||||
const summary = await summarizeControlledCampaign(f.manifest);
|
||||
assert.equal(summary.incompleteAttempts, 1);
|
||||
assert.equal(summary.heldUsd, '0.014000000');
|
||||
assert.equal(summary.operatingReconciliation, 'pending');
|
||||
});
|
||||
|
||||
|
||||
test('model readiness rejects zero model requests and keeps the full allocation', async t => {
|
||||
const f = await fixture(t, { requests: 0 });
|
||||
f.manifest.overhead.accountingReadiness = 'model_requests';
|
||||
for (const component of f.manifest.overhead.components) delete component.provisionalBound;
|
||||
await f.save();
|
||||
const result = await f.run();
|
||||
assert.equal(result.report.failure, 'request_accounting_unbounded');
|
||||
assert.equal(result.report.accounting.requests.length, 0);
|
||||
const summary = await summarizeControlledCampaign(f.manifest);
|
||||
assert.equal(summary.incompleteAttempts, 1); assert.equal(summary.heldUsd, '0.014000000');
|
||||
});
|
||||
|
||||
test('mixed-policy private histories keep the pending operating reconciliation warning', async t => {
|
||||
const model = await fixture(t, { requests: 1 }), strict = await fixture(t, { requests: 1 });
|
||||
model.manifest.campaignId = 'model-readiness-history';
|
||||
model.manifest.overhead.accountingReadiness = 'model_requests';
|
||||
for (const component of model.manifest.overhead.components) delete component.provisionalBound;
|
||||
await model.save(); await model.run(); await strict.run();
|
||||
const bundles = await Promise.all([model, strict].map(f => packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-0', environment: f.environment })));
|
||||
const history = await summarizeControlledEvidence(bundles);
|
||||
assert.equal(history.incompleteAttempts, 0); assert.equal(history.operatingHeldUsd, '0.008000000');
|
||||
assert.equal(history.accountingReadiness, undefined); assert.equal(history.operatingReconciliation, 'pending');
|
||||
});
|
||||
@@ -23,7 +23,9 @@ export function prepareHostedManifest({ catalog, catalogSha256, prepared, policy
|
||||
|| !Number.isSafeInteger(policy.maxCells) || policy.maxCells < 1 || policy.maxCells > 512
|
||||
|| !Array.isArray(policy.allocations) || !policy.allocations.length) throw new Error('A bounded hosted policy with tracing is required');
|
||||
if (!/^[a-f0-9]{64}$/.test(policy.controllerLockSha256 ?? '')) throw new Error('Invalid hosted controller lock binding');
|
||||
costRuntime.reservationBound('operating_allowance', policy.capacityBound, hostedPolicyConfiguration(policy));
|
||||
if (policy.overhead?.accountingReadiness !== 'model_requests' || policy.capacityBound !== undefined) {
|
||||
costRuntime.reservationBound('operating_allowance', policy.capacityBound, hostedPolicyConfiguration(policy));
|
||||
}
|
||||
fields(policy.observability, ['projectId', 'capture', 'nativeLimits']);
|
||||
const runId = environment.GITHUB_RUN_ID, paths = hostedPaths(runId);
|
||||
if (prepared.schema !== 'paperclip.runner-eval.prepared-requests/v1' || prepared.githubRunId !== runId || prepared.githubRunAttempt !== 1
|
||||
@@ -65,7 +67,7 @@ export function prepareHostedManifest({ catalog, catalogSha256, prepared, policy
|
||||
models: policy.models, overhead: policy.overhead, observability: { ...policy.observability },
|
||||
sourceRevision: prepared.sourceRevision, definitionsRevision: prepared.definitionsRevision, ...artifacts,
|
||||
outputDirectory: paths.evidence, hosted: { repository: environment.GITHUB_REPOSITORY, runId, runAttempt: 1, workflowRevision: environment.GITHUB_SHA, controllerLockSha256: policy.controllerLockSha256, policySha256: sha256(canonical(policy)) }, attempts };
|
||||
manifest.observability.provisionalBound = { ...policy.capacityBound, configurationSha256: sha256(canonical(observabilityBoundConfiguration(manifest))) };
|
||||
if (policy.capacityBound !== undefined) manifest.observability.provisionalBound = { ...policy.capacityBound, configurationSha256: sha256(canonical(observabilityBoundConfiguration(manifest))) };
|
||||
const validated = validateManifest(manifest, now);
|
||||
assertHostedAdmission(manifest, attempts[0], { ...environment, PAPERCLIP_RUNNER_EVAL_EXECUTION_ID: attempts[0].executionId });
|
||||
return validated;
|
||||
|
||||
@@ -119,3 +119,52 @@ test('hosted CLI reports safe admission reasons without echoing private paths or
|
||||
assert.match(result.stderr, /A required hosted input file is missing/);
|
||||
assert.doesNotMatch(result.stderr + result.stdout, /SENSITIVE_SENTINEL/);
|
||||
});
|
||||
|
||||
|
||||
test('model-readiness policy retains operating holds without requiring vendor cost proofs', async t => {
|
||||
const c = await campaign(t);
|
||||
c.input.policy.overhead.accountingReadiness = 'model_requests';
|
||||
for (const component of c.input.policy.overhead.components) delete component.provisionalBound;
|
||||
delete c.input.policy.capacityBound;
|
||||
const result = c.build();
|
||||
assert.equal(result.manifest.overhead.accountingReadiness, 'model_requests');
|
||||
assert.equal(result.manifest.observability.provisionalBound, undefined);
|
||||
Object.assign(c.f.manifest, result.manifest); await c.f.save();
|
||||
const missing = await summarizeControlledManifest(result.manifest, []);
|
||||
assert.equal(missing.incompleteAttempts, 2);
|
||||
assert.equal(missing.operatingHeldUsd, '0.004000000');
|
||||
Object.assign(c.f.environment, c.environment, { PAPERCLIP_RUNNER_EVAL_EXECUTION_ID: 'execution-0' });
|
||||
assert.equal((await c.f.run(0, () => success())).code, 0);
|
||||
const first = await packControlledAttempt({ manifest: result.manifest, attemptId: 'attempt-0', environment: c.f.environment });
|
||||
Object.assign(c.f.environment, { PAPERCLIP_RUNNER_EVAL_EXECUTION_ID: 'execution-1' });
|
||||
assert.equal((await c.f.run(1, () => success())).code, 0);
|
||||
const second = await packControlledAttempt({ manifest: result.manifest, attemptId: 'attempt-1', environment: c.f.environment });
|
||||
const complete = await summarizeControlledManifest(result.manifest, [first, second]);
|
||||
assert.equal(complete.incompleteAttempts, 0);
|
||||
assert.equal(complete.operatingHeldUsd, '0.004000000');
|
||||
assert.equal(complete.accountingReadiness, 'model_requests');
|
||||
assert.equal(complete.operatingReconciliation, 'pending');
|
||||
delete c.input.policy.overhead.accountingReadiness;
|
||||
assert.throws(() => c.build(), /cost bound|operating bound/);
|
||||
});
|
||||
|
||||
|
||||
test('model readiness still validates supplied operating and capacity proofs', async t => {
|
||||
const c = await campaign(t), policy = c.input.policy;
|
||||
policy.overhead.accountingReadiness = 'model_requests';
|
||||
for (const component of policy.overhead.components) {
|
||||
component.provisionalBound.configurationSha256 = sha256(canonical({ accountingReadiness: 'model_requests',
|
||||
reference: policy.overhead.reference, runner: policy.overhead.runner, verifiedAt: policy.overhead.verifiedAt,
|
||||
expiresAt: policy.overhead.expiresAt, component: { component: component.component, maxUsd: component.maxUsd, reference: component.reference } }));
|
||||
}
|
||||
policy.capacityBound.configurationSha256 = sha256(canonical(hostedPolicyConfiguration(policy)));
|
||||
assert.equal(c.build().manifest.overhead.accountingReadiness, 'model_requests');
|
||||
const mismatch = structuredClone(c.input); mismatch.policy.capacityBound.evidenceSha256 = '0'.repeat(64);
|
||||
assert.throws(() => prepareHostedManifest(mismatch), /Observability capacity requires its operating bound evidence/);
|
||||
const stale = structuredClone(c.input); stale.policy.capacityBound.configurationSha256 = '0'.repeat(64);
|
||||
assert.throws(() => prepareHostedManifest(stale), /Reviewed cost bound configuration changed/);
|
||||
const absent = structuredClone(c.input);
|
||||
for (const component of absent.policy.overhead.components) delete component.provisionalBound;
|
||||
absent.policy.capacityBound.configurationSha256 = sha256(canonical(hostedPolicyConfiguration(absent.policy)));
|
||||
assert.throws(() => prepareHostedManifest(absent), /Observability capacity requires its operating bound evidence/);
|
||||
});
|
||||
Loaded 100 of 398 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user