fix: retain cloud cleanup IDs before provider lookup

doc/benchmarks/native-event-ingestion-2026-11-03/cloud-stop/README.md

@@ -18,9 +18,9 @@ In a shell with no inherited API keys/tokens, start the owned production control
 
 In a separate shell, bind `DAYTONA_API_KEY` securely from your credential manager without printing it or placing it in arguments/files. Run the app checkout's Node+TSX loader on `helpers/run.mjs NEW_EVIDENCE_DIRECTORY/launch.json`. It sends the key once through the normal encrypted company-secret API, removes inherited credential environment variables before downstream work, and retains only secret references. No public controller tunnel is used. The helper installs the first-party plugin, creates an owned environment, creates a fresh task through the ordinary UI, observes the real process and actual normalized text, clicks the ordinary Stop button, and requires native/shared/provider/SDK/reload/cleanup gates.
 
-Keep the isolated app/database alive for independent verification with the repository's `scripts/native-ingestion/verify-database.mjs` against the explicit owned loopback database. Then gracefully stop the recorded launcher, verify its HTTP/DB ports and process are gone, and remove only its recorded temporary data directory. Scan retained files and decompressed trace entries for the actual credential before sharing. The public helper adapts environment-supplied credentials, a one-resource cap, and the process-age guard to use the provider's own clock. It records all already-admitted owned IDs before enforcing the cap so failure cleanup can retire them. Exact executed private-helper hashes are recorded separately. These public adaptations passed syntax checks and an offline actual-source cleanup regression; they were not used for a second live acceptance claim.
+Keep the isolated app/database alive for independent verification with the repository's `scripts/native-ingestion/verify-database.mjs` against the explicit owned loopback database. Then gracefully stop the recorded launcher, verify its HTTP/DB ports and process are gone, and remove only its recorded temporary data directory. Scan retained files and decompressed trace entries for the actual credential before sharing. The public helper adapts environment-supplied credentials, a one-resource cap, and the process-age guard to use the provider's own clock. It records all host-observed provider IDs before the first SDK lookup or cap assertion so failure cleanup can retire them. SDK label checks remain before remote execution and every deletion. Exact executed private-helper hashes are recorded separately. These public adaptations passed syntax checks and offline actual-source cleanup regressions; they were not used for a second live acceptance claim.
 
-Run `node helpers/verify-failure-cleanup.mjs` from this directory for the offline regression. It exercises the reproducer's actual admission and cleanup sections with two already-admitted owned IDs and a cap of one. Qualification must fail while both IDs reach the watchdog ledger and exact SDK cleanup. All external effects are stubbed; it creates no sandbox and reads no credential.
+Run `node helpers/verify-failure-cleanup.mjs` from this directory for the offline regressions. They exercise the actual admission, first provider lookup and cleanup sections with two observed IDs. Both an exceeded cap and an injected first SDK lookup failure must leave both IDs in the watchdog ledger and reachable by exact SDK cleanup. All external effects are stubbed; the regressions create no sandbox and read no credential.
 
 Example commands after setting the path variables (never include credentials in arguments):
 

doc/benchmarks/native-event-ingestion-2026-11-03/cloud-stop/helper-checks.json

@@ -106,5 +106,34 @@
       "exactOwnedCleanupCalls": 2
     },
     "sourceSha256": "026fe92d5452062ddd5840a0bc9aeba9f6b4f7fda57040eaadb76ca80b960409"
+  },
+  "providerLookupFailureCleanup": {
+    "scope": "Offline actual-source admission, first provider lookup, and finally-cleanup sections; SDK and filesystem effects stubbed; no provider calls",
+    "red": {
+      "exitCode": 1,
+      "reason": "First provider lookup threw before any of two host-observed IDs reached the ledger",
+      "sourceSha256": "026fe92d5452062ddd5840a0bc9aeba9f6b4f7fda57040eaadb76ca80b960409"
+    },
+    "green": {
+      "status": "passed",
+      "cloudCalls": 0,
+      "cases": [
+        {
+          "failure": "resource-cap",
+          "observedOwnedResources": 2,
+          "declaredCap": 1,
+          "ledgeredBeforeFailure": 2,
+          "exactOwnedCleanupCalls": 2
+        },
+        {
+          "failure": "first-provider-lookup",
+          "observedOwnedResources": 2,
+          "declaredCap": 2,
+          "ledgeredBeforeFailure": 2,
+          "exactOwnedCleanupCalls": 2
+        }
+      ]
+    },
+    "sourceSha256": "753ea9b830bbaa9be6fa8510fe2b7bdcc1099248939f7aa28d51d29fb854e4e9"
   }
 }

doc/benchmarks/native-event-ingestion-2026-11-03/cloud-stop/helpers/run.mjs

@@ -121,12 +121,12 @@ try {
     // Ledger every admitted host-owned ID before further observer work.
     for (const lease of leases) if (lease.heartbeatRunId && lease.providerLeaseId && !ledger.resources.some(row => row.providerLeaseId === lease.providerLeaseId)) {
       const row = { providerLeaseId: lease.providerLeaseId, leaseId: lease.id, runId: lease.heartbeatRunId, expectedLabels: expectedLabels(lease.heartbeatRunId), admittedAt: Date.now() };
-      const sandbox = await daytona.get(row.providerLeaseId); assertOwned(sandbox, row);
       ledger.resources.push(row); await writeFile(ledgerPath, JSON.stringify(ledger, null, 2));
       console.log(JSON.stringify({ resourceAdmitted: row.providerLeaseId, leaseId: row.leaseId, runId: row.runId, ttlMinutes: 30 }));
     }
-    // These resources already exist. Record every observed owned ID before a
-    // bound failure so finally/watchdog cleanup can retire all of them.
+    // These scoped host-observed resources already exist. Record every ID
+    // before the first provider lookup or bound failure. The post-admission
+    // lookup and every cleanup still verify labels before execution/deletion.
     assert.ok(priorResourceCount + ledger.resources.length <= RESOURCE_CAP, 'cloud_resource_bound');
     if (runs.length !== 1) return null;
     if (!['running', 'queued'].includes(runs[0].status)) throw new Error('native_run_terminal_before_admission');

doc/benchmarks/native-event-ingestion-2026-11-03/cloud-stop/helpers/verify-failure-cleanup.mjs

@@ -12,39 +12,53 @@ function section(start, end) {
   return source.slice(offset, stop);
 }
 const admission = section('    for (const lease of leases)', '    if (runs.length !== 1)');
+const lookup = section('\n  const sandbox = await daytona.get(row.providerLeaseId); assertOwned(sandbox, row);', '\n  const cwd = lease.metadata.remoteCwd;');
 const cleanup = section('  for (const row of ledger.resources)', '  if (!summary.cleanup.providerCleanupFailed)');
 const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor;
-const ledger = { resources: [] };
-const leases = [1, 2].map(index => ({ heartbeatRunId: `run-${index}`, providerLeaseId: `provider-${index}`, id: `lease-${index}` }));
-const expectedLabels = runId => ({ owner: 'offline-test', runId });
-const states = new Map(leases.map(lease => [lease.providerLeaseId, 'active']));
-const deletions = [];
-let persistedLedger;
-const daytona = {
-  async get(id) {
-    const lease = leases.find(lease => lease.providerLeaseId === id);
-    assert.ok(lease, 'Only the exact test resources can be read');
-    return { id, labels: expectedLabels(lease.heartbeatRunId) };
-  },
-  async delete(sandbox) { deletions.push(sandbox.id); states.set(sandbox.id, 'absent'); },
-};
-const assertOwned = (sandbox, row) => {
-  assert.equal(sandbox.id, row.providerLeaseId);
-  assert.deepEqual(sandbox.labels, row.expectedLabels);
-};
-const writeFile = async (_path, text) => { persistedLedger = JSON.parse(text); };
-await assert.rejects(new AsyncFunction(
-  'leases', 'ledger', 'priorResourceCount', 'RESOURCE_CAP', 'daytona', 'expectedLabels',
-  'assertOwned', 'writeFile', 'ledgerPath', 'assert', 'console', admission,
-)(leases, ledger, 0, 1, daytona, expectedLabels, assertOwned, writeFile, 'offline-ledger', assert, { log() {} }), /cloud_resource_bound/);
-assert.equal(persistedLedger.resources.length, 2, 'Every observed owned ID must reach the watchdog ledger');
-const summary = { cleanup: {} };
-const providerState = async row => ({ id: row.providerLeaseId, state: states.get(row.providerLeaseId) });
-const poll = async (_label, load, accept) => { const value = await load(); assert.ok(accept(value)); return value; };
-await new AsyncFunction(
-  'ledger', 'summary', 'providerState', 'daytona', 'assertOwned', 'poll', 'safeError', cleanup,
-)(ledger, summary, providerState, daytona, assertOwned, poll, error => ({ message: error.message }));
-assert.deepEqual(deletions, leases.map(lease => lease.providerLeaseId));
-assert.ok(summary.cleanup.providerAbsence.every(row => row.state === 'absent'));
-assert.equal(summary.cleanup.providerCleanupFailed, undefined);
-console.log(JSON.stringify({ status: 'passed', cloudCalls: 0, observedOwnedResources: 2, declaredCap: 1, ledgeredBeforeFailure: 2, exactOwnedCleanupCalls: 2 }));
+async function scenario(failFirstLookup) {
+  const ledger = { resources: [] };
+  const leases = [1, 2].map(index => ({ heartbeatRunId: `run-${index}`, providerLeaseId: `provider-${index}`, id: `lease-${index}` }));
+  const expectedLabels = runId => ({ owner: 'offline-test', runId });
+  const states = new Map(leases.map(lease => [lease.providerLeaseId, 'active']));
+  const deletions = [];
+  let persistedLedger;
+  let lookupAttempts = 0;
+  const lookupFailure = new Error('Injected first provider lookup failure');
+  const daytona = {
+    async get(id) {
+      if (failFirstLookup && lookupAttempts++ === 0) throw lookupFailure;
+      const lease = leases.find(lease => lease.providerLeaseId === id);
+      assert.ok(lease, 'Only the exact test resources can be read');
+      return { id, labels: expectedLabels(lease.heartbeatRunId) };
+    },
+    async delete(sandbox) { deletions.push(sandbox.id); states.set(sandbox.id, 'absent'); },
+  };
+  const assertOwned = (sandbox, row) => {
+    assert.equal(sandbox.id, row.providerLeaseId);
+    assert.deepEqual(sandbox.labels, row.expectedLabels);
+  };
+  const writeFile = async (_path, text) => { persistedLedger = JSON.parse(text); };
+  const admit = () => new AsyncFunction(
+    'leases', 'ledger', 'priorResourceCount', 'RESOURCE_CAP', 'daytona', 'expectedLabels',
+    'assertOwned', 'writeFile', 'ledgerPath', 'assert', 'console', admission,
+  )(leases, ledger, 0, failFirstLookup ? 2 : 1, daytona, expectedLabels, assertOwned, writeFile, 'offline-ledger', assert, { log() {} });
+  if (failFirstLookup) {
+    await assert.rejects(async () => {
+      await admit();
+      await new AsyncFunction('daytona', 'row', 'assertOwned', lookup)(daytona, ledger.resources[0], assertOwned);
+    }, error => error === lookupFailure);
+  } else await assert.rejects(admit, /cloud_resource_bound/);
+  assert.equal(persistedLedger?.resources.length, 2, 'Every observed owned ID must reach the watchdog ledger');
+  const summary = { cleanup: {} };
+  const providerState = async row => ({ id: row.providerLeaseId, state: states.get(row.providerLeaseId) });
+  const poll = async (_label, load, accept) => { const value = await load(); assert.ok(accept(value)); return value; };
+  await new AsyncFunction(
+    'ledger', 'summary', 'providerState', 'daytona', 'assertOwned', 'poll', 'safeError', cleanup,
+  )(ledger, summary, providerState, daytona, assertOwned, poll, error => ({ message: error.message }));
+  assert.deepEqual(deletions, leases.map(lease => lease.providerLeaseId));
+  assert.ok(summary.cleanup.providerAbsence.every(row => row.state === 'absent'));
+  assert.equal(summary.cleanup.providerCleanupFailed, undefined);
+  return { failure: failFirstLookup ? 'first-provider-lookup' : 'resource-cap', observedOwnedResources: 2,
+    declaredCap: failFirstLookup ? 2 : 1, ledgeredBeforeFailure: 2, exactOwnedCleanupCalls: 2 };
+}
+console.log(JSON.stringify({ status: 'passed', cloudCalls: 0, cases: [await scenario(false), await scenario(true)] }));

doc/benchmarks/native-event-ingestion-2026-11-03/cloud-stop/manifest.json

@@ -5,8 +5,8 @@
   "files": [
     {
       "path": "README.md",
-      "bytes": 6340,
-      "sha256": "1b5f3dfd236c083c212003d0bafd4b39bec6282f72de6a3f62bdd72f7bf22664"
+      "bytes": 6471,
+      "sha256": "2e501c60a59686809cd1302539cf0945c97be6d0adec93d96c918c994db02978"
     },
     {
       "path": "after-stop-reload.png",
@@ -35,8 +35,8 @@
     },
     {
       "path": "helper-checks.json",
-      "bytes": 3750,
-      "sha256": "3af2aaa7073a720b999c48efeee20f367f1fb1d063801f3c53c9a443a43fdc1c"
+      "bytes": 4772,
+      "sha256": "b34e33f2df0fa63589630fd1e425cf0d3f04e01b002eed49f74e8f8cd406e43f"
     },
     {
       "path": "helpers/cloud-ui.mjs",
@@ -55,8 +55,8 @@
     },
     {
       "path": "helpers/run.mjs",
-      "bytes": 26810,
-      "sha256": "026fe92d5452062ddd5840a0bc9aeba9f6b4f7fda57040eaadb76ca80b960409"
+      "bytes": 26801,
+      "sha256": "753ea9b830bbaa9be6fa8510fe2b7bdcc1099248939f7aa28d51d29fb854e4e9"
     },
     {
       "path": "helpers/watchdog.mjs",
@@ -65,13 +65,13 @@
     },
     {
       "path": "qualification.json",
-      "bytes": 10731,
-      "sha256": "f810543654f373ad4472dd441da3c8d7ea9ad6a8ebfa49a285a0d1ca8cf1b83e"
+      "bytes": 10815,
+      "sha256": "47e68bd12c90f2fb0bee58166b0e3ea63fc67233dd9a6a9bbebce79f04658610"
     },
     {
       "path": "helpers/verify-failure-cleanup.mjs",
-      "bytes": 3135,
-      "sha256": "5b32c076598087cbba2fb51bee16514d3be163bcdc428563385dbd2b46ca9996"
+      "bytes": 4045,
+      "sha256": "3ae73e3146ddb65051568e7b25e7b9f87fdd35ee6a649f9bc29b434ee8086d83"
     }
   ],
   "checks": {
@@ -82,7 +82,7 @@
     "reproductionHelperSyntax": "all6 passed after public guard/cleanup corrections",
     "terminalLeaseAndClockCorrections": "passed",
     "localControllerDatabaseDataCleanup": "passed",
-    "offlineFailureCleanupRegression": "passed; old helper failed; current helper ledgered and cleaned both already-admitted owned IDs despite cap1; zero cloud calls"
+    "offlineFailureCleanupRegression": "passed; previous helper failed injected first SDK lookup; both cap failure and first-lookup failure ledger and clean all two observed IDs; zero cloud calls"
   },
   "excluded": [
     "Private credential bootstrap identifiers",

doc/benchmarks/native-event-ingestion-2026-11-03/cloud-stop/qualification.json

@@ -233,7 +233,7 @@
   "sourceDurabilityLimit": "This verifies database-retained events and derived cursors. It does not equate every wire ACK with a durable ordinary streaming event.",
   "provenance": {
     "executedPrivateHelperSha256": "a1a9b14f3718a448dbd566a92de9e14306df165fbacad5f3b6371eaaae3e005b",
-    "publicReproductionAdaptation": "Environment-supplied DAYTONA_API_KEY, one-resource cap, OP metadata omitted, same-provider-clock process-age guard, and ledger all observed owned resources before enforcing the cap. Public adaptations passed syntax checks and an offline actual-source failure-cleanup regression; they were not used for a second live acceptance claim. The historical same-provider-clock process age is independently retained in this proof."
+    "publicReproductionAdaptation": "Environment-supplied DAYTONA_API_KEY, one-resource cap, OP metadata omitted, same-provider-clock process-age guard, and ledger all host-observed resource IDs before any SDK lookup or cap assertion. Provider label checks remain before execution and every deletion. Public adaptations passed syntax and offline actual-source cap/lookup-failure cleanup regressions; they were not used for a second live acceptance claim. The historical same-provider-clock process age is independently retained in this proof."
   },
   "terminalLease": {
     "id": "1b1aca64-e936-47d4-8509-43b7fe49f75f",