test(observability): cover lease waits and use shared state key

Co-Authored-By: Paperclip <noreply@paperclip.ing>

packages/adapter-utils/src/acpx-engine/startup-timing.ts

@@ -67,6 +67,8 @@ export const EXECUTION_PREPARATION_SPAN_ATTRS = {
  * command, a path, an argument, or an environment value can never ride a span.
  */
 export const SANDBOX_STARTUP_SPAN_ATTRS = {
+  /** Provider state before a lease start or recovery; the host clamps values. */
+  state: `${SANDBOX_STARTUP_SPAN_ATTR_PREFIX}state`,
   /** The low-cardinality provider family (through `normalizeProviderFamily`). */
   provider: `${SANDBOX_STARTUP_SPAN_ATTR_PREFIX}provider`,
   /** The step or execution outcome: `ok`, `skipped`, or `failed`. */

packages/plugins/sandbox-providers/daytona/src/lease-tracing.test.ts

@@ -24,8 +24,10 @@ type Receipt = {
 
 function sandbox(id = "fixture-sandbox", state = "archived") {
   return {
-    id, state, target: "us", recoverable: true,
+    id, state, target: "us", recoverable: true, errorReason: null as string | null,
     start: vi.fn().mockResolvedValue(undefined),
+    stop: vi.fn().mockResolvedValue(undefined),
+    refreshData: vi.fn().mockResolvedValue(undefined),
     recover: vi.fn().mockResolvedValue(undefined),
     delete: vi.fn().mockResolvedValue(undefined),
     getWorkDir: vi.fn().mockResolvedValue("/home/daytona"),
@@ -142,6 +144,65 @@ describe("Daytona lease worker tracing", () => {
     } finally { release(); worker.close(); }
   });
 
+  it("waits for a pending teardown and records the wait under the resume worker", async () => {
+    const resource = sandbox("fixture-sandbox", "started"); get.mockResolvedValue(resource);
+    let release!: () => void, stopping!: () => void;
+    const waiting = new Promise<void>(resolve => { release = resolve; });
+    const entered = new Promise<void>(resolve => { stopping = resolve; });
+    resource.stop.mockImplementation(async () => { stopping(); await waiting; resource.state = "stopped"; });
+    const worker = workerHarness();
+    try {
+      await worker.initialize();
+      const teardown = worker.call("environmentReleaseLease", params, invocation("release"));
+      await entered;
+      let settled = false;
+      const resume = worker.call("environmentResumeLease", params, invocation("resume"))
+        .then(result => { settled = true; return result; });
+      // Process the resume request while the actual provider stop is held open.
+      await new Promise<void>(resolve => setImmediate(resolve));
+      expect(settled).toBe(false);
+      expect(get).toHaveBeenCalledTimes(1);
+      expect(resource.start).not.toHaveBeenCalled();
+      release();
+      await teardown;
+      expect(await resume).toMatchObject({ providerLeaseId: resource.id });
+      const root = worker.receipts.find(receipt => receipt.invocationId === "resume")!.span;
+      expect(root.name).toBe("plugin.worker");
+      expect(root.children.filter(span => span.name === "lease.wait_for_teardown")).toHaveLength(1);
+      expect(root.children[0].name).toBe("lease.wait_for_teardown");
+      expect(root.children[0].endTimeMs).toBeLessThanOrEqual(root.children[1].startTimeMs);
+      expect(get).toHaveBeenCalledTimes(2);
+      expect(resource.start).toHaveBeenCalledTimes(1);
+    } finally { release(); worker.close(); }
+  });
+
+  it.each([false, true])("records missing-container confirmation and preserves refresh failures: %s", async fails => {
+    const resource = sandbox("fixture-sandbox", "error");
+    resource.recoverable = false;
+    resource.errorReason = `not found: failed to inspect sandbox container ${resource.id}: Error response from daemon: No such container: ${resource.id}`;
+    get.mockResolvedValue(resource);
+    resource.refreshData.mockImplementation(async () => {
+      if (fails) throw new Error("private-refresh-failure");
+      resource.state = "started";
+    });
+    const worker = workerHarness();
+    try {
+      await worker.initialize();
+      const result = worker.call("environmentResumeLease", params, invocation("resume"));
+      if (fails) await expect(result).rejects.toThrow("private-refresh-failure");
+      else expect(await result).toMatchObject({ providerLeaseId: resource.id });
+      expect(resource.refreshData).toHaveBeenCalledTimes(1);
+      const root = worker.receipts[0].span;
+      expect(root.children.map(span => span.name)).toEqual(fails
+        ? ["lease.lookup", "lease.refresh"] : ["lease.lookup", "lease.refresh", "lease.verify"]);
+      expect(root.children[1].status).toEqual(fails ? { code: 2 } : undefined);
+      expect(resource.start).not.toHaveBeenCalled();
+      expect(resource.delete).not.toHaveBeenCalled();
+      expect(create).not.toHaveBeenCalled();
+      expect(JSON.stringify(worker.receipts)).not.toContain("private-refresh-failure");
+    } finally { worker.close(); }
+  });
+
   it.each(["started", "error"])("traces a %s sandbox without adding a provider start", async state => {
     const resource = sandbox("fixture-sandbox", state); get.mockResolvedValue(resource);
     const worker = workerHarness();

server/src/services/plugin-host-services.ts

@@ -567,7 +567,7 @@ const PROVIDER_SPAN_ATTR_ALLOWLIST: ReadonlySet<string> = new Set<string>([
   SPAN_ATTRS.transferWallMs,
   SPAN_ATTRS.transferGuardCount,
   SPAN_ATTRS.transferDirection,
-  "paperclip.sandbox.startup.state",
+  SPAN_ATTRS.state,
   "plugin.operation", "plugin.reused", "plugin.item_count", "plugin.bytes", "plugin.call_count",
 ]);
 
@@ -599,7 +599,7 @@ export function clampProviderSpanAttributes(
   if (!raw) return clamped;
   for (const [key, value] of Object.entries(raw)) {
     if (!PROVIDER_SPAN_ATTR_ALLOWLIST.has(key)) continue;
-    if (key === "paperclip.sandbox.startup.state") {
+    if (key === SPAN_ATTRS.state) {
       if (typeof value === "string" && ["started", "stopped", "archived", "error", "other"].includes(value)) clamped[key] = value;
       continue;
     }