Merge origin/master into master

Co-Authored-By: Paperclip <noreply@paperclip.ing>

server/src/__tests__/cleanup-removal-service.test.ts

@@ -9,6 +9,7 @@ import {
   createDb,
   documents,
   documentRevisions,
+  heartbeatRunEvents,
   heartbeatRuns,
   issueComments,
   issueDocuments,
@@ -42,6 +43,7 @@ describeEmbeddedPostgres("cleanup removal services", () => {
   }, 20_000);
 
   afterEach(async () => {
+    await db.delete(heartbeatRunEvents);
     await db.delete(activityLog);
     await db.delete(issueReadStates);
     await db.delete(issueComments);
@@ -228,4 +230,32 @@ describeEmbeddedPostgres("cleanup removal services", () => {
     await expect(db.select().from(issueReadStates).where(eq(issueReadStates.companyId, companyId))).resolves.toHaveLength(0);
     await expect(db.select().from(activityLog).where(eq(activityLog.companyId, companyId))).resolves.toHaveLength(0);
   });
+
+  it("removes heartbeat events by run id before deleting company-owned runs", async () => {
+    const { agentId, companyId, runId } = await seedFixture();
+    const otherCompanyId = randomUUID();
+
+    await db.insert(companies).values({
+      id: otherCompanyId,
+      name: "Other Company",
+      issuePrefix: `O${otherCompanyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`,
+      requireBoardApprovalForNewAgents: false,
+    });
+
+    await db.insert(heartbeatRunEvents).values({
+      companyId: otherCompanyId,
+      runId,
+      agentId,
+      seq: 1,
+      eventType: "output",
+      message: "event with mismatched company scope",
+    });
+
+    const removed = await companyService(db).remove(companyId);
+
+    expect(removed?.id).toBe(companyId);
+    await expect(db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId))).resolves.toHaveLength(0);
+    await expect(db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, runId))).resolves.toHaveLength(0);
+    await expect(db.select().from(companies).where(eq(companies.id, otherCompanyId))).resolves.toHaveLength(1);
+  });
 });

server/src/__tests__/cloud-upstreams.test.ts

@@ -1,7 +1,28 @@
-import { describe, expect, it } from "vitest";
+import { generateKeyPairSync, randomUUID } from "node:crypto";
+import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
+import { companies, cloudUpstreamConnections, cloudUpstreamRuns, companySkills, createDb } from "@paperclipai/db";
 
 import { HttpError } from "../errors.js";
-import { cloudUpstreamRemoteFailureReport } from "../services/cloud-upstreams.js";
+import {
+  cloudUpstreamRemoteFailureReport,
+  cloudUpstreamService,
+  reconcileCloudUpstreamRunsOnStartup,
+  sealCloudUpstreamCredential,
+  unsealCloudUpstreamCredential,
+} from "../services/cloud-upstreams.js";
+import {
+  getEmbeddedPostgresTestSupport,
+  startEmbeddedPostgresTestDatabase,
+} from "./helpers/embedded-postgres.js";
+
+const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
+const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
+
+if (!embeddedPostgresSupport.supported) {
+  console.warn(
+    `Skipping embedded Postgres cloud upstream tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`,
+  );
+}
 
 describe("cloud upstream remote failures", () => {
   it("preserves the cloud response body and message on run reports", () => {
@@ -24,3 +45,290 @@ describe("cloud upstream remote failures", () => {
     });
   });
 });
+
+describe("cloud upstream credential storage", () => {
+  const previousMasterKey = process.env.PAPERCLIP_SECRETS_MASTER_KEY;
+
+  afterEach(() => {
+    if (previousMasterKey === undefined) {
+      delete process.env.PAPERCLIP_SECRETS_MASTER_KEY;
+    } else {
+      process.env.PAPERCLIP_SECRETS_MASTER_KEY = previousMasterKey;
+    }
+  });
+
+  it("stores new credentials as encrypted envelopes and preserves legacy plaintext reads", async () => {
+    process.env.PAPERCLIP_SECRETS_MASTER_KEY = "12345678901234567890123456789012";
+    const sealed = await sealCloudUpstreamCredential("cloud-access-token");
+
+    expect(sealed).toMatch(/^paperclip-cloud-credential:/);
+    expect(sealed).not.toContain("cloud-access-token");
+    await expect(unsealCloudUpstreamCredential(sealed)).resolves.toBe("cloud-access-token");
+    await expect(unsealCloudUpstreamCredential("legacy-plaintext-token")).resolves.toBe("legacy-plaintext-token");
+  });
+});
+
+describeEmbeddedPostgres("cloud upstream persistence", () => {
+  let db!: ReturnType<typeof createDb>;
+  let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
+  const previousMasterKey = process.env.PAPERCLIP_SECRETS_MASTER_KEY;
+
+  beforeAll(async () => {
+    process.env.PAPERCLIP_SECRETS_MASTER_KEY = "12345678901234567890123456789012";
+    tempDb = await startEmbeddedPostgresTestDatabase("paperclip-cloud-upstreams-");
+    db = createDb(tempDb.connectionString);
+  }, 20_000);
+
+  afterEach(async () => {
+    vi.restoreAllMocks();
+    await db.delete(cloudUpstreamRuns);
+    await db.delete(cloudUpstreamConnections);
+    await db.delete(companySkills);
+    await db.delete(companies);
+  });
+
+  afterAll(async () => {
+    if (previousMasterKey === undefined) {
+      delete process.env.PAPERCLIP_SECRETS_MASTER_KEY;
+    } else {
+      process.env.PAPERCLIP_SECRETS_MASTER_KEY = previousMasterKey;
+    }
+    await tempDb?.cleanup();
+  });
+
+  it("encrypts stored upstream credentials while keeping connection flows usable", async () => {
+    const companyId = randomUUID();
+    await seedCompany(companyId);
+    const tokenUrl = "https://cloud.example.test/oauth/token";
+    vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => {
+      const url = String(input);
+      if (url.startsWith("https://cloud.example.test/.well-known/paperclip-upstream")) {
+        return jsonResponse({
+          product: "Paperclip Cloud",
+          stack: {
+            id: "stack-1",
+            companyId: "cloud-company-1",
+            origin: "https://cloud.example.test",
+            primaryHost: "cloud.example.test",
+          },
+          transfer: {
+            supportedSchemaMajor: 1,
+            maxChunkBytes: 8192,
+          },
+          auth: {
+            scopes: ["upstream_import:write"],
+            pkce: {
+              authorizeUrl: "https://cloud.example.test/oauth/authorize",
+              tokenUrl,
+            },
+          },
+        });
+      }
+      if (url === tokenUrl && init?.method === "POST") {
+        const payload = JSON.parse(String(init.body));
+        expect(payload.codeVerifier).toEqual(expect.any(String));
+        expect(payload.codeVerifier).not.toContain("paperclip-cloud-credential:");
+        return jsonResponse({
+          accessToken: "cloud-access-token",
+          token: {
+            id: "token-1",
+            expiresAt: "2026-05-22T13:00:00.000Z",
+            globalUserId: "user-1",
+          },
+        });
+      }
+      throw new Error(`Unexpected fetch: ${url}`);
+    });
+
+    const service = cloudUpstreamService(db, { instanceId: "test" });
+    const started = await service.startConnect({
+      companyId,
+      remoteUrl: "https://cloud.example.test",
+      redirectUri: "http://localhost:3100/callback",
+    });
+    await service.finishConnect({
+      pendingConnectionId: started.pendingConnectionId,
+      code: "auth-code",
+      state: new URL(started.authorizationUrl).searchParams.get("state") ?? "",
+    });
+
+    const [row] = await db.select().from(cloudUpstreamConnections);
+    expect(row.privateKeyPem).toMatch(/^paperclip-cloud-credential:/);
+    expect(row.privateKeyPem).not.toContain("BEGIN PRIVATE KEY");
+    expect(row.accessToken).toMatch(/^paperclip-cloud-credential:/);
+    expect(row.accessToken).not.toContain("cloud-access-token");
+  });
+
+  it("marks orphaned running runs failed during startup reconciliation", async () => {
+    const companyId = randomUUID();
+    const connectionId = randomUUID();
+    const runningRunId = randomUUID();
+    const succeededRunId = randomUUID();
+    const reconciledAt = new Date("2026-05-22T13:00:00.000Z");
+    await seedCompany(companyId);
+    await db.insert(cloudUpstreamConnections).values({
+      id: connectionId,
+      companyId,
+      remoteUrl: "https://cloud.example.test",
+      sourceInstanceId: "source-1",
+      sourceInstanceFingerprint: "sha256:test",
+      sourcePublicKey: "public-key",
+      privateKeyPem: "legacy-private-key",
+      tokenStatus: "connected",
+      scopes: ["upstream_import:write"],
+      authorizedGlobalUserId: "user-1",
+      accessToken: "legacy-token",
+      tokenId: "token-1",
+      targetStackId: "stack-1",
+      targetCompanyId: "cloud-company-1",
+      targetOrigin: "https://cloud.example.test",
+      targetPrimaryHost: "cloud.example.test",
+      targetProduct: "Paperclip Cloud",
+      targetSchemaMajor: 1,
+      targetMaxChunkBytes: 8192,
+    });
+    await db.insert(cloudUpstreamRuns).values([
+      cloudRunRow({ id: runningRunId, connectionId, companyId, status: "running" }),
+      cloudRunRow({ id: succeededRunId, connectionId, companyId, status: "succeeded", completedAt: reconciledAt }),
+    ]);
+
+    await expect(reconcileCloudUpstreamRunsOnStartup(db, reconciledAt)).resolves.toEqual({ reconciled: 1 });
+
+    const rows = await db.select().from(cloudUpstreamRuns);
+    const running = rows.find((row) => row.id === runningRunId);
+    const succeeded = rows.find((row) => row.id === succeededRunId);
+    expect(running?.status).toBe("failed");
+    expect(running?.completedAt?.toISOString()).toBe(reconciledAt.toISOString());
+    expect(running?.events.at(-1)?.message).toContain("server startup");
+    expect(running?.report).toMatchObject({
+      error: "orphaned_running_run",
+      reconciledAt: reconciledAt.toISOString(),
+    });
+    expect(succeeded?.status).toBe("succeeded");
+  });
+
+  it("rejects a new run when the connection already has a running run", async () => {
+    const companyId = randomUUID();
+    const connectionId = randomUUID();
+    const runningRunId = randomUUID();
+    await seedCompany(companyId);
+    await db.insert(cloudUpstreamConnections).values(cloudConnectionRow({ id: connectionId, companyId }));
+    await db.insert(cloudUpstreamRuns).values(
+      cloudRunRow({ id: runningRunId, connectionId, companyId, status: "running" }),
+    );
+
+    await expect(cloudUpstreamService(db).createRun({ connectionId, companyId })).rejects.toMatchObject({
+      status: 409,
+      details: { runId: runningRunId },
+    });
+  });
+
+  it("preserves a cancelled run when an in-flight createRun tries to finish", async () => {
+    const companyId = randomUUID();
+    const connectionId = randomUUID();
+    await seedCompany(companyId);
+    await db.insert(cloudUpstreamConnections).values(cloudConnectionRow({ id: connectionId, companyId }));
+
+    const service = cloudUpstreamService(db);
+    const remoteCalls: string[] = [];
+    globalThis.fetch = vi.fn(async (input) => {
+      const path = new URL(String(input)).pathname;
+      remoteCalls.push(path);
+      if (path.endsWith("/upstream-imports/runs")) {
+        return jsonResponse({ run: { id: "remote-run-1" } });
+      }
+      if (path.endsWith("/chunks")) {
+        const run = await db.select().from(cloudUpstreamRuns).then((rows) => rows[0]);
+        expect(run?.status).toBe("running");
+        await service.cancelRun(connectionId, run.id, companyId);
+        return jsonResponse({ ok: true });
+      }
+      if (path.endsWith("/cancel")) {
+        return jsonResponse({ ok: true });
+      }
+      if (path.endsWith("/apply")) {
+        return jsonResponse({ ok: true });
+      }
+      if (path.endsWith("/events")) {
+        return jsonResponse({ events: [] });
+      }
+      return jsonResponse({ error: "not_found" }, 404);
+    }) as typeof fetch;
+
+    const result = await service.createRun({ connectionId, companyId });
+
+    expect(result.status).toBe("cancelled");
+    expect(remoteCalls.some((path) => path.endsWith("/apply"))).toBe(false);
+    const rows = await db.select().from(cloudUpstreamRuns);
+    expect(rows).toHaveLength(1);
+    expect(rows[0]?.status).toBe("cancelled");
+  });
+
+  async function seedCompany(companyId: string) {
+    await db.insert(companies).values({
+      id: companyId,
+      name: "Paperclip",
+      issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`,
+      requireBoardApprovalForNewAgents: false,
+    });
+  }
+});
+
+function jsonResponse(body: unknown): Response {
+  return new Response(JSON.stringify(body), {
+    status: 200,
+    headers: { "Content-Type": "application/json" },
+  });
+}
+
+function cloudConnectionRow(input: { id: string; companyId: string }) {
+  const { privateKey } = generateKeyPairSync("ed25519");
+  return {
+    id: input.id,
+    companyId: input.companyId,
+    remoteUrl: "https://cloud.example.test",
+    sourceInstanceId: "source-1",
+    sourceInstanceFingerprint: "sha256:test",
+    sourcePublicKey: "public-key",
+    privateKeyPem: privateKey.export({ type: "pkcs8", format: "pem" }).toString(),
+    tokenStatus: "connected",
+    scopes: ["upstream_import:write"],
+    authorizedGlobalUserId: "user-1",
+    accessToken: "legacy-token",
+    tokenId: "token-1",
+    targetStackId: "stack-1",
+    targetCompanyId: "cloud-company-1",
+    targetOrigin: "https://cloud.example.test",
+    targetPrimaryHost: "cloud.example.test",
+    targetProduct: "Paperclip Cloud",
+    targetSchemaMajor: 1,
+    targetMaxChunkBytes: 8192,
+  };
+}
+
+function cloudRunRow(input: {
+  id: string;
+  connectionId: string;
+  companyId: string;
+  status: string;
+  completedAt?: Date;
+}) {
+  return {
+    id: input.id,
+    connectionId: input.connectionId,
+    companyId: input.companyId,
+    status: input.status,
+    activeStep: "push",
+    progressPercent: input.status === "running" ? 45 : 100,
+    dryRun: false,
+    summary: [],
+    warnings: [],
+    conflicts: [],
+    events: [],
+    report: {},
+    idempotencyKey: `key-${input.id}`,
+    manifestHash: `sha256:${input.id.replace(/-/g, "")}`,
+    targetUrl: "https://cloud.example.test",
+    completedAt: input.completedAt,
+  };
+}

server/src/__tests__/company-portability-routes.test.ts

@@ -507,6 +507,7 @@ describe.sequential("company portability routes", () => {
     expect(succeeded.status).toBe(200);
     expect(succeeded.body.job.status).toBe("succeeded");
     expect(succeeded.body.job.result.companyId).toBe(companyId);
+    expect(succeeded.body.retryAfterMs).toBeUndefined();
     expect(mockLogActivity).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({
       action: "company.imported",
       companyId,
@@ -516,6 +517,15 @@ describe.sequential("company portability routes", () => {
         companyAction: "updated",
       }),
     }));
+
+    const nowSpy = vi.spyOn(Date, "now").mockReturnValue(Date.parse(succeeded.body.job.completedAt) + (5 * 60 * 1000) + 1);
+    try {
+      const expired = await request(app).get(accepted.body.statusUrl).set(cloudHeaders);
+      expect(expired.status).toBe(404);
+      expect(expired.body.error).toBe("Import job not found");
+    } finally {
+      nowSpy.mockRestore();
+    }
   });
 
   it.sequential("reports trusted Cloud async import job failures with the tenant error message", async () => {
@@ -534,6 +544,7 @@ describe.sequential("company portability routes", () => {
     expect(failed.status).toBe(200);
     expect(failed.body.job.status).toBe("failed");
     expect(failed.body.job.error.message).toBe("tenant import exploded");
+    expect(failed.body.retryAfterMs).toBeUndefined();
     expect(failed.body.message).toBe("tenant import exploded");
     expect(mockLogActivity).not.toHaveBeenCalled();
   });

server/src/__tests__/fixtures/plugin-worker-invocation-scope.cjs

@@ -68,13 +68,13 @@ rl.on("line", (line) => {
       id: message.id,
       result: {
         ok: true,
-        supportedMethods: ["getData"],
+        supportedMethods: ["getData", "performAction"],
       },
     });
     return;
   }
 
-  if (method === "getData") {
+  if (method === "getData" || method === "performAction") {
     sendNestedHostRequest(message, message.paperclipInvocation?.id);
     return;
   }

server/src/__tests__/plugin-routes-authz.test.ts

@@ -715,6 +715,29 @@ describe.sequential("plugin tool and bridge authz", () => {
     });
   });
 
+  it("uses null for board actor userId when no authenticated user id is present", async () => {
+    readyPlugin();
+    const call = vi.fn().mockResolvedValue({ ok: true });
+    const { app } = await createApp(boardActor({ userId: undefined }), {}, {
+      bridgeDeps: {
+        workerManager: { call },
+      },
+    });
+
+    const res = await request(app)
+      .post(`/api/plugins/${pluginId}/actions/sync`)
+      .send({ companyId: companyA });
+
+    expect(res.status).toBe(200);
+    expect(call).toHaveBeenCalledWith(pluginId, "performAction", expect.objectContaining({
+      actorContext: expect.objectContaining({
+        type: "user",
+        userId: null,
+        companyId: companyA,
+      }),
+    }));
+  });
+
   it("allows agent-scoped plugin actions with authenticated actor context", async () => {
     readyPlugin();
     const call = vi.fn().mockResolvedValue({ ok: true });

server/src/__tests__/plugin-worker-manager.test.ts

@@ -186,6 +186,58 @@ describe("plugin-worker-manager stderr failure context", () => {
     }
   });
 
+  it("passes performAction invocation scope to nested worker host calls", async () => {
+    const companiesGet = vi.fn(async (
+      params: { companyId: string },
+      context?: { invocationScope?: { companyId?: string | null } | null },
+    ) => ({
+      id: params.companyId,
+      scopedCompanyId: context?.invocationScope?.companyId ?? null,
+    }));
+    const handle = createPluginWorkerHandle("test.plugin", {
+      entrypointPath: INVOCATION_SCOPE_WORKER_ENTRYPOINT,
+      manifest: TEST_MANIFEST,
+      config: {},
+      instanceInfo: {
+        instanceId: "instance-1",
+        hostVersion: "1.0.0",
+      },
+      apiVersion: 1,
+      hostHandlers: {
+        "companies.get": companiesGet as never,
+      },
+    });
+
+    try {
+      await handle.start();
+
+      await expect(handle.call("performAction", {
+        key: "probe",
+        params: {
+          mode: "echo",
+          requestedCompanyId: "company-a",
+        },
+        actorContext: {
+          type: "agent",
+          userId: null,
+          agentId: "agent-1",
+          runId: "run-1",
+          companyId: "company-a",
+        },
+        renderEnvironment: null,
+      })).resolves.toEqual({
+        id: "company-a",
+        scopedCompanyId: "company-a",
+      });
+      expect(companiesGet).toHaveBeenCalledWith(
+        { companyId: "company-a" },
+        { invocationScope: { companyId: "company-a" } },
+      );
+    } finally {
+      await handle.stop().catch(() => undefined);
+    }
+  });
+
   it("passes echoed invocation scope to worker-to-host handlers", async () => {
     const companiesGet = vi.fn(async () => ({ id: "company-1" }));
     const handle = createPluginWorkerHandle("test.plugin", {
@@ -223,6 +275,104 @@ describe("plugin-worker-manager stderr failure context", () => {
     }
   });
 
+  it("rejects performAction nested host calls that omit the invocation id", async () => {
+    const handlers = createHostClientHandlers({
+      pluginId: "test.plugin",
+      capabilities: ["companies.read"],
+      services: {
+        companies: {
+          list: vi.fn(async () => []),
+          get: vi.fn(async (params: { companyId: string }) => ({ id: params.companyId })),
+        },
+      } as unknown as HostServices,
+    });
+    const handle = createPluginWorkerHandle("test.plugin", {
+      entrypointPath: INVOCATION_SCOPE_WORKER_ENTRYPOINT,
+      manifest: TEST_MANIFEST,
+      config: {},
+      instanceInfo: {
+        instanceId: "instance-1",
+        hostVersion: "1.0.0",
+      },
+      apiVersion: 1,
+      hostHandlers: handlers,
+    });
+
+    try {
+      await handle.start();
+
+      await expect(handle.call("performAction", {
+        key: "probe",
+        params: {
+          requestedCompanyId: "company-b",
+        },
+        actorContext: {
+          type: "agent",
+          userId: null,
+          agentId: "agent-1",
+          runId: "run-1",
+          companyId: "company-a",
+        },
+        renderEnvironment: null,
+      })).rejects.toMatchObject({
+        code: PLUGIN_RPC_ERROR_CODES.INVOCATION_SCOPE_DENIED,
+        message: expect.stringContaining("unknown invocation scope"),
+      });
+    } finally {
+      await handle.stop().catch(() => undefined);
+    }
+  });
+
+  it("rejects nested worker host calls that forge an unknown invocation id", async () => {
+    const companiesGet = vi.fn(async (params: { companyId: string }) => ({ id: params.companyId }));
+    const handlers = createHostClientHandlers({
+      pluginId: "test.plugin",
+      capabilities: ["companies.read"],
+      services: {
+        companies: {
+          get: companiesGet,
+        },
+      } as unknown as HostServices,
+    });
+    const handle = createPluginWorkerHandle("test.plugin", {
+      entrypointPath: INVOCATION_SCOPE_WORKER_ENTRYPOINT,
+      manifest: TEST_MANIFEST,
+      config: {},
+      instanceInfo: {
+        instanceId: "instance-1",
+        hostVersion: "1.0.0",
+      },
+      apiVersion: 1,
+      hostHandlers: handlers,
+    });
+
+    try {
+      await handle.start();
+
+      await expect(handle.call("performAction", {
+        key: "probe",
+        params: {
+          mode: "unknown",
+          requestedCompanyId: "company-a",
+        },
+        actorContext: {
+          type: "agent",
+          userId: null,
+          agentId: "agent-1",
+          runId: "run-1",
+          companyId: "company-a",
+        },
+        renderEnvironment: null,
+      })).rejects.toMatchObject({
+        code: PLUGIN_RPC_ERROR_CODES.INVOCATION_SCOPE_DENIED,
+        message: expect.stringContaining("unknown invocation scope"),
+      });
+      expect(companiesGet).not.toHaveBeenCalled();
+    } finally {
+      await handle.stop().catch(() => undefined);
+    }
+  });
+
   it("rejects missing or unknown invocation ids while a company invocation is active", async () => {
     const companiesGet = vi.fn(async () => ({ id: "company-2" }));
     const hostHandlers = createHostClientHandlers({
@@ -258,7 +408,7 @@ describe("plugin-worker-manager stderr failure context", () => {
             requestedCompanyId: "company-2",
           },
         } as HostToWorkerMethods["getData"][0])).rejects.toMatchObject({
-          code: PLUGIN_RPC_ERROR_CODES.CAPABILITY_DENIED,
+          code: PLUGIN_RPC_ERROR_CODES.INVOCATION_SCOPE_DENIED,
         });
       }
 

server/src/__tests__/server-startup-feedback-export.test.ts

@@ -166,6 +166,7 @@ vi.mock("../services/index.js", () => ({
       },
     })),
   })),
+  reconcileCloudUpstreamRunsOnStartup: vi.fn(async () => ({ reconciled: 0 })),
   reconcilePersistedRuntimeServicesOnStartup: vi.fn(async () => ({ reconciled: 0 })),
   routineService: vi.fn(() => ({
     tickScheduledTriggers: vi.fn(async () => ({ triggered: 0 })),

server/src/index.ts

@@ -34,6 +34,7 @@ import {
   backfillPrincipalAccessCompatibility,
   heartbeatService,
   instanceSettingsService,
+  reconcileCloudUpstreamRunsOnStartup,
   reconcilePersistedRuntimeServicesOnStartup,
   routineService,
 } from "./services/index.js";
@@ -701,6 +702,19 @@ export async function startServer(): Promise<StartedServer> {
     .catch((err) => {
       logger.error({ err }, "startup reconciliation of persisted runtime services failed");
     });
+
+  void reconcileCloudUpstreamRunsOnStartup(db as any)
+    .then((result) => {
+      if (result.reconciled > 0) {
+        logger.warn(
+          { reconciled: result.reconciled },
+          "reconciled cloud upstream runs from a previous server process",
+        );
+      }
+    })
+    .catch((err) => {
+      logger.error({ err }, "startup reconciliation of cloud upstream runs failed");
+    });
   
   if (config.heartbeatSchedulerEnabled) {
     const heartbeat = heartbeatService(db as any, { pluginWorkerManager });

server/src/routes/cloud-upstreams.ts

@@ -45,7 +45,7 @@ export function cloudUpstreamRoutes(db: Db, options: { instanceId?: string } = {
   router.post("/cloud-upstreams/:connectionId/push-runs/preview", async (req, res) => {
     assertBoardOrgAccess(req);
     await assertEnabled();
-    res.json(await service.preview(req.params.connectionId));
+    res.json(await service.preview(req.params.connectionId, stringBody(req.body, "companyId")));
   });
 
   router.post("/cloud-upstreams/:connectionId/push-runs", async (req, res) => {
@@ -53,6 +53,7 @@ export function cloudUpstreamRoutes(db: Db, options: { instanceId?: string } = {
     await assertEnabled();
     res.json(await service.createRun({
       connectionId: req.params.connectionId,
+      companyId: stringBody(req.body, "companyId"),
       retryOfRunId: optionalString(req.body?.retryOfRunId),
     }));
   });
@@ -60,13 +61,13 @@ export function cloudUpstreamRoutes(db: Db, options: { instanceId?: string } = {
   router.get("/cloud-upstreams/:connectionId/push-runs/:runId", async (req, res) => {
     assertBoardOrgAccess(req);
     await assertEnabled();
-    res.json(await service.readRun(req.params.connectionId, req.params.runId));
+    res.json(await service.readRun(req.params.connectionId, req.params.runId, stringQuery(req.query.companyId, "companyId")));
   });
 
   router.post("/cloud-upstreams/:connectionId/push-runs/:runId/cancel", async (req, res) => {
     assertBoardOrgAccess(req);
     await assertEnabled();
-    res.json(await service.cancelRun(req.params.connectionId, req.params.runId));
+    res.json(await service.cancelRun(req.params.connectionId, req.params.runId, stringBody(req.body, "companyId")));
   });
 
   router.post("/cloud-upstreams/:connectionId/push-runs/:runId/activation", async (req, res) => {
@@ -75,6 +76,7 @@ export function cloudUpstreamRoutes(db: Db, options: { instanceId?: string } = {
     res.json(await service.activateRunEntities({
       connectionId: req.params.connectionId,
       runId: req.params.runId,
+      companyId: stringBody(req.body, "companyId"),
       entityType: activationEntityTypeBody(req.body),
     }));
   });

server/src/routes/companies.ts

@@ -37,6 +37,7 @@ export function companyRoutes(db: Db, storage?: StorageService) {
   const budgets = budgetService(db);
   const feedback = feedbackService(db);
   const importJobs = new Map<string, ImportJobRecord>();
+  const importJobTerminalRetentionMs = 5 * 60 * 1000;
 
   function parseBooleanQuery(value: unknown) {
     return value === true || value === "true" || value === "1";
@@ -181,6 +182,7 @@ export function companyRoutes(db: Db, storage?: StorageService) {
 
   router.get("/import/jobs/:jobId", async (req, res) => {
     assertCloudTenantCaller(req);
+    cleanupTerminalImportJobs(importJobs, importJobTerminalRetentionMs);
     const job = importJobs.get(req.params.jobId as string);
     if (!job || job.cloudTenantKey !== cloudTenantRequestKey(req)) {
       res.status(404).json({ error: "Import job not found" });
@@ -196,6 +198,7 @@ export function companyRoutes(db: Db, storage?: StorageService) {
     const boardUserId = req.actor.type === "board" ? req.actor.userId : null;
     if (req.header("x-paperclip-cloud-async-import") === "1") {
       assertCloudTenantCaller(req);
+      cleanupTerminalImportJobs(importJobs, importJobTerminalRetentionMs);
       const job = createImportJob(cloudTenantRequestKey(req));
       importJobs.set(job.id, job);
       const operation = async () => {
@@ -568,6 +571,7 @@ function importJobAcceptedResponse(job: ImportJobRecord) {
 }
 
 function importJobResponse(job: ImportJobRecord) {
+  const isTerminal = job.status === "succeeded" || job.status === "failed";
   const response: Record<string, unknown> = {
     job: {
       id: job.id,
@@ -578,7 +582,7 @@ function importJobResponse(job: ImportJobRecord) {
       ...(job.error ? { error: job.error } : {}),
       ...(job.result ? { result: job.result } : {}),
     },
-    retryAfterMs: 1000,
+    ...(isTerminal ? {} : { retryAfterMs: 1000 }),
   };
   if (job.error?.message) {
     response.error = job.error.message;
@@ -588,6 +592,16 @@ function importJobResponse(job: ImportJobRecord) {
   return response;
 }
 
+function cleanupTerminalImportJobs(importJobs: Map<string, ImportJobRecord>, terminalRetentionMs: number) {
+  const now = Date.now();
+  for (const [jobId, job] of importJobs) {
+    if (job.status === "running" || !job.completedAt) continue;
+    if (now - Date.parse(job.completedAt) > terminalRetentionMs) {
+      importJobs.delete(jobId);
+    }
+  }
+}
+
 function errorMessage(error: unknown) {
   return error instanceof Error && error.message.trim() ? error.message : String(error);
 }

server/src/routes/plugins.ts

@@ -586,7 +586,7 @@ export function pluginRoutes(
     if (req.actor.type === "board") {
       return {
         type: "user",
-        userId: req.actor.userId ?? "board",
+        userId: req.actor.userId ?? null,
         agentId: null,
         runId: req.actor.runId ?? null,
         companyId: scopedCompanyId,
@@ -1021,6 +1021,12 @@ export function pluginRoutes(
             message: err.message,
             details: err.data,
           };
+        case PLUGIN_RPC_ERROR_CODES.INVOCATION_SCOPE_DENIED:
+          return {
+            code: "INVOCATION_SCOPE_DENIED",
+            message: err.message,
+            details: err.data,
+          };
         case PLUGIN_RPC_ERROR_CODES.TIMEOUT:
           return {
             code: "TIMEOUT",
@@ -1254,7 +1260,6 @@ export function pluginRoutes(
         "performAction",
         {
           key: body.key,
-          ...(companyId ? { companyId } : {}),
           params: actionParamsWithAuthorizedCompanyScope(body.params, companyId),
           actorContext: performActionActorContext(req, companyId),
           renderEnvironment: body.renderEnvironment ?? null,
@@ -1439,7 +1444,6 @@ export function pluginRoutes(
         "performAction",
         {
           key,
-          ...(companyId ? { companyId } : {}),
           params: actionParamsWithAuthorizedCompanyScope(body?.params, companyId),
           actorContext: performActionActorContext(req, companyId),
           renderEnvironment: body?.renderEnvironment ?? null,
@@ -1645,7 +1649,10 @@ export function pluginRoutes(
     } catch (err) {
       const status = typeof (err as { status?: unknown }).status === "number"
         ? (err as { status: number }).status
-        : err instanceof JsonRpcCallError && err.code === PLUGIN_RPC_ERROR_CODES.CAPABILITY_DENIED
+        : err instanceof JsonRpcCallError && (
+          err.code === PLUGIN_RPC_ERROR_CODES.CAPABILITY_DENIED ||
+          err.code === PLUGIN_RPC_ERROR_CODES.INVOCATION_SCOPE_DENIED
+        )
           ? 403
           : err instanceof JsonRpcCallError && err.code === PLUGIN_RPC_ERROR_CODES.METHOD_NOT_IMPLEMENTED
             ? 501

server/src/services/cloud-upstreams.ts

@@ -1,5 +1,5 @@
 import crypto, { sign } from "node:crypto";
-import { and, count, desc, eq } from "drizzle-orm";
+import { and, count, desc, eq, sql } from "drizzle-orm";
 import type {
   CloudUpstreamConnectStartResponse,
   CloudUpstreamActivationDecision,
@@ -28,8 +28,9 @@ import {
   projects,
   routines,
 } from "@paperclipai/db";
-import { badRequest, HttpError, notFound } from "../errors.js";
+import { badRequest, conflict, HttpError, notFound } from "../errors.js";
 import { companyPortabilityService } from "./company-portability.js";
+import { localEncryptedProvider } from "../secrets/local-encrypted-provider.js";
 
 const DEFAULT_SCOPES = ["upstream_import:preview", "upstream_import:write", "upstream_import:read"];
 const TRANSFER_SCHEMA = {
@@ -39,6 +40,9 @@ const TRANSFER_SCHEMA = {
   minor: 0,
 } as const;
 const DEFAULT_MAX_ENTITIES_PER_CHUNK = 100;
+const DISCOVERY_FETCH_TIMEOUT_MS = 30_000;
+const REMOTE_FETCH_TIMEOUT_MS = 120_000;
+const CLOUD_CREDENTIAL_PREFIX = "paperclip-cloud-credential:";
 
 type NormalizedSha256 = `sha256:${string}`;
 
@@ -75,7 +79,6 @@ type LocalUpstreamExportChunk = {
   totalChunks: number;
   byteLength: number;
   sha256: NormalizedSha256;
-  manifestHash: NormalizedSha256;
   payload: {
     entityKeys: SourceEntityKey[];
   };
@@ -102,7 +105,7 @@ type UpstreamTransferManifest = {
   entityCount: number;
   perEntityTypeCounts: Record<string, number>;
   entities: UpstreamTransferEntityRecord[];
-  chunks: Array<Omit<LocalUpstreamExportChunk, "payload">>;
+  chunks: Array<Omit<LocalUpstreamExportChunk, "payload"> & { manifestHash: NormalizedSha256 }>;
   warnings: UpstreamTransferWarning[];
   featureFlags: string[];
   manifestHash: NormalizedSha256;
@@ -133,7 +136,8 @@ export function cloudUpstreamService(db: Db, options: { instanceId?: string } =
           .select()
           .from(cloudUpstreamRuns)
           .where(eq(cloudUpstreamRuns.companyId, companyId))
-          .orderBy(desc(cloudUpstreamRuns.createdAt)),
+          .orderBy(desc(cloudUpstreamRuns.createdAt))
+          .limit(50),
       ]);
       return {
         connections: connectionRows.map(connectionFromRow),
@@ -170,7 +174,7 @@ export function cloudUpstreamService(db: Db, options: { instanceId?: string } =
         sourceInstanceId,
         sourceInstanceFingerprint,
         sourcePublicKey,
-        privateKeyPem: privateKey.export({ type: "pkcs8", format: "pem" }).toString(),
+        privateKeyPem: await sealCloudUpstreamCredential(privateKey.export({ type: "pkcs8", format: "pem" }).toString()),
         tokenStatus: "pending",
         scopes: scopesFromDiscovery(discovery),
         targetStackId: target.stackId,
@@ -183,7 +187,7 @@ export function cloudUpstreamService(db: Db, options: { instanceId?: string } =
         targetSchemaMajor: target.schemaMajor,
         targetMaxChunkBytes: target.maxChunkBytes,
         pendingState: state,
-        pendingCodeVerifier: codeVerifier,
+        pendingCodeVerifier: await sealCloudUpstreamCredential(codeVerifier),
         pendingRedirectUri: input.redirectUri,
         pendingTokenUrl: tokenUrlFromDiscovery(discovery),
       }).returning();
@@ -221,7 +225,7 @@ export function cloudUpstreamService(db: Db, options: { instanceId?: string } =
         grantType: "authorization_code",
         code: input.code,
         redirectUri: pending.pendingRedirectUri,
-        codeVerifier: pending.pendingCodeVerifier,
+        codeVerifier: await unsealCloudUpstreamCredential(pending.pendingCodeVerifier),
       });
       const accessToken = stringField(tokenResponse, "accessToken");
       const token = objectField(tokenResponse, "token");
@@ -231,7 +235,7 @@ export function cloudUpstreamService(db: Db, options: { instanceId?: string } =
         .set({
           tokenStatus: "connected",
           authorizedGlobalUserId: optionalString(token.globalUserId),
-          accessToken,
+          accessToken: await sealCloudUpstreamCredential(accessToken),
           tokenId: optionalString(token.id),
           tokenExpiresAt: expiresAt ? new Date(expiresAt) : null,
           pendingState: null,
@@ -246,8 +250,8 @@ export function cloudUpstreamService(db: Db, options: { instanceId?: string } =
       return connectionFromRow(updated);
     },
 
-    preview: async (connectionId: string): Promise<CloudUpstreamPreview> => {
-      const connection = await getConnectionRow(connectionId);
+    preview: async (connectionId: string, companyId: string): Promise<CloudUpstreamPreview> => {
+      const connection = await getConnectionRow(connectionId, companyId);
       const basePreview = await localPreview(connection);
       if (!basePreview.schemaCompatible || connection.tokenStatus !== "connected") {
         return basePreview;
@@ -271,11 +275,12 @@ export function cloudUpstreamService(db: Db, options: { instanceId?: string } =
       };
     },
 
-    createRun: async (input: { connectionId: string; retryOfRunId?: string | null }): Promise<CloudUpstreamRun> => {
-      const connection = await getConnectionRow(input.connectionId);
+    createRun: async (input: { connectionId: string; companyId: string; retryOfRunId?: string | null }): Promise<CloudUpstreamRun> => {
+      const connection = await getConnectionRow(input.connectionId, input.companyId);
       if (connection.tokenStatus !== "connected") {
         throw badRequest("Cloud upstream connection is not connected");
       }
+      await assertNoRunningRun(input.connectionId, input.companyId, db);
       const preview = await localPreview(connection);
       if (!preview.schemaCompatible) {
         throw badRequest("Cloud stack schema is not compatible with this local Paperclip version");
@@ -292,26 +297,33 @@ export function cloudUpstreamService(db: Db, options: { instanceId?: string } =
           ? [event(now.toISOString(), "push", "retrying", `Retrying run ${input.retryOfRunId} with the same import ledger idempotency key.`)]
           : []),
       ];
-      const [created] = await db.insert(cloudUpstreamRuns).values({
-        id: runId,
-        connectionId: connection.id,
-        companyId: connection.companyId,
-        status: "running",
-        activeStep: "push",
-        progressPercent: 45,
-        dryRun: false,
-        retryOfRunId: input.retryOfRunId ?? null,
-        summary: preview.summary,
-        warnings: preview.warnings,
-        conflicts: preview.conflicts,
-        events: initialEvents,
-        report: {},
-        idempotencyKey: bundle.manifest.idempotencyKey,
-        manifestHash: bundle.manifest.manifestHash,
-        targetUrl: connection.targetOrigin,
-        createdAt: now,
-        updatedAt: now,
-      }).returning();
+      const created = await db.transaction(async (tx) => {
+        await tx.execute(
+          sql`select ${cloudUpstreamConnections.id} from ${cloudUpstreamConnections} where ${cloudUpstreamConnections.id} = ${connection.id} and ${cloudUpstreamConnections.companyId} = ${connection.companyId} for update`,
+        );
+        await assertNoRunningRun(input.connectionId, input.companyId, tx);
+        const [row] = await tx.insert(cloudUpstreamRuns).values({
+          id: runId,
+          connectionId: connection.id,
+          companyId: connection.companyId,
+          status: "running",
+          activeStep: "push",
+          progressPercent: 45,
+          dryRun: false,
+          retryOfRunId: input.retryOfRunId ?? null,
+          summary: preview.summary,
+          warnings: preview.warnings,
+          conflicts: preview.conflicts,
+          events: initialEvents,
+          report: {},
+          idempotencyKey: bundle.manifest.idempotencyKey,
+          manifestHash: bundle.manifest.manifestHash,
+          targetUrl: connection.targetOrigin,
+          createdAt: now,
+          updatedAt: now,
+        }).returning();
+        return row;
+      });
       if (!created) throw badRequest("Failed to create cloud upstream run");
 
       try {
@@ -321,7 +333,7 @@ export function cloudUpstreamService(db: Db, options: { instanceId?: string } =
           entities: bundle.entities,
         });
         const remoteRunId = remoteRunIdFromResponse(remoteRun);
-        await updateRun(runId, {
+        const pushedRun = await updateRunIfRunning(runId, {
           remoteRunId,
           activeStep: "push",
           progressPercent: 60,
@@ -330,11 +342,12 @@ export function cloudUpstreamService(db: Db, options: { instanceId?: string } =
             event(new Date().toISOString(), "push", "updated", "Created or resumed the cloud import ledger run."),
           ],
         });
+        if (pushedRun.status !== "running") return pushedRun;
 
         for (const chunk of bundle.chunks) {
           await remotePost(connection, `/api/upstream-import-runs/${encodeURIComponent(remoteRunId)}/chunks`, chunk);
         }
-        await updateRun(runId, {
+        const verifiedRun = await updateRunIfRunning(runId, {
           activeStep: "verify",
           progressPercent: 82,
           events: [
@@ -342,6 +355,7 @@ export function cloudUpstreamService(db: Db, options: { instanceId?: string } =
             event(new Date().toISOString(), "push", "completed", `Uploaded ${bundle.chunks.length} manifest chunk${bundle.chunks.length === 1 ? "" : "s"}.`),
           ],
         });
+        if (verifiedRun.status !== "running") return verifiedRun;
 
         const applied = await remotePost(connection, `/api/upstream-import-runs/${encodeURIComponent(remoteRunId)}/apply`, {});
         const remoteEvents = await remoteGet(connection, `/api/upstream-import-runs/${encodeURIComponent(remoteRunId)}/events`).catch(() => null);
@@ -353,7 +367,7 @@ export function cloudUpstreamService(db: Db, options: { instanceId?: string } =
           event(completedAt.toISOString(), "activate", "completed", "Activation checklist is ready for manual unpause decisions."),
           ...eventsFromRemote(remoteEvents),
         ];
-        const finalRun = await updateRun(runId, {
+        const finalRun = await updateRunIfRunning(runId, {
           remoteRunId,
           status: "succeeded",
           activeStep: "activate",
@@ -373,15 +387,17 @@ export function cloudUpstreamService(db: Db, options: { instanceId?: string } =
           },
           completedAt,
         });
-        await db
-          .update(cloudUpstreamConnections)
-          .set({ lastRunId: finalRun.id, updatedAt: new Date() })
-          .where(eq(cloudUpstreamConnections.id, connection.id));
+        if (finalRun.status === "succeeded") {
+          await db
+            .update(cloudUpstreamConnections)
+            .set({ lastRunId: finalRun.id, updatedAt: new Date() })
+            .where(eq(cloudUpstreamConnections.id, connection.id));
+        }
         return finalRun;
       } catch (error) {
         const failedAt = new Date();
         const failure = cloudUpstreamRemoteFailureReport(error);
-        return updateRun(runId, {
+        return updateRunIfRunning(runId, {
           status: "failed",
           activeStep: "push",
           progressPercent: 100,
@@ -402,15 +418,15 @@ export function cloudUpstreamService(db: Db, options: { instanceId?: string } =
       }
     },
 
-    readRun: async (connectionId: string, runId: string): Promise<CloudUpstreamRun> => {
-      const row = await getRunRow(connectionId, runId);
+    readRun: async (connectionId: string, runId: string, companyId: string): Promise<CloudUpstreamRun> => {
+      const row = await getRunRow(connectionId, runId, companyId);
       return runFromRow(row);
     },
 
-    cancelRun: async (connectionId: string, runId: string): Promise<CloudUpstreamRun> => {
-      const row = await getRunRow(connectionId, runId);
+    cancelRun: async (connectionId: string, runId: string, companyId: string): Promise<CloudUpstreamRun> => {
+      const row = await getRunRow(connectionId, runId, companyId);
       if (row.status !== "running") return runFromRow(row);
-      const connection = await getConnectionRow(connectionId);
+      const connection = await getConnectionRow(connectionId, companyId);
       if (row.remoteRunId) {
         await remotePost(connection, `/api/upstream-import-runs/${encodeURIComponent(row.remoteRunId)}/cancel`, {}).catch(() => null);
       }
@@ -429,9 +445,10 @@ export function cloudUpstreamService(db: Db, options: { instanceId?: string } =
     activateRunEntities: async (input: {
       connectionId: string;
       runId: string;
+      companyId: string;
       entityType: CloudUpstreamActivationEntityType;
     }): Promise<CloudUpstreamRun> => {
-      const row = await getRunRow(input.connectionId, input.runId);
+      const row = await getRunRow(input.connectionId, input.runId, input.companyId);
       assertActivationEntityType(input.entityType);
       if (row.status !== "succeeded") {
         throw badRequest("Only succeeded cloud upstream runs can activate imported entities");
@@ -470,26 +487,51 @@ export function cloudUpstreamService(db: Db, options: { instanceId?: string } =
     if (!row) throw notFound("Company was not found");
   }
 
-  async function getConnectionRow(connectionId: string): Promise<ConnectionRow> {
+  async function getConnectionRow(connectionId: string, companyId?: string): Promise<ConnectionRow> {
     const row = await db
       .select()
       .from(cloudUpstreamConnections)
-      .where(eq(cloudUpstreamConnections.id, connectionId))
+      .where(companyId
+        ? and(eq(cloudUpstreamConnections.id, connectionId), eq(cloudUpstreamConnections.companyId, companyId))
+        : eq(cloudUpstreamConnections.id, connectionId))
       .then((rows) => rows[0]);
     if (!row) throw notFound("Cloud upstream connection was not found");
     return row;
   }
 
-  async function getRunRow(connectionId: string, runId: string): Promise<RunRow> {
+  async function getRunRow(connectionId: string, runId: string, companyId: string): Promise<RunRow> {
     const row = await db
       .select()
       .from(cloudUpstreamRuns)
-      .where(and(eq(cloudUpstreamRuns.id, runId), eq(cloudUpstreamRuns.connectionId, connectionId)))
+      .where(and(
+        eq(cloudUpstreamRuns.id, runId),
+        eq(cloudUpstreamRuns.connectionId, connectionId),
+        eq(cloudUpstreamRuns.companyId, companyId),
+      ))
       .then((rows) => rows[0]);
     if (!row) throw notFound("Cloud upstream run was not found");
     return row;
   }
 
+  async function assertNoRunningRun(
+    connectionId: string,
+    companyId: string,
+    database: Pick<Db, "select">,
+  ) {
+    const [running] = await database
+      .select({ id: cloudUpstreamRuns.id })
+      .from(cloudUpstreamRuns)
+      .where(and(
+        eq(cloudUpstreamRuns.connectionId, connectionId),
+        eq(cloudUpstreamRuns.companyId, companyId),
+        eq(cloudUpstreamRuns.status, "running"),
+      ))
+      .limit(1);
+    if (running) {
+      throw conflict("A cloud upstream run is already running for this connection", { runId: running.id });
+    }
+  }
+
   async function updateRun(runId: string, patch: Partial<typeof cloudUpstreamRuns.$inferInsert>): Promise<CloudUpstreamRun> {
     const [updated] = await db
       .update(cloudUpstreamRuns)
@@ -500,6 +542,23 @@ export function cloudUpstreamService(db: Db, options: { instanceId?: string } =
     return runFromRow(updated);
   }
 
+  async function updateRunIfRunning(runId: string, patch: Partial<typeof cloudUpstreamRuns.$inferInsert>): Promise<CloudUpstreamRun> {
+    const [updated] = await db
+      .update(cloudUpstreamRuns)
+      .set({ ...patch, updatedAt: new Date() })
+      .where(and(eq(cloudUpstreamRuns.id, runId), eq(cloudUpstreamRuns.status, "running")))
+      .returning();
+    if (updated) return runFromRow(updated);
+
+    const [current] = await db
+      .select()
+      .from(cloudUpstreamRuns)
+      .where(eq(cloudUpstreamRuns.id, runId))
+      .limit(1);
+    if (!current) throw notFound("Cloud upstream run was not found");
+    return runFromRow(current);
+  }
+
   async function localPreview(connection: ConnectionRow): Promise<CloudUpstreamPreview> {
     return {
       connectionId: connection.id,
@@ -596,13 +655,53 @@ async function fetchDiscovery(remoteUrl: string): Promise<Record<string, unknown
   if (stackId) {
     discoveryUrl.searchParams.set("stackId", stackId);
   }
-  const response = await fetch(discoveryUrl);
+  const response = await fetchWithTimeout(discoveryUrl, undefined, DISCOVERY_FETCH_TIMEOUT_MS);
   if (!response.ok) {
     throw badRequest(`Cloud upstream discovery failed: ${response.status}`);
   }
   return await response.json() as Record<string, unknown>;
 }
 
+export async function reconcileCloudUpstreamRunsOnStartup(db: Db, now = new Date()): Promise<{ reconciled: number }> {
+  const runningRows = await db
+    .select()
+    .from(cloudUpstreamRuns)
+    .where(eq(cloudUpstreamRuns.status, "running"));
+  if (runningRows.length === 0) return { reconciled: 0 };
+
+  for (const row of runningRows) {
+    const report = asRecord(row.report);
+    await db
+      .update(cloudUpstreamRuns)
+      .set({
+        status: "failed",
+        activeStep: row.activeStep,
+        progressPercent: 100,
+        completedAt: now,
+        updatedAt: now,
+        events: [
+          ...safeRunEvents(row.events),
+          event(
+            now.toISOString(),
+            cloudUpstreamStep(row.activeStep),
+            "failed",
+            "Marked failed on server startup because the previous process stopped while the cloud upstream run was in progress.",
+          ),
+        ],
+        report: {
+          ...report,
+          error: optionalString(report.error) ?? "orphaned_running_run",
+          errorMessage: optionalString(report.errorMessage)
+            ?? "The server restarted while this cloud upstream run was running, so Paperclip marked it failed instead of leaving it stuck.",
+          reconciledAt: now.toISOString(),
+        },
+      })
+      .where(eq(cloudUpstreamRuns.id, row.id));
+  }
+
+  return { reconciled: runningRows.length };
+}
+
 function firstPathSegment(pathname: string): string | null {
   const segment = pathname.split("/").find(Boolean);
   return segment && segment.toLowerCase() !== "dashboard" ? segment : null;
@@ -818,20 +917,19 @@ function buildLocalUpstreamExportBundle(input: {
     entityCount: entities.length,
     perEntityTypeCounts: countEntityTypesForManifest(entities),
     entities: entities.map((entity) => entity.record),
-    chunks: chunksWithoutManifestHash.map(({ payload: _payload, manifestHash: _manifestHash, ...chunk }) => chunk),
+    chunks: chunksWithoutManifestHash.map(({ payload: _payload, ...chunk }) => chunk),
     warnings: input.warnings ?? [],
     featureFlags: (input.featureFlags ?? ["cloud_sync"]).slice().sort(),
   };
   const manifestHash = normalizedContentHash(manifestWithoutHash);
-  const chunks = chunksWithoutManifestHash.map((chunk) => ({ ...chunk, manifestHash }));
   return {
     manifest: {
       ...manifestWithoutHash,
-      chunks: chunks.map(({ payload: _payload, ...chunk }) => chunk),
+      chunks: chunksWithoutManifestHash.map(({ payload: _payload, ...chunk }) => ({ ...chunk, manifestHash })),
       manifestHash,
     },
     entities,
-    chunks,
+    chunks: chunksWithoutManifestHash,
   };
 }
 
@@ -864,36 +962,37 @@ function buildLocalChunks(entities: LocalUpstreamExportEntity[], maxEntitiesPerC
       totalChunks: groups.length,
       byteLength: Buffer.byteLength(canonicalJson(payload)),
       sha256: normalizedContentHash(payload),
-      manifestHash: "sha256:" as NormalizedSha256,
       payload,
     };
   });
 }
 
 async function remoteGet(connection: ConnectionRow, path: string): Promise<unknown> {
-  const response = await fetch(`${connection.targetOrigin}${path}`, {
+  const response = await fetchWithTimeout(`${connection.targetOrigin}${path}`, {
     method: "GET",
-    headers: proofHeaders(connection, "GET", path),
-  });
+    headers: await proofHeaders(connection, "GET", path),
+  }, REMOTE_FETCH_TIMEOUT_MS);
   return parseRemoteResponse(response);
 }
 
 async function remotePost(connection: ConnectionRow, path: string, body: unknown): Promise<unknown> {
-  const response = await fetch(`${connection.targetOrigin}${path}`, {
+  const response = await fetchWithTimeout(`${connection.targetOrigin}${path}`, {
     method: "POST",
     headers: {
       "Content-Type": "application/json",
-      ...proofHeaders(connection, "POST", path),
+      ...await proofHeaders(connection, "POST", path),
     },
     body: JSON.stringify(body),
-  });
+  }, REMOTE_FETCH_TIMEOUT_MS);
   return parseRemoteResponse(response);
 }
 
-function proofHeaders(connection: ConnectionRow, method: string, pathAndSearch: string): Record<string, string> {
+async function proofHeaders(connection: ConnectionRow, method: string, pathAndSearch: string): Promise<Record<string, string>> {
   if (!connection.accessToken || !connection.tokenId) {
     throw badRequest("Cloud upstream connection is missing an import token");
   }
+  const accessToken = await unsealCloudUpstreamCredential(connection.accessToken);
+  const privateKeyPem = await unsealCloudUpstreamCredential(connection.privateKeyPem);
   const timestamp = new Date().toISOString();
   const nonce = crypto.randomUUID();
   const payload = [
@@ -906,14 +1005,14 @@ function proofHeaders(connection: ConnectionRow, method: string, pathAndSearch:
     nonce,
   ].join("\n");
   return {
-    Authorization: `Bearer ${connection.accessToken}`,
+    Authorization: `Bearer ${accessToken}`,
     "X-Paperclip-Upstream-Source-Instance-Id": connection.sourceInstanceId,
     "X-Paperclip-Upstream-Proof-Timestamp": timestamp,
     "X-Paperclip-Upstream-Proof-Nonce": nonce,
     "X-Paperclip-Upstream-Proof-Signature": sign(
       null,
       Buffer.from(payload, "utf8"),
-      connection.privateKeyPem,
+      privateKeyPem,
     ).toString("base64url"),
   };
 }
@@ -930,6 +1029,32 @@ async function parseRemoteResponse(response: Response): Promise<unknown> {
   return parsed;
 }
 
+async function fetchWithTimeout(input: RequestInfo | URL, init: RequestInit | undefined, timeoutMs: number): Promise<Response> {
+  return fetch(input, {
+    ...init,
+    signal: AbortSignal.timeout(timeoutMs),
+  });
+}
+
+export async function sealCloudUpstreamCredential(value: string): Promise<string> {
+  const prepared = await localEncryptedProvider.createSecret({ value });
+  return `${CLOUD_CREDENTIAL_PREFIX}${JSON.stringify(prepared.material)}`;
+}
+
+export async function unsealCloudUpstreamCredential(value: string): Promise<string> {
+  if (!value.startsWith(CLOUD_CREDENTIAL_PREFIX)) return value;
+  const encoded = value.slice(CLOUD_CREDENTIAL_PREFIX.length);
+  const parsed = safeParseJson(encoded);
+  const material = optionalObject(parsed);
+  if (!material) {
+    throw badRequest("Invalid encrypted cloud upstream credential material");
+  }
+  return localEncryptedProvider.resolveVersion({
+    material,
+    externalRef: null,
+  });
+}
+
 export function cloudUpstreamRemoteFailureReport(error: unknown): {
   error: string;
   errorMessage?: string;
@@ -957,11 +1082,11 @@ function remoteErrorBody(details: unknown): { error?: string; message?: string }
 }
 
 async function postJson<T>(url: string, body: unknown): Promise<T> {
-  const response = await fetch(url, {
+  const response = await fetchWithTimeout(url, {
     method: "POST",
     headers: { "Content-Type": "application/json" },
     body: JSON.stringify(body),
-  });
+  }, DISCOVERY_FETCH_TIMEOUT_MS);
   const payload = await response.json().catch(() => null);
   if (!response.ok) {
     throw badRequest((payload as { error?: string } | null)?.error ?? `Cloud upstream request failed: ${response.status}`);
@@ -1026,6 +1151,10 @@ function eventsFromRemote(value: unknown): CloudUpstreamRunEvent[] {
   });
 }
 
+function safeRunEvents(value: unknown): CloudUpstreamRunEvent[] {
+  return Array.isArray(value) ? value as CloudUpstreamRunEvent[] : [];
+}
+
 function assertActivationEntityType(value: string): asserts value is CloudUpstreamActivationEntityType {
   if (value !== "agents" && value !== "routines" && value !== "monitors") {
     throw badRequest("entityType must be agents, routines, or monitors");

server/src/services/companies.ts

@@ -265,7 +265,17 @@ export function companyService(db: Db) {
     remove: (id: string) =>
       db.transaction(async (tx) => {
         // Delete from child tables in dependency order
+        const companyRunIds = await tx
+          .select({ id: heartbeatRuns.id })
+          .from(heartbeatRuns)
+          .where(eq(heartbeatRuns.companyId, id));
+
         await tx.delete(heartbeatRunEvents).where(eq(heartbeatRunEvents.companyId, id));
+        if (companyRunIds.length > 0) {
+          await tx
+            .delete(heartbeatRunEvents)
+            .where(inArray(heartbeatRunEvents.runId, companyRunIds.map((run) => run.id)));
+        }
         await tx.delete(agentTaskSessions).where(eq(agentTaskSessions.companyId, id));
         await tx.delete(activityLog).where(eq(activityLog.companyId, id));
         await tx.delete(heartbeatRuns).where(eq(heartbeatRuns.companyId, id));

server/src/services/index.ts

@@ -59,7 +59,7 @@ export type {
 } from "./authorization.js";
 export { boardAuthService } from "./board-auth.js";
 export { instanceSettingsService } from "./instance-settings.js";
-export { cloudUpstreamService } from "./cloud-upstreams.js";
+export { cloudUpstreamService, reconcileCloudUpstreamRunsOnStartup } from "./cloud-upstreams.js";
 export { companyPortabilityService } from "./company-portability.js";
 export { environmentService } from "./environments.js";
 export { executionWorkspaceService } from "./execution-workspaces.js";

server/src/services/plugin-worker-manager.ts

@@ -437,6 +437,14 @@ export function createPluginWorkerHandle(
     childProcess.stdin.write(serialized);
   }
 
+  function errorCodeForWorkerHostError(err: unknown): number {
+    const code = (err as { code?: unknown } | null)?.code;
+    const pluginErrorCodes: readonly number[] = Object.values(PLUGIN_RPC_ERROR_CODES);
+    return typeof code === "number" && pluginErrorCodes.includes(code)
+      ? code
+      : JSONRPC_ERROR_CODES.INTERNAL_ERROR;
+  }
+
   // -----------------------------------------------------------------------
   // Incoming message handling
   // -----------------------------------------------------------------------
@@ -505,6 +513,11 @@ export function createPluginWorkerHandle(
     const directCompanyId = readNonEmptyString(params.companyId);
     if (directCompanyId) return { companyId: directCompanyId };
 
+    if (method === "performAction" && isRecord(params.actorContext)) {
+      const companyId = readNonEmptyString(params.actorContext.companyId);
+      return companyId ? { companyId } : null;
+    }
+
     if (method === "executeTool" && isRecord(params.runContext)) {
       const companyId = readNonEmptyString(params.runContext.companyId);
       return companyId ? { companyId } : null;
@@ -589,15 +602,12 @@ export function createPluginWorkerHandle(
       });
     } catch (err) {
       const errorMessage = err instanceof Error ? err.message : String(err);
-      const errorCode = typeof (err as { code?: unknown }).code === "number"
-        ? (err as { code: number }).code
-        : JSONRPC_ERROR_CODES.INTERNAL_ERROR;
       log.error({ method, err: errorMessage }, "host handler error");
       try {
         sendMessage(
           createErrorResponse(
             request.id,
-            errorCode,
+            errorCodeForWorkerHostError(err),
             errorMessage,
           ),
         );