From fb41c8ce491d5d0dd675b23a982cfd4e4d779e90 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sun, 8 Nov 2026 21:24:51 -0600 Subject: [PATCH] test(evals): refresh stale SDK public-proof pins that blocked plugin cells The six plugin-SDK chat cells scan the built packages/shared and plugin-sdk output before the web server starts. packages/shared/dist/validators/agent.d.ts contains `OPENROUTER_API_KEY: z.ZodOptional`, which matches the OpenRouter key shape (detector 3). The scanner proves such matches are public zod schema types only when the pinned public inputs match the tree. The pins were stale (18 changed and 5 new shared sources, the reviewed lock moved to 2360d5..., zod moved to 4.6.7), so the proof was unavailable, the match stayed unresolved_context, and startup aborted before the agent got the request. Regenerate the source pins with source-sdk-public-inputs.mjs and move the lock and zod runtime pins to the reviewed lock (report-lock.json output) and the zod 4.6.7 it resolves. Add a regression test with the shipped declaration that also keeps literal values at that position strict. Co-Authored-By: Claude Opus 5.5 --- .../runner-e2e/source-sdk-public-inputs.json | 55 ++++++++++--------- .../runner-e2e/source-sdk-public-type.test.ts | 14 +++++ 2 files changed, 44 insertions(+), 25 deletions(-) diff --git a/tests/runner-e2e/source-sdk-public-inputs.json b/tests/runner-e2e/source-sdk-public-inputs.json index f3ab5a28f12..7fd31cbd883 100644 --- a/tests/runner-e2e/source-sdk-public-inputs.json +++ b/tests/runner-e2e/source-sdk-public-inputs.json @@ -1,6 +1,6 @@ { "inputs": { - "package.json": "3eb157776316aa11ae2023ac6b56933303f4fc81", + "package.json": "55889627a043b8ac82c32dee1200f576500d0d1c", "packages/plugins/sdk/package.json": "b7a4936e2803fd0d3b32dd7f17fc1ea86609ead0", "packages/plugins/sdk/src/bundlers.ts": "883bf3d9f2bd42877cfc0c3707cd882a6f6248d0", "packages/plugins/sdk/src/define-plugin.ts": "18edbc1db9da4a731364023c2673be2dc201bceb", @@ -22,7 +22,7 @@ "packages/plugins/sdk/src/protocol.duplex-channel.test.ts": "18255eb3d63038dc459240c111cbcb29fcba25cc", "packages/plugins/sdk/src/protocol.ts": "0ad0814b2fdb48de33504d171bffec1ca46c214b", "packages/plugins/sdk/src/testing.ts": "eb496adf22c62b6d68ef467c25465afd0973d236", - "packages/plugins/sdk/src/types.ts": "e6be5116e5dd1526414fee50b82a0f46c6a26993", + "packages/plugins/sdk/src/types.ts": "9555522abfd796c59b1439dd433be356ec344bb5", "packages/plugins/sdk/src/ui/clipboard.ts": "1f42069c97d731e636a6ad5481dad22d20c0ed63", "packages/plugins/sdk/src/ui/components.ts": "3f0594408a1bb5ac3a8d00e709f91877dd260359", "packages/plugins/sdk/src/ui/hooks.ts": "686d02cf4b35f1cddb79a83a4c4d81fd52f0100e", @@ -31,7 +31,7 @@ "packages/plugins/sdk/src/ui/types.ts": "5a5644acf6554d2e5e1d000bfd03b97d57001eb2", "packages/plugins/sdk/src/worker-rpc-host.ts": "ec3f1ff2aa52d00eef7e9cc3a3755c2a61a3c404", "packages/plugins/sdk/src/workspace-operation-error.ts": "2cd247d002d83b163239229e631bca4e519e5075", - "packages/plugins/sdk/src/workspace-provider.ts": "e46a3c4e98eaa0440fc30164afbedf0124c6566e", + "packages/plugins/sdk/src/workspace-provider.ts": "c212c3baffd4151010060c9ec7393bd989a9e833", "packages/plugins/sdk/tsconfig.json": "d62ebdff94bc4b75dabf1fcad187e1e7a90fc794", "packages/shared/package.json": "6417ef8baef0cb9b836b14f453dc8bc5025709e0", "packages/shared/src/__snapshots__/slack-app-manifest.test.ts.snap": "79020ba3aa6662923f70aa44e2c483844e9901d9", @@ -54,15 +54,15 @@ "packages/shared/src/agent-eligibility.ts": "bf658da7e9de17719fd1cf1e151c5d231e766805", "packages/shared/src/agent-observability.test.ts": "36cd74a8a8d1947364ec96fa30deabff14658df2", "packages/shared/src/agent-observability.ts": "4e71f2ca92ff7eb5720b01b5568b26f46f21c5e3", - "packages/shared/src/agent-runner.test.ts": "b430f3355cfad53213109663afd0bb2a07443c67", - "packages/shared/src/agent-runner.ts": "e9d02159f801ef773ecac53b1e6302276d15f00d", + "packages/shared/src/agent-runner.test.ts": "d02d0bba0cf57838011d4ec5f1304b65c15cff56", + "packages/shared/src/agent-runner.ts": "c4e9a7206d8c5cba7d9d95e7b02389cc351f3ecb", "packages/shared/src/agent-url-key.ts": "d0ec20b3ea1636a960bc5372a2c1e5a124bc5511", "packages/shared/src/aggregator-app-catalog.test.ts": "39ee377e55cc9565fce004b928bbe0998268e004", "packages/shared/src/aggregator-app-catalog.ts": "d6778b07a41356878c549d039ae316553ca9a22b", "packages/shared/src/aggregator-apps.ts": "2a69a54ee0cb305170275b6f4e97689c107defb1", "packages/shared/src/ai-connection-router.ts": "9fbd89c385a4d5827bc68e1c7f6a66e00453d413", "packages/shared/src/ai-connection-usage.ts": "4b51573ee92f8a2ae0f80dd15288a0e59b4dddc7", - "packages/shared/src/ai-connections.ts": "fb784919866402874a18c7e4022cb8becd0deef3", + "packages/shared/src/ai-connections.ts": "d16164c63068884655993ba3b92e1817d8651e7c", "packages/shared/src/ai-provider-routing.ts": "519c93bd489100125e9a4ad096611f25fd60af4e", "packages/shared/src/announcements.test.ts": "121ed97b560e0dd271fe6602c97e2415ee4e4bf7", "packages/shared/src/announcements.ts": "1405e849eeeeb67239bacc1bbb3903385a5faa6f", @@ -224,7 +224,10 @@ "packages/shared/src/connector-availability.ts": "82aec0dfb2e9b8cb04e21c4bcd9aab74b087c76a", "packages/shared/src/connector-identity.test.ts": "d58357707faefa7c98bb906cdaeb4db39d31081a", "packages/shared/src/connector-identity.ts": "5ede83c9c5225dbe8af9cf3a49c74e477ef5b955", - "packages/shared/src/constants.ts": "341858b027a843960e1493cb7a07d6915266c2cd", + "packages/shared/src/constants.ts": "ec1cd7fca0cb50157024563d20f12fcaf2ce4f22", + "packages/shared/src/crash-point-registry.ts": "708b1e431032ecf370fd259ab180d11e4c0b5cf3", + "packages/shared/src/crash-points.test.ts": "804a3acfd7a51010f31dbc3fb9a4dca771562c99", + "packages/shared/src/crash-points.ts": "0ad6fdaeb6bb09196399881522beec6fffcb8c87", "packages/shared/src/customer-success.ts": "a2d02aaa8223164c0eae4267b2a6da82242eaa6f", "packages/shared/src/decision-models.ts": "a07aefd87d2fae34283ce21a0334d57e2108aa7e", "packages/shared/src/decision.test.ts": "d0783d18613c73102f662e8a5da795ff6e7faed5", @@ -260,7 +263,9 @@ "packages/shared/src/home-paths.ts": "fa4f1c440653b76b3dbb7a51026d557287deb45d", "packages/shared/src/humanize-connection.test.ts": "ba9e90a44c63452b0bf9f733adb90241921e8eca", "packages/shared/src/humanize-connection.ts": "fbd414476a0ef7106a7c80d8572208781b091699", - "packages/shared/src/index.ts": "4f351647499e91ef474791ef08a66043307eeec6", + "packages/shared/src/index.ts": "9ab71a2cf39a010b2364f3200699ed3c461cc4dc", + "packages/shared/src/issue-activity-identity.test.ts": "3dc294640f54c2be67943e3e0149cf4eff2c95ca", + "packages/shared/src/issue-activity-identity.ts": "1b0f07ec3fe0506aaaf80a4fa9893f3baf8f0075", "packages/shared/src/issue-attribution.test.ts": "84563685e7f3c48029243cf1cc24a3a09ee3cde9", "packages/shared/src/issue-attribution.ts": "9c85194fb8739aeb5d70c8e128db6127a3057569", "packages/shared/src/issue-context-usage.ts": "2de0de824da2e4bce3c248335cdb5694d2a926ce", @@ -353,7 +358,7 @@ "packages/shared/src/types/activity.ts": "9d383f85775bf3657a7f9bdfb32a6e58e58c12a3", "packages/shared/src/types/adapter-registry.ts": "bd9b0a1f7980bf444f7f3d75a4687874014c4bca", "packages/shared/src/types/adapter-skills.ts": "ac77f7e013e8fb8de6fcbe916b7f7923554de815", - "packages/shared/src/types/agent-lifecycle.ts": "bfb9b6533a92ab61fc3743a99f491252feb8978f", + "packages/shared/src/types/agent-lifecycle.ts": "fca9e2df1dee7837f116b55b8809151585de55d3", "packages/shared/src/types/agent.adapter-auth-session.test.ts": "2b6af7a2cf8e7898077224ad819c05b19fdfdfe6", "packages/shared/src/types/agent.ts": "ec6079f87cb6d4d13d2feebb036a12c3cb11e215", "packages/shared/src/types/app-definition.ts": "f55fad80e9c765e16a3a59f694bdb0764940d968", @@ -376,9 +381,9 @@ "packages/shared/src/types/decision.ts": "b0d40077d481ce6180de6b02bad6df5bb0fbf977", "packages/shared/src/types/document-annotation.ts": "7769a42f1ccd707a2259774e7fe6aa536b5d183f", "packages/shared/src/types/email.ts": "556ff6a0dcefa01eb704d6d00d3b191de81ff4dc", - "packages/shared/src/types/environment.ts": "9f929fcb71be9dbfb66baa3186e62b1d84f34f60", - "packages/shared/src/types/execution-continuation.ts": "0546b3b4707cabcd69e055f12ca792095ba8feaf", - "packages/shared/src/types/execution-projection.ts": "7818cc27f979404a0bcb29fac4abe74c9c3c0f7e", + "packages/shared/src/types/environment.ts": "f28a839dd385bef5549a96b4c575e67371043f2d", + "packages/shared/src/types/execution-continuation.ts": "545998e33ab18d38268558d1a367f4aa12c2d5d3", + "packages/shared/src/types/execution-projection.ts": "052ad125886cef671b8cdea4328113b049277ccd", "packages/shared/src/types/external-object.ts": "7a225e0f5a6777d449019a30530c858b753eb4c1", "packages/shared/src/types/feedback.ts": "c5eb31545c7f67c04ad9afd9db45a6b4774bcb35", "packages/shared/src/types/finance.ts": "5c2208c7950020f1005aa0dd86a1680c54dd4354", @@ -389,10 +394,10 @@ "packages/shared/src/types/inbox-agent-policy.ts": "cb9d41d9681e80b834889268ad96015a39df92f6", "packages/shared/src/types/inbox-dismissal.ts": "5d2c71ce44ad77266d286ec1b1119ab1f0390a9d", "packages/shared/src/types/inbox-mine.ts": "1d302b531d34842189704abd2f75f8743d09490d", - "packages/shared/src/types/index.ts": "884c44fd39c7875ad9e9576d4cfc907e983ec752", + "packages/shared/src/types/index.ts": "c2e73fb144a392b89d9cdc52d37a83c4b1f8bace", "packages/shared/src/types/instance.ts": "2c40a18c05010991d93d719937faa4a43e47104f", "packages/shared/src/types/issue-tree-control.ts": "75489fffd491b8350b05caf57d7796419797d9df", - "packages/shared/src/types/issue.ts": "cb54060b60ef33173b606b1b024fdafcf4570491", + "packages/shared/src/types/issue.ts": "d053454552abe48df5336cb0fd5f540e03f1abb6", "packages/shared/src/types/live.ts": "b90e55e4de8600c6fc17d7fa8d89079d4ae655ec", "packages/shared/src/types/native-finalization.ts": "c026a73003ae232fea4f7624d561493b383df358", "packages/shared/src/types/plugin.ts": "07c409d7daca3fc9ad550538d4a7ecac03176a07", @@ -464,12 +469,12 @@ "packages/shared/src/validators/folder.ts": "b486444ab09ec3d5a9afec7dbd2e71c1bc2beb67", "packages/shared/src/validators/goal.ts": "602724134915e1e83c50115abba36b288c0abd3d", "packages/shared/src/validators/inbox-agent-policy.ts": "29cef157f4d9b564fd813868038aa8c813df4a7b", - "packages/shared/src/validators/index.ts": "6bf6703588c6bebdaba1ec7c35ac9974550a4dfb", + "packages/shared/src/validators/index.ts": "d62b0678d5f30c0ee6a500e0c7594d4775bdc73c", "packages/shared/src/validators/instance.test.ts": "3faa9593a9a05f74429d9b1d2251906a03e9a903", "packages/shared/src/validators/instance.ts": "d94ffda92ccacd3d9410562e2898410db1a100be", "packages/shared/src/validators/issue-tree-control.ts": "7144ef86a83e008ac0626e1d62a1d6164c46f438", - "packages/shared/src/validators/issue.test.ts": "80328c7d2c2220867a5ada7df76d482eec6a2144", - "packages/shared/src/validators/issue.ts": "60b75f0ce6cf5c25546bdf1bce69aad832fc732d", + "packages/shared/src/validators/issue.test.ts": "2ffd5488cdac8faf1ffc61b0f2a981641e9f64ec", + "packages/shared/src/validators/issue.ts": "68ed55331c807313b6e36f2b7fc0e8ede26f7f36", "packages/shared/src/validators/native-finalization.test.ts": "40c68bdd1d0e8a77cd70c4576fae53334fe5be0d", "packages/shared/src/validators/native-finalization.ts": "d4e674de6ac7d36385e6f6c2172fa16ffc9b1830", "packages/shared/src/validators/onboarding-seed.ts": "de302a8400d4199053f379d9ed853522a3ac6d32", @@ -492,7 +497,7 @@ "packages/shared/src/validators/skill-source.ts": "94e418769a493e75b58fba2e17c4e11826b114c5", "packages/shared/src/validators/smoke-lab.ts": "9672d163b711124cdfd0d32d919ef0122301d557", "packages/shared/src/validators/status-card.test.ts": "26194a5d39ab8924a8676af85cc93fa8bf159245", - "packages/shared/src/validators/status-card.ts": "0dccd38bed2678f9309f1cc9ab28ac4e928af127", + "packages/shared/src/validators/status-card.ts": "9095a835a40b829a83089ddfbe4969db3119b3a4", "packages/shared/src/validators/summary-slot.ts": "44005a160d02756fb34025d57f9d6ac85d4bb5fb", "packages/shared/src/validators/task-workspace.ts": "3237763bc3639da2786d3ae91592b726b3f54a51", "packages/shared/src/validators/teams-catalog.ts": "7bddf40c091a52aaa54e79de784bea5ae496f9aa", @@ -519,20 +524,20 @@ "packages/shared/src/worktree-seed-source.test.ts": "852856b1b5b8f4d0d3dada6501886fe50a2c2dde", "packages/shared/src/worktree-seed-source.ts": "3d1a09bed2d969feabef3400ff84fd499c1408f4", "packages/shared/tsconfig.json": "5a24989cd30dcf3749ca7e60d4e83d6ddbb14714", - "pnpm-lock.yaml": "a87f8e749f565800be6f28776d2894fa2bcb65f4", + "pnpm-lock.yaml": "851e65af7af027e3db99e759ece3d89e4a4700c2", "scripts/ensure-plugin-build-deps.mjs": "356e0e0f80e50890067027da3003485cf5d2ccbe", "tsconfig.base.json": "2d0e41f723add0c678936eb9ae1c60eab710f8e0", "tsconfig.json": "994ee490fb97be7cc0954f4dfd88cf0723290e6c" }, - "lockSha256": "54a81b2ed3f1f85eaa085c034560ac248e384db34fbc7aba8c5a3846c29b3ce8", + "lockSha256": "2360d5e93876ce3cb6e810257d92d669ac2b109027b0e01af36f2048861648c0", "compilerPackageSha256": "3722b30210616a13a3213ded11575ba6b2dbab10c32a5ef67afca8513e27017e", "parserPackageSha256": "6d6a65b879cd7cd249a4367c0b57cce00e78c728de45056ac2a5ca1d1f528d1a", "parserEntrySha256": "6969920ae0610df927b6b3e675d1309372c268e36d391652af8e3e0183cbe8f8", - "zodPackageSha256": "c630bd10b52dcf71c112a2bf78dbf2734b9db58d62de663b8d86c2ec2c8cda2e", - "zodEntrySha256": "2a3455cecff4f7021c92d0a3e2e5fc170c0448dc5cc261160127e08f4888a9c2", + "zodPackageSha256": "c2d190d9ee4e0bdb60332aaa2130a6a98d65ddb903fe1d29248b15de57832c45", + "zodEntrySha256": "d93a8730d52c0fedab13619c8bc971cc99934c13d8dce7c97ee4d03a98d4096e", "zodTypeFiles": { - "index.d.cts": "29f823cbe0166e10e7176a94afe609a24b9e5af3858628c541ff8ce1727023cd", - "v4/classic/external.d.cts": "268fd6d9f2e807a39a6c5aa654b00f949feb63d3faa7dd0f9bba7dde9172159c", - "v4/classic/schemas.d.cts": "aef26cf95593c8ace1c62c4724f9afac77bdfa756fb8a00613cd152117cb2f43" + "index.d.cts": "31506fd6e72c0d09e082b2f0b3741e6bbe90670361c9560ef998bdb1e56edc3e", + "v4/classic/external.d.cts": "88c4f1da1a1a6f8ca005fa1a4241897da754b7120fdb731b51a4434232bbcb42", + "v4/classic/schemas.d.cts": "9943ae429279a7a4bab6bacc828f9416ee0eed92a370d4c22d5b4235b0c9bd66" } } diff --git a/tests/runner-e2e/source-sdk-public-type.test.ts b/tests/runner-e2e/source-sdk-public-type.test.ts index 9b8f7b701a9..552d5fdc7f6 100644 --- a/tests/runner-e2e/source-sdk-public-type.test.ts +++ b/tests/runner-e2e/source-sdk-public-type.test.ts @@ -52,6 +52,20 @@ it("classifies varied genuinely public schema types while retaining the raw obse const audit = await scanner.audit(); expect(audit.suppressedFindings).toBe(0); expect(audit.observations).toHaveLength(1); expect(audit.observations[0]!.classification).toBe("public_schema_type"); } }); +it("classifies the shipped agent env schema declaration and keeps a literal at that position strict", async () => { + // Plugin cells retain packages/shared/dist/validators/agent.d.ts, whose + // `OPENROUTER_API_KEY: z.ZodOptional` matches the OpenRouter key + // shape. Stale default pins left it unresolved and blocked web-server startup. + const repository = await realpath(originalRoot), agentFile = "packages/shared/dist/validators/agent.d.ts"; + const shipped = 'import { z } from "zod";\nexport declare const env: z.ZodObject<{\n OPENROUTER_API_KEY: z.ZodOptional;\n}>;\n'; + const scanner = createSdkArtifactScanner(repository, []); expect(await scanner.scan(agentFile, Buffer.from(shipped))).toBe(true); + expect((await scanner.audit()).observations.map(row => row.classification)).toEqual(["public_schema_type"]); + for (const value of ['"syntheticCredentialValue123456789"', "syntheticCredentialValue123456789", 'z.ZodOptional | "syntheticCredential1"']) { + const strict = createSdkArtifactScanner(repository, []); + expect(await strict.scan(agentFile, Buffer.from(shipped.replace("z.ZodOptional", value)))).toBe(false); + expect((await strict.audit()).observations.map(row => row.classification)).toContain("unresolved_context"); + } +}); it("protects an exact private value even when equal to a public schema symbol", async () => { const scanner = createSdkArtifactScanner(root, ["ZodString"], await fixture()); expect(await scanner.scan(file, Buffer.from(declaration))).toBe(false); expect((await scanner.audit()).observations[0]!.classification).toBe("private_exact");