fix(connections): parse template capture corpus

scripts/ingest-app-definitions.mjs

@@ -4,7 +4,7 @@ const out=path.join(root,"packages/shared/src/app-definitions"); const favicon=d
 const field=(key,label,placeholder)=>({key,label,type:"password",required:true,placeholder,secret:true});
 const method=(key,transport,auth,defaults,riskTier,guidanceMd,extra={})=>({key,transport,auth,ownershipModes:auth==="oauth"?["customer","dcr"]:["customer"],whenToUse:transport==="mcp_remote"?"Use the provider-hosted connection for the quickest setup.":"Use credentials from your provider account.",defaults,guidanceMd,riskTier,...extra});
 const apps=[
-["zapier","Zapier","Reach thousands of apps through your Zapier account.","productivity","zapier.com",["https://mcp.zapier.com/*"],method("mcp-key","mcp_remote","api_key",{serverUrl:"https://mcp.zapier.com/api/mcp"},"S3","Create a Zapier MCP connection, then paste its token here.",{credentialFields:[field("authorization","Zapier token","Paste your Zapier token")],keyPlacement:{location:"header",name:"Authorization",prefix:"Bearer "}})],
+["zapier","Zapier","Reach thousands of apps through your Zapier account.","productivity","zapier.com",["https://mcp.zapier.com/*"],method("mcp-key","mcp_remote","api_key",{serverUrl:"https://mcp.zapier.com/api/mcp"},"S3","Create a Zapier MCP connection, then paste its token here.",{credentialFields:[field("authorization","Zapier MCP token","Paste your Zapier token")],keyPlacement:{location:"header",name:"Authorization",prefix:"Bearer "}})],
 ["github","GitHub","Read code and pull requests, and coordinate repository work.","developer","github.com",["https://api.githubcopilot.com/mcp/*"],method("mcp-key","mcp_remote","api_key",{serverUrl:"https://api.githubcopilot.com/mcp/"},"S3","Create a fine-grained token limited to the repositories agents should use.",{credentialFields:[field("authorization","GitHub token","github_pat_...")],keyPlacement:{location:"header",name:"Authorization",prefix:"Bearer "},requiredResourceFilters:["organization","repository"]})],
 ["slack","Slack","Search channels and coordinate team communication.","communication","slack.com",["https://mcp.slack.com/*"],method("mcp-oauth","mcp_remote","oauth",{serverUrl:"https://mcp.slack.com/mcp",authorizationEndpoint:"https://slack.com/oauth/v2/authorize",tokenEndpoint:"https://slack.com/api/oauth.v2.access",scopesHint:["channels:read","chat:write","search:read"]},"S3","Connect a Slack workspace and limit access to the channels agents need.",{requiredResourceFilters:["workspace","channel"]})],
 ["notion","Notion","Read and update pages in your Notion workspace.","content","notion.so",["https://mcp.notion.com/*"],method("mcp-oauth","mcp_remote","oauth",{serverUrl:"https://mcp.notion.com/mcp",authorizationEndpoint:"https://api.notion.com/v1/oauth/authorize",tokenEndpoint:"https://api.notion.com/v1/oauth/token"},"S3","Connect Notion for workspace content. Share only the pages and databases agents should use.",{requiredResourceFilters:["workspace","page","database"]})],
@@ -17,8 +17,48 @@ const apps=[
 ["vercel","Vercel","Inspect projects, deployments, and runtime logs.","developer","vercel.com",["https://mcp.vercel.com/*"],method("mcp-oauth","mcp_remote","oauth",{serverUrl:"https://mcp.vercel.com/mcp"},"S3","Connect the Vercel team and projects agents should operate.",{requiredResourceFilters:["team","project","environment"]})],
 ["anthropic","Anthropic","Use Anthropic APIs with a restricted key.","ai","anthropic.com",["https://api.anthropic.com/*"],method("api-key","rest_api","api_key",{serviceHost:"api.anthropic.com"},"S3","Create a key in the Anthropic Console and rotate it if it has been exposed.",{credentialFields:[field("apiKey","API key","sk-ant-api03-...")],keyPlacement:{location:"header",name:"x-api-key"}})],
 ].map(([slug,name,description,category,domain,urlPatterns,m])=>({schemaVersion:1,slug,name,description,categories:[category],featured:["zapier","github","slack","notion","linear"].includes(slug),branding:{logoUrl:favicon(domain)},urlPatterns,methods:[m]}));
-const captures=fs.readdirSync(corpus).filter(f=>f.endsWith(".md")&&f!=="INDEX.md"); if(captures.length!==99) throw new Error(`Expected 99 captures, found ${captures.length}`);
+const parseTableRow=(line)=>line.slice(1,-1).split("|").map((cell)=>cell.trim());
+const parseCapture=(fileName)=>{
+ const markdown=fs.readFileSync(path.join(corpus,fileName),"utf8");
+ const stateMatches=[...markdown.matchAll(/^## State: (.+)$/gm)];
+ if(stateMatches.length===0) throw new Error(`${fileName}: no captured states`);
+ return stateMatches.map((match,index)=>{
+  const body=markdown.slice(match.index+match[0].length,stateMatches[index+1]?.index??markdown.length);
+  const inputsBlock=body.match(/### Inputs\n([\s\S]*?)(?=\n### |$)/)?.[1]??"";
+  const inputRows=inputsBlock.split("\n").filter((line)=>line.startsWith("|")).slice(2).map(parseTableRow);
+  const fields=inputRows.map(([label,tagType,required,placeholder,prefilledValue,checked])=>({label,tagType,required:required.toLowerCase()==="yes",placeholder:placeholder||null,prefilledValue:prefilledValue||null,checked:checked.toLowerCase()==="true"}));
+  const linksBlock=body.match(/### Links\n([\s\S]*?)(?=\n## |$)/)?.[1]??"";
+  const links=linksBlock.split("\n").map((line)=>line.match(/^(.+?) → (https?:\/\/\S+)$/)).filter(Boolean).map((link)=>({label:link[1].trim(),href:link[2]}));
+  return {label:match[1].trim(),fields,links};
+ });
+};
+const inferState=(slug,state)=>{
+ const label=state.label.toLowerCase();
+ const fieldText=state.fields.map((field)=>field.label.toLowerCase()).join(" ");
+ const transport=slug==="oauth-generic"||slug==="api-key-generic"||label.includes("path: api")||label.includes("api key form")?"rest_api":"mcp_remote";
+ const auth=slug==="oauth-generic"||label.includes("oauth")||fieldText.includes("client id")?"oauth":slug==="api-key-generic"||label.includes("api key")||fieldText.includes("api key")?"api_key":null;
+ const ownershipModes=[];
+ if(label.includes("managed")) ownershipModes.push("platform_shared");
+ if(label.includes("your own credentials")||label.includes("manual")||label.includes("api key")) ownershipModes.push("customer");
+ if(slug==="oauth-generic"&&!label.includes("manually")) ownershipModes.push("dcr");
+ return {label:state.label,transport,auth,ownershipModes:[...new Set(ownershipModes)],fieldCount:state.fields.length,linkCount:state.links.length};
+};
+const validateApp=(app)=>{
+ if(app.schemaVersion!==1||!app.slug||!app.name||!Array.isArray(app.methods)||app.methods.length===0) throw new Error(`${app.slug||"unknown"}: invalid AppDefinition`);
+ for(const connectionMethod of app.methods){
+  if(connectionMethod.auth==="api_key"&&!connectionMethod.keyPlacement) throw new Error(`${app.slug}/${connectionMethod.key}: api_key requires keyPlacement`);
+  if(connectionMethod.auth==="oauth"&&connectionMethod.ownershipModes.length===0) throw new Error(`${app.slug}/${connectionMethod.key}: oauth requires ownershipModes`);
+  for(const connectionField of [...connectionMethod.tenantFields??[],...connectionMethod.extensionFields??[],...connectionMethod.credentialFields??[]]) if(connectionField.required&&connectionField.type!=="checkbox"&&!connectionField.placeholder) throw new Error(`${app.slug}/${connectionMethod.key}/${connectionField.key}: required field needs placeholder`);
+ }
+};
+const captureFiles=fs.readdirSync(corpus).filter((fileName)=>fileName.endsWith(".md")&&fileName!=="INDEX.md").sort();
+if(captureFiles.length!==99) throw new Error(`Expected 99 captures, found ${captureFiles.length}`);
+const parsedCaptures=Object.fromEntries(captureFiles.map((fileName)=>[path.basename(fileName,".md"),parseCapture(fileName)]));
+const reviewReport={schemaVersion:1,corpusSize:captureFiles.length,providers:captureFiles.map((fileName)=>{const slug=path.basename(fileName,".md");const states=parsedCaptures[slug].map((state)=>inferState(slug,state));return {slug,stateCount:states.length,states,ambiguities:states.filter((state)=>!state.auth).map((state)=>`Auth is not explicit in capture state: ${state.label}`)};})};
+for(const app of apps){validateApp(app);if(parsedCaptures[app.slug]&&parsedCaptures[app.slug].length===0) throw new Error(`${app.slug}: capture has no states`);}
 fs.mkdirSync(out,{recursive:true}); for(const app of apps) fs.writeFileSync(path.join(out,`${app.slug}.json`),JSON.stringify(app,null,2)+"\n");
+fs.writeFileSync(path.join(root,"packages/shared/src/app-definitions.ingestion-report.json"),JSON.stringify(reviewReport,null,2)+"\n");
 const imports=apps.map((a,i)=>`import a${i} from "./app-definitions/${a.slug}.json" with { type: "json" };`).join("\n");
 fs.writeFileSync(path.join(root,"packages/shared/src/app-definitions.generated.ts"),`${imports}\nimport type { AppDefinition } from "./types/app-definition.js";\nexport const APP_DEFINITIONS=[${apps.map((_,i)=>`a${i}`).join(",")}] as AppDefinition[];\n`);
-console.log(`Parsed ${captures.length} captures; emitted ${apps.length} Wave 1 definitions.`);
+const ambiguityCount=reviewReport.providers.reduce((total,provider)=>total+provider.ambiguities.length,0);
+console.log(`Parsed ${captureFiles.length} captures and ${reviewReport.providers.reduce((total,provider)=>total+provider.stateCount,0)} states; emitted ${apps.length} Wave 1 definitions and flagged ${ambiguityCount} states for review.`);