diff --git a/doc/AGENT-LIFECYCLE.md b/doc/AGENT-LIFECYCLE.md index 87e0967316f..4302e23247d 100644 --- a/doc/AGENT-LIFECYCLE.md +++ b/doc/AGENT-LIFECYCLE.md @@ -203,6 +203,7 @@ Return the same `operationId` and `version` with `status: "complete"` or Return `complete` only after the requested effect is complete. Return `pending` while external work continues. Throw an error when the step fails. + The host stores a fixed error message. It does not store the provider error text. Server logs identify the failed step, phase, operation, and plugin when available. They include only known error codes. Unknown codes become `unclassified`. @@ -218,6 +219,41 @@ It must also prevent an older operation from creating resources after cleanup. Serialize external effects or remove resources from a late completion. Do not treat a host timeout as cancellation of the external effect. +### Local instance secrets + +Lifecycle requests include `instanceSecrets`, a map of this plugin's private +credentials for this company and agent. A plugin can return an `instanceSecrets` +map with its result to save values. A null value removes the key. Omitted keys +remain unchanged. Writes are allowed during preparation and resume; removals can +also occur during stop and cleanup. Use at most 16 keys per result, with names +matching `[a-z][a-z0-9_]{0,63}` and values no larger than 64 KiB each. + +The tenant persists the changes before accepting the result. It locks the agent +and checks its company, phase, operation, revision, required plugin selection, +and the captured attempt's current lease owner and unexpired deadline. +A stale reply or a failed secret write cannot complete preparation. Calls after +a restart receive the saved values. Stop and cleanup requests receive an empty +map, so a damaged credential cannot block removal. Return removals even while +cleanup is pending. An older host omits the field; plugins that require durable +credentials must reject that host before resource effects. + +This uses the existing secret service, `company_secrets`, secret versions, and +`local_encrypted` provider. There is no new encryption system or remote API. +The `instance` scope requires an owning agent and plugin. Agent/plugin deletion +cascades the native secret records. These secrets are excluded from company +catalogs, normal secret references, agent access, plugin state, and public agent +records. The general SDK `ctx.secrets.resolve()` remains read-only. Company vault imports +ignore instance-owned names and keys. Device-login cleanup checks instance-owned +account-home claims privately and keeps the directory if a value cannot be read. + +Trusted server runtime code can resolve values with +`agentInstanceSecretService(db).read({ companyId, agentId, pluginId })` after its +normal execution admission checks. Use the installed plugin's database ID. +Never place returned values in an agent environment, task payload, log, or UI +response. Preserve the existing secret master key through restarts and backups. +Migration `0339_luxuriant_iron_monger.sql` adds native secret ownership and its +scope constraint; keep these columns when reverting application code. + ## Verification and recovery The harness verification service uses the existing environment-test code. diff --git a/maps/agent-access-map/secrets.md b/maps/agent-access-map/secrets.md index de4a5426219..551f7046987 100644 --- a/maps/agent-access-map/secrets.md +++ b/maps/agent-access-map/secrets.md @@ -136,6 +136,13 @@ credentials/run context; inspect its result and applicable persistence/attributi Assert the relevant denied actor, owner, company or mode from Policy. Review the remaining operations before qualifying the whole capability. +Server-only instance secrets are excluded from this path. They belong to the +company, owning agent, and installed lifecycle plugin. The existing local lifecycle +callback can save them through the native secret service; there is no agent API +for their values. Source: [native secret coordinator](../../server/src/services/agent-instance-secrets.ts) +and [scope/denial tests](../../server/src/__tests__/agent-instance-secrets.test.ts). +No live agent execution is claimed for this exclusion. + Gaps: Values remain restricted to the bound agent and active run. Credential responses stay inline and use encrypted mutation receipts; they are not saved as plaintext response assets. Dedicated scenario read_secret_value remains unbound. diff --git a/maps/feature-map/secrets.md b/maps/feature-map/secrets.md index 34082ab4ba9..68658519cd9 100644 --- a/maps/feature-map/secrets.md +++ b/maps/feature-map/secrets.md @@ -12,7 +12,7 @@ Agent access: [companion](../agent-access-map/secrets.md). - `user-secrets`: define required user credentials and set a value for the current user. - `proposals`: review proposed changes with the appropriate authority. - `vault-import`: import eligible external-vault references through the offered flow. -- `runtime-delivery`: deliver only the secret references authorized for the run/actor. +- `runtime-delivery`: deliver only the secret references authorized for the run/actor. Agent-owned instance secrets stay in the trusted server runtime and are never delivered to an agent. ## How to get to it (user POV) @@ -30,7 +30,7 @@ Preconditions: follow the [baseline](./README.md#before-driving-a-journey). Use ### `secret-management` -Automated: [secrets service](../../server/src/__tests__/secrets-service.test.ts) and [user ownership](../../server/src/__tests__/secrets-service-user-secret-owner-scoped.test.ts) cover storage/access contracts. +Automated: [secrets service](../../server/src/__tests__/secrets-service.test.ts) and [user ownership](../../server/src/__tests__/secrets-service-user-secret-owner-scoped.test.ts) cover storage/access contracts. [Instance-secret tests](../../server/src/__tests__/agent-instance-secrets.test.ts) cover local encrypted storage, owner isolation, and exclusion from ordinary bindings. Manual: Create a dummy secret reference, bind it through a supported field, and verify its use in a harmless run without logging the value. Update it and reload. Switch users and confirm personal values remain owner-scoped and missing credentials produce an actionable state. diff --git a/packages/db/src/migrations/0339_luxuriant_iron_monger.sql b/packages/db/src/migrations/0339_luxuriant_iron_monger.sql new file mode 100644 index 00000000000..4064f5e6c87 --- /dev/null +++ b/packages/db/src/migrations/0339_luxuriant_iron_monger.sql @@ -0,0 +1,21 @@ +ALTER TABLE "company_secrets" DROP CONSTRAINT "company_secrets_scope_shape_check";--> statement-breakpoint +ALTER TABLE "company_secrets" ADD COLUMN "owner_agent_id" uuid;--> statement-breakpoint +ALTER TABLE "company_secrets" ADD COLUMN "owner_plugin_id" uuid;--> statement-breakpoint +ALTER TABLE "company_secrets" ADD CONSTRAINT "company_secrets_owner_plugin_id_plugins_id_fk" FOREIGN KEY ("owner_plugin_id") REFERENCES "public"."plugins"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "company_secrets" ADD CONSTRAINT "company_secrets_company_id_owner_agent_id_agents_company_id_id_fk" FOREIGN KEY ("company_id","owner_agent_id") REFERENCES "public"."agents"("company_id","id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +CREATE UNIQUE INDEX "company_secrets_instance_owner_key_uq" ON "company_secrets" USING btree ("company_id","owner_agent_id","owner_plugin_id","key") WHERE "company_secrets"."scope" = 'instance' and "company_secrets"."deleted_at" is null;--> statement-breakpoint +ALTER TABLE "company_secrets" ADD CONSTRAINT "company_secrets_scope_shape_check" CHECK (( + "company_secrets"."scope" = 'company' + and "company_secrets"."owner_user_id" is null + and "company_secrets"."user_secret_definition_id" is null + and "company_secrets"."owner_agent_id" is null and "company_secrets"."owner_plugin_id" is null + ) or ( + "company_secrets"."scope" = 'user' + and "company_secrets"."owner_user_id" is not null + and "company_secrets"."user_secret_definition_id" is not null + and "company_secrets"."owner_agent_id" is null and "company_secrets"."owner_plugin_id" is null + ) or ( + "company_secrets"."scope" = 'instance' + and "company_secrets"."owner_user_id" is null and "company_secrets"."user_secret_definition_id" is null + and "company_secrets"."owner_agent_id" is not null and "company_secrets"."owner_plugin_id" is not null + )); \ No newline at end of file diff --git a/packages/db/src/migrations/meta/0334_snapshot.json b/packages/db/src/migrations/meta/0339_snapshot.json similarity index 98% rename from packages/db/src/migrations/meta/0334_snapshot.json rename to packages/db/src/migrations/meta/0339_snapshot.json index cfd478f7890..bad9c2d8d2e 100644 --- a/packages/db/src/migrations/meta/0334_snapshot.json +++ b/packages/db/src/migrations/meta/0339_snapshot.json @@ -1,6 +1,6 @@ { - "id": "39d82f2c-8fd5-4606-84df-f51880dbd6a7", - "prevId": "fb1a65b0-7eff-4e5a-a023-fecec51f938c", + "id": "045fe4f7-1636-4e1b-b304-11b6128fefcf", + "prevId": "108e07e7-19ab-4f16-8400-fd733f77afdc", "version": "7", "dialect": "postgresql", "tables": { @@ -9858,6 +9858,12 @@ "notNull": true, "default": false }, + "execution_defaults": { + "name": "execution_defaults", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, "metadata": { "name": "metadata", "type": "jsonb", @@ -10098,6 +10104,12 @@ "notNull": true, "default": "''" }, + "execution_defaults": { + "name": "execution_defaults", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, "allow_direct_messages": { "name": "allow_direct_messages", "type": "boolean", @@ -12433,6 +12445,189 @@ }, "isRLSEnabled": false }, + "public.chat_slack_manager_grants": { + "name": "chat_slack_manager_grants", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "slack_user_id": { + "name": "slack_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_name": { + "name": "workspace_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "manager_app_id": { + "name": "manager_app_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "access_secret_id": { + "name": "access_secret_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "refresh_secret_id": { + "name": "refresh_secret_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "rate_limits": { + "name": "rate_limits", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "chat_slack_manager_grants_company_id_idx": { + "name": "chat_slack_manager_grants_company_id_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "chat_slack_manager_grants_owner_idx": { + "name": "chat_slack_manager_grants_owner_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "manager_app_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "chat_slack_manager_grants_company_id_companies_id_fk": { + "name": "chat_slack_manager_grants_company_id_companies_id_fk", + "tableFrom": "chat_slack_manager_grants", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "chat_slack_manager_grants_status_check": { + "name": "chat_slack_manager_grants_status_check", + "value": "\"chat_slack_manager_grants\".\"status\" in ('active', 'revoked', 'reauthorize')" + } + }, + "isRLSEnabled": false + }, "public.chat_slack_registrations": { "name": "chat_slack_registrations", "schema": "", @@ -12449,6 +12644,12 @@ "primaryKey": false, "notNull": true }, + "manager_grant_id": { + "name": "manager_grant_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, "request_id": { "name": "request_id", "type": "uuid", @@ -12539,6 +12740,21 @@ }, "indexes": {}, "foreignKeys": { + "chat_slack_registrations_company_id_manager_grant_id_chat_slack_manager_grants_company_id_id_fk": { + "name": "chat_slack_registrations_company_id_manager_grant_id_chat_slack_manager_grants_company_id_id_fk", + "tableFrom": "chat_slack_registrations", + "tableTo": "chat_slack_manager_grants", + "columnsFrom": [ + "company_id", + "manager_grant_id" + ], + "columnsTo": [ + "company_id", + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, "chat_slack_registrations_company_id_endpoint_id_chat_endpoints_company_id_id_fk": { "name": "chat_slack_registrations_company_id_endpoint_id_chat_endpoints_company_id_id_fk", "tableFrom": "chat_slack_registrations", @@ -16108,6 +16324,18 @@ "primaryKey": false, "notNull": false }, + "owner_agent_id": { + "name": "owner_agent_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "owner_plugin_id": { + "name": "owner_plugin_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, "user_secret_definition_id": { "name": "user_secret_definition_id", "type": "uuid", @@ -16336,6 +16564,40 @@ "method": "btree", "with": {} }, + "company_secrets_instance_owner_key_uq": { + "name": "company_secrets_instance_owner_key_uq", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner_agent_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner_plugin_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"company_secrets\".\"scope\" = 'instance' and \"company_secrets\".\"deleted_at\" is null", + "concurrently": false, + "method": "btree", + "with": {} + }, "company_secrets_provider_config_idx": { "name": "company_secrets_provider_config_idx", "columns": [ @@ -16438,6 +16700,19 @@ "onDelete": "no action", "onUpdate": "no action" }, + "company_secrets_owner_plugin_id_plugins_id_fk": { + "name": "company_secrets_owner_plugin_id_plugins_id_fk", + "tableFrom": "company_secrets", + "tableTo": "plugins", + "columnsFrom": [ + "owner_plugin_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, "company_secrets_user_secret_definition_id_user_secret_definitions_id_fk": { "name": "company_secrets_user_secret_definition_id_user_secret_definitions_id_fk", "tableFrom": "company_secrets", @@ -16476,6 +16751,21 @@ ], "onDelete": "set null", "onUpdate": "no action" + }, + "company_secrets_company_id_owner_agent_id_agents_company_id_id_fk": { + "name": "company_secrets_company_id_owner_agent_id_agents_company_id_id_fk", + "tableFrom": "company_secrets", + "tableTo": "agents", + "columnsFrom": [ + "company_id", + "owner_agent_id" + ], + "columnsTo": [ + "company_id", + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" } }, "compositePrimaryKeys": {}, @@ -16484,7 +16774,7 @@ "checkConstraints": { "company_secrets_scope_shape_check": { "name": "company_secrets_scope_shape_check", - "value": "(\n \"company_secrets\".\"scope\" = 'company'\n and \"company_secrets\".\"owner_user_id\" is null\n and \"company_secrets\".\"user_secret_definition_id\" is null\n ) or (\n \"company_secrets\".\"scope\" = 'user'\n and \"company_secrets\".\"owner_user_id\" is not null\n and \"company_secrets\".\"user_secret_definition_id\" is not null\n )" + "value": "(\n \"company_secrets\".\"scope\" = 'company'\n and \"company_secrets\".\"owner_user_id\" is null\n and \"company_secrets\".\"user_secret_definition_id\" is null\n and \"company_secrets\".\"owner_agent_id\" is null and \"company_secrets\".\"owner_plugin_id\" is null\n ) or (\n \"company_secrets\".\"scope\" = 'user'\n and \"company_secrets\".\"owner_user_id\" is not null\n and \"company_secrets\".\"user_secret_definition_id\" is not null\n and \"company_secrets\".\"owner_agent_id\" is null and \"company_secrets\".\"owner_plugin_id\" is null\n ) or (\n \"company_secrets\".\"scope\" = 'instance'\n and \"company_secrets\".\"owner_user_id\" is null and \"company_secrets\".\"user_secret_definition_id\" is null\n and \"company_secrets\".\"owner_agent_id\" is not null and \"company_secrets\".\"owner_plugin_id\" is not null\n )" } }, "isRLSEnabled": false @@ -26377,6 +26667,476 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.execution_workspace_repositories": { + "name": "execution_workspace_repositories", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "execution_workspace_id": { + "name": "execution_workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "requested_by_issue_id": { + "name": "requested_by_issue_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "repository_identity": { + "name": "repository_identity", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "catalog_repository_id": { + "name": "catalog_repository_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "repo_url": { + "name": "repo_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "relative_path": { + "name": "relative_path", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "requested_ref": { + "name": "requested_ref", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'HEAD'" + }, + "pinned_commit": { + "name": "pinned_commit", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "branch_name": { + "name": "branch_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "request_key": { + "name": "request_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "request_keys": { + "name": "request_keys", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "failure_code": { + "name": "failure_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "execution_workspace_repositories_identity_idx": { + "name": "execution_workspace_repositories_identity_idx", + "columns": [ + { + "expression": "execution_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "repository_identity", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "execution_workspace_repositories_path_idx": { + "name": "execution_workspace_repositories_path_idx", + "columns": [ + { + "expression": "execution_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "relative_path", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "execution_workspace_repositories_request_idx": { + "name": "execution_workspace_repositories_request_idx", + "columns": [ + { + "expression": "execution_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "request_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "execution_workspace_repositories_company_workspace_idx": { + "name": "execution_workspace_repositories_company_workspace_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "execution_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "execution_workspace_repositories_company_id_companies_id_fk": { + "name": "execution_workspace_repositories_company_id_companies_id_fk", + "tableFrom": "execution_workspace_repositories", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "execution_workspace_repositories_execution_workspace_id_execution_workspaces_id_fk": { + "name": "execution_workspace_repositories_execution_workspace_id_execution_workspaces_id_fk", + "tableFrom": "execution_workspace_repositories", + "tableTo": "execution_workspaces", + "columnsFrom": [ + "execution_workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "execution_workspace_repositories_requested_by_issue_id_issues_id_fk": { + "name": "execution_workspace_repositories_requested_by_issue_id_issues_id_fk", + "tableFrom": "execution_workspace_repositories", + "tableTo": "issues", + "columnsFrom": [ + "requested_by_issue_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.execution_workspace_repository_access": { + "name": "execution_workspace_repository_access", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "root_key": { + "name": "root_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "canonical_cwd": { + "name": "canonical_cwd", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "scope_id": { + "name": "scope_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "issue_id": { + "name": "issue_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "agent_id": { + "name": "agent_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "responsible_user_id": { + "name": "responsible_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "execution_workspace_repository_access_scope_idx": { + "name": "execution_workspace_repository_access_scope_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "root_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "scope_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "execution_workspace_repository_access_company_id_companies_id_fk": { + "name": "execution_workspace_repository_access_company_id_companies_id_fk", + "tableFrom": "execution_workspace_repository_access", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.execution_workspace_repository_roots": { + "name": "execution_workspace_repository_roots", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "canonical_cwd": { + "name": "canonical_cwd", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "root_key": { + "name": "root_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "execution_workspace_repository_roots_company_root_idx": { + "name": "execution_workspace_repository_roots_company_root_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "root_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "execution_workspace_repository_roots_company_id_companies_id_fk": { + "name": "execution_workspace_repository_roots_company_id_companies_id_fk", + "tableFrom": "execution_workspace_repository_roots", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.execution_workspace_runtime_leases": { "name": "execution_workspace_runtime_leases", "schema": "", @@ -26632,7 +27392,7 @@ "name": "project_id", "type": "uuid", "primaryKey": false, - "notNull": true + "notNull": false }, "project_workspace_id": { "name": "project_workspace_id", @@ -26910,7 +27670,7 @@ "columnsTo": [ "id" ], - "onDelete": "cascade", + "onDelete": "restrict", "onUpdate": "no action" }, "execution_workspaces_project_workspace_id_project_workspaces_id_fk": { @@ -37698,6 +38458,25 @@ "primaryKey": false, "notNull": false }, + "workspace_binding_revision": { + "name": "workspace_binding_revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "workspace_selection": { + "name": "workspace_selection", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "workspace_pending_selection": { + "name": "workspace_pending_selection", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, "execution_workspace_preference": { "name": "execution_workspace_preference", "type": "text", diff --git a/packages/db/src/migrations/meta/_journal.json b/packages/db/src/migrations/meta/_journal.json index cf354e19e94..4e82cc85923 100644 --- a/packages/db/src/migrations/meta/_journal.json +++ b/packages/db/src/migrations/meta/_journal.json @@ -2360,6 +2360,13 @@ "when": 1791837431016, "tag": "0338_absurd_vengeance", "breakpoints": true + }, + { + "idx": 339, + "version": "7", + "when": 1791903526575, + "tag": "0339_luxuriant_iron_monger", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/db/src/schema/company_secrets.ts b/packages/db/src/schema/company_secrets.ts index 6d3a9ddc29d..ac6da0af2fd 100644 --- a/packages/db/src/schema/company_secrets.ts +++ b/packages/db/src/schema/company_secrets.ts @@ -1,9 +1,10 @@ import { sql } from "drizzle-orm"; -import { check, pgTable, uuid, text, timestamp, integer, jsonb, index, uniqueIndex } from "drizzle-orm/pg-core"; +import { check, pgTable, uuid, text, timestamp, integer, jsonb, index, uniqueIndex, foreignKey } from "drizzle-orm/pg-core"; import { companies } from "./companies.js"; import { agents } from "./agents.js"; import { companySecretProviderConfigs } from "./company_secret_provider_configs.js"; import { userSecretDefinitions } from "./user_secret_definitions.js"; +import { plugins } from "./plugins.js"; export const companySecrets = pgTable( "company_secrets", @@ -12,6 +13,8 @@ export const companySecrets = pgTable( companyId: uuid("company_id").notNull().references(() => companies.id), scope: text("scope").notNull().default("company"), ownerUserId: text("owner_user_id"), + ownerAgentId: uuid("owner_agent_id"), + ownerPluginId: uuid("owner_plugin_id").references(() => plugins.id, { onDelete: "cascade" }), userSecretDefinitionId: uuid("user_secret_definition_id").references(() => userSecretDefinitions.id, { onDelete: "set null" }), key: text("key").notNull(), name: text("name").notNull(), @@ -42,6 +45,9 @@ export const companySecrets = pgTable( table.ownerUserId, ), companyProviderIdx: index("company_secrets_company_provider_idx").on(table.companyId, table.provider), + agentOwner: foreignKey({ columns: [table.companyId, table.ownerAgentId], foreignColumns: [agents.companyId, agents.id] }).onDelete("cascade"), + instanceOwnerKeyUq: uniqueIndex("company_secrets_instance_owner_key_uq").on(table.companyId, table.ownerAgentId, table.ownerPluginId, table.key) + .where(sql`${table.scope} = 'instance' and ${table.deletedAt} is null`), providerConfigIdx: index("company_secrets_provider_config_idx").on(table.providerConfigId), companyNameUq: uniqueIndex("company_secrets_company_name_uq") .on(table.companyId, table.name) @@ -58,10 +64,16 @@ export const companySecrets = pgTable( ${table.scope} = 'company' and ${table.ownerUserId} is null and ${table.userSecretDefinitionId} is null + and ${table.ownerAgentId} is null and ${table.ownerPluginId} is null ) or ( ${table.scope} = 'user' and ${table.ownerUserId} is not null and ${table.userSecretDefinitionId} is not null + and ${table.ownerAgentId} is null and ${table.ownerPluginId} is null + ) or ( + ${table.scope} = 'instance' + and ${table.ownerUserId} is null and ${table.userSecretDefinitionId} is null + and ${table.ownerAgentId} is not null and ${table.ownerPluginId} is not null )`, ), }), diff --git a/packages/shared/src/constants.ts b/packages/shared/src/constants.ts index 70840a8fd64..f943b187d8e 100644 --- a/packages/shared/src/constants.ts +++ b/packages/shared/src/constants.ts @@ -732,7 +732,7 @@ export type SecretProviderConfigHealthStatus = export const SECRET_STATUSES = ["active", "disabled", "archived", "deleted"] as const; export type SecretStatus = (typeof SECRET_STATUSES)[number]; -export const SECRET_SCOPES = ["company", "user"] as const; +export const SECRET_SCOPES = ["company", "user", "instance"] as const; export type SecretScope = (typeof SECRET_SCOPES)[number]; export const SECRET_MANAGED_MODES = ["paperclip_managed", "external_reference"] as const; diff --git a/packages/shared/src/types/agent-lifecycle.ts b/packages/shared/src/types/agent-lifecycle.ts index a1b03e88e39..13a7a829c80 100644 --- a/packages/shared/src/types/agent-lifecycle.ts +++ b/packages/shared/src/types/agent-lifecycle.ts @@ -23,9 +23,11 @@ export interface AgentLifecycleRequest { operationId: string; version: number; phase: AgentLifecycleState; + instanceSecrets?: Record; } export interface AgentLifecycleResult { operationId: string; version: number; status: "complete" | "pending"; + instanceSecrets?: Record; } diff --git a/server/src/__tests__/agent-device-login-routes.test.ts b/server/src/__tests__/agent-device-login-routes.test.ts index 61b617d2127..c9e1841964f 100644 --- a/server/src/__tests__/agent-device-login-routes.test.ts +++ b/server/src/__tests__/agent-device-login-routes.test.ts @@ -77,6 +77,7 @@ const mockSecretService = vi.hoisted(() => ({ // idempotent-repeat-login and secret-creation-failure paths. getByName: vi.fn(async (_companyId: string, _name: string) => null as Record | null), create: vi.fn(async (_companyId: string, _input: Record) => ({ id: "secret-1" })), + hasInstanceAccountHomeClaim: vi.fn(async (_companyId: string, _path: string) => false), // The company-wide secret listing the cleanup-safety scan reads. Empty by // default, so a cleanup that finds no other claimant removes the // directory; a test overrides it with a differently named secret to prove @@ -1404,6 +1405,27 @@ describe("adapter device-login routes", () => { await expect(lstat(accountHomeDir)).resolves.toBeDefined(); }); + it("keeps a newly promoted account home claimed by an instance secret when company secret creation fails", async () => { + const accountHomeDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-codex-account-home-test-")); + accountHomeTestDirs.push(accountHomeDir); + mockDeviceLoginPromotion.mockResolvedValueOnce({ + outcome: "promoted", accountId: "acct-instance-owner", accountHomeDir, accountHomeCreated: true, + }); + mockSecretService.create.mockRejectedValueOnce(new Error("db unavailable")); + mockSecretService.hasInstanceAccountHomeClaim.mockResolvedValueOnce(true); + const app = await createApp(); + const start = await request(app).post(loginPath(COMPANY_1)).send({ environmentId: SANDBOX_ENV_1 }); + expect(start.status, JSON.stringify(start.body)).toBe(201); + harness.releaseGate(); + await vi.waitFor(async () => { + const status = await request(app).get(`${loginPath(COMPANY_1)}/${start.body.sessionId}`); + expect(status.body.status).toBe("failed"); + expect(status.body.failure?.reason).toBe("promotion_failed"); + }); + expect(mockSecretService.hasInstanceAccountHomeClaim).toHaveBeenCalledWith(COMPANY_1, accountHomeDir); + await expect(lstat(accountHomeDir)).resolves.toBeDefined(); + }); + it("does not remove an account home a concurrent login already claimed", async () => { // This call created the account home directory (`accountHomeCreated: // true`), but a concurrent login for the same account can still create diff --git a/server/src/__tests__/agent-instance-secrets.test.ts b/server/src/__tests__/agent-instance-secrets.test.ts new file mode 100644 index 00000000000..7cd3af24911 --- /dev/null +++ b/server/src/__tests__/agent-instance-secrets.test.ts @@ -0,0 +1,240 @@ +import { randomBytes, randomUUID } from "node:crypto"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { eq } from "drizzle-orm"; +import { agents, companies, companySecrets, companySecretVersions, activityLog, plugins, createDb, type Db } from "@paperclipai/db"; +import type { AgentLifecycleRequest } from "@paperclipai/shared"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { agentInstanceSecretService } from "../services/agent-instance-secrets.js"; +import { secretService } from "../services/secrets.js"; +import { localEncryptedProvider } from "../secrets/local-encrypted-provider.js"; +import { createLifecycleDriver } from "../services/agent-lifecycle-driver.js"; +import { createLifecycleStore } from "../modules/agent-lifecycle/adapters/postgres.js"; +import { logger } from "../middleware/logger.js"; +import { resolveCodexAuthCacheDir } from "@paperclipai/adapter-codex-local/server"; +import path from "node:path"; +import { mkdir, rm } from "node:fs/promises"; +import { awsSecretsManagerProvider } from "../secrets/aws-secrets-manager-provider.js"; + +const support = await getEmbeddedPostgresTestSupport(); +(support.supported ? describe : describe.skip)("agent instance secrets use the tenant secret system", () => { + let database: Awaited>; + let db: Db; + let companyId: string; + let pluginId: string; + const runtime = JSON.stringify({ url: "https://runtime.example.test", token: "e".repeat(64) }); + beforeAll(async () => { + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", randomBytes(32).toString("base64")); + database = await startEmbeddedPostgresTestDatabase("paperclip-instance-secrets-"); + db = createDb(database.connectionString); + companyId = randomUUID(); pluginId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Secret fixture", issuePrefix: "SCR" }); + await db.insert(plugins).values({ id: pluginId, pluginKey: pluginId, packageName: "fixture-lifecycle", version: "1.0.0", status: "ready", + manifestJson: { agentLifecycle: true, capabilities: ["agents.lifecycle.manage"] } as never }); + }, 90_000); + afterAll(async () => { await database?.cleanup(); vi.unstubAllEnvs(); }); + async function fixture(ownerPluginId = pluginId) { + const operationId = randomUUID(); + const owner = randomUUID(); + const now = new Date(); + const [agent] = await db.insert(agents).values({ companyId, name: "Secret owner", lifecycleState: "preparing", lifecycleVersion: 1, + lifecycleRequiredPluginIds: [ownerPluginId], lifecycleOperation: { id: operationId, hostComplete: true, completedPluginIds: [], attempts: 0, + leaseOwner: owner, leaseUntil: new Date(now.getTime() + 120_000).toISOString() } }).returning(); + const request: AgentLifecycleRequest = { companyId, agentId: agent.id, operationId, version: 1, phase: "preparing" }; + const scope = { companyId, agentId: agent.id, pluginId: ownerPluginId }; + const native = agentInstanceSecretService(db); + const service = { read: native.read, apply: (input: AgentLifecycleRequest, id: string, changes: unknown, attemptOwner = owner) => native.apply(input, id, changes, attemptOwner) }; + return { agent, request, scope, service, owner }; + } + + it("persists and rotates native secret versions, survives restart, and isolates agent, company and plugin", async () => { + const f = await fixture(); + await f.service.apply(f.request, pluginId, { runtime }); + const [stored] = await db.select().from(companySecrets).where(eq(companySecrets.ownerAgentId, f.agent.id)); + expect(stored).toMatchObject({ scope: "instance", ownerPluginId: pluginId, provider: "local_encrypted", latestVersion: 1 }); + const versions = await db.select().from(companySecretVersions).where(eq(companySecretVersions.secretId, stored.id)); + expect(versions).toHaveLength(1); + expect(JSON.stringify(versions)).not.toContain("e".repeat(64)); + expect(JSON.stringify(versions)).not.toContain("runtime.example.test"); + expect(await agentInstanceSecretService(createDb(database.connectionString)).read(f.scope)).toEqual({ runtime }); + for (const scope of [{ ...f.scope, companyId: randomUUID() }, { ...f.scope, agentId: randomUUID() }, { ...f.scope, pluginId: randomUUID() }]) { + expect(await f.service.read(scope)).toEqual({}); + } + const replacement = runtime.replace("e".repeat(64), "f".repeat(64)); + await f.service.apply(f.request, pluginId, { runtime: replacement }); + expect(await f.service.read(f.scope)).toEqual({ runtime: replacement }); + expect((await secretService(db).getById(stored.id))!.latestVersion).toBe(2); + expect(JSON.stringify(await db.select().from(activityLog).where(eq(activityLog.entityId, f.agent.id)))).not.toContain(replacement); + }); + + it("excludes instance secrets from company catalogs, ordinary resolution, bindings and agent access", async () => { + const f = await fixture(); await f.service.apply(f.request, pluginId, { runtime }); + const [stored] = await db.select().from(companySecrets).where(eq(companySecrets.ownerAgentId, f.agent.id)); + const secrets = secretService(db); + expect((await secrets.list(companyId)).some(secret => secret.id === stored.id)).toBe(false); + expect(await secrets.getByName(companyId, "runtime")).toBeNull(); + expect(await secrets.getByKey(companyId, "runtime")).toBeNull(); + await expect(secrets.resolveSecretValue(companyId, stored.id, "latest")).rejects.toThrow("secret declarations"); + await expect(secrets.createBinding({ companyId, secretId: stored.id, targetType: "agent", targetId: f.agent.id, configPath: "env.RUNTIME" })).rejects.toThrow("company-scoped"); + }); + + it("fences stale writes and removals, and cascades credentials when the owning agent is deleted", async () => { + const f = await fixture(); await f.service.apply(f.request, pluginId, { runtime }); + for (const patch of [{ version: 2 }, { operationId: randomUUID() }, { phase: "cleaning_up" }, { companyId: randomUUID() }, { agentId: randomUUID() }]) { + await expect(f.service.apply({ ...f.request, ...patch } as AgentLifecycleRequest, pluginId, { runtime: null })).rejects.toThrow("stale or outside scope"); + } + await expect(f.service.apply(f.request, randomUUID(), { runtime })).rejects.toThrow("stale or outside scope"); + await db.update(agents).set({ lifecycleState: "cleaning_up", lifecycleVersion: 2 }).where(eq(agents.id, f.agent.id)); + await expect(f.service.apply(f.request, pluginId, { runtime })).rejects.toThrow("stale or outside scope"); + const cleanup = { ...f.request, version: 2, phase: "cleaning_up" as const }; + await expect(f.service.apply(cleanup, pluginId, { runtime })).rejects.toThrow("preparation or resume"); + await f.service.apply(cleanup, pluginId, { runtime: null }); + expect(await f.service.read(f.scope)).toEqual({}); + await db.delete(agents).where(eq(agents.id, f.agent.id)); + expect(await db.select().from(companySecrets).where(eq(companySecrets.ownerAgentId, f.agent.id))).toEqual([]); + }); + + it("fails without acknowledging storage errors and validates changes before writes", async () => { + const f = await fixture(); + const spy = vi.spyOn(localEncryptedProvider, "createSecret").mockRejectedValueOnce(new Error(runtime)); + const warnings = vi.spyOn(logger, "warn").mockImplementation(() => undefined); + await expect(f.service.apply(f.request, pluginId, { runtime })).rejects.toThrow("storage unavailable"); + expect(JSON.stringify(warnings.mock.calls)).not.toContain("e".repeat(64)); + warnings.mockRestore(); + spy.mockRestore(); + expect(await f.service.read(f.scope)).toEqual({}); + for (const changes of [null, [], { invalid: 1 }, { runtime: "" }, { "bad/key": runtime }, { runtime: "x".repeat(65_537) }]) { + await expect(f.service.apply(f.request, pluginId, changes)).rejects.toThrow("Invalid agent instance secret changes"); + } + }); + + it("saves the existing local lifecycle reply before completion and supplies secrets on the next invocation", async () => { + const f = await fixture(); + const call = vi.fn(async (_id, _method, input) => ({ operationId: input.operationId, version: input.version, status: "complete", instanceSecrets: { runtime } })); + const driver = createLifecycleDriver(db, { call } as never); + const store = createLifecycleStore(db, {} as never); + const snapshot = (await store.get(f.agent.id))!; + expect(await driver.runPlugin(snapshot, pluginId)).toBe("complete"); + expect(await f.service.read(f.scope)).toEqual({ runtime }); + expect(call.mock.calls[0][2].instanceSecrets).toEqual({}); + await driver.runPlugin(snapshot, pluginId); + expect(call.mock.calls[1][2].instanceSecrets).toEqual({ runtime }); + call.mockImplementationOnce(async (_id, _method, input) => { + await db.update(agents).set({ lifecycleVersion: 2, lifecycleState: "terminating" }).where(eq(agents.id, f.agent.id)); + return { operationId: input.operationId, version: input.version, status: "complete", instanceSecrets: { runtime: "stale credentials" } }; + }); + await expect(driver.runPlugin(snapshot, pluginId)).rejects.toThrow("stale or outside scope"); + expect(await f.service.read(f.scope)).toEqual({ runtime }); + }); + + it("preserves the native guard against missing account-home directories", async () => { + const f = await fixture(); + const missing = path.join(resolveCodexAuthCacheDir(undefined, companyId), "missing-instance-secret-fixture"); + await expect(f.service.apply(f.request, pluginId, { runtime: missing })).rejects.toThrow("storage unavailable"); + expect(await f.service.read(f.scope)).toEqual({}); + }); + + it("rejects a previous attempt after lease takeover and rejects an expired owner's writes", async () => { + const f = await fixture(); await f.service.apply(f.request, pluginId, { runtime }); + const snapshot = (await createLifecycleStore(db, {} as never).get(f.agent.id))!; + const successor = randomUUID(); + await db.update(agents).set({ lifecycleOperation: { ...f.agent.lifecycleOperation!, leaseOwner: successor } }).where(eq(agents.id, f.agent.id)); + await f.service.apply(f.request, pluginId, { runtime: "successor credentials" }, successor); + await expect(f.service.apply(f.request, pluginId, { runtime })).rejects.toThrow("stale or outside scope"); + await expect(f.service.apply(f.request, pluginId, { runtime: null })).rejects.toThrow("stale or outside scope"); + const call = vi.fn(async (_id, _method, input) => ({ operationId: input.operationId, version: input.version, status: "complete", instanceSecrets: { runtime } })); + await expect(createLifecycleDriver(db, { call } as never).runPlugin(snapshot, pluginId)).rejects.toThrow("stale or outside scope"); + expect(await f.service.read(f.scope)).toEqual({ runtime: "successor credentials" }); + await db.update(agents).set({ lifecycleOperation: { ...f.agent.lifecycleOperation!, leaseOwner: successor, leaseUntil: new Date(0).toISOString() } }).where(eq(agents.id, f.agent.id)); + await expect(f.service.apply(f.request, pluginId, { runtime }, successor)).rejects.toThrow("stale or outside scope"); + expect(await f.service.read(f.scope)).toEqual({ runtime: "successor credentials" }); + }); + + it("deletes live secret metadata and versions with the owning agent or plugin", async () => { + for (const deleting of ["agent", "plugin"] as const) { + const ownerPluginId = randomUUID(); + await db.insert(plugins).values({ id: ownerPluginId, pluginKey: ownerPluginId, packageName: "fixture-cascade", version: "1.0.0", status: "ready", + manifestJson: { agentLifecycle: true, capabilities: ["agents.lifecycle.manage"] } as never }); + const f = await fixture(ownerPluginId); + await f.service.apply(f.request, ownerPluginId, { runtime }); + const [stored] = await db.select().from(companySecrets).where(eq(companySecrets.ownerAgentId, f.agent.id)); + expect(await db.select().from(companySecretVersions).where(eq(companySecretVersions.secretId, stored.id))).toHaveLength(1); + if (deleting === "agent") await db.delete(agents).where(eq(agents.id, f.agent.id)); + else await db.delete(plugins).where(eq(plugins.id, ownerPluginId)); + expect(await secretService(db).getById(stored.id)).toBeNull(); + expect(await db.select().from(companySecretVersions).where(eq(companySecretVersions.secretId, stored.id))).toEqual([]); + } + }); + + it("can clean up after a lost master key without decrypting credentials", async () => { + const f = await fixture(); await f.service.apply(f.request, pluginId, { runtime }); + const masterKey = process.env.PAPERCLIP_SECRETS_MASTER_KEY!; + try { + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", randomBytes(32).toString("base64")); + await expect(f.service.read(f.scope)).rejects.toThrow("Agent instance secret unavailable"); + await db.update(agents).set({ lifecycleState: "cleaning_up", lifecycleVersion: 2 }).where(eq(agents.id, f.agent.id)); + const call = vi.fn(async (_id, _method, input) => ({ operationId: input.operationId, version: input.version, status: "complete", instanceSecrets: { runtime: null } })); + const driver = createLifecycleDriver(db, { call } as never); + const snapshot = (await createLifecycleStore(db, {} as never).get(f.agent.id))!; + expect(await driver.runPlugin(snapshot, pluginId)).toBe("complete"); + expect(call.mock.calls[0][2].instanceSecrets).toEqual({}); + expect(await f.service.read(f.scope)).toEqual({}); + } finally { vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", masterKey); } + }); + + it("allows company vault imports with the same name and key as an instance secret", async () => { + const f = await fixture(); + await f.service.apply(f.request, pluginId, { runtime }); + const secrets = secretService(db); + const vault = await secrets.createProviderConfig(companyId, { + provider: "aws_secrets_manager", displayName: "Import fixture", config: { region: "us-east-1" }, + }); + const externalRef = "arn:aws:secretsmanager:us-east-1:123456789012:secret:runtime"; + const list = vi.spyOn(awsSecretsManagerProvider, "listRemoteSecrets").mockResolvedValue({ + secrets: [{ externalRef, name: "runtime", providerVersionRef: null, metadata: {} }], + }); + try { + const preview = await secrets.previewRemoteImport(companyId, { providerConfigId: vault.id }); + expect(preview.candidates[0]).toMatchObject({ status: "ready", importable: true, conflicts: [] }); + const result = await secrets.importRemoteSecrets(companyId, { + providerConfigId: vault.id, secrets: [{ externalRef, name: "runtime", key: "runtime" }], + }); + expect(result).toMatchObject({ importedCount: 1, skippedCount: 0, errorCount: 0 }); + expect(await f.service.read(f.scope)).toEqual({ runtime }); + const imported = await secrets.getByName(companyId, "runtime"); + expect(imported).toMatchObject({ scope: "company", externalRef }); + await secrets.remove(imported!.id); + } finally { list.mockRestore(); } + }); + + it("protects instance-owned account homes without exposing their values in the company catalog", async () => { + const f = await fixture(); + const accountHome = path.join(resolveCodexAuthCacheDir(undefined, companyId), randomUUID()); + await mkdir(accountHome, { recursive: true }); + const secrets = secretService(db); + try { + await f.service.apply(f.request, pluginId, { account_home: accountHome }); + expect(await secrets.hasInstanceAccountHomeClaim(companyId, accountHome)).toBe(true); + expect(await secrets.hasInstanceAccountHomeClaim(randomUUID(), accountHome)).toBe(false); + expect(await secrets.hasInstanceAccountHomeClaim(companyId, accountHome + "-other")).toBe(false); + expect((await secrets.list(companyId)).some(secret => secret.ownerAgentId === f.agent.id)).toBe(false); + const resolve = vi.spyOn(localEncryptedProvider, "resolveVersion").mockRejectedValueOnce(new Error("unavailable")); + try { expect(await secrets.hasInstanceAccountHomeClaim(companyId, accountHome)).toBe(true); } + finally { resolve.mockRestore(); } + await f.service.apply(f.request, pluginId, { account_home: null }); + expect(await secrets.hasInstanceAccountHomeClaim(companyId, accountHome)).toBe(false); + } finally { await rm(accountHome, { recursive: true, force: true }); } + }); + + it("keeps same-named company and other-agent secrets independent", async () => { + const f = await fixture(); const other = await fixture(); + const companySecret = await secretService(db).create(companyId, { name: "runtime", provider: "local_encrypted", value: "ordinary company secret" }); + await f.service.apply(f.request, pluginId, { runtime }); + await other.service.apply(other.request, pluginId, { runtime: "other agent connection" }); + expect(await f.service.read(f.scope)).toEqual({ runtime }); + expect(await other.service.read(other.scope)).toEqual({ runtime: "other agent connection" }); + expect(await secretService(db).resolveSecretValue(companyId, companySecret.id, "latest")).toBe("ordinary company secret"); + await f.service.apply(f.request, pluginId, { runtime: null }); + expect(await other.service.read(other.scope)).toEqual({ runtime: "other agent connection" }); + expect(await secretService(db).getByName(companyId, "runtime")).toMatchObject({ id: companySecret.id }); + }); +}); diff --git a/server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts b/server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts index aaf1198d1c7..b40d4897a76 100644 --- a/server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts +++ b/server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts @@ -481,7 +481,7 @@ const support = await getEmbeddedPostgresTestSupport(); await db.update(pluginCompanySettings).set({ enabled: true }).where(eq(pluginCompanySettings.pluginId, pluginId)); expect(await driver.runPlugin(snapshot, pluginId)).toBe("complete"); expect(call).toHaveBeenCalledWith(pluginId, "agentLifecycle", { companyId, agentId: agent.id, - operationId: snapshot.lifecycleOperation!.id, version: snapshot.lifecycleVersion, phase: "paused" }, 30_000); + operationId: snapshot.lifecycleOperation!.id, version: snapshot.lifecycleVersion, phase: "paused", instanceSecrets: {} }, 30_000); call.mockResolvedValueOnce({ operationId: "stale", version: snapshot.lifecycleVersion, status: "complete" }); await expect(driver.runPlugin(snapshot, pluginId)).rejects.toThrow("Invalid lifecycle result"); }); diff --git a/server/src/routes/agents.ts b/server/src/routes/agents.ts index e481c661e27..5eb5eece028 100644 --- a/server/src/routes/agents.ts +++ b/server/src/routes/agents.ts @@ -420,6 +420,7 @@ const ACCOUNT_HOME_CLAIM_SCAN_MAX_PASSES = 5; async function anySecretNamesAccountHome( secretsSvc: { + hasInstanceAccountHomeClaim: (companyId: string, accountHomeDir: string) => Promise; list: (companyId: string) => Promise>; resolveSecretValueForDeviceLoginCheck: ( companyId: string, @@ -430,6 +431,7 @@ async function anySecretNamesAccountHome( companyId: string, accountHomeDir: string, ): Promise { + if (await secretsSvc.hasInstanceAccountHomeClaim(companyId, accountHomeDir)) return true; const checkedSecretIds = new Set(); for (let pass = 0; pass < ACCOUNT_HOME_CLAIM_SCAN_MAX_PASSES; pass += 1) { const secrets = await secretsSvc.list(companyId); diff --git a/server/src/services/agent-instance-secrets.ts b/server/src/services/agent-instance-secrets.ts new file mode 100644 index 00000000000..73edbffcf93 --- /dev/null +++ b/server/src/services/agent-instance-secrets.ts @@ -0,0 +1,41 @@ +import { and, eq } from "drizzle-orm"; +import { agents, type Db } from "@paperclipai/db"; +import type { AgentLifecycleRequest } from "@paperclipai/shared"; +import { secretService } from "./secrets.js"; +import { persistActivity } from "./activity-log.js"; +import { withAccountHomeSecretMutationLock } from "@paperclipai/adapter-codex-local/server"; + +type Scope = { companyId: string; agentId: string; pluginId: string }; + +export function agentInstanceSecretService(db: Db) { + return { + async read(scope: Scope): Promise> { + try { return await secretService(db).agentInstance.read(scope); } + catch { throw new Error("Agent instance secret unavailable"); } + }, + async apply(request: AgentLifecycleRequest, pluginId: string, changes: unknown, attemptOwner: string | undefined): Promise { + if (!changes || typeof changes !== "object" || Array.isArray(changes)) throw new Error("Invalid agent instance secret changes"); + const entries = Object.entries(changes); + if (entries.length > 16 || entries.some(([key, value]) => !/^[a-z][a-z0-9_]{0,63}$/.test(key) || + (value !== null && (typeof value !== "string" || !value.trim() || Buffer.byteLength(value) > 65_536)))) throw new Error("Invalid agent instance secret changes"); + const scope = { companyId: request.companyId, agentId: request.agentId, pluginId }; + await withAccountHomeSecretMutationLock(undefined, request.companyId, () => db.transaction(async tx => { + const [agent] = await tx.select().from(agents).where(and(eq(agents.companyId, request.companyId), eq(agents.id, request.agentId))).for("update"); + const leaseUntil = Date.parse(agent?.lifecycleOperation?.leaseUntil ?? ""); + if (!agent || agent.lifecycleVersion !== request.version || agent.lifecycleState !== request.phase || + agent.lifecycleOperation?.id !== request.operationId || !agent.lifecycleRequiredPluginIds?.includes(pluginId) || + !attemptOwner || agent.lifecycleOperation.leaseOwner !== attemptOwner || !Number.isFinite(leaseUntil) || leaseUntil <= Date.now()) { + throw new Error("Agent instance secret operation is stale or outside scope"); + } + if (entries.some(([, value]) => value !== null) && !["preparing", "resuming"].includes(request.phase)) throw new Error("Agent instance secrets require preparation or resume"); + for (const [key, value] of entries) { + try { await secretService(tx).agentInstance.set(scope, key, value as string | null); } + catch { throw new Error("Agent instance secret storage unavailable"); } + await persistActivity(tx as unknown as Db, { companyId: request.companyId, actorType: "plugin", actorId: pluginId, + action: value === null ? "agent.instance_secret.removed" : "agent.instance_secret.saved", entityType: "agent", entityId: request.agentId, + details: { key } }); + } + })); + }, + }; +} diff --git a/server/src/services/agent-lifecycle-driver.ts b/server/src/services/agent-lifecycle-driver.ts index 11963d997fa..f10c562bf4c 100644 --- a/server/src/services/agent-lifecycle-driver.ts +++ b/server/src/services/agent-lifecycle-driver.ts @@ -5,6 +5,7 @@ import { listInvalidOrgChainDescendantIds } from "./agent-invokability.js"; import type { PluginWorkerManager } from "./plugin-worker-manager.js"; import type { LifecycleDriver } from "../modules/agent-lifecycle/index.js"; import { agentHarnessVerificationService } from "./agent-harness-verification.js"; +import { agentInstanceSecretService } from "./agent-instance-secrets.js"; export function createLifecycleDriver(db: Db, manager: PluginWorkerManager): LifecycleDriver { const { verify } = agentHarnessVerificationService(db, manager); @@ -29,11 +30,16 @@ export function createLifecycleDriver(db: Db, manager: PluginWorkerManager): Lif throw Object.assign(new Error("A required lifecycle plugin is unavailable"), { code: "required_plugin_unavailable" }); } const operationId = agent.lifecycleOperation!.id; - const result = await manager.call(plugin.id, "agentLifecycle", { companyId: agent.companyId, - agentId: agent.id, operationId, version: agent.lifecycleVersion, phase: agent.lifecycleState }, 30_000); + const secrets = agentInstanceSecretService(db); + const instanceSecrets = ["preparing", "resuming"].includes(agent.lifecycleState) + ? await secrets.read({ companyId: agent.companyId, agentId: agent.id, pluginId }) : {}; + const request = { companyId: agent.companyId, agentId: agent.id, operationId, version: agent.lifecycleVersion, phase: agent.lifecycleState, + instanceSecrets }; + const result = await manager.call(plugin.id, "agentLifecycle", request, 30_000); if (!result || result.operationId !== operationId || result.version !== agent.lifecycleVersion || !["complete", "pending"].includes(result.status)) { throw Object.assign(new Error("Invalid lifecycle result"), { code: "invalid_lifecycle_result" }); } + if (result.instanceSecrets !== undefined) await secrets.apply(request, pluginId, result.instanceSecrets, agent.lifecycleOperation?.leaseOwner); return result.status; }, async runHost(agent) { diff --git a/server/src/services/secrets.ts b/server/src/services/secrets.ts index 62338a9dc4d..75164ed3102 100644 --- a/server/src/services/secrets.ts +++ b/server/src/services/secrets.ts @@ -338,7 +338,6 @@ function remoteProviderHttpError(error: unknown, context: { if (isSecretProviderClientError(error)) { logger.warn( { - err: error, companyId: context.companyId, provider: context.provider, providerConfigId: context.providerConfigId, @@ -352,7 +351,6 @@ function remoteProviderHttpError(error: unknown, context: { if (error instanceof HttpError) return error; logger.warn( { - err: error, companyId: context.companyId, provider: context.provider, providerConfigId: context.providerConfigId, @@ -649,6 +647,7 @@ type SecretResolutionOptions = { bindingContext?: SecretBindingContext; accessContext?: SecretConsumerContext; allowUserSecretScope?: boolean; + instanceOwner?: { agentId: string; pluginId: string }; }; export type AgentSecretReadContext = { @@ -1361,7 +1360,9 @@ export function secretService(db: Db | DbTransaction) { const secret = await getById(secretId); if (!secret) throw notFound("Secret not found"); if (secret.companyId !== companyId) throw unprocessable("Secret must belong to same company"); - if (secret.scope !== "company" && !options?.allowUserSecretScope) { + const instanceOwned = secret.scope === "instance" && options?.instanceOwner && + secret.ownerAgentId === options.instanceOwner.agentId && secret.ownerPluginId === options.instanceOwner.pluginId; + if (secret.scope !== "company" && !(secret.scope === "user" && options?.allowUserSecretScope) && !instanceOwned) { throw unprocessable("User-scoped secrets must be resolved through user secret declarations", { code: "secret_scope_invalid", }); @@ -2186,12 +2187,13 @@ export function secretService(db: Db | DbTransaction) { providerMetadata?: Record | null; }, actor?: { userId?: string | null; agentId?: string | null }, + instanceOwner?: { agentId: string; pluginId: string }, ) { - const existing = await getByName(companyId, input.name); + const existing = instanceOwner ? null : await getByName(companyId, input.name); if (existing) throw conflict(`Secret already exists: ${input.name}`); const key = normalizeSecretKey(input.key ?? input.name); if (!key) throw unprocessable("Secret key is required"); - const duplicateKey = await db + const duplicateKey = instanceOwner ? null : await db .select() .from(companySecrets) .where(and( @@ -2229,6 +2231,9 @@ export function secretService(db: Db | DbTransaction) { .insert(companySecrets) .values({ companyId, + scope: instanceOwner ? "instance" : "company", + ownerAgentId: instanceOwner?.agentId ?? null, + ownerPluginId: instanceOwner?.pluginId ?? null, key, name: input.name, provider: input.provider, @@ -2890,7 +2895,7 @@ export function secretService(db: Db | DbTransaction) { status: companySecrets.status, }) .from(companySecrets) - .where(and(eq(companySecrets.companyId, companyId), ne(companySecrets.status, "deleted"))); + .where(and(eq(companySecrets.companyId, companyId), eq(companySecrets.scope, "company"), ne(companySecrets.status, "deleted"))); return { byProviderConfigExternalRef: new Map( activeSecrets @@ -3468,6 +3473,56 @@ export function secretService(db: Db | DbTransaction) { return { listProviders: () => listSecretProviders(), + async hasInstanceAccountHomeClaim(companyId: string, accountHomeDir: string): Promise { + try { + const rows = await db.select().from(companySecrets).where(and(eq(companySecrets.companyId, companyId), + eq(companySecrets.scope, "instance"), ne(companySecrets.status, "deleted"))); + for (const row of rows) { + if (!row.ownerAgentId || !row.ownerPluginId) return true; + const resolved = await resolveSecretValueInternal(companyId, row.id, "latest", { + instanceOwner: { agentId: row.ownerAgentId, pluginId: row.ownerPluginId }, + accessContext: { consumerType: "system", consumerId: DEVICE_LOGIN_SECRET_CHECK_CONSUMER_ID, + actorType: "system", actorId: null }, + }); + if (resolved.value === accountHomeDir) return true; + } + return false; + } catch { return true; } + }, + + agentInstance: { + async read(scope: { companyId: string; agentId: string; pluginId: string }): Promise> { + const rows = await db.select().from(companySecrets).where(and(eq(companySecrets.companyId, scope.companyId), + eq(companySecrets.scope, "instance"), eq(companySecrets.ownerAgentId, scope.agentId), + eq(companySecrets.ownerPluginId, scope.pluginId), ne(companySecrets.status, "deleted"))); + const values: Record = Object.create(null); + for (const row of rows) { + const resolved = await resolveSecretValueInternal(scope.companyId, row.id, "latest", { + instanceOwner: scope, accessContext: { consumerType: "plugin_worker", consumerId: scope.pluginId, + actorType: "plugin", actorId: scope.pluginId, pluginId: scope.pluginId }, + }); + values[row.name] = resolved.value; + } + return values; + }, + async set(scope: { companyId: string; agentId: string; pluginId: string }, key: string, value: string | null) { + return withAccountHomeSecretMutationLock(undefined, scope.companyId, async () => { + if (value) await assertAccountHomeCacheDirStillValid(undefined, scope.companyId, value); + const normalizedKey = normalizeSecretKey(key); + const [existing] = await db.select().from(companySecrets).where(and(eq(companySecrets.companyId, scope.companyId), + eq(companySecrets.scope, "instance"), eq(companySecrets.ownerAgentId, scope.agentId), + eq(companySecrets.ownerPluginId, scope.pluginId), eq(companySecrets.key, normalizedKey), ne(companySecrets.status, "deleted"))); + if (value === null) { + if (existing) await removeSecretUnlocked(existing.id); + } else if (existing) { + await rotateUnlocked(existing.id, { value, expectedLatestVersion: existing.latestVersion }); + } else { + await createSecretUnlocked(scope.companyId, { name: key, key, provider: "local_encrypted", value }, undefined, scope); + } + }); + }, + }, + checkProviders: () => checkSecretProviders(), previewProviderConfigDiscovery: async (