diff --git a/packages/db/src/client.ts b/packages/db/src/client.ts index eaea1067bb6..7a362b0ae71 100644 --- a/packages/db/src/client.ts +++ b/packages/db/src/client.ts @@ -691,6 +691,7 @@ const MIGRATIONS_WITH_TOP_LEVEL_STATEMENTS = new Set([ "0470_heartbeat_runs_queued_created_index.sql", "0472_runner_api_capture_pending_deletes.sql", "0476_wakeup_request_run_index.sql", + "0486_heartbeat_runs_company_finished_index.sql", "0469_zippy_photon.sql", diff --git a/packages/db/src/migrations/0485_moaning_ultimatum.sql b/packages/db/src/migrations/0485_moaning_ultimatum.sql new file mode 100644 index 00000000000..abdde7a9b72 --- /dev/null +++ b/packages/db/src/migrations/0485_moaning_ultimatum.sql @@ -0,0 +1,16 @@ +CREATE TABLE "automatic_run_brakes" ( + "company_id" uuid PRIMARY KEY NOT NULL, + "state" text DEFAULT 'closed' NOT NULL, + "reason" jsonb DEFAULT '{}'::jsonb NOT NULL, + "state_changed_at" timestamp with time zone DEFAULT now() NOT NULL, + "opened_at" timestamp with time zone, + "next_probe_at" timestamp with time zone, + "probe_run_id" uuid, + "probe_successes" integer DEFAULT 0 NOT NULL, + "release_window_started_at" timestamp with time zone, + "release_window_starts" integer DEFAULT 0 NOT NULL, + "next_creation_at" timestamp with time zone, + "updated_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +ALTER TABLE "automatic_run_brakes" ADD CONSTRAINT "automatic_run_brakes_company_id_companies_id_fk" FOREIGN KEY ("company_id") REFERENCES "public"."companies"("id") ON DELETE cascade ON UPDATE no action; \ No newline at end of file diff --git a/packages/db/src/migrations/0486_heartbeat_runs_company_finished_index.sql b/packages/db/src/migrations/0486_heartbeat_runs_company_finished_index.sql new file mode 100644 index 00000000000..9d1995ff234 --- /dev/null +++ b/packages/db/src/migrations/0486_heartbeat_runs_company_finished_index.sql @@ -0,0 +1 @@ +CREATE INDEX CONCURRENTLY IF NOT EXISTS "heartbeat_runs_company_finished_idx" ON "heartbeat_runs" USING btree ("company_id","finished_at" DESC NULLS LAST) WHERE "heartbeat_runs"."finished_at" is not null; \ No newline at end of file diff --git a/packages/db/src/migrations/meta/0481_snapshot.json b/packages/db/src/migrations/meta/0485_snapshot.json similarity index 99% rename from packages/db/src/migrations/meta/0481_snapshot.json rename to packages/db/src/migrations/meta/0485_snapshot.json index c4ad44771b7..fcf5987ed8c 100644 --- a/packages/db/src/migrations/meta/0481_snapshot.json +++ b/packages/db/src/migrations/meta/0485_snapshot.json @@ -1,6 +1,6 @@ { - "id": "41eacc6a-6eab-4f01-965a-62aa0cd6778f", - "prevId": "55a8c37c-453c-4be4-b657-b703932836a0", + "id": "78d10bc7-b440-4c00-a02e-90111c50071d", + "prevId": "9389124c-379a-4b64-9c20-c7b37538e7c8", "version": "7", "dialect": "postgresql", "tables": { @@ -6582,6 +6582,111 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.automatic_run_brakes": { + "name": "automatic_run_brakes", + "schema": "", + "columns": { + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'closed'" + }, + "reason": { + "name": "reason", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "state_changed_at": { + "name": "state_changed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "opened_at": { + "name": "opened_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "next_probe_at": { + "name": "next_probe_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "probe_run_id": { + "name": "probe_run_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "probe_successes": { + "name": "probe_successes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "release_window_started_at": { + "name": "release_window_started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "release_window_starts": { + "name": "release_window_starts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "next_creation_at": { + "name": "next_creation_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "automatic_run_brakes_company_id_companies_id_fk": { + "name": "automatic_run_brakes_company_id_companies_id_fk", + "tableFrom": "automatic_run_brakes", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.board_api_keys": { "name": "board_api_keys", "schema": "", @@ -18092,6 +18197,12 @@ "notNull": true, "default": "'company'" }, + "system_purpose": { + "name": "system_purpose", + "type": "text", + "primaryKey": false, + "notNull": false + }, "owner_user_id": { "name": "owner_user_id", "type": "text", @@ -59601,6 +59712,12 @@ "primaryKey": false, "notNull": false }, + "progress": { + "name": "progress", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, "created_at": { "name": "created_at", "type": "timestamp with time zone", @@ -60107,6 +60224,277 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.tool_mcp_elicitations": { + "name": "tool_mcp_elicitations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "issue_id": { + "name": "issue_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "invocation_id": { + "name": "invocation_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "connection_id": { + "name": "connection_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "grant_id": { + "name": "grant_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "catalog_tool_name": { + "name": "catalog_tool_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "call_identity": { + "name": "call_identity", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "era": { + "name": "era", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sealed_request_state": { + "name": "sealed_request_state", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "questions": { + "name": "questions", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "arguments_identity": { + "name": "arguments_identity", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "dedupe_key": { + "name": "dedupe_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "claimed_until": { + "name": "claimed_until", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "sent_at": { + "name": "sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "tool_mcp_elicitations_call_idx": { + "name": "tool_mcp_elicitations_call_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "connection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "call_identity", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "tool_mcp_elicitations_invocation_idx": { + "name": "tool_mcp_elicitations_invocation_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "invocation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "tool_mcp_elicitations_waiting_uq": { + "name": "tool_mcp_elicitations_waiting_uq", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "dedupe_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"tool_mcp_elicitations\".\"status\" = 'waiting'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "tool_mcp_elicitations_company_id_companies_id_fk": { + "name": "tool_mcp_elicitations_company_id_companies_id_fk", + "tableFrom": "tool_mcp_elicitations", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "tool_mcp_elicitations_invocation_id_tool_invocations_id_fk": { + "name": "tool_mcp_elicitations_invocation_id_tool_invocations_id_fk", + "tableFrom": "tool_mcp_elicitations", + "tableTo": "tool_invocations", + "columnsFrom": [ + "invocation_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "tool_mcp_elicitations_connection_id_tool_connections_id_fk": { + "name": "tool_mcp_elicitations_connection_id_tool_connections_id_fk", + "tableFrom": "tool_mcp_elicitations", + "tableTo": "tool_connections", + "columnsFrom": [ + "connection_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.tool_mcp_gateway_tokens": { "name": "tool_mcp_gateway_tokens", "schema": "", diff --git a/packages/db/src/migrations/meta/0480_snapshot.json b/packages/db/src/migrations/meta/0486_snapshot.json similarity index 99% rename from packages/db/src/migrations/meta/0480_snapshot.json rename to packages/db/src/migrations/meta/0486_snapshot.json index b4ce9431fde..e0af95ce1b3 100644 --- a/packages/db/src/migrations/meta/0480_snapshot.json +++ b/packages/db/src/migrations/meta/0486_snapshot.json @@ -1,6 +1,6 @@ { - "id": "55a8c37c-453c-4be4-b657-b703932836a0", - "prevId": "f5f852bd-0568-4b2d-bc35-0dac5d9c8c5b", + "id": "0c3e74be-7936-4c97-9798-505daf53c16b", + "prevId": "78d10bc7-b440-4c00-a02e-90111c50071d", "version": "7", "dialect": "postgresql", "tables": { @@ -6582,6 +6582,111 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.automatic_run_brakes": { + "name": "automatic_run_brakes", + "schema": "", + "columns": { + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'closed'" + }, + "reason": { + "name": "reason", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "state_changed_at": { + "name": "state_changed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "opened_at": { + "name": "opened_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "next_probe_at": { + "name": "next_probe_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "probe_run_id": { + "name": "probe_run_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "probe_successes": { + "name": "probe_successes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "release_window_started_at": { + "name": "release_window_started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "release_window_starts": { + "name": "release_window_starts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "next_creation_at": { + "name": "next_creation_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "automatic_run_brakes_company_id_companies_id_fk": { + "name": "automatic_run_brakes_company_id_companies_id_fk", + "tableFrom": "automatic_run_brakes", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.board_api_keys": { "name": "board_api_keys", "schema": "", @@ -18092,6 +18197,12 @@ "notNull": true, "default": "'company'" }, + "system_purpose": { + "name": "system_purpose", + "type": "text", + "primaryKey": false, + "notNull": false + }, "owner_user_id": { "name": "owner_user_id", "type": "text", @@ -36023,6 +36134,28 @@ "method": "btree", "with": {} }, + "heartbeat_runs_company_finished_idx": { + "name": "heartbeat_runs_company_finished_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "finished_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"heartbeat_runs\".\"finished_at\" is not null", + "concurrently": true, + "method": "btree", + "with": {} + }, "heartbeat_runs_company_created_at_desc_idx": { "name": "heartbeat_runs_company_created_at_desc_idx", "columns": [ @@ -55989,6 +56122,18 @@ "primaryKey": false, "notNull": false }, + "credential_version": { + "name": "credential_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "credential_hash": { + "name": "credential_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, "created_by_user_id": { "name": "created_by_user_id", "type": "text", @@ -59589,6 +59734,12 @@ "primaryKey": false, "notNull": false }, + "progress": { + "name": "progress", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, "created_at": { "name": "created_at", "type": "timestamp with time zone", @@ -60095,6 +60246,277 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.tool_mcp_elicitations": { + "name": "tool_mcp_elicitations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "issue_id": { + "name": "issue_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "invocation_id": { + "name": "invocation_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "connection_id": { + "name": "connection_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "grant_id": { + "name": "grant_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "catalog_tool_name": { + "name": "catalog_tool_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "call_identity": { + "name": "call_identity", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "era": { + "name": "era", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sealed_request_state": { + "name": "sealed_request_state", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "questions": { + "name": "questions", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "arguments_identity": { + "name": "arguments_identity", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "dedupe_key": { + "name": "dedupe_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "claimed_until": { + "name": "claimed_until", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "sent_at": { + "name": "sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "tool_mcp_elicitations_call_idx": { + "name": "tool_mcp_elicitations_call_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "connection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "call_identity", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "tool_mcp_elicitations_invocation_idx": { + "name": "tool_mcp_elicitations_invocation_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "invocation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "tool_mcp_elicitations_waiting_uq": { + "name": "tool_mcp_elicitations_waiting_uq", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "dedupe_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"tool_mcp_elicitations\".\"status\" = 'waiting'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "tool_mcp_elicitations_company_id_companies_id_fk": { + "name": "tool_mcp_elicitations_company_id_companies_id_fk", + "tableFrom": "tool_mcp_elicitations", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "tool_mcp_elicitations_invocation_id_tool_invocations_id_fk": { + "name": "tool_mcp_elicitations_invocation_id_tool_invocations_id_fk", + "tableFrom": "tool_mcp_elicitations", + "tableTo": "tool_invocations", + "columnsFrom": [ + "invocation_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "tool_mcp_elicitations_connection_id_tool_connections_id_fk": { + "name": "tool_mcp_elicitations_connection_id_tool_connections_id_fk", + "tableFrom": "tool_mcp_elicitations", + "tableTo": "tool_connections", + "columnsFrom": [ + "connection_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.tool_mcp_gateway_tokens": { "name": "tool_mcp_gateway_tokens", "schema": "", diff --git a/packages/db/src/migrations/meta/_journal.json b/packages/db/src/migrations/meta/_journal.json index d9e6e48b3f5..f3774187bbb 100644 --- a/packages/db/src/migrations/meta/_journal.json +++ b/packages/db/src/migrations/meta/_journal.json @@ -3382,6 +3382,20 @@ "when": 1794621009918, "tag": "0484_certain_zemo", "breakpoints": true + }, + { + "idx": 485, + "version": "7", + "when": 1794623155354, + "tag": "0485_moaning_ultimatum", + "breakpoints": true + }, + { + "idx": 486, + "version": "7", + "when": 1794623177597, + "tag": "0486_heartbeat_runs_company_finished_index", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/db/src/schema/automatic_run_brakes.ts b/packages/db/src/schema/automatic_run_brakes.ts new file mode 100644 index 00000000000..d8e7ac4d49c --- /dev/null +++ b/packages/db/src/schema/automatic_run_brakes.ts @@ -0,0 +1,17 @@ +import { integer, jsonb, pgTable, text, timestamp, uuid } from "drizzle-orm/pg-core"; +import { companies } from "./companies.js"; + +export const automaticRunBrakes = pgTable("automatic_run_brakes", { + companyId: uuid("company_id").primaryKey().references(() => companies.id, { onDelete: "cascade" }), + state: text("state").$type<"closed" | "open" | "half_open">().notNull().default("closed"), + reason: jsonb("reason").$type>().notNull().default({}), + stateChangedAt: timestamp("state_changed_at", { withTimezone: true }).notNull().defaultNow(), + openedAt: timestamp("opened_at", { withTimezone: true }), + nextProbeAt: timestamp("next_probe_at", { withTimezone: true }), + probeRunId: uuid("probe_run_id"), + probeSuccesses: integer("probe_successes").notNull().default(0), + releaseWindowStartedAt: timestamp("release_window_started_at", { withTimezone: true }), + releaseWindowStarts: integer("release_window_starts").notNull().default(0), + nextCreationAt: timestamp("next_creation_at", { withTimezone: true }), + updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), +}); diff --git a/packages/db/src/schema/heartbeat_runs.ts b/packages/db/src/schema/heartbeat_runs.ts index b2979d52b30..0b424df776d 100644 --- a/packages/db/src/schema/heartbeat_runs.ts +++ b/packages/db/src/schema/heartbeat_runs.ts @@ -248,6 +248,8 @@ export const heartbeatRuns = pgTable( table.status, table.processStartedAt, ), + companyFinishedIdx: index("heartbeat_runs_company_finished_idx").on(table.companyId, table.finishedAt.desc()) + .where(sql`${table.finishedAt} is not null`).concurrently(), companyCreatedAtDescIdx: index("heartbeat_runs_company_created_at_desc_idx").on( table.companyId, table.createdAt.desc(), diff --git a/packages/db/src/schema/index.ts b/packages/db/src/schema/index.ts index 7eef891d0d3..fd8c162f7ee 100644 --- a/packages/db/src/schema/index.ts +++ b/packages/db/src/schema/index.ts @@ -26,6 +26,7 @@ export { budgetResumeClaims } from "./budget_resume_claims.js"; export { agentConfigRevisions } from "./agent_config_revisions.js"; export { agentApiKeys } from "./agent_api_keys.js"; export { agentRuntimeState } from "./agent_runtime_state.js"; +export { automaticRunBrakes } from "./automatic_run_brakes.js"; export { agentTaskSessions, agentSessionGoalActions } from "./agent_task_sessions.js"; export { agentWakeupRequests } from "./agent_wakeup_requests.js"; export { projects } from "./projects.js"; diff --git a/server/src/__tests__/assigned-work-stalled-queued-run.test.ts b/server/src/__tests__/assigned-work-stalled-queued-run.test.ts index 794809a50a9..f812d312305 100644 --- a/server/src/__tests__/assigned-work-stalled-queued-run.test.ts +++ b/server/src/__tests__/assigned-work-stalled-queued-run.test.ts @@ -1,7 +1,7 @@ import { randomUUID } from "node:crypto"; import { and, eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; -import { activityLog, agents, agentWakeupRequests, companies, createDb, heartbeatRuns, issues, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db"; +import { activityLog, agents, agentWakeupRequests, automaticRunBrakes, companies, createDb, heartbeatRuns, issues, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { createHeartbeatQueueTestDependencies } from "./helpers/heartbeat-queue.js"; import { createHeartbeatQueue, shouldDeferFollowupWakeForSameIssue, shouldQueueFollowupForRunningIssueWake } from "../services/heartbeat/queue.js"; @@ -13,6 +13,7 @@ import { CHAT_CONTROL_RECOVERY_ADMISSION_KEY } from "../services/chat-control-re import { ASSIGNED_WORK_CONTINUED_ACTION, assignedWorkSweep } from "../services/recovery/assigned-work-sweep.js"; import { CONTINUATION_CAP_WINDOW_MS } from "../services/recovery/assigned-work-invariant.js"; import { logger } from "../middleware/logger.js"; +import { automaticRunBrake, automaticRunCreationHeldUntil } from "../services/recovery/automatic-run-brake.js"; const support = await getEmbeddedPostgresTestSupport(); const MINUTE = 60_000; @@ -25,7 +26,7 @@ describe.skipIf(!support.supported)("a queued run the dispatcher keeps passing o db = createDb(database.connectionString); }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); afterAll(async () => { await db?.$client.end(); await database?.cleanup(); }); - afterEach(() => { vi.restoreAllMocks(); }); + afterEach(() => { vi.restoreAllMocks(); automaticRunBrake.reset(); }); async function seed() { const prefix = `S${randomUUID().slice(0, 5)}`; @@ -112,6 +113,32 @@ describe.skipIf(!support.supported)("a queued run the dispatcher keeps passing o return runs.filter(run => run.id !== stalledId); } + it("a replacement spends the paused company's creation slot", async () => { + const f = await seed(); + const stalled = await f.queuedRun(); + await f.otherStarts(OLDEST_DISPATCH_SHARE + 1, QUEUED_RUN_AGING_MS / 2); + await db.insert(automaticRunBrakes).values({ companyId: f.companyId, state: "open", openedAt: new Date(), stateChangedAt: new Date() }); + expect(await automaticRunCreationHeldUntil(db, f.companyId, { issueId: f.issue.id, sourceRunId: stalled.id })).toBeNull(); + + const check = await f.sweep().checkIssue({ companyId: f.companyId, issueId: f.issue.id }); + expect(check).toMatchObject({ outcome: "replaced_stalled_run" }); + const [brake] = await db.select().from(automaticRunBrakes).where(eq(automaticRunBrakes.companyId, f.companyId)); + expect(brake!.nextCreationAt!.getTime()).toBeGreaterThan(Date.now()); + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); + + it("an old finished request from a person does not exempt a replacement from the pause", async () => { + const f = await seed(); + await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agentId, source: "on_demand", triggerDetail: "manual", + reason: "task_retry", payload: { issueId: f.issue.id }, status: "completed", requestedByActorType: "user", requestedByActorId: "person-1", + requestedAt: new Date(Date.now() - 4 * 60 * MINUTE), finishedAt: new Date(Date.now() - 4 * 60 * MINUTE) }); + const stalled = await f.queuedRun(); + await f.otherStarts(OLDEST_DISPATCH_SHARE + 1, QUEUED_RUN_AGING_MS / 2); + await db.insert(automaticRunBrakes).values({ companyId: f.companyId, state: "open", openedAt: new Date(), stateChangedAt: new Date() }); + expect(await f.sweep().checkIssue({ companyId: f.companyId, issueId: f.issue.id })).toMatchObject({ outcome: "replaced_stalled_run" }); + const [brake] = await db.select().from(automaticRunBrakes).where(eq(automaticRunBrakes.companyId, f.companyId)); + expect(brake!.nextCreationAt!.getTime()).toBeGreaterThan(Date.now()); + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); + it("cancels an automatic run the claim keeps refusing and queues exactly one claimable replacement", async () => { const f = await seed(); const stalled = await f.queuedRun(); diff --git a/server/src/__tests__/automatic-run-brake-import.test.ts b/server/src/__tests__/automatic-run-brake-import.test.ts new file mode 100644 index 00000000000..a46adb46a18 --- /dev/null +++ b/server/src/__tests__/automatic-run-brake-import.test.ts @@ -0,0 +1,11 @@ +import { describe, expect, it } from "vitest"; + +describe("automatic-run brake module graph", () => { + it("loads with the server's heartbeat and recovery modules in the order the server imports them", async () => { + await expect(import("../services/heartbeat.js")).resolves.toBeDefined(); + const brake = await import("../services/recovery/automatic-run-brake.js"); + const rules = await brake.automaticRunOutcomeRules(); + expect(rules.isTransientSetupCode("environment_provider_unavailable")).toBe(true); + expect(rules.isPersonOwnedFailureCode("budget_hard_stop")).toBe(true); + }, 600_000); +}); diff --git a/server/src/__tests__/automatic-run-brake-recovery-passes.test.ts b/server/src/__tests__/automatic-run-brake-recovery-passes.test.ts new file mode 100644 index 00000000000..a8830f57ea7 --- /dev/null +++ b/server/src/__tests__/automatic-run-brake-recovery-passes.test.ts @@ -0,0 +1,495 @@ +import { randomUUID } from "node:crypto"; +import { and, eq, inArray, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { + activityLog, agentWakeupRequests, agents, automaticRunBrakes, companies, createDb, heartbeatRuns, issueRecoveryActions, issues, + nativeRunFinalizations, +} from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { AUTOMATIC_RUN_BRAKE, automaticRunBrake } from "../services/recovery/automatic-run-brake.js"; +import { recoveryService } from "../services/recovery/service.js"; +import { ASSIGNED_WORK_CONTINUED_ACTION } from "../services/recovery/assigned-work-sweep.js"; +import { taskAutomaticAttempts } from "../services/recovery/task-automatic-attempts.js"; +import { CONTINUATION_CAP } from "../services/recovery/assigned-work-invariant.js"; +import { deliverReconciledExecutions } from "../services/execution-recovery-resolution.js"; +import { continueHeldExecutions, resetBacklogContinuationPacingForTests } from "../services/fresh-continuation-fallback.js"; + +const support = await getEmbeddedPostgresTestSupport(); +const suite = support.supported ? describe : describe.skip; +const MINUTE = 60_000; +const HOUR = 60 * MINUTE; + +suite("automatic-run brake in the recovery passes", () => { + let db!: ReturnType; + let temporary: Awaited> | null = null; + let companyId = "", agentId = ""; + + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("paperclip-automatic-run-brake-passes-"); + db = createDb(temporary.connectionString); + }, 60_000); + + afterEach(async () => { + automaticRunBrake.reset(); + resetBacklogContinuationPacingForTests(db); + await db.execute(sql`truncate table automatic_run_brakes, issue_recovery_actions, native_run_finalizations, activity_log, + agent_wakeup_requests, heartbeat_runs, issues, agents, companies cascade`); + }, 120_000); + + afterAll(async () => { await db?.$client.end(); await temporary?.cleanup(); }, 120_000); + + async function seedCompany() { + companyId = randomUUID(); agentId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Passes", issuePrefix: `P${companyId.replaceAll("-", "").slice(0, 6)}`, + requireBoardApprovalForNewAgents: false }); + await db.insert(agents).values({ id: agentId, companyId, name: "Builder", role: "engineer", status: "idle", adapterType: "codex_local", + adapterConfig: {}, runtimeConfig: { heartbeat: { maxConcurrentRuns: 1 } }, permissions: {} }); + } + + const ago = (ms: number) => new Date(Date.now() - ms); + + async function openBrake(options: { startingAutomaticRun?: boolean } = { startingAutomaticRun: true }) { + const failures = await db.insert(heartbeatRuns).values(Array.from({ length: 20 }, (_, index) => ({ companyId, agentId, + invocationSource: "automation", triggerDetail: "system", runtimeMode: "legacy", status: "failed", + errorCode: "environment_provider_unavailable", executionStage: "preparing", createdAt: ago(4 * MINUTE + index), + startedAt: ago(3 * MINUTE + index), finishedAt: ago(MINUTE - index * 100), + resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }, + contextSnapshot: { wakeReason: "transient_failure_retry", issueId: randomUUID() } }))).returning(); + if (options.startingAutomaticRun) { + await db.insert(heartbeatRuns).values({ companyId, agentId, invocationSource: "automation", triggerDetail: "system", + status: "running", runtimeMode: "legacy", executionStage: "preparing", startedAt: ago(20_000), + contextSnapshot: { wakeReason: "issue_continuation_needed", previousRunId: failures[0]!.id, issueId: randomUUID() } }); + } + expect((await automaticRunBrake.refresh(db, companyId)).state).toBe("open"); + return failures; + } + + async function recover(failures: Array<{ id: string }>) { + await db.update(heartbeatRuns).set({ createdAt: sql`created_at - interval '1 hour'`, startedAt: sql`started_at - interval '1 hour'`, + finishedAt: sql`finished_at - interval '1 hour'` }).where(inArray(heartbeatRuns.id, failures.map(run => run.id))); + await db.insert(heartbeatRuns).values(Array.from({ length: 2 }, () => ({ companyId, agentId, invocationSource: "automation", + triggerDetail: "system", runtimeMode: "legacy", status: "succeeded", executionStage: "dispatching", createdAt: ago(MINUTE), + startedAt: ago(50_000), finishedAt: ago(1_000), contextSnapshot: { wakeReason: "transient_failure_retry", issueId: randomUUID() } }))); + await db.update(automaticRunBrakes).set({ stateChangedAt: sql`state_changed_at - ${`${AUTOMATIC_RUN_BRAKE.minOpenMs} milliseconds`}::interval` }) + .where(eq(automaticRunBrakes.companyId, companyId)); + expect((await automaticRunBrake.refresh(db, companyId)).state).toBe("half_open"); + } + + async function strandedTask() { + const issueId = randomUUID(); + await db.insert(issues).values({ id: issueId, companyId, title: "Continue the work", status: "in_progress", assigneeAgentId: agentId, + updatedAt: ago(3 * HOUR), createdAt: ago(5 * HOUR) }); + await db.insert(heartbeatRuns).values({ companyId, agentId, invocationSource: "on_demand", status: "interrupted", + errorCode: "server_shutdown_interrupted", startedAt: ago(4 * HOUR), finishedAt: ago(3 * HOUR), livenessState: "needs_followup", + createdAt: ago(4 * HOUR), contextSnapshot: { issueId, taskId: issueId }, + resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } }); + return issueId; + } + + function wakeQueue() { + const enqueueWakeup = vi.fn(async (wakeAgentId: string, opts?: { payload?: Record | null; reason?: string | null }) => { + const [wake] = await db.insert(agentWakeupRequests).values({ companyId, agentId: wakeAgentId, source: "automation", + triggerDetail: "system", reason: opts?.reason ?? null, payload: opts?.payload ?? null, status: "queued" }).returning(); + return { id: wake!.id } as never; + }); + return enqueueWakeup; + } + + const ledger = async (issueId: string) => (await taskAutomaticAttempts(db, companyId, issueId, new Date())).all.length; + const continuedActivity = async (issueId: string) => db.select().from(activityLog) + .where(and(eq(activityLog.entityId, issueId), eq(activityLog.action, ASSIGNED_WORK_CONTINUED_ACTION))); + + it("the assigned-work sweep defers continuations while open without spending the task's attempts, and resumes by itself", async () => { + await seedCompany(); + const issueId = await strandedTask(); + const failures = await openBrake(); + const enqueueWakeup = wakeQueue(); + const recovery = recoveryService(db, { enqueueWakeup, scheduleRecoveryRetry: async () => null }); + for (let pass = 0; pass < 3; pass += 1) { + const check = await recovery.assignedWork.checkIssue({ companyId, issueId }); + expect(check).toMatchObject({ outcome: "deferred_rate_limit" }); + expect(check!.retryAt!.getTime()).toBeGreaterThan(Date.now() + 30_000); + } + expect(enqueueWakeup).not.toHaveBeenCalled(); + expect(await continuedActivity(issueId)).toHaveLength(0); + expect(await ledger(issueId)).toBe(0); + expect(await recovery.assignedWork.report(companyId)).toMatchObject({ automaticWork: { state: "open", + summary: expect.stringMatching(/^Automatic work is paused for this company since .* because the last 5 automatic runs could not get a sandbox\.$/) } }); + + await recover(failures); + expect(await recovery.assignedWork.checkIssue({ companyId, issueId })).toMatchObject({ outcome: "repaired" }); + expect(enqueueWakeup).toHaveBeenCalledTimes(1); + expect(await ledger(issueId)).toBe(1); + + await db.update(automaticRunBrakes).set({ stateChangedAt: sql`state_changed_at + - ${`${AUTOMATIC_RUN_BRAKE.releaseSteps * AUTOMATIC_RUN_BRAKE.releaseIntervalMs} milliseconds`}::interval` }) + .where(eq(automaticRunBrakes.companyId, companyId)); + expect((await automaticRunBrake.refresh(db, companyId)).state).toBe("closed"); + expect((await recovery.assignedWork.report(companyId)).automaticWork).toMatchObject({ state: "closed", + summary: "Automatic work is running normally." }); + }, 120_000); + + it("the sweep still continues one task a minute while open when no automatic run is starting", async () => { + await seedCompany(); + const [first, second] = [await strandedTask(), await strandedTask()]; + await openBrake({ startingAutomaticRun: false }); + const enqueueWakeup = wakeQueue(); + const recovery = recoveryService(db, { enqueueWakeup, scheduleRecoveryRetry: async () => null }); + expect(await recovery.assignedWork.checkIssue({ companyId, issueId: first })).toMatchObject({ outcome: "repaired" }); + expect(await recovery.assignedWork.checkIssue({ companyId, issueId: second })).toMatchObject({ outcome: "deferred_rate_limit" }); + expect(enqueueWakeup).toHaveBeenCalledTimes(1); + }, 120_000); + + it("a task that loses the creation slot to another sweep check gains no extra wait", async () => { + await seedCompany(); + const [first, second] = [await strandedTask(), await strandedTask()]; + await openBrake({ startingAutomaticRun: false }); + const enqueueWakeup = wakeQueue(); + const recovery = recoveryService(db, { enqueueWakeup, scheduleRecoveryRetry: async () => null }); + const checks = await Promise.all([first, second].map(issueId => recovery.assignedWork.checkIssue({ companyId, issueId }))); + expect(checks.map(check => check!.outcome).sort()).toEqual(["deferred_rate_limit", "repaired"]); + expect(enqueueWakeup).toHaveBeenCalledTimes(1); + const loser = checks[0]!.outcome === "deferred_rate_limit" ? first : second; + expect(await ledger(loser)).toBe(0); + }, 120_000); + + it("leaves every pass exactly as before when the company has no pause record", async () => { + await seedCompany(); + const issueId = await strandedTask(); + const action = await pendingContinuation(); + const enqueueWakeup = wakeQueue(); + const recovery = recoveryService(db, { enqueueWakeup, scheduleRecoveryRetry: async () => null }); + const wake = vi.fn(async (wakeAgentId: string, opts: { contextSnapshot: Record }) => { + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: wakeAgentId, invocationSource: "automation", + triggerDetail: "system", status: "queued", contextSnapshot: opts.contextSnapshot }).returning(); + return run!; + }); + expect(await recovery.assignedWork.checkIssue({ companyId, issueId })).toMatchObject({ outcome: "repaired" }); + await deliverReconciledExecutions(db, wake as never); + expect(enqueueWakeup).toHaveBeenCalledTimes(1); + expect(wake).toHaveBeenCalledTimes(1); + expect(wake.mock.calls[0]![1]).toMatchObject({ reason: "issue_recovery_action_restored", contextSnapshot: { recoveryActionId: action.id } }); + expect(await db.select().from(automaticRunBrakes)).toHaveLength(0); + }, 120_000); + + it("tries again on another agent when the waiting automatic run cannot start, and closes once it works", async () => { + await seedCompany(); + const otherAgent = randomUUID(); + await db.insert(agents).values({ id: otherAgent, companyId, name: "Reviewer", role: "engineer", status: "idle", adapterType: "codex_local", + adapterConfig: {}, runtimeConfig: { heartbeat: { maxConcurrentRuns: 1 } }, permissions: {} }); + const failures = await openBrake({ startingAutomaticRun: false }); + await db.insert(heartbeatRuns).values({ companyId, agentId, invocationSource: "automation", triggerDetail: "system", status: "queued", + createdAt: ago(10 * MINUTE), contextSnapshot: { wakeReason: "issue_continuation_needed", previousRunId: failures[0]!.id, issueId: randomUUID() } }); + const issueId = randomUUID(); + await db.insert(issues).values({ id: issueId, companyId, title: "Review the work", status: "in_progress", assigneeAgentId: otherAgent, + updatedAt: ago(3 * HOUR), createdAt: ago(5 * HOUR) }); + await db.insert(heartbeatRuns).values({ companyId, agentId: otherAgent, invocationSource: "on_demand", status: "interrupted", + errorCode: "server_shutdown_interrupted", startedAt: ago(4 * HOUR), finishedAt: ago(3 * HOUR), livenessState: "needs_followup", + createdAt: ago(4 * HOUR), contextSnapshot: { issueId, taskId: issueId }, + resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } }); + const enqueueWakeup = wakeQueue(); + const recovery = recoveryService(db, { enqueueWakeup, scheduleRecoveryRetry: async () => null }); + expect(await recovery.assignedWork.checkIssue({ companyId, issueId })).toMatchObject({ outcome: "repaired" }); + expect(enqueueWakeup.mock.calls[0]![0]).toBe(otherAgent); + + await db.update(heartbeatRuns).set({ finishedAt: sql`finished_at - interval '1 hour'` }).where(inArray(heartbeatRuns.id, failures.map(run => run.id))); + await db.insert(heartbeatRuns).values({ companyId, agentId: otherAgent, invocationSource: "automation", triggerDetail: "system", + status: "succeeded", runtimeMode: "legacy", executionStage: "dispatching", startedAt: ago(40_000), finishedAt: ago(1_000), + contextSnapshot: { wakeReason: "issue_continuation_needed", previousRunId: randomUUID(), issueId } }); + await db.update(automaticRunBrakes).set({ stateChangedAt: sql`state_changed_at - ${`${AUTOMATIC_RUN_BRAKE.minOpenMs} milliseconds`}::interval` }) + .where(eq(automaticRunBrakes.companyId, companyId)); + expect((await automaticRunBrake.refresh(db, companyId)).state).toBe("half_open"); + await db.update(automaticRunBrakes).set({ stateChangedAt: sql`state_changed_at + - ${`${AUTOMATIC_RUN_BRAKE.releaseSteps * AUTOMATIC_RUN_BRAKE.releaseIntervalMs} milliseconds`}::interval` }) + .where(eq(automaticRunBrakes.companyId, companyId)); + expect((await automaticRunBrake.refresh(db, companyId)).state).toBe("closed"); + }, 120_000); + + it("a delivery its agent refuses gives the creation slot back to the next continuation in the same pass", async () => { + await seedCompany(); + const otherAgent = randomUUID(); + await db.insert(agents).values({ id: otherAgent, companyId, name: "Reviewer", role: "engineer", status: "idle", adapterType: "codex_local", + adapterConfig: {}, runtimeConfig: { heartbeat: { maxConcurrentRuns: 1 } }, permissions: {} }); + const refused = await pendingContinuation(); + const accepted = await pendingContinuation(otherAgent, 3 * MINUTE); + await openBrake({ startingAutomaticRun: false }); + const wake = vi.fn(async (wakeAgentId: string, opts: { contextSnapshot: Record }) => { + if (wakeAgentId === agentId) return null; + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: wakeAgentId, invocationSource: "automation", + triggerDetail: "system", status: "queued", contextSnapshot: opts.contextSnapshot }).returning(); + return run!; + }); + await deliverReconciledExecutions(db, wake as never); + expect(wake.mock.calls.map(call => call[0])).toEqual([agentId, otherAgent]); + const delivery = async (id: string) => (await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, id)))[0]! + .evidence.continuationDelivery; + expect(await delivery(refused.id)).toBe("pending"); + expect(await delivery(accepted.id)).toBe("delivered"); + expect((await db.select().from(automaticRunBrakes))[0]!.nextCreationAt).not.toBeNull(); + }, 120_000); + + it("a capped task gives the creation slot back to the next task the sweep checks", async () => { + await seedCompany(); + const [capped, next] = [await strandedTask(), await strandedTask()]; + await openBrake({ startingAutomaticRun: false }); + await db.insert(activityLog).values(Array.from({ length: CONTINUATION_CAP }, (_, index) => ({ companyId, actorType: "system", + actorId: "assigned-work-invariant", action: ASSIGNED_WORK_CONTINUED_ACTION, entityType: "issue", entityId: capped, + createdAt: ago(10 * MINUTE + index), details: {} }))); + const enqueueWakeup = wakeQueue(); + const recovery = recoveryService(db, { enqueueWakeup, scheduleRecoveryRetry: async () => null }); + expect(await recovery.assignedWork.checkIssue({ companyId, issueId: capped })).toMatchObject({ outcome: "deferred_cap" }); + expect((await db.select().from(automaticRunBrakes))[0]!.nextCreationAt).toBeNull(); + expect(await recovery.assignedWork.checkIssue({ companyId, issueId: next })).toMatchObject({ outcome: "repaired" }); + expect(enqueueWakeup).toHaveBeenCalledTimes(1); + }, 120_000); + + for (const createdBeforeError of [true, false]) { + it(`a delivery whose wake fails ${createdBeforeError ? "after creating its run keeps" : "before creating a run gives back"} the creation slot`, async () => { + await seedCompany(); + const otherAgent = randomUUID(); + await db.insert(agents).values({ id: otherAgent, companyId, name: "Reviewer", role: "engineer", status: "idle", adapterType: "codex_local", + adapterConfig: {}, runtimeConfig: { heartbeat: { maxConcurrentRuns: 1 } }, permissions: {} }); + await pendingContinuation(); + await pendingContinuation(otherAgent, 3 * MINUTE); + await openBrake({ startingAutomaticRun: false }); + const wake = vi.fn(async (wakeAgentId: string, opts: { contextSnapshot: Record }) => { + if (wakeAgentId === agentId && !createdBeforeError) throw new Error("wake refused before creating a run"); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: wakeAgentId, invocationSource: "automation", + triggerDetail: "system", status: "queued", contextSnapshot: opts.contextSnapshot }).returning(); + if (wakeAgentId === agentId) throw new Error("dispatch failed after the run was committed"); + return run!; + }); + await deliverReconciledExecutions(db, wake as never); + expect(wake.mock.calls.map(call => call[0])).toEqual(createdBeforeError ? [agentId] : [agentId, otherAgent]); + expect((await db.select().from(automaticRunBrakes))[0]!.nextCreationAt).not.toBeNull(); + }, 120_000); + + it(`a sweep whose wake fails ${createdBeforeError ? "after creating its run keeps" : "before creating a run gives back"} the creation slot`, async () => { + await seedCompany(); + const [first, second] = [await strandedTask(), await strandedTask()]; + await openBrake({ startingAutomaticRun: false }); + const enqueueWakeup = vi.fn(async (wakeAgentId: string, opts?: { payload?: Record | null; contextSnapshot?: Record }) => { + const issueId = String(opts?.payload?.issueId); + if (issueId === first && !createdBeforeError) throw new Error("wake refused before creating a run"); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: wakeAgentId, invocationSource: "automation", + triggerDetail: "system", status: "queued", contextSnapshot: opts?.contextSnapshot ?? { issueId } }).returning(); + if (issueId === first) throw new Error("dispatch failed after the run was committed"); + return { id: run!.id } as never; + }); + const recovery = recoveryService(db, { enqueueWakeup, scheduleRecoveryRetry: async () => null }); + await recovery.assignedWork.checkIssue({ companyId, issueId: first }); + expect(await recovery.assignedWork.checkIssue({ companyId, issueId: second })) + .toMatchObject({ outcome: createdBeforeError ? "deferred_rate_limit" : "repaired" }); + }, 120_000); + } + + async function pendingContinuation(ownerAgentId = agentId, endedAgoMs = 2 * MINUTE) { + const issueId = randomUUID(); + await db.insert(issues).values({ id: issueId, companyId, title: "Continue the work", status: "todo", assigneeAgentId: ownerAgentId }); + const [source] = await db.insert(heartbeatRuns).values({ companyId, agentId: ownerAgentId, issueId, invocationSource: "assignment", status: "failed", + errorCode: "server_shutdown_interrupted", startedAt: ago(endedAgoMs + MINUTE), finishedAt: ago(endedAgoMs), createdAt: ago(endedAgoMs + 2 * MINUTE), + contextSnapshot: { issueId, wakeReason: "issue_assigned" } }).returning(); + const recordedAt = new Date().toISOString(); + const [action] = await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: issueId, kind: "active_run_watchdog", + status: "resolved", outcome: "handed_back", resolvedAt: new Date(), ownerType: "board", returnOwnerAgentId: ownerAgentId, + cause: "legacy_execution_requires_reconciliation", fingerprint: `legacy-execution:${source!.id}`, nextAction: "Continue the task.", + attemptCount: 1, evidence: { runId: source!.id, originalFailureCode: "server_shutdown_interrupted", continuationDelivery: "pending", + freshContinuation: { schema: "paperclip.fresh-continuation.v1", reason: "exact_resumption_unavailable", recordedAt }, + freshContinuationSettlement: { recordedAt }, + executionReconciliation: { runId: source!.id, observedOutcome: "unknown", disposition: "continue_fresh", + actorId: "execution-recovery", recordedAt } } }).returning(); + return action!; + } + + it("the continuation delivery leaves pending continuations pending while open and delivers them after", async () => { + await seedCompany(); + const action = await pendingContinuation(); + const failures = await openBrake(); + const wake = vi.fn(async (wakeAgentId: string, opts: { contextSnapshot: Record }) => { + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: wakeAgentId, invocationSource: "automation", + triggerDetail: "system", status: "queued", contextSnapshot: opts.contextSnapshot }).returning(); + return run!; + }); + const delivery = async () => (await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, action.id)))[0]! + .evidence.continuationDelivery; + await deliverReconciledExecutions(db, wake as never); + await deliverReconciledExecutions(db, wake as never); + expect(wake).not.toHaveBeenCalled(); + expect(await delivery()).toBe("pending"); + await recover(failures); + await db.delete(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.status, "running"))); + await deliverReconciledExecutions(db, wake as never); + expect(wake).toHaveBeenCalledTimes(1); + expect(await delivery()).toBe("delivered"); + }, 120_000); + + async function heldBacklog() { + const issueId = randomUUID(), runId = randomUUID(); + const finishedAt = ago(HOUR); + await db.insert(issues).values({ id: issueId, companyId, title: "Ship the notes", status: "blocked", assigneeAgentId: agentId, + statusVersion: 4, blockedTransitionAt: finishedAt }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "assignment", issueId, nativeIssueId: issueId, + runtimeMode: "native", status: "failed", errorCode: "native_restart_recovery_blocked", error: "native_restart_recovery_blocked", + startedAt: new Date(finishedAt.getTime() - 10 * MINUTE), finishedAt, createdAt: new Date(finishedAt.getTime() - 11 * MINUTE), + contextSnapshot: { issueId, wakeReason: "issue_assigned" }, nativePhase: "terminal_failure" }); + await db.insert(nativeRunFinalizations).values({ runId, companyId, issueId, phase: "terminal_failure", recoveryState: "blocked", + failureCode: "native_continuation_requires_reconciliation", leaseOwner: null }); + const [action] = await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: issueId, kind: "active_run_watchdog", + status: "resolved", outcome: "blocked", resolvedAt: finishedAt, ownerType: "board", returnOwnerAgentId: agentId, + cause: "native_continuation_requires_reconciliation", fingerprint: `native-continuation:${runId}`, nextAction: "Continue.", + maxAttempts: 3, createdAt: finishedAt, evidence: { runId, originalFailureCode: "native_restart_recovery_blocked", + nativeFailureBlock: { runId, statusVersion: 4 }, automaticRecovery: { policy: "preserve_without_replay_v1", runId, replay: "blocked", + actionOutcome: "unknown", recordedAt: finishedAt.toISOString() } } }).returning(); + return { runId, actionId: action!.id }; + } + + function runWake() { + return vi.fn(async (wakeAgentId: string, opts: { contextSnapshot: Record }) => { + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: wakeAgentId, invocationSource: "automation", + triggerDetail: "system", status: "queued", contextSnapshot: opts.contextSnapshot }).returning(); + return run!; + }); + } + + const continuationRecords = async (runId: string) => db.select().from(issueRecoveryActions) + .where(sql`${issueRecoveryActions.evidence}#>>'{freshContinuation,sourceRunId}' = ${runId}`); + + it("the held backlog records its continuation for free and the delivery creates the run only after the pause", async () => { + await seedCompany(); + const held = await heldBacklog(); + const failures = await openBrake(); + expect(await continueHeldExecutions(db, new Date())).toMatchObject({ continued: 1 }); + expect(await continuationRecords(held.runId)).toHaveLength(1); + const wake = runWake(); + await deliverReconciledExecutions(db, wake as never); + expect(wake).not.toHaveBeenCalled(); + expect((await db.select().from(automaticRunBrakes))[0]!.nextCreationAt).toBeNull(); + await recover(failures); + await db.delete(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.status, "running"))); + await deliverReconciledExecutions(db, wake as never); + expect(wake).toHaveBeenCalledTimes(1); + }, 120_000); + + it("one held task spends one creation slot while open", async () => { + await seedCompany(); + await heldBacklog(); + await openBrake({ startingAutomaticRun: false }); + expect(await continueHeldExecutions(db, new Date())).toMatchObject({ continued: 1 }); + expect((await db.select().from(automaticRunBrakes))[0]!.nextCreationAt).toBeNull(); + const wake = runWake(); + await deliverReconciledExecutions(db, wake as never); + expect(wake).toHaveBeenCalledTimes(1); + expect((await db.select().from(automaticRunBrakes))[0]!.nextCreationAt).not.toBeNull(); + }, 120_000); + + const userWake = (issueId: string, values: Partial) => db.insert(agentWakeupRequests).values({ + companyId, agentId, source: "on_demand", triggerDetail: "manual", reason: "task_retry", status: "deferred_issue_execution", + requestedByActorType: "user", requestedByActorId: "board", payload: { issueId }, ...values }); + + const deliveryShapes: Array<[string, (issueId: string, sourceRunId: string) => Promise]> = [ + ["a Retry waiting behind the task", issueId => userWake(issueId, {})], + ["a move-forward Retry queued for the task", issueId => userWake(issueId, { status: "queued", + payload: { issueId, taskRetry: { schema: "paperclip.task-retry.v1", sourceRunId: randomUUID() } } })], + ["a Retry folded into the continuation", issueId => userWake(issueId, { status: "coalesced", finishedAt: new Date(), + payload: { issueId, coalescedIntoFreshContinuationRun: randomUUID() } })], + ["a person's message", issueId => userWake(issueId, { reason: "issue_commented", status: "skipped", finishedAt: new Date() })], + ["a Retry refused at dispatch while the task was held", issueId => db.insert(heartbeatRuns).values({ companyId, agentId, issueId, + invocationSource: "on_demand", triggerDetail: "manual", status: "cancelled", errorCode: "execution_reconciliation_required", + createdAt: ago(60_000), startedAt: null, finishedAt: ago(59_000), + contextSnapshot: { issueId, wakeReason: "task_retry", taskRetryRequestId: randomUUID() } })], + ["a continuation of a person's Retry", (_issueId, sourceRunId) => db.update(heartbeatRuns) + .set({ contextSnapshot: sql`${heartbeatRuns.contextSnapshot} || '{"wakeReason":"task_retry"}'::jsonb` }).where(eq(heartbeatRuns.id, sourceRunId))], + ]; + + for (const [shape, apply] of deliveryShapes) { + it(`delivers at once while paused: ${shape}`, async () => { + await seedCompany(); + const action = await pendingContinuation(); + await openBrake(); + const sourceRunId = (action.evidence.executionReconciliation as { runId: string }).runId; + await apply(action.sourceIssueId, sourceRunId); + const wake = runWake(); + await deliverReconciledExecutions(db, wake as never); + expect(wake).toHaveBeenCalledTimes(1); + expect((await db.select().from(automaticRunBrakes))[0]).toMatchObject({ state: "open", nextCreationAt: null }); + }, 120_000); + } + + it("a held automatic continuation does not hide a person's continuation for the same agent", async () => { + await seedCompany(); + const personAction = await pendingContinuation(); + await userWake(personAction.sourceIssueId, {}); + const automaticAction = await pendingContinuation(); + await openBrake(); + const wake = runWake(); + await deliverReconciledExecutions(db, wake as never); + expect(wake).toHaveBeenCalledTimes(1); + expect(wake.mock.calls[0]![1]).toMatchObject({ payload: { issueId: personAction.sourceIssueId } }); + const delivery = async (id: string) => (await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, id)))[0]! + .evidence.continuationDelivery; + expect(await delivery(personAction.id)).toBe("delivered"); + expect(await delivery(automaticAction.id)).toBe("pending"); + }, 120_000); + + it("held continuations take turns across passes so they cannot fill every page", async () => { + await seedCompany(); + const recent = await pendingContinuation(agentId, 2 * HOUR); + const older = await pendingContinuation(agentId, 3 * HOUR); + await openBrake(); + const wake = runWake(); + const attempted = async (id: string) => (await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, id)))[0]! + .evidence.continuationDeliveryAttemptedAt; + await deliverReconciledExecutions(db, wake as never); + await deliverReconciledExecutions(db, wake as never); + expect(wake).not.toHaveBeenCalled(); + expect(await attempted(recent.id)).toBeTruthy(); + expect(await attempted(older.id)).toBeTruthy(); + }, 120_000); + + it("still pauses the delivery of a continuation no person asked for", async () => { + await seedCompany(); + const action = await pendingContinuation(); + await openBrake(); + await userWake(action.sourceIssueId, { status: "completed", requestedAt: ago(HOUR), finishedAt: ago(HOUR) }); + const wake = runWake(); + await deliverReconciledExecutions(db, wake as never); + expect(wake).not.toHaveBeenCalled(); + }, 120_000); + + const sweepShapes: Array<[string, (issueId: string) => Promise]> = [ + ["the task's last run was a person's Retry", async issueId => { + const [run] = await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`); + await db.update(heartbeatRuns).set({ contextSnapshot: { issueId, taskId: issueId, wakeReason: "task_retry" } }).where(eq(heartbeatRuns.id, run!.id)); + }], + ["a person assigned the task after its last run", issueId => userWake(issueId, { reason: "issue_assigned", status: "failed", finishedAt: new Date() })], + ["a person commented after its last run", issueId => userWake(issueId, { reason: "issue_commented", status: "skipped", finishedAt: new Date() })], + ]; + + for (const [shape, apply] of sweepShapes) { + it(`the sweep continues at once while paused: ${shape}`, async () => { + await seedCompany(); + const issueId = await strandedTask(); + await openBrake(); + await apply(issueId); + const enqueueWakeup = wakeQueue(); + const recovery = recoveryService(db, { enqueueWakeup, scheduleRecoveryRetry: async () => null }); + expect(await recovery.assignedWork.checkIssue({ companyId, issueId })).toMatchObject({ outcome: "repaired" }); + expect(enqueueWakeup).toHaveBeenCalledTimes(1); + }, 120_000); + } + + it("the held backlog continues a task a person asked for while paused, and its delivery is not paused", async () => { + await seedCompany(); + const held = await heldBacklog(); + await openBrake(); + const [source] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, held.runId)); + await userWake(source!.issueId!, {}); + expect(await continueHeldExecutions(db, new Date())).toMatchObject({ continued: 1 }); + const wake = runWake(); + await deliverReconciledExecutions(db, wake as never); + const records = await continuationRecords(held.runId); + expect(records.map(record => [record.evidence.freshContinuation ? 'record' : 'hold', record.evidence.continuationDelivery ?? null])) + .not.toContainEqual(['record', 'pending']); + expect((await db.select().from(automaticRunBrakes))[0]!.nextCreationAt).toBeNull(); + }, 120_000); +}); diff --git a/server/src/__tests__/automatic-run-brake.test.ts b/server/src/__tests__/automatic-run-brake.test.ts new file mode 100644 index 00000000000..48afc8a7c2a --- /dev/null +++ b/server/src/__tests__/automatic-run-brake.test.ts @@ -0,0 +1,441 @@ +import { randomUUID } from "node:crypto"; +import { eq, inArray, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { agentWakeupRequests, agents, automaticRunBrakes, companies, createDb, heartbeatRuns } from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { + AUTOMATIC_RUN_BRAKE, automaticRunOutcomeRules, automaticWorkStatusText, classifyAutomaticRunOutcome, createAutomaticRunBrake, + isAutomaticRun, +} from "../services/recovery/automatic-run-brake.js"; +import { taskAutomaticAttempts } from "../services/recovery/task-automatic-attempts.js"; + +const support = await getEmbeddedPostgresTestSupport(); +const suite = support.supported ? describe : describe.skip; +const MINUTE = 60_000; + +type Run = typeof heartbeatRuns.$inferSelect; + +suite("automatic-run brake", () => { + let db!: ReturnType; + let temporary: Awaited> | null = null; + + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("paperclip-automatic-run-brake-"); + db = createDb(temporary.connectionString); + }, 60_000); + + afterEach(async () => { + await db.execute(sql`truncate table automatic_run_brakes, agent_wakeup_requests, heartbeat_runs, agents, companies cascade`); + }, 120_000); + + afterAll(async () => { await db?.$client.end(); await temporary?.cleanup(); }, 120_000); + + async function seedCompany() { + const companyId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Brake", issuePrefix: `B${companyId.replaceAll("-", "").slice(0, 6)}`, + requireBoardApprovalForNewAgents: false }); + const agentIds: string[] = []; + for (let index = 0; index < 2; index += 1) { + const agentId = randomUUID(); + await db.insert(agents).values({ id: agentId, companyId, name: `Worker ${index}`, role: "engineer", status: "idle", + adapterType: "codex_local", adapterConfig: {}, runtimeConfig: { heartbeat: { maxConcurrentRuns: 1 } }, permissions: {} }); + agentIds.push(agentId); + } + return { companyId, agentIds }; + } + + async function insertRuns(companyId: string, agentId: string, count: number, values: (index: number) => Partial) { + if (count === 0) return [] as Run[]; + return db.insert(heartbeatRuns).values(Array.from({ length: count }, (_, index) => ({ companyId, agentId, + invocationSource: "automation", triggerDetail: "system", runtimeMode: "legacy", + contextSnapshot: { wakeReason: "transient_failure_retry", issueId: randomUUID() }, ...values(index) }))).returning(); + } + + const ago = (ms: number) => new Date(Date.now() - ms); + + function failedBeforeStarting(companyId: string, agentId: string, count: number, errorCode = "environment_provider_unavailable") { + return insertRuns(companyId, agentId, count, index => ({ status: "failed", errorCode, executionStage: "preparing", + createdAt: ago(4 * MINUTE + index), startedAt: ago(3 * MINUTE + index), finishedAt: ago(MINUTE - index * 100), + resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } })); + } + + function interruptedWhileWorking(companyId: string, agentId: string, count: number) { + const codes = ["controller_lease_lost", "server_shutdown_interrupted", "process_lost", "native_session_interrupted"]; + return insertRuns(companyId, agentId, count, index => ({ status: index % 2 ? "failed" : "interrupted", errorCode: codes[index % codes.length], + executionStage: "dispatching", lastOutputAt: ago(40_000), createdAt: ago(20 * MINUTE), startedAt: ago(19 * MINUTE), + finishedAt: ago(30_000 - index * 500), contextSnapshot: { wakeReason: "issue_assigned", issueId: randomUUID() } })); + } + + function healthy(companyId: string, agentId: string, count: number, status: "succeeded" | "failed" = "succeeded") { + return insertRuns(companyId, agentId, count, index => ({ status, errorCode: status === "failed" ? "adapter_failed" : null, + executionStage: "dispatching", lastOutputAt: ago(10_000), createdAt: ago(5 * MINUTE), startedAt: ago(4 * MINUTE), + finishedAt: ago(5_000 - index * 10) })); + } + + function queuedAutomatic(companyId: string, agentId: string, count: number) { + return insertRuns(companyId, agentId, count, index => ({ status: "queued", createdAt: ago(30_000 - index), + contextSnapshot: { wakeReason: "issue_continuation_needed", previousRunId: randomUUID(), issueId: randomUUID() } })); + } + + async function forgetRuns(ids: string[]) { + await db.update(heartbeatRuns).set({ createdAt: sql`created_at - interval '1 hour'`, startedAt: sql`started_at - interval '1 hour'`, + finishedAt: sql`finished_at - interval '1 hour'` }).where(inArray(heartbeatRuns.id, ids)); + } + + async function ageBrake(companyId: string, ms: number) { + const shift = `${ms} milliseconds`; + await db.update(automaticRunBrakes).set({ + stateChangedAt: sql`state_changed_at - ${shift}::interval`, openedAt: sql`opened_at - ${shift}::interval`, + nextProbeAt: sql`next_probe_at - ${shift}::interval`, nextCreationAt: sql`next_creation_at - ${shift}::interval`, + releaseWindowStartedAt: sql`release_window_started_at - ${shift}::interval`, + }).where(eq(automaticRunBrakes.companyId, companyId)); + } + + const brakeRow = async (companyId: string) => (await db.select().from(automaticRunBrakes).where(eq(automaticRunBrakes.companyId, companyId)))[0]; + + async function openBrake(companyId: string, agentId: string, brake = createAutomaticRunBrake()) { + const failures = await failedBeforeStarting(companyId, agentId, 20); + expect((await brake.refresh(db, companyId)).state).toBe("open"); + return failures; + } + + async function heldCount(brake: ReturnType, runs: Run[], options: { personOrigin?: boolean } = {}) { + let started = 0; + for (const run of runs) if (!await brake.holdAutomaticRunStart(db, run, options)) started += 1; + return started; + } + + function countingDb() { + let queries = 0; + const counted = new Set(["select", "insert", "update", "delete", "execute"]); + const proxy = new Proxy(db, { + get(target, property, receiver) { + const value = Reflect.get(target, property, receiver); + if (typeof value !== "function") return value; + return counted.has(String(property)) ? (...args: unknown[]) => { queries += 1; return value.apply(target, args); } : value.bind(target); + }, + }); + return { db: proxy as typeof db, queries: () => queries }; + } + + it("classifies automatic runs and outcomes with the shared person and restart rules", async () => { + const rules = await automaticRunOutcomeRules(); + expect(isAutomaticRun({ id: "a", companyId: "c", contextSnapshot: { wakeReason: "issue_continuation_needed" } })).toBe(true); + expect(isAutomaticRun({ id: "a", companyId: "c", scheduledRetryReason: "transient_failure" })).toBe(true); + expect(isAutomaticRun({ id: "a", companyId: "c", contextSnapshot: { wakeReason: "task_retry", previousRunId: "x" } })).toBe(false); + expect(isAutomaticRun({ id: "a", companyId: "c", contextSnapshot: { wakeReason: "issue_assigned" } })).toBe(false); + const base = { status: "failed", runtimeMode: "legacy", startedAt: new Date(), hasOutput: false, providerWorkStarted: null }; + expect(classifyAutomaticRunOutcome({ ...base, errorCode: "environment_provider_unavailable", executionStage: "preparing" }, rules)).toBe("failed_before_starting"); + expect(classifyAutomaticRunOutcome({ ...base, errorCode: "controller_lease_lost", executionStage: "dispatching" }, rules)).toBe("neutral"); + expect(classifyAutomaticRunOutcome({ ...base, errorCode: "process_lost", executionStage: "preparing" }, rules)).toBe("neutral"); + expect(classifyAutomaticRunOutcome({ ...base, errorCode: "budget_hard_stop", executionStage: "preparing" }, rules)).toBe("healthy"); + expect(classifyAutomaticRunOutcome({ ...base, errorCode: "adapter_failed", executionStage: "dispatching" }, rules)).toBe("healthy"); + expect(classifyAutomaticRunOutcome({ ...base, errorCode: "adapter_failed", executionStage: null, providerWorkStarted: "false" }, rules)).toBe("failed_before_starting"); + expect(classifyAutomaticRunOutcome({ ...base, status: "running", errorCode: null, executionStage: "preparing" }, rules)).toBe("neutral"); + expect(classifyAutomaticRunOutcome({ ...base, status: "cancelled", errorCode: "environment_provider_unavailable", executionStage: "preparing" }, rules)).toBe("neutral"); + const preDispatch = { version: 1, providerWorkStarted: false }; + for (const runtimeMode of ["legacy", "native"]) { + const shape = { ...base, runtimeMode, executionStage: runtimeMode === "legacy" ? "preparing" : null, + providerWorkStarted: runtimeMode === "legacy" ? "false" : null, nativePreDispatchFailure: runtimeMode === "native" ? preDispatch : undefined }; + for (const errorCode of ["environment_provider_unavailable", "setup_failed", "adapter_failed"]) { + expect(classifyAutomaticRunOutcome({ ...shape, errorCode }, rules)).toBe("failed_before_starting"); + } + for (const errorCode of ["controller_lease_lost", "process_lost", "native_session_interrupted", "server_shutdown_interrupted"]) { + expect(classifyAutomaticRunOutcome({ ...shape, errorCode }, rules)).toBe("neutral"); + } + const working = { ...base, runtimeMode, executionStage: runtimeMode === "legacy" ? "dispatching" : null, hasOutput: true }; + expect(classifyAutomaticRunOutcome({ ...working, errorCode: "adapter_failed" }, rules)).toBe("healthy"); + expect(classifyAutomaticRunOutcome({ ...working, errorCode: "environment_provider_unavailable" }, rules)).toBe("failed_before_starting"); + } + }); + + it("opens on native runs that failed before dispatch", async () => { + const { companyId, agentIds: [agentId] } = await seedCompany(); + await insertRuns(companyId, agentId!, 20, index => ({ runtimeMode: "native", status: "failed", + errorCode: index % 2 ? "adapter_failed" : "setup_failed", createdAt: ago(4 * MINUTE), startedAt: ago(3 * MINUTE), + finishedAt: ago(MINUTE - index * 100), nativePhase: "terminal_failure", + resultJson: { nativePreDispatchFailure: { version: 1, providerWorkStarted: false }, failureCause: index % 2 ? "40P01" : "adapter_reported_error" } })); + expect((await createAutomaticRunBrake().refresh(db, companyId))).toMatchObject({ state: "open", + reason: { recentAutomaticRuns: 20, failedBeforeStarting: 20, trigger: "failed_before_starting" } }); + }, 60_000); + + it("sees failures that finished in the window behind more than 500 newer runs", async () => { + const { companyId, agentIds: [agentId, otherAgent] } = await seedCompany(); + const failures = await failedBeforeStarting(companyId, agentId!, 20, "setup_failed"); + await db.update(heartbeatRuns).set({ createdAt: sql`now() - interval '90 minutes'`, startedAt: sql`now() - interval '3 minutes'` }) + .where(inArray(heartbeatRuns.id, failures.map(run => run.id))); + await queuedAutomatic(companyId, otherAgent!, 600); + await insertRuns(companyId, otherAgent!, 50, () => ({ status: "succeeded", createdAt: ago(MINUTE), contextSnapshot: { wakeReason: "issue_assigned" } })); + expect((await createAutomaticRunBrake().refresh(db, companyId)).state).toBe("open"); + }, 120_000); + + it("opens when automatic runs fail before starting, holds all but one probe, and never holds a person's work", async () => { + const { companyId, agentIds: [agentId, otherAgent] } = await seedCompany(); + const brake = createAutomaticRunBrake(); + await failedBeforeStarting(companyId, agentId!, 20); + const snapshot = await brake.refresh(db, companyId); + expect(snapshot).toMatchObject({ state: "open", reason: { recentAutomaticRuns: 20, failedBeforeStarting: 20 } }); + expect(automaticWorkStatusText(snapshot)).toMatch(/^Automatic work is paused for this company since .* because the last 5 automatic runs could not get a sandbox/); + expect(automaticWorkStatusText(snapshot)).not.toMatch(/brake|probe|_/); + + const queued = [...await queuedAutomatic(companyId, agentId!, 25), ...await queuedAutomatic(companyId, otherAgent!, 25)]; + const attemptsBefore = await Promise.all(queued.map(run => taskAutomaticAttempts(db, companyId, String(run.contextSnapshot!.issueId), new Date()))); + expect(await heldCount(brake, queued, { personOrigin: false })).toBe(1); + expect(await heldCount(createAutomaticRunBrake(), queued, { personOrigin: false })).toBe(0); + const after = await db.select().from(heartbeatRuns).where(inArray(heartbeatRuns.id, queued.map(run => run.id))); + expect(after.every(run => run.status === "queued" && run.errorCode === null)).toBe(true); + const attemptsAfter = await Promise.all(queued.map(run => taskAutomaticAttempts(db, companyId, String(run.contextSnapshot!.issueId), new Date()))); + expect(attemptsAfter.map(attempts => attempts.all.length)).toEqual(attemptsBefore.map(attempts => attempts.all.length)); + + const [retry] = await insertRuns(companyId, agentId!, 1, () => ({ status: "queued", contextSnapshot: { wakeReason: "task_retry", previousRunId: queued[0]!.id } })); + const [assignment] = await insertRuns(companyId, agentId!, 1, () => ({ status: "queued", contextSnapshot: { wakeReason: "issue_assigned" } })); + const [joinedByPerson] = await insertRuns(companyId, agentId!, 1, () => ({ status: "queued", + contextSnapshot: { wakeReason: "issue_continuation_needed", previousRunId: randomUUID() } })); + await db.insert(agentWakeupRequests).values({ companyId, agentId: agentId!, source: "on_demand", triggerDetail: "manual", + reason: "issue_commented", status: "queued", requestedByActorType: "user", requestedByActorId: "board", runId: joinedByPerson!.id }); + const [continuationOfPersonRetry] = await insertRuns(companyId, agentId!, 1, () => ({ status: "queued", + contextSnapshot: { wakeReason: "transient_failure_retry", previousRunId: retry!.id } })); + for (const run of [retry!, assignment!, joinedByPerson!, continuationOfPersonRetry!]) { + expect(await brake.holdAutomaticRunStart(db, run)).toBe(false); + } + expect(await brake.holdAutomaticRunStart(db, queued[3]!, { personOrigin: true })).toBe(false); + }, 60_000); + + it("stays closed for mixed outcomes, person-owned failures and task failures after the agent started", async () => { + const { companyId, agentIds: [agentId] } = await seedCompany(); + const brake = createAutomaticRunBrake(); + expect((await brake.refresh(db, companyId)).state).toBe("closed"); + expect(await brakeRow(companyId)).toBeUndefined(); + await failedBeforeStarting(companyId, agentId!, 8, "setup_failed"); + await healthy(companyId, agentId!, 4, "failed"); + expect((await brake.refresh(db, companyId)).state).toBe("closed"); + await insertRuns(companyId, agentId!, 30, index => ({ status: "failed", errorCode: index % 2 ? "budget_hard_stop" : "claude_auth_required", + executionStage: "preparing", createdAt: ago(3 * MINUTE), startedAt: ago(2 * MINUTE), finishedAt: ago(MINUTE), + resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } })); + await healthy(companyId, agentId!, 30, "failed"); + expect((await brake.refresh(db, companyId)).state).toBe("closed"); + expect(await brakeRow(companyId)).toBeUndefined(); + }, 60_000); + + it("ignores failures older than the window", async () => { + const { companyId, agentIds: [agentId] } = await seedCompany(); + const failures = await failedBeforeStarting(companyId, agentId!, 40); + await db.update(heartbeatRuns).set({ finishedAt: sql`now() - interval '11 minutes'` }).where(inArray(heartbeatRuns.id, failures.map(run => run.id))); + expect((await createAutomaticRunBrake().refresh(db, companyId)).state).toBe("closed"); + }, 60_000); + + it("stays closed through a release swap that interrupts working runs, and opens when their continuations cannot get a sandbox", async () => { + const { companyId, agentIds: [agentId, otherAgent] } = await seedCompany(); + const brake = createAutomaticRunBrake(); + await interruptedWhileWorking(companyId, agentId!, 40); + expect((await brake.refresh(db, companyId)).state).toBe("closed"); + const continuations = [...await queuedAutomatic(companyId, agentId!, 20), ...await queuedAutomatic(companyId, otherAgent!, 20)]; + expect(await heldCount(createAutomaticRunBrake(), continuations)).toBe(40); + expect((await brake.reserveCreation(db, companyId, { issueId: randomUUID(), sourceRunId: randomUUID() })).heldUntil).toBeNull(); + + await db.update(heartbeatRuns).set({ status: "failed", errorCode: "environment_provider_unavailable", executionStage: "preparing", + startedAt: sql`now() - interval '20 seconds'`, finishedAt: sql`now() - interval '5 seconds'`, + resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } }) + .where(inArray(heartbeatRuns.id, continuations.map(run => run.id))); + expect((await brake.refresh(db, companyId)).state).toBe("open"); + }, 60_000); + + it("opens on five sandbox refusals in a row, below the run count", async () => { + const { companyId, agentIds: [agentId] } = await seedCompany(); + const brake = createAutomaticRunBrake(); + await failedBeforeStarting(companyId, agentId!, 4); + expect((await brake.refresh(db, companyId)).state).toBe("closed"); + await failedBeforeStarting(companyId, agentId!, 1); + expect((await brake.refresh(db, companyId)).state).toBe("open"); + }, 60_000); + + it("waits out a failed probe, then releases at a doubling rate after two probes reach the agent, and closes", async () => { + const { companyId, agentIds: [agentId, otherAgent] } = await seedCompany(); + const failures = await openBrake(companyId, agentId!); + await forgetRuns(failures.map(run => run.id)); + const queued = [...await queuedAutomatic(companyId, agentId!, 30), ...await queuedAutomatic(companyId, otherAgent!, 30)]; + const probe = async () => { + const brake = createAutomaticRunBrake(); + for (const run of queued) if (!await brake.holdAutomaticRunStart(db, run, { personOrigin: false })) return run; + return null; + }; + const first = await probe(); + expect(first).not.toBeNull(); + await db.update(heartbeatRuns).set({ status: "failed", errorCode: "environment_provider_unavailable", executionStage: "preparing", + startedAt: sql`now()`, finishedAt: sql`now()`, resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } }) + .where(eq(heartbeatRuns.id, first!.id)); + queued.splice(queued.indexOf(first!), 1); + expect(await probe()).toBeNull(); + expect((await brakeRow(companyId))).toMatchObject({ state: "open", probeSuccesses: 0, probeRunId: null }); + await ageBrake(companyId, AUTOMATIC_RUN_BRAKE.probeIntervalMs); + + for (let success = 0; success < AUTOMATIC_RUN_BRAKE.probeSuccessesToRelease; success += 1) { + const run = await probe(); + expect(run).not.toBeNull(); + await db.update(heartbeatRuns).set({ status: "running", startedAt: sql`now()`, executionStage: "dispatching" }) + .where(eq(heartbeatRuns.id, run!.id)); + queued.splice(queued.indexOf(run!), 1); + await createAutomaticRunBrake().refresh(db, companyId); + } + expect(await brakeRow(companyId)).toMatchObject({ state: "open", probeSuccesses: 2 }); + await ageBrake(companyId, AUTOMATIC_RUN_BRAKE.minOpenMs); + expect((await createAutomaticRunBrake().refresh(db, companyId)).state).toBe("half_open"); + + const startedPerWindow: number[] = []; + for (let step = 0; step < AUTOMATIC_RUN_BRAKE.releaseSteps; step += 1) { + const startedNow = await heldCount(createAutomaticRunBrake(), queued, { personOrigin: false }) + + await heldCount(createAutomaticRunBrake(), queued, { personOrigin: false }); + startedPerWindow.push(startedNow); + await ageBrake(companyId, AUTOMATIC_RUN_BRAKE.releaseIntervalMs); + } + expect(startedPerWindow).toEqual([2, 4, 8, 16, 32, 64]); + expect((await createAutomaticRunBrake().refresh(db, companyId)).state).toBe("closed"); + expect(await heldCount(createAutomaticRunBrake(), queued, { personOrigin: false })).toBe(queued.length); + }, 120_000); + + it("stays open when runs that started fine are followed by newer failures before starting", async () => { + const { companyId, agentIds: [agentId] } = await seedCompany(); + const failures = await openBrake(companyId, agentId!); + await forgetRuns(failures.map(run => run.id)); + await insertRuns(companyId, agentId!, 2, () => ({ status: "running", executionStage: "dispatching", lastOutputAt: ago(80_000), + startedAt: ago(90_000), createdAt: ago(100_000) })); + await failedBeforeStarting(companyId, agentId!, 10); + await ageBrake(companyId, AUTOMATIC_RUN_BRAKE.minOpenMs); + expect((await createAutomaticRunBrake().refresh(db, companyId)).state).toBe("open"); + }, 60_000); + + it("reopens from a gradual release when runs fail before starting again", async () => { + const { companyId, agentIds: [agentId] } = await seedCompany(); + const failures = await openBrake(companyId, agentId!); + await forgetRuns(failures.map(run => run.id)); + const recovered = await healthy(companyId, agentId!, 2); + await ageBrake(companyId, AUTOMATIC_RUN_BRAKE.minOpenMs); + expect((await createAutomaticRunBrake().refresh(db, companyId)).state).toBe("half_open"); + await forgetRuns(recovered.map(run => run.id)); + const again = await failedBeforeStarting(companyId, agentId!, 5); + await db.update(heartbeatRuns).set({ finishedAt: sql`now()` }).where(inArray(heartbeatRuns.id, again.map(run => run.id))); + expect((await createAutomaticRunBrake().refresh(db, companyId)).state).toBe("open"); + }, 60_000); + + it("frees the probe slot when the probe is cancelled, deleted or never starts", async () => { + const { companyId, agentIds: [agentId, otherAgent] } = await seedCompany(); + await openBrake(companyId, agentId!); + const queued = [...await queuedAutomatic(companyId, agentId!, 3), ...await queuedAutomatic(companyId, otherAgent!, 3)]; + const brake = createAutomaticRunBrake(); + expect(await brake.holdAutomaticRunStart(db, queued[0]!, { personOrigin: false })).toBe(false); + expect(await heldCount(createAutomaticRunBrake(), queued.slice(1), { personOrigin: false })).toBe(0); + await ageBrake(companyId, AUTOMATIC_RUN_BRAKE.probeIntervalMs); + expect(await createAutomaticRunBrake().holdAutomaticRunStart(db, queued[3]!, { personOrigin: false })).toBe(false); + expect((await brakeRow(companyId))?.probeRunId).toBe(queued[3]!.id); + + await db.update(heartbeatRuns).set({ status: "cancelled", finishedAt: sql`now()` }).where(eq(heartbeatRuns.id, queued[3]!.id)); + await createAutomaticRunBrake().refresh(db, companyId); + expect((await brakeRow(companyId))?.probeRunId).toBeNull(); + await ageBrake(companyId, AUTOMATIC_RUN_BRAKE.probeIntervalMs); + expect(await createAutomaticRunBrake().holdAutomaticRunStart(db, queued[4]!, { personOrigin: false })).toBe(false); + await db.delete(heartbeatRuns).where(eq(heartbeatRuns.id, queued[4]!.id)); + await createAutomaticRunBrake().refresh(db, companyId); + expect((await brakeRow(companyId))?.probeRunId).toBeNull(); + }, 60_000); + + it("lets another run take the probe when the reserved probe never starts within the claim bound", async () => { + const { companyId, agentIds: [agentId, otherAgent] } = await seedCompany(); + await openBrake(companyId, agentId!); + const [first] = await queuedAutomatic(companyId, agentId!, 1); + const [second] = await queuedAutomatic(companyId, otherAgent!, 1); + expect(await createAutomaticRunBrake().holdAutomaticRunStart(db, first!, { personOrigin: false })).toBe(false); + expect(await createAutomaticRunBrake().holdAutomaticRunStart(db, second!, { personOrigin: false })).toBe(true); + await ageBrake(companyId, AUTOMATIC_RUN_BRAKE.probeClaimBoundMs + 1_000); + expect(await createAutomaticRunBrake().holdAutomaticRunStart(db, second!, { personOrigin: false })).toBe(false); + expect((await brakeRow(companyId))?.probeRunId).toBe(second!.id); + await db.update(heartbeatRuns).set({ status: "running", startedAt: sql`now()` }).where(eq(heartbeatRuns.id, second!.id)); + await ageBrake(companyId, AUTOMATIC_RUN_BRAKE.probeClaimBoundMs + 1_000); + expect(await createAutomaticRunBrake().holdAutomaticRunStart(db, first!, { personOrigin: false })).toBe(true); + }, 60_000); + + it("closes by itself when no automatic run started or finished for the quiet bound, even with runs waiting", async () => { + const { companyId, agentIds: [agentId] } = await seedCompany(); + const failures = await openBrake(companyId, agentId!); + await ageBrake(companyId, AUTOMATIC_RUN_BRAKE.quietCloseMs - MINUTE); + expect((await createAutomaticRunBrake().refresh(db, companyId)).state).toBe("open"); + await forgetRuns(failures.map(run => run.id)); + await queuedAutomatic(companyId, agentId!, 5); + await ageBrake(companyId, 2 * MINUTE); + expect((await createAutomaticRunBrake().refresh(db, companyId))).toMatchObject({ state: "closed", reason: { trigger: "no_automatic_work" } }); + }, 60_000); + + it("lets two servers agree on the state and start one probe between them", async () => { + const { companyId, agentIds: [agentId, otherAgent] } = await seedCompany(); + await failedBeforeStarting(companyId, agentId!, 20); + const [left, right] = [createAutomaticRunBrake(), createAutomaticRunBrake()]; + const [leftState, rightState] = await Promise.all([left.refresh(db, companyId), right.refresh(db, companyId)]); + expect([leftState.state, rightState.state]).toEqual(["open", "open"]); + expect(leftState.stateChangedAt?.getTime()).toBe(rightState.stateChangedAt?.getTime()); + const queued = [...await queuedAutomatic(companyId, agentId!, 25), ...await queuedAutomatic(companyId, otherAgent!, 25)]; + const results = await Promise.all(queued.map((run, index) => (index % 2 ? left : right).holdAutomaticRunStart(db, run, { personOrigin: false }))); + expect(results.filter(held => !held)).toHaveLength(1); + }, 60_000); + + it("allows one new automatic run a minute while open and no automatic run is starting, across servers", async () => { + const { companyId, agentIds: [agentId] } = await seedCompany(); + await openBrake(companyId, agentId!); + const work = { issueId: randomUUID(), sourceRunId: randomUUID() }; + const [left, right] = [createAutomaticRunBrake(), createAutomaticRunBrake()]; + const decisions = await Promise.all([left, right, left, right].map(async brake => (await brake.reserveCreation(db, companyId, work)).heldUntil)); + expect(decisions.filter(held => held === null)).toHaveLength(1); + const later = decisions.find(Boolean)!; + expect(later.getTime()).toBeGreaterThan(Date.now() + 50_000); + await queuedAutomatic(companyId, agentId!, 3); + await ageBrake(companyId, AUTOMATIC_RUN_BRAKE.creationIntervalMs); + expect((await createAutomaticRunBrake().reserveCreation(db, companyId, work)).heldUntil).toBeNull(); + await insertRuns(companyId, agentId!, 1, () => ({ status: "running", executionStage: "preparing", startedAt: ago(20_000) })); + await ageBrake(companyId, AUTOMATIC_RUN_BRAKE.creationIntervalMs); + expect((await createAutomaticRunBrake().reserveCreation(db, companyId, work)).heldUntil).not.toBeNull(); + }, 60_000); + + it("keeps a pass over hundreds of held runs at a bounded number of queries", async () => { + const { companyId, agentIds: [agentId, otherAgent] } = await seedCompany(); + await openBrake(companyId, agentId!); + const queued = [...await queuedAutomatic(companyId, agentId!, 125), ...await queuedAutomatic(companyId, otherAgent!, 125)]; + const counting = countingDb(); + const brake = createAutomaticRunBrake(); + let started = 0; + for (let pass = 0; pass < 100; pass += 1) { + for (const run of queued) { + if (await brake.holdAutomaticRunStart(counting.db, run, { personOrigin: false })) continue; + started += 1; + await db.update(heartbeatRuns).set({ status: "running", startedAt: sql`now()` }).where(eq(heartbeatRuns.id, run.id)); + } + } + expect(started).toBe(1); + expect(counting.queries()).toBeLessThanOrEqual(6); + + const fallback = countingDb(); + const second = createAutomaticRunBrake(); + for (let pass = 0; pass < 100; pass += 1) for (const run of queued) await second.holdAutomaticRunStart(fallback.db, run); + expect(fallback.queries()).toBeLessThanOrEqual(queued.length + 5); + }, 120_000); + + it("reads as closed and logs once when its run rules cannot load", async () => { + const { companyId, agentIds: [agentId] } = await seedCompany(); + await failedBeforeStarting(companyId, agentId!, 20); + const loadRules = vi.fn(async () => { throw new Error("module failed to load"); }); + const brake = createAutomaticRunBrake({ loadRules }); + const queued = await queuedAutomatic(companyId, agentId!, 3); + expect((await brake.refresh(db, companyId)).state).toBe("closed"); + expect((await brake.reserveCreation(db, companyId, { issueId: randomUUID(), sourceRunId: randomUUID() })).heldUntil).toBeNull(); + expect(await heldCount(brake, queued)).toBe(3); + expect(await brakeRow(companyId)).toBeUndefined(); + }, 60_000); + + it("keeps its in-process memory bounded across many companies and runs", async () => { + const brake = createAutomaticRunBrake(); + const { companyId, agentIds: [agentId] } = await seedCompany(); + for (let index = 0; index < AUTOMATIC_RUN_BRAKE.maxCompanies + 100; index += 1) await brake.read(db, randomUUID()); + await openBrake(companyId, agentId!, brake); + for (let index = 0; index < AUTOMATIC_RUN_BRAKE.maxRuns + 200; index += 1) { + await brake.holdAutomaticRunStart(db, { id: randomUUID(), companyId, contextSnapshot: { wakeReason: "issue_continuation_needed" } }); + } + expect(brake.cacheSizes()).toEqual({ companies: AUTOMATIC_RUN_BRAKE.maxCompanies, runs: AUTOMATIC_RUN_BRAKE.maxRuns }); + }, 180_000); +}); diff --git a/server/src/services/execution-recovery-resolution.ts b/server/src/services/execution-recovery-resolution.ts index d87006be042..250c985095d 100644 --- a/server/src/services/execution-recovery-resolution.ts +++ b/server/src/services/execution-recovery-resolution.ts @@ -39,6 +39,8 @@ import { hasRemoteTerminationReceipt } from "./remote-execution-termination.js"; import { readSystemRecovery } from "./system-recovery-state.js"; import { isReattachPhaseActive } from "./heartbeat/run-admission.js"; import { provenGoneLegacyControllerBootId } from "./legacy-controller-lease.js"; +import { reserveAutomaticRunCreation } from "./recovery/automatic-run-brake.js"; +import { recordContinuationDeliveryAttempt } from "./recovery/continuation-delivery-attempt.js"; import { continueFreshInTransaction, continueHeldExecutions, freshContinuationCheckDue, recordFreshContinuationCheck, settleCoalescedFreshContinuationHolds } from "./fresh-continuation-fallback.js"; import { alias } from "drizzle-orm/pg-core"; @@ -395,6 +397,7 @@ export async function deliverReconciledExecutions( ] : [ sql`${pastBound} desc`, sql`${overdueTurn} asc nulls last`, + sql`${attemptedAt} asc nulls first`, sql`${sourceRun.finishedAt} desc nulls last`, sql`${issueRecoveryActions.updatedAt} asc`, ]; @@ -445,21 +448,14 @@ export async function deliverReconciledExecutions( ] : [ sql`${candidates.pastBound} desc`, sql`${candidates.heldAt} asc nulls last`, + sql`${candidates.attemptedAt} asc nulls first`, sql`${candidates.sourceFinishedAt} desc nulls last`, sql`${candidates.actionUpdatedAt} asc`, ])) .limit(25); return rows.map(row => row.action); }; - const deferRefusedContinuation = (action: typeof issueRecoveryActions.$inferSelect) => db.update(issueRecoveryActions) - .set({ evidence: sql`${issueRecoveryActions.evidence} || ${JSON.stringify( - { continuationDeliveryAttemptedAt: new Date().toISOString() })}::jsonb` }) - .where(and( - eq(issueRecoveryActions.companyId, action.companyId), - eq(issueRecoveryActions.id, action.id), - eq(issueRecoveryActions.status, "resolved"), - sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`, - )); + const deferRefusedContinuation = (action: typeof issueRecoveryActions.$inferSelect) => recordContinuationDeliveryAttempt(db, action); const agentHasRoom = async (action: typeof issueRecoveryActions.$inferSelect, agentId: string, kind: ContinuationClass) => { if ((admittedByCompany.get(action.companyId) ?? 0) >= companyLimit) return false; // A few at a time per agent, fresh or not: the rest waits here, durably, @@ -553,6 +549,8 @@ export async function deliverReconciledExecutions( seenAgents.add(action.returnOwnerAgentId); continue; } + const slot = await reserveAutomaticRunCreation(db, action.companyId, { issueId: task.id, sourceRunId: decision.runId }); + if (slot.heldUntil) { await deferRefusedContinuation(action); continue; } const run = await wake(action.returnOwnerAgentId, { source: "automation", triggerDetail: "system", @@ -573,9 +571,10 @@ export async function deliverReconciledExecutions( ...((action.evidence.freshContinuation as { isolation?: unknown } | undefined)?.isolation === "fresh_isolated_workspace" ? { freshContinuationIsolation: action.id } : {}), }, - }); + }).catch(async error => { await slot.release(); throw error; }); seenAgents.add(action.returnOwnerAgentId); if (!run) { + await slot.release(); await deferRefusedContinuation(action); continue; } diff --git a/server/src/services/recovery/assigned-work-sweep.ts b/server/src/services/recovery/assigned-work-sweep.ts index 6e69d610eb7..c7ac8ea5525 100644 --- a/server/src/services/recovery/assigned-work-sweep.ts +++ b/server/src/services/recovery/assigned-work-sweep.ts @@ -35,7 +35,8 @@ import { HELD_RECOVERY_WAKE_KEYS, queuedRecoveryRunsOf } from "../recovery-queue import { listPersonOriginSourceRunIds } from "../heartbeat/queued-run-line.js"; import { aliasedRun, queuedRunStalledSql } from "./stalled-queued-run-sql.js"; import { attributeBlock, BLOCK_ATTRIBUTION_WINDOW_MS, blockedStatusDetailsSql } from "../interruption-block-release.js"; -import { ASSIGNED_WORK_CONTINUED_ACTION, taskAutomaticAttempts } from "./task-automatic-attempts.js"; +import { ASSIGNED_WORK_CONTINUED_ACTION, PAUSED_FOR_AUTOMATIC_WORK, taskAutomaticAttempts } from "./task-automatic-attempts.js"; +import { automaticRunCreationHeldUntil, readAutomaticWorkStatus, reserveAutomaticRunCreation } from "./automatic-run-brake.js"; import { isReattachPhaseActive, reattachPhaseRecheckAt } from "../heartbeat/run-admission.js"; import { ensureFreshContinuation, type FreshContinuationResult } from "../fresh-continuation-fallback.js"; import { canContinueRecoveryRunAutomatically } from "../continuation-cause.js"; @@ -627,6 +628,9 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { if (repairTimes.length >= ASSIGNED_WORK_REPAIRS_PER_MINUTE) { return { verdict, outcome: "deferred_rate_limit", retryAt: new Date(repairTimes[0] + 60_000 + Math.floor(Math.random() * 5_000)) }; } + const heldUntil = facts.latestRun + ? await automaticRunCreationHeldUntil(db, issue.companyId, { issueId: issue.id, sourceRunId: facts.latestRun.id }) : null; + if (heldUntil) return { verdict, outcome: "deferred_rate_limit", retryAt: heldUntil }; const claimed = await claimContinuation(issue, at); if (claimed.kind !== "claimed") return { verdict, outcome: claimed.kind === "capped" ? "deferred_cap" : claimed.kind === "changed" ? "changed" : "deferred_rate_limit", retryAt: claimed.kind === "changed" ? null : claimed.retryAt }; @@ -648,6 +652,15 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { await settleContinuationActivity(claimed, claimed.reason, "not_started", { waitingUntil: fresh.retryAt.toISOString() }); return { verdict: claimed.verdict, outcome: fresh.kind === "backoff" ? "deferred_cap" : "waiting_on_recovery", retryAt: fresh.retryAt }; } + const slot = facts.latestRun + ? await reserveAutomaticRunCreation(db, issue.companyId, { issueId: issue.id, sourceRunId: facts.latestRun.id }) : null; + if (slot?.heldUntil) { + if (claimed.restoredTo) await revertRestoredBlock(issue, claimed); + await settleContinuationActivity(claimed, claimed.reason, "not_started", + { waitingUntil: slot.heldUntil.toISOString(), [PAUSED_FOR_AUTOMATIC_WORK]: "true" }); + return { verdict: claimed.verdict, outcome: "deferred_rate_limit", retryAt: slot.heldUntil }; + } + const release = async () => { if (slot && !slot.heldUntil) await slot.release(); }; const run = await deps.enqueueWakeup(issue.assigneeAgentId!, { source: facts.latestRun ? "automation" : "assignment", triggerDetail: "system", reason: facts.latestRun ? "issue_continuation_needed" : "issue_assigned", @@ -663,6 +676,7 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { // A refused continuation stays a violation; the claim spaces the next try. // A block lifted for it is put back, so the task never sits unblocked // with nothing pending. + if (!run) await release(); if (!run && claimed.restoredTo) await revertRestoredBlock(issue, claimed); await settleContinuationActivity(claimed, claimed.reason, run ? "queued" : "not_started", fresh?.kind === "person_owned" ? { waitingFor: fresh.reason } : {}); @@ -680,12 +694,19 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { } async function replaceStalledRun(issue: IssueRow, claimed: Claimed, runId: string): Promise { + const slot = await reserveAutomaticRunCreation(db, issue.companyId, { issueId: issue.id, sourceRunId: runId }); + if (slot.heldUntil) { + await settleContinuationActivity(claimed, claimed.reason, "not_started", + { waitingUntil: slot.heldUntil.toISOString(), [PAUSED_FOR_AUTOMATIC_WORK]: "true" }); + return { verdict: claimed.verdict, outcome: "deferred_rate_limit", retryAt: slot.heldUntil }; + } const cancelled = !claimed.stalledFence ? false : await deps.cancelStalledQueuedRun!({ companyId: issue.companyId, runId, ...claimed.stalledFence }).catch(error => { logger.warn({ err: error, companyId: issue.companyId, issueId: issue.id, runId }, "assigned-work sweep could not cancel a stalled queued run"); return false; }); if (!cancelled) { + await slot.release(); await db.delete(activityLog).where(eq(activityLog.id, claimed.activityId)); return { verdict: claimed.verdict, outcome: "changed", retryAt: null }; } @@ -699,6 +720,7 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { logger.warn({ err: error, companyId: issue.companyId, issueId: issue.id }, "assigned-work replacement for a stalled run was not admitted"); return null; }); + if (!run) await slot.release(); await settleContinuationActivity(claimed, claimed.reason, run ? "queued" : "not_started"); return { verdict: claimed.verdict, outcome: run ? "replaced_stalled_run" : "enqueue_refused", retryAt: claimed.nextCheckAt }; } @@ -942,7 +964,7 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { const queuedByAgent = new Map(queuedRows.map(row => [row.agentId, Number(row.queuedRuns)])); const slotRows = stalledRows.map(row => ({ agentName: row.agentName, stalledRuns: Number(row.stalledRuns), queuedRuns: queuedByAgent.get(row.agentId) ?? 0 })); - return { ...summary, agentsWithStalledSlots: slotRows }; + return { ...summary, agentsWithStalledSlots: slotRows, automaticWork: await readAutomaticWorkStatus(db, companyId) }; } return { checkIssue, listSuspects, report, loadFacts, flushPassLog: passLog.flush }; diff --git a/server/src/services/recovery/automatic-run-brake.ts b/server/src/services/recovery/automatic-run-brake.ts new file mode 100644 index 00000000000..fa345482221 --- /dev/null +++ b/server/src/services/recovery/automatic-run-brake.ts @@ -0,0 +1,565 @@ +import { eq, sql } from "drizzle-orm"; +import { automaticRunBrakes, type Db } from "@paperclipai/db"; +import { logger } from "../../middleware/logger.js"; +import { PERSON_WAKE_REASONS, RECOVERY_WAKE_REASONS } from "../heartbeat/queued-run-dispatch-order.js"; +import { ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE } from "../environment-provider-unavailable-error.js"; + +export const AUTOMATIC_RUN_BRAKE = { + windowMs: 10 * 60_000, + startProgressMs: 2 * 60_000, + minCounted: 12, + failureShare: 0.8, + consecutiveProviderRefusals: 5, + minOpenMs: 2 * 60_000, + quietCloseMs: 15 * 60_000, + probeIntervalMs: 60_000, + probeClaimBoundMs: 10_000, + probeRunningBoundMs: 10 * 60_000, + probeSuccessesToRelease: 2, + releaseIntervalMs: 60_000, + releaseInitialRate: 2, + releaseSteps: 6, + creationIntervalMs: 60_000, + cacheTtlMs: 5_000, + maxCompanies: 256, + maxRuns: 1_024, + rowLimit: 500, + runningLimit: 200, +} as const; + +export type AutomaticRunBrakeState = "closed" | "open" | "half_open"; + +export type AutomaticRunFacts = { + id: string; + companyId: string; + retryOfRunId?: string | null; + scheduledRetryReason?: string | null; + contextSnapshot?: Record | null; +}; + +const EXTRA_AUTOMATIC_WAKE_REASONS: ReadonlySet = new Set(["issue_assignment_recovery", "transient_failure_retry"]); + +export function isAutomaticRun(run: AutomaticRunFacts): boolean { + const context = run.contextSnapshot ?? {}; + const reason = typeof context.wakeReason === "string" ? context.wakeReason : null; + if (context.explicitUserContinuation || context.taskRetryRequestId || (reason && PERSON_WAKE_REASONS.has(reason))) return false; + return Boolean(run.retryOfRunId || run.scheduledRetryReason || context.previousRunId || context.retryReason + || context.freshContinuationIsolation + || (reason && (RECOVERY_WAKE_REASONS.has(reason) || EXTRA_AUTOMATIC_WAKE_REASONS.has(reason)))); +} + +export type AutomaticRunOutcome = "failed_before_starting" | "healthy" | "neutral"; + +export type AutomaticRunOutcomeFacts = { + status: string; + errorCode: string | null; + runtimeMode: string | null; + executionStage: string | null; + startedAt: Date | null; + hasOutput: boolean; + providerWorkStarted: string | null; + nativePreDispatchFailure?: unknown; +}; + +export type AutomaticRunOutcomeRules = { + isPersonOwnedFailureCode: (code: string | null) => boolean; + restartCodes: readonly string[]; + isTransientSetupCode: (code: string | null) => boolean; + hasNativePreDispatchFailureRecord: (resultJson: unknown) => boolean; + listPersonOriginSourceRunIds: (db: Db, companyId: string, runIds: string[]) => Promise; +}; + +let outcomeRules: Promise | null = null; + +export function automaticRunOutcomeRules(): Promise { + outcomeRules ??= Promise.all([import("../fresh-continuation-fallback.js"), import("../heartbeat/retries.js"), + import("../heartbeat/run-failure-cause.js"), import("../heartbeat/queued-run-line.js")]) + .then(([fallback, retries, failureCause, queuedRunLine]) => ({ + isPersonOwnedFailureCode: fallback.isPersonOwnedFailureCode, + restartCodes: fallback.BACKLOG_CONTINUATION_SOURCE_CODES, + isTransientSetupCode: retries.isTransientWorkspacePreparationCode, + hasNativePreDispatchFailureRecord: failureCause.hasNativePreDispatchFailureRecord, + listPersonOriginSourceRunIds: queuedRunLine.listPersonOriginSourceRunIds, + })) + .catch(error => { outcomeRules = null; throw error; }); + return outcomeRules; +} + +export function classifyAutomaticRunOutcome(run: AutomaticRunOutcomeFacts, rules: AutomaticRunOutcomeRules): AutomaticRunOutcome { + if (run.status === "running") return run.executionStage === "dispatching" || run.hasOutput ? "healthy" : "neutral"; + if (!run.startedAt || run.status === "cancelled" || !["succeeded", "failed", "timed_out", "interrupted"].includes(run.status)) return "neutral"; + if (run.status === "succeeded" || rules.isPersonOwnedFailureCode(run.errorCode)) return "healthy"; + if (run.errorCode && rules.restartCodes.includes(run.errorCode)) return "neutral"; + const startPhase = run.providerWorkStarted === "false" || rules.isTransientSetupCode(run.errorCode) + || rules.hasNativePreDispatchFailureRecord({ nativePreDispatchFailure: run.nativePreDispatchFailure }) + || (run.runtimeMode === "legacy" && run.executionStage === "preparing"); + return startPhase ? "failed_before_starting" : "healthy"; +} + +type BrakeRow = typeof automaticRunBrakes.$inferSelect; + +type SignalRow = { + db_now: string | Date; + id: string; + status: string; + error_code: string | null; + runtime_mode: string | null; + execution_stage: string | null; + started_at: string | Date | null; + finished_at: string | Date | null; + created_at: string | Date; + has_output: boolean; + provider_work_started: string | null; + native_pre_dispatch: unknown; + retry_of_run_id: string | null; + scheduled_retry_reason: string | null; + context: Record | null; +}; + +type Evidence = { + counted: number; + failedBeforeStarting: number; + healthy: number; + providerRefusalStreak: boolean; + recoveredSince: boolean; + lastStartAt: number | null; + lastActivityAt: number | null; +}; + +export type AutomaticRunBrakeSnapshot = { + companyId: string; + state: AutomaticRunBrakeState; + pausedSince: Date | null; + stateChangedAt: Date | null; + reason: Record; + nextProbeAt: Date | null; + nextCreationAt: Date | null; + probeRunId: string | null; + probeSuccesses: number; + dbNow: Date; + evidence: Evidence; +}; + +const toDate = (value: string | Date | null | undefined) => value == null ? null : value instanceof Date ? value : new Date(value); + +function boundedMap(max: number) { + const map = new Map(); + return { + get(key: K) { + const value = map.get(key); + if (value !== undefined) { map.delete(key); map.set(key, value); } + return value; + }, + set(key: K, value: V) { + map.delete(key); + map.set(key, value); + while (map.size > max) map.delete(map.keys().next().value as K); + }, + delete(key: K) { map.delete(key); }, + clear() { map.clear(); }, + get size() { return map.size; }, + }; +} + +function rampStep(stateChangedAt: Date, now: Date) { + return Math.max(0, Math.floor((now.getTime() - stateChangedAt.getTime()) / AUTOMATIC_RUN_BRAKE.releaseIntervalMs)); +} + +const outcomeFacts = (row: SignalRow): AutomaticRunOutcomeFacts => ({ status: row.status, errorCode: row.error_code, + runtimeMode: row.runtime_mode, executionStage: row.execution_stage, startedAt: toDate(row.started_at), hasOutput: row.has_output, + providerWorkStarted: row.provider_work_started, nativePreDispatchFailure: row.native_pre_dispatch }); + +const latest = (values: Array) => { + const known = values.filter((value): value is number => typeof value === "number"); + return known.length ? Math.max(...known) : null; +}; + +function evaluate(rows: SignalRow[], state: AutomaticRunBrakeState, lower: number, now: Date, rules: AutomaticRunOutcomeRules): Evidence { + const automatic = rows.filter(row => isAutomaticRun({ id: row.id, companyId: "", retryOfRunId: row.retry_of_run_id, + scheduledRetryReason: row.scheduled_retry_reason, contextSnapshot: row.context })); + const outcomes = automatic.flatMap(row => { + const finishedAt = toDate(row.finished_at); + if (row.status !== "running" && (!finishedAt || finishedAt.getTime() < lower)) return []; + const startedAt = toDate(row.started_at); + if (row.status === "running" && (!startedAt || startedAt.getTime() < lower)) return []; + const outcome = classifyAutomaticRunOutcome(outcomeFacts(row), rules); + return outcome === "neutral" ? [] : [{ outcome, row, at: (finishedAt ?? startedAt!).getTime() }]; + }).sort((left, right) => right.at - left.at); + const failedBeforeStarting = outcomes.filter(entry => entry.outcome === "failed_before_starting").length; + const streak = outcomes.slice(0, AUTOMATIC_RUN_BRAKE.consecutiveProviderRefusals); + const lastStartAt = latest(automatic.map(row => toDate(row.started_at)?.getTime())); + const lastActivityAt = latest([lastStartAt, ...automatic.map(row => toDate(row.finished_at)?.getTime())]); + return { + counted: outcomes.length, + failedBeforeStarting, + healthy: outcomes.length - failedBeforeStarting, + providerRefusalStreak: streak.length === AUTOMATIC_RUN_BRAKE.consecutiveProviderRefusals + && streak.every(entry => entry.row.error_code === ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE), + recoveredSince: state === "open" && outcomes.length > 0 && (outcomes[0]!.row.status === "succeeded" + || (outcomes.length >= AUTOMATIC_RUN_BRAKE.probeSuccessesToRelease + && outcomes.slice(0, AUTOMATIC_RUN_BRAKE.probeSuccessesToRelease).every(entry => entry.outcome === "healthy"))), + lastStartAt, + lastActivityAt, + }; +} + +function trips(evidence: Evidence) { + return evidence.providerRefusalStreak || (evidence.counted >= AUTOMATIC_RUN_BRAKE.minCounted + && evidence.failedBeforeStarting / evidence.counted >= AUTOMATIC_RUN_BRAKE.failureShare); +} + +export type AutomaticWork = { issueId: string; sourceRunId: string | null }; + +export type AutomaticRunCreationSlot = { heldUntil: Date } | { heldUntil: null; release: () => Promise }; + +const NO_RESERVATION = async () => undefined; + +type CacheEntry = { snapshot: AutomaticRunBrakeSnapshot; readAt: number; skewMs: number; heldUntil: number }; + +export function createAutomaticRunBrake(options: { loadRules?: () => Promise } = {}) { + const loadRules = options.loadRules ?? automaticRunOutcomeRules; + const companies = boundedMap(AUTOMATIC_RUN_BRAKE.maxCompanies); + const personOrigin = boundedMap(AUTOMATIC_RUN_BRAKE.maxRuns); + let rulesFailureLogged = false; + + async function rulesOrNull() { + try { + return await loadRules(); + } catch (error) { + if (!rulesFailureLogged) { + rulesFailureLogged = true; + logger.error({ err: error, event: "automatic_run_brake.rules_unavailable" }, + "Automatic work pause could not load its run rules; automatic work is not paused"); + } + return null; + } + } + + async function readRow(db: Db, companyId: string): Promise { + const [row] = await db.select().from(automaticRunBrakes).where(eq(automaticRunBrakes.companyId, companyId)); + return row ?? null; + } + + async function readSignal(db: Db, companyId: string, row: BrakeRow | null) { + const state = row?.state ?? "closed"; + const changedAt = row?.stateChangedAt ?? null; + const lower = state === "open" + ? sql`coalesce(${changedAt?.toISOString() ?? null}::timestamptz, now() - ${`${AUTOMATIC_RUN_BRAKE.windowMs} milliseconds`}::interval)` + : sql`greatest(now() - ${`${AUTOMATIC_RUN_BRAKE.windowMs} milliseconds`}::interval, ${changedAt?.toISOString() ?? null}::timestamptz)`; + const columns = sql`r.id, r.status, r.error_code, r.runtime_mode, r.execution_stage, r.started_at, r.finished_at, r.created_at, + r.last_output_at is not null as has_output, r.result_json #>> '{executionRecovery,providerWorkStarted}' as provider_work_started, + r.result_json -> 'nativePreDispatchFailure' as native_pre_dispatch, r.retry_of_run_id, r.scheduled_retry_reason, + jsonb_build_object('wakeReason', r.context_snapshot->'wakeReason', 'previousRunId', r.context_snapshot->'previousRunId', + 'retryReason', r.context_snapshot->'retryReason', 'freshContinuationIsolation', r.context_snapshot->'freshContinuationIsolation', + 'explicitUserContinuation', r.context_snapshot->'explicitUserContinuation', + 'taskRetryRequestId', r.context_snapshot->'taskRetryRequestId') as context`; + const rows = [...await db.execute(sql` + with clock as (select now() as db_now, ${lower} as lower_at) + select clock.db_now, clock.lower_at, runs.* from clock left join lateral ( + (select ${columns} from heartbeat_runs r + where r.company_id = ${companyId} and r.finished_at is not null and r.finished_at >= clock.lower_at + order by r.finished_at desc limit ${AUTOMATIC_RUN_BRAKE.rowLimit}) + union all + (select ${columns} from heartbeat_runs r + where r.company_id = ${companyId} and r.status = 'running' + order by r.started_at desc nulls last limit ${AUTOMATIC_RUN_BRAKE.runningLimit}) + ) runs on true`)]; + return { rows: rows.filter(row => row.id), now: toDate(rows[0]!.db_now)!, lower: toDate(rows[0]!.lower_at)!.getTime() }; + } + + async function transition(db: Db, companyId: string, row: BrakeRow | null, to: AutomaticRunBrakeState, + reason: Record): Promise { + const from = row?.state ?? "closed"; + const values = { + state: to, + reason, + stateChangedAt: sql`date_trunc('milliseconds', now())`, + openedAt: to === "open" ? (from === "half_open" && row?.openedAt ? row.openedAt : sql`date_trunc('milliseconds', now())`) : to === "closed" ? null : row?.openedAt ?? null, + nextProbeAt: null, + probeRunId: null, + probeSuccesses: 0, + releaseWindowStartedAt: null, + releaseWindowStarts: 0, + nextCreationAt: null, + updatedAt: sql`now()`, + }; + const expectedChangedAt = row?.stateChangedAt?.toISOString() ?? null; + const [updated] = await db.insert(automaticRunBrakes).values({ companyId, ...values }) + .onConflictDoUpdate({ + target: automaticRunBrakes.companyId, + set: values, + setWhere: sql`${automaticRunBrakes.state} = ${from} + and date_trunc('milliseconds', ${automaticRunBrakes.stateChangedAt}) is not distinct from ${expectedChangedAt}::timestamptz`, + }).returning(); + if (!updated) return readRow(db, companyId); + const event = to === "open" ? (from === "half_open" ? "reopened" : "opened") : to; + logger[to === "open" ? "warn" : "info"]({ event: `automatic_run_brake.${event}`, companyId, from, to, ...reason }, + to === "open" ? "Automatic work paused for a company while its runs fail before starting" + : to === "half_open" ? "Automatic work for a company is resuming gradually" : "Automatic work for a company is running normally again"); + return updated; + } + + async function settleProbe(db: Db, row: BrakeRow, rows: SignalRow[], rules: AutomaticRunOutcomeRules): Promise { + if (row.state !== "open" || !row.probeRunId) return row; + const known = rows.find(candidate => candidate.id === row.probeRunId); + const [probe] = known ? [known] : [...await db.execute(sql` + select r.id, r.status, r.error_code, r.runtime_mode, r.execution_stage, r.started_at, r.finished_at, r.created_at, + r.last_output_at is not null as has_output, r.result_json #>> '{executionRecovery,providerWorkStarted}' as provider_work_started, + r.result_json -> 'nativePreDispatchFailure' as native_pre_dispatch + from heartbeat_runs r where r.company_id = ${row.companyId} and r.id = ${row.probeRunId}`)]; + if (probe && (probe.status === "queued" || probe.status === "scheduled_retry")) return row; + const outcome = probe ? classifyAutomaticRunOutcome(outcomeFacts(probe), rules) : "neutral"; + if (probe?.status === "running" && outcome === "neutral") return row; + const completed = probe?.status === "succeeded"; + const successes = outcome === "healthy" ? row.probeSuccesses + (completed ? AUTOMATIC_RUN_BRAKE.probeSuccessesToRelease : 1) + : outcome === "failed_before_starting" ? 0 : row.probeSuccesses; + const [updated] = await db.update(automaticRunBrakes).set({ + probeRunId: null, probeSuccesses: successes, updatedAt: sql`now()`, + ...(outcome === "healthy" ? { nextProbeAt: sql`now()` } : {}), + }).where(sql`${automaticRunBrakes.companyId} = ${row.companyId} and ${automaticRunBrakes.state} = 'open' + and ${automaticRunBrakes.probeRunId} = ${row.probeRunId}`).returning(); + return updated ?? (await readRow(db, row.companyId)) ?? row; + } + + async function refresh(db: Db, companyId: string): Promise { + const rules = await rulesOrNull(); + if (!rules) { + const now = new Date(); + const entry: CacheEntry = { readAt: now.getTime(), skewMs: 0, heldUntil: 0, snapshot: { companyId, state: "closed", pausedSince: null, + stateChangedAt: null, reason: {}, nextProbeAt: null, nextCreationAt: null, probeRunId: null, probeSuccesses: 0, dbNow: now, + evidence: { counted: 0, failedBeforeStarting: 0, healthy: 0, providerRefusalStreak: false, recoveredSince: false, + lastStartAt: null, lastActivityAt: null } } }; + companies.set(companyId, entry); + return entry; + } + let row = await readRow(db, companyId); + let signal = await readSignal(db, companyId, row); + if (row) row = await settleProbe(db, row, signal.rows, rules); + let state: AutomaticRunBrakeState = row?.state ?? "closed"; + let evidence = evaluate(signal.rows, state, signal.lower, signal.now, rules); + const numbers = { windowMinutes: AUTOMATIC_RUN_BRAKE.windowMs / 60_000, recentAutomaticRuns: evidence.counted, + failedBeforeStarting: evidence.failedBeforeStarting, healthy: evidence.healthy }; + const changedAt = row?.stateChangedAt?.getTime() ?? 0; + const elapsed = signal.now.getTime() - changedAt; + let target: { to: AutomaticRunBrakeState; reason: Record } | null = null; + if (state === "closed" && trips(evidence)) { + target = { to: "open", reason: { ...numbers, trigger: evidence.providerRefusalStreak ? "sandbox_refusals" : "failed_before_starting" } }; + } else if (state === "half_open" && trips(evidence)) { + target = { to: "open", reason: { ...numbers, trigger: evidence.providerRefusalStreak ? "sandbox_refusals" : "failed_before_starting" } }; + } else if (state === "half_open" && rampStep(row!.stateChangedAt, signal.now) >= AUTOMATIC_RUN_BRAKE.releaseSteps) { + target = { to: "closed", reason: { ...numbers, trigger: "released" } }; + } else if (state === "open" && elapsed >= AUTOMATIC_RUN_BRAKE.minOpenMs + && (evidence.recoveredSince || (row?.probeSuccesses ?? 0) >= AUTOMATIC_RUN_BRAKE.probeSuccessesToRelease)) { + target = { to: "half_open", reason: { ...numbers, trigger: "runs_starting_again" } }; + } else if (state === "open" && elapsed >= AUTOMATIC_RUN_BRAKE.quietCloseMs + && (evidence.lastActivityAt ?? 0) < signal.now.getTime() - AUTOMATIC_RUN_BRAKE.quietCloseMs) { + target = { to: "closed", reason: { ...numbers, trigger: "no_automatic_work" } }; + } + if (target) { + row = await transition(db, companyId, row, target.to, target.reason); + state = row?.state ?? "closed"; + if (state !== "closed") { + signal = await readSignal(db, companyId, row); + evidence = evaluate(signal.rows, state, signal.lower, signal.now, rules); + } + } + const snapshot: AutomaticRunBrakeSnapshot = { + companyId, state, pausedSince: row?.openedAt ?? null, stateChangedAt: row?.stateChangedAt ?? null, + reason: row?.reason ?? {}, nextProbeAt: row?.nextProbeAt ?? null, nextCreationAt: row?.nextCreationAt ?? null, + probeRunId: row?.probeRunId ?? null, probeSuccesses: row?.probeSuccesses ?? 0, dbNow: signal.now, evidence, + }; + const readAt = Date.now(); + const entry = { snapshot, readAt, skewMs: signal.now.getTime() - readAt, heldUntil: companies.get(companyId)?.heldUntil ?? 0 }; + companies.set(companyId, entry); + return entry; + } + + async function current(db: Db, companyId: string): Promise { + const entry = companies.get(companyId); + if (entry && Date.now() - entry.readAt < AUTOMATIC_RUN_BRAKE.cacheTtlMs) return entry; + return refresh(db, companyId); + } + + async function read(db: Db, companyId: string): Promise { + return (await current(db, companyId)).snapshot; + } + + async function isPersonWork(db: Db, companyId: string, work: AutomaticWork) { + if (!work.sourceRunId) return true; + const personReasons = sql.join([...PERSON_WAKE_REASONS].map(reason => sql`${reason}`), sql`, `); + const [asked] = [...await db.execute<{ asked: boolean }>(sql` + with source as (select max(coalesce(r.finished_at, r.created_at)) as finished_at from heartbeat_runs r + where r.company_id = ${companyId} and r.id = ${work.sourceRunId}::uuid) + select exists (select 1 from agent_wakeup_requests w, source + where w.company_id = ${companyId} and w.requested_by_actor_type = 'user' and w.payload->>'issueId' = ${work.issueId} + and ((w.run_id is null and w.status in ('queued', 'deferred_issue_execution', 'claimed')) + or w.requested_at > coalesce(source.finished_at, '-infinity'::timestamptz))) + or exists (select 1 from heartbeat_runs h, source + where h.company_id = ${companyId} and h.context_snapshot->>'issueId' = ${work.issueId} + and h.created_at > coalesce(source.finished_at, '-infinity'::timestamptz) + and (h.context_snapshot->>'wakeReason' in (${personReasons}) or h.context_snapshot ? 'taskRetryRequestId' + or jsonb_typeof(h.context_snapshot->'explicitUserContinuation') = 'object')) as asked`)]; + if (asked?.asked) return true; + const rules = await rulesOrNull(); + return !rules || (await rules.listPersonOriginSourceRunIds(db, companyId, [work.sourceRunId])).includes(work.sourceRunId); + } + + async function creationHeldUntil(db: Db, companyId: string, work: AutomaticWork): Promise { + const entry = await current(db, companyId); + if (entry.snapshot.state !== "open") return null; + const now = Date.now(); + const lastStartAt = entry.snapshot.evidence.lastStartAt; + const startHold = lastStartAt !== null && lastStartAt - entry.skewMs > now - AUTOMATIC_RUN_BRAKE.startProgressMs + ? now + AUTOMATIC_RUN_BRAKE.creationIntervalMs : 0; + const slotHold = entry.snapshot.nextCreationAt ? entry.snapshot.nextCreationAt.getTime() - entry.skewMs : 0; + const until = Math.max(startHold, slotHold); + if (until <= now) return null; + return await isPersonWork(db, companyId, work) ? null : new Date(until); + } + + async function reserveCreation(db: Db, companyId: string, work: AutomaticWork): Promise { + const entry = await current(db, companyId); + if (entry.snapshot.state !== "open") return { heldUntil: null, release: NO_RESERVATION }; + if (await isPersonWork(db, companyId, work)) return { heldUntil: null, release: NO_RESERVATION }; + const recheck = new Date(Date.now() + AUTOMATIC_RUN_BRAKE.creationIntervalMs); + const lastStartAt = entry.snapshot.evidence.lastStartAt; + if (lastStartAt !== null && lastStartAt - entry.skewMs > Date.now() - AUTOMATIC_RUN_BRAKE.startProgressMs) return { heldUntil: recheck }; + const [seeded] = await db.update(automaticRunBrakes).set({ + nextCreationAt: sql`date_trunc('milliseconds', now()) + ${`${AUTOMATIC_RUN_BRAKE.creationIntervalMs} milliseconds`}::interval`, + updatedAt: sql`now()`, + }).where(sql`${automaticRunBrakes.companyId} = ${companyId} and ${automaticRunBrakes.state} = 'open' + and (${automaticRunBrakes.nextCreationAt} is null or ${automaticRunBrakes.nextCreationAt} <= now())`) + .returning({ nextCreationAt: automaticRunBrakes.nextCreationAt }); + if (seeded?.nextCreationAt) { + const token = seeded.nextCreationAt.toISOString(); + entry.snapshot = { ...entry.snapshot, nextCreationAt: seeded.nextCreationAt }; + return { heldUntil: null, release: async () => { + await db.update(automaticRunBrakes).set({ nextCreationAt: null, updatedAt: sql`now()` }) + .where(sql`${automaticRunBrakes.companyId} = ${companyId} and ${automaticRunBrakes.nextCreationAt} = ${token}::timestamptz + and not exists (select 1 from heartbeat_runs r where r.company_id = ${companyId} + and r.context_snapshot->>'issueId' = ${work.issueId} + and r.created_at >= ${token}::timestamptz - ${`${AUTOMATIC_RUN_BRAKE.creationIntervalMs} milliseconds`}::interval)`); + companies.delete(companyId); + } }; + } + const row = await readRow(db, companyId); + if (row?.state !== "open") { companies.delete(companyId); return { heldUntil: null, release: NO_RESERVATION }; } + const at = row.nextCreationAt ? row.nextCreationAt.getTime() - entry.skewMs : recheck.getTime(); + return { heldUntil: new Date(Math.max(Date.now(), at)) }; + } + + async function isPersonOrigin(db: Db, run: AutomaticRunFacts) { + const known = personOrigin.get(run.id); + if (known !== undefined) return known; + const rules = await rulesOrNull(); + if (!rules) return true; + const origin = (await rules.listPersonOriginSourceRunIds(db, run.companyId, [run.id])).includes(run.id); + personOrigin.set(run.id, origin); + return origin; + } + + async function grantProbe(db: Db, entry: CacheEntry, run: AutomaticRunFacts) { + const [granted] = await db.update(automaticRunBrakes).set({ + probeRunId: run.id, updatedAt: sql`now()`, + nextProbeAt: sql`now() + ${`${AUTOMATIC_RUN_BRAKE.probeIntervalMs} milliseconds`}::interval`, + }).where(sql`${automaticRunBrakes.companyId} = ${run.companyId} and ${automaticRunBrakes.state} = 'open' + and (((${automaticRunBrakes.nextProbeAt} is null or ${automaticRunBrakes.nextProbeAt} <= now()) + and (${automaticRunBrakes.probeRunId} is null or not exists (select 1 from heartbeat_runs p + where p.company_id = ${automaticRunBrakes.companyId} and p.id = ${automaticRunBrakes.probeRunId} and p.status = 'running' + and p.started_at > now() - ${`${AUTOMATIC_RUN_BRAKE.probeRunningBoundMs} milliseconds`}::interval))) + or (${automaticRunBrakes.probeRunId} is not null + and ${automaticRunBrakes.nextProbeAt} <= now() + ${`${AUTOMATIC_RUN_BRAKE.probeIntervalMs - AUTOMATIC_RUN_BRAKE.probeClaimBoundMs} milliseconds`}::interval + and not exists (select 1 from heartbeat_runs p where p.company_id = ${automaticRunBrakes.companyId} + and p.id = ${automaticRunBrakes.probeRunId} and p.started_at is not null)))`) + .returning({ nextProbeAt: automaticRunBrakes.nextProbeAt }); + const now = Date.now(); + if (granted) { + entry.heldUntil = now + AUTOMATIC_RUN_BRAKE.probeClaimBoundMs; + entry.snapshot = { ...entry.snapshot, probeRunId: run.id, nextProbeAt: granted.nextProbeAt }; + return true; + } + const nextProbeAt = entry.snapshot.nextProbeAt ? entry.snapshot.nextProbeAt.getTime() - entry.skewMs : 0; + entry.heldUntil = Math.max(now + AUTOMATIC_RUN_BRAKE.cacheTtlMs, nextProbeAt); + return false; + } + + async function grantRelease(db: Db, entry: CacheEntry, companyId: string) { + const [granted] = [...await db.execute<{ starts: number; window_ends_at: string | Date }>(sql` + with ramp as ( + select b.company_id, b.state_changed_at + floor(extract(epoch from now() - b.state_changed_at) * 1000 + / ${AUTOMATIC_RUN_BRAKE.releaseIntervalMs}) * ${`${AUTOMATIC_RUN_BRAKE.releaseIntervalMs} milliseconds`}::interval as window_start, + ${AUTOMATIC_RUN_BRAKE.releaseInitialRate} * power(2, floor(extract(epoch from now() - b.state_changed_at) * 1000 + / ${AUTOMATIC_RUN_BRAKE.releaseIntervalMs}))::int as rate + from automatic_run_brakes b where b.company_id = ${companyId} and b.state = 'half_open' + ) + update automatic_run_brakes b set + release_window_starts = case when b.release_window_started_at = ramp.window_start then b.release_window_starts + 1 else 1 end, + release_window_started_at = ramp.window_start, updated_at = now() + from ramp + where b.company_id = ramp.company_id and b.state = 'half_open' + and (b.release_window_started_at is distinct from ramp.window_start or b.release_window_starts < ramp.rate) + returning b.release_window_starts as starts, ramp.window_start + ${`${AUTOMATIC_RUN_BRAKE.releaseIntervalMs} milliseconds`}::interval as window_ends_at`)]; + if (granted) return true; + const step = rampStep(entry.snapshot.stateChangedAt ?? entry.snapshot.dbNow, new Date(Date.now() + entry.skewMs)); + const windowEnds = (entry.snapshot.stateChangedAt?.getTime() ?? 0) + (step + 1) * AUTOMATIC_RUN_BRAKE.releaseIntervalMs; + entry.heldUntil = Math.max(Date.now() + 1_000, windowEnds - entry.skewMs); + return false; + } + + async function holdAutomaticRunStart(db: Db, run: AutomaticRunFacts, options: { personOrigin?: boolean } = {}): Promise { + if (!isAutomaticRun(run)) return false; + const cached = companies.get(run.companyId); + if (cached && cached.snapshot.state === "closed" && Date.now() - cached.readAt < AUTOMATIC_RUN_BRAKE.cacheTtlMs) return false; + if (options.personOrigin ?? await isPersonOrigin(db, run)) return false; + if (cached && cached.heldUntil > Date.now()) return true; + const entry = await current(db, run.companyId); + if (entry.snapshot.state === "closed") return false; + if (entry.heldUntil > Date.now()) return true; + const allowed = entry.snapshot.state === "open" ? await grantProbe(db, entry, run) : await grantRelease(db, entry, run.companyId); + return !allowed; + } + + return { + read, + refresh: async (db: Db, companyId: string) => (await refresh(db, companyId)).snapshot, + creationHeldUntil, + reserveCreation, + holdAutomaticRunStart, + cacheSizes: () => ({ companies: companies.size, runs: personOrigin.size }), + reset: () => { companies.clear(); personOrigin.clear(); }, + }; +} + +export const automaticRunBrake = createAutomaticRunBrake(); + +export function automaticRunCreationHeldUntil(db: Db, companyId: string, work: AutomaticWork) { + return automaticRunBrake.creationHeldUntil(db, companyId, work); +} + +export function reserveAutomaticRunCreation(db: Db, companyId: string, work: AutomaticWork) { + return automaticRunBrake.reserveCreation(db, companyId, work); +} + +export function automaticWorkStatusText(snapshot: AutomaticRunBrakeSnapshot) { + if (snapshot.state === "closed") return "Automatic work is running normally."; + const since = (snapshot.pausedSince ?? snapshot.stateChangedAt)?.toISOString() ?? "recently"; + if (snapshot.state === "half_open") return `Automatic work is resuming gradually since ${snapshot.stateChangedAt?.toISOString() ?? since} because recent runs started normally again.`; + const reason = snapshot.reason as { trigger?: string; recentAutomaticRuns?: number; failedBeforeStarting?: number }; + const because = reason.trigger === "sandbox_refusals" + ? `the last ${AUTOMATIC_RUN_BRAKE.consecutiveProviderRefusals} automatic runs could not get a sandbox` + : `${reason.failedBeforeStarting ?? 0} of ${reason.recentAutomaticRuns ?? 0} recent automatic runs failed before starting`; + const next = snapshot.nextProbeAt ?? snapshot.nextCreationAt; + return `Automatic work is paused for this company since ${since} because ${because}${next ? `; next try at ${next.toISOString()}` : ""}.`; +} + +export async function readAutomaticWorkStatus(db: Db, companyId: string) { + const snapshot = await automaticRunBrake.read(db, companyId); + return { + state: snapshot.state, + pausedSince: snapshot.pausedSince, + nextTryAt: snapshot.state === "open" ? snapshot.nextProbeAt ?? snapshot.nextCreationAt : null, + recentAutomaticRuns: snapshot.evidence.counted, + failedBeforeStarting: snapshot.evidence.failedBeforeStarting, + summary: automaticWorkStatusText(snapshot), + }; +} diff --git a/server/src/services/recovery/continuation-delivery-attempt.ts b/server/src/services/recovery/continuation-delivery-attempt.ts new file mode 100644 index 00000000000..ce3b7c2d221 --- /dev/null +++ b/server/src/services/recovery/continuation-delivery-attempt.ts @@ -0,0 +1,15 @@ +import { and, eq, sql } from "drizzle-orm"; +import { issueRecoveryActions, type Db } from "@paperclipai/db"; + +/** Records a delivery attempt for a pending continuation, so the next pass tries the records not yet tried first. */ +export function recordContinuationDeliveryAttempt(db: Db, action: Pick) { + return db.update(issueRecoveryActions) + .set({ evidence: sql`${issueRecoveryActions.evidence} || ${JSON.stringify( + { continuationDeliveryAttemptedAt: new Date().toISOString() })}::jsonb` }) + .where(and( + eq(issueRecoveryActions.companyId, action.companyId), + eq(issueRecoveryActions.id, action.id), + eq(issueRecoveryActions.status, "resolved"), + sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`, + )); +} diff --git a/server/src/services/recovery/task-automatic-attempts.ts b/server/src/services/recovery/task-automatic-attempts.ts index 20a2aa266f7..e1e47d17673 100644 --- a/server/src/services/recovery/task-automatic-attempts.ts +++ b/server/src/services/recovery/task-automatic-attempts.ts @@ -4,13 +4,15 @@ import { ASSIGNED_WORK_CONTINUED_ACTION, AUTOMATIC_RETRY_REASONS, SCHEDULER_TRAN import { CONTINUATION_CAP, CONTINUATION_CAP_WINDOW_MS } from "./assigned-work-invariant.js"; export { ASSIGNED_WORK_CONTINUED_ACTION, SCHEDULER_TRANSIENT_RETRY_REASON }; +export const PAUSED_FOR_AUTOMATIC_WORK = "pausedForAutomaticWork"; export async function taskAutomaticAttempts(db: Db, companyId: string, issueId: string, at: Date) { const since = new Date(at.getTime() - CONTINUATION_CAP_WINDOW_MS); const [continuations, transientRetries] = await Promise.all([ db.select({ createdAt: activityLog.createdAt, outcome: sql`${activityLog.details}->>'outcome'` }).from(activityLog).where(and( eq(activityLog.companyId, companyId), eq(activityLog.entityType, "issue"), eq(activityLog.entityId, issueId), - eq(activityLog.action, ASSIGNED_WORK_CONTINUED_ACTION), gte(activityLog.createdAt, since))) + eq(activityLog.action, ASSIGNED_WORK_CONTINUED_ACTION), gte(activityLog.createdAt, since), + sql`coalesce(${activityLog.details}->>${PAUSED_FOR_AUTOMATIC_WORK}, '') <> 'true'`)) .orderBy(desc(activityLog.createdAt)).limit(64), db.select({ createdAt: heartbeatRuns.createdAt }).from(heartbeatRuns).where(and( eq(heartbeatRuns.companyId, companyId), sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`, diff --git a/server/src/services/saved-work-recovery-overdue.test.ts b/server/src/services/saved-work-recovery-overdue.test.ts index eb583d68320..fd3f17941e2 100644 --- a/server/src/services/saved-work-recovery-overdue.test.ts +++ b/server/src/services/saved-work-recovery-overdue.test.ts @@ -1,6 +1,6 @@ import { and, eq, sql } from "drizzle-orm"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { activityLog, heartbeatRuns, issueRecoveryActions, issues } from "@paperclipai/db"; +import { activityLog, automaticRunBrakes, heartbeatRuns, issueRecoveryActions, issues } from "@paperclipai/db"; import { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; import { registerIssueRecoveryWork } from "./recovery/work.js"; import type { AdapterSandboxExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; @@ -11,6 +11,7 @@ import { heartbeatService } from "./heartbeat.js"; import { deliverReconciledExecutions } from "./execution-recovery-resolution.js"; import { assignedWorkSweep, ASSIGNED_WORK_CONTINUED_ACTION } from "./recovery/assigned-work-sweep.js"; import { SAVED_WORK_RECOVERY_START_BOUND_MS } from "./recovery/assigned-work-invariant.js"; +import { automaticRunBrake } from "./recovery/automatic-run-brake.js"; import { action, connectionString, db, failedRemotePreparation, useWorkspaceRecoveryFixture } from "./__tests__/workspace-recovery-fixture.js"; const targets = vi.hoisted(() => new Map()); @@ -90,6 +91,17 @@ describe("a saved-work recovery the queue has not reached", () => { expect(successor).toBeDefined(); }, 90_000); + it("starts the overdue recovery while automatic work is paused, without using the pause's creation slot", async () => { + const f = await stagingShape(45 * 60_000); + await db.insert(automaticRunBrakes).values({ companyId: f.companyId, state: "open", openedAt: new Date() }); + automaticRunBrake.reset(); + expect(await f.sweep.checkIssue({ companyId: f.companyId, issueId: f.issueId })) + .toMatchObject({ verdict: { reason: "saved_work_recovery_overdue" }, outcome: "started_saved_work_recovery" }); + expect((await db.select().from(automaticRunBrakes).where(eq(automaticRunBrakes.companyId, f.companyId)))[0]) + .toMatchObject({ state: "open", nextCreationAt: null }); + automaticRunBrake.reset(); + }); + it("counts a recovery the queue has time to start as the task's pending path, and looks again when its bound ends", async () => { let clock = Date.now(); const f = await stagingShape(2 * 60_000, { now: () => new Date(clock) });