mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-26 02:00:52 +01:00
feat(workspaces): verify seeds and add managed repair
Replace binary readiness markers with an evidence-bearing seed manifest and enforce a shared validation contract for minimal and full clones. Add an exclusive audited database repair path with recoverable backup, bounded phase diagnostics, and healthy restart validation. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
4b968d8c05
commit
53a0aa3020
21 files changed
+1568
-154
No files matched your search
@@ -8,13 +8,16 @@ import { eq } from "drizzle-orm";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
agents,
|
||||
authAccounts,
|
||||
authUsers,
|
||||
companies,
|
||||
companyMemberships,
|
||||
createDb,
|
||||
executionWorkspaces,
|
||||
issueComments,
|
||||
issues,
|
||||
projectWorkspaces,
|
||||
instanceUserRoles,
|
||||
projects,
|
||||
routines,
|
||||
routineTriggers,
|
||||
@@ -27,6 +30,7 @@ import {
|
||||
markWorktreeSeedPending,
|
||||
pauseSeededScheduledRoutines,
|
||||
quarantineSeededWorktreeExecutionState,
|
||||
readWorktreeSeedManifest,
|
||||
readSourceAttachmentBody,
|
||||
rebindWorkspaceCwd,
|
||||
resolveSourceConfigPath,
|
||||
@@ -64,6 +68,88 @@ const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
|
||||
const itEmbeddedPostgres = embeddedPostgresSupport.supported ? it : it.skip;
|
||||
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
|
||||
|
||||
function mockVerifiedSeedResult() {
|
||||
return {
|
||||
backupSummary: "snapshot.sql",
|
||||
snapshotAt: "2026-08-18T00:00:00.000Z",
|
||||
migrationRevision: "0142_test.sql",
|
||||
pausedScheduledRoutines: 0,
|
||||
executionQuarantine: {
|
||||
disabledTimerHeartbeats: 0,
|
||||
resetRunningAgents: 0,
|
||||
quarantinedInProgressIssues: 0,
|
||||
unassignedTodoIssues: 0,
|
||||
unassignedReviewIssues: 0,
|
||||
stoppedProjectWorkspaceRuntimes: 0,
|
||||
stoppedExecutionWorkspaceRuntimes: 0,
|
||||
stoppedRuntimeServices: 0,
|
||||
},
|
||||
reboundWorkspaces: [],
|
||||
validation: {
|
||||
authUserCount: 1,
|
||||
credentialAccountCount: 1,
|
||||
instanceAdminCount: 1,
|
||||
activeMembershipCount: 1,
|
||||
companyCount: 1,
|
||||
issueCount: 1,
|
||||
representativeCompanyId: "00000000-0000-4000-8000-000000000001",
|
||||
representativeIssueId: "00000000-0000-4000-8000-000000000002",
|
||||
migrationRevision: "0142_test.sql",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function seedValidWorktreeSource(connectionString: string) {
|
||||
const db = createDb(connectionString);
|
||||
const companyId = randomUUID();
|
||||
const issueId = randomUUID();
|
||||
const now = new Date();
|
||||
await db.insert(authUsers).values({
|
||||
id: "user-existing",
|
||||
email: "existing@paperclip.ing",
|
||||
name: "Existing User",
|
||||
emailVerified: true,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
});
|
||||
await db.insert(authAccounts).values({
|
||||
id: "credential-existing",
|
||||
accountId: "existing@paperclip.ing",
|
||||
providerId: "credential",
|
||||
userId: "user-existing",
|
||||
password: "fixture-password-hash",
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
});
|
||||
await db.insert(instanceUserRoles).values({
|
||||
userId: "user-existing",
|
||||
role: "instance_admin",
|
||||
});
|
||||
await db.insert(companies).values({
|
||||
id: companyId,
|
||||
name: "Seed Source",
|
||||
issuePrefix: "SEED",
|
||||
requireBoardApprovalForNewAgents: false,
|
||||
});
|
||||
await db.insert(companyMemberships).values({
|
||||
companyId,
|
||||
principalType: "user",
|
||||
principalId: "user-existing",
|
||||
status: "active",
|
||||
});
|
||||
await db.insert(issues).values({
|
||||
id: issueId,
|
||||
companyId,
|
||||
title: "Representative seed issue",
|
||||
status: "backlog",
|
||||
priority: "medium",
|
||||
issueNumber: 1,
|
||||
identifier: "SEED-1",
|
||||
});
|
||||
await db.$client.end({ timeout: 5 });
|
||||
return { companyId, issueId };
|
||||
}
|
||||
|
||||
if (!embeddedPostgresSupport.supported) {
|
||||
console.warn(
|
||||
`Skipping embedded Postgres worktree CLI tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`,
|
||||
@@ -400,7 +486,7 @@ describe("worktree helpers", () => {
|
||||
expect(full.nullifyColumns).toEqual({});
|
||||
});
|
||||
|
||||
it("ensure-seeded seeds once and fast-exits on the seed-complete marker", async () => {
|
||||
it("ensure-seeded seeds once and fast-exits on the verified manifest", async () => {
|
||||
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-worktree-ensure-seeded-"));
|
||||
try {
|
||||
const sourceConfigPath = path.join(tempRoot, "source", "config.json");
|
||||
@@ -429,7 +515,7 @@ describe("worktree helpers", () => {
|
||||
markWorktreeSeedPending({ configPath: targetConfigPath, sourceConfigPath });
|
||||
|
||||
const seedDatabase = vi.fn().mockResolvedValue({
|
||||
backupSummary: "snapshot.sql",
|
||||
...mockVerifiedSeedResult(),
|
||||
pausedScheduledRoutines: 2,
|
||||
executionQuarantine: {
|
||||
disabledTimerHeartbeats: 1,
|
||||
@@ -441,7 +527,6 @@ describe("worktree helpers", () => {
|
||||
stoppedExecutionWorkspaceRuntimes: 0,
|
||||
stoppedRuntimeServices: 0,
|
||||
},
|
||||
reboundWorkspaces: [],
|
||||
});
|
||||
|
||||
await expect(
|
||||
@@ -449,7 +534,7 @@ describe("worktree helpers", () => {
|
||||
).resolves.toMatchObject({ seeded: true, reason: "seeded" });
|
||||
await expect(
|
||||
ensureWorktreeSeeded({ config: targetConfigPath }, { seedDatabase }),
|
||||
).resolves.toEqual({ seeded: false, reason: "complete_marker" });
|
||||
).resolves.toEqual({ seeded: false, reason: "verified_manifest" });
|
||||
|
||||
expect(seedDatabase).toHaveBeenCalledTimes(1);
|
||||
expect(seedDatabase).toHaveBeenCalledWith(expect.objectContaining({
|
||||
@@ -458,7 +543,14 @@ describe("worktree helpers", () => {
|
||||
instanceId: "ensure-seeded-test",
|
||||
}));
|
||||
expect(fs.existsSync(path.join(targetRoot, ".paperclip", "seed-pending"))).toBe(false);
|
||||
expect(fs.existsSync(path.join(targetRoot, ".paperclip", "seed-complete"))).toBe(true);
|
||||
expect(fs.existsSync(path.join(targetRoot, ".paperclip", "seed-complete"))).toBe(false);
|
||||
expect(readWorktreeSeedManifest(targetConfigPath)).toMatchObject({
|
||||
version: 2,
|
||||
state: "verified",
|
||||
phase: "complete",
|
||||
migrationRevision: "0142_test.sql",
|
||||
targetInstanceId: "ensure-seeded-test",
|
||||
});
|
||||
} finally {
|
||||
fs.rmSync(tempRoot, { recursive: true, force: true });
|
||||
}
|
||||
@@ -499,7 +591,11 @@ describe("worktree helpers", () => {
|
||||
),
|
||||
).rejects.toThrow("seed failed");
|
||||
|
||||
expect(fs.existsSync(path.join(targetRoot, ".paperclip", "seed-pending"))).toBe(true);
|
||||
expect(readWorktreeSeedManifest(targetConfigPath)).toMatchObject({
|
||||
state: "failed",
|
||||
phase: "pending",
|
||||
});
|
||||
expect(fs.existsSync(path.join(targetRoot, ".paperclip", "seed-pending"))).toBe(false);
|
||||
expect(fs.existsSync(path.join(targetRoot, ".paperclip", "seed-complete"))).toBe(false);
|
||||
expect(fs.existsSync(path.join(targetRoot, ".paperclip", "seed.lock"))).toBe(false);
|
||||
} finally {
|
||||
@@ -537,21 +633,7 @@ describe("worktree helpers", () => {
|
||||
|
||||
const seedDatabase = vi.fn(async () => {
|
||||
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||
return {
|
||||
backupSummary: "snapshot.sql",
|
||||
pausedScheduledRoutines: 0,
|
||||
executionQuarantine: {
|
||||
disabledTimerHeartbeats: 0,
|
||||
resetRunningAgents: 0,
|
||||
quarantinedInProgressIssues: 0,
|
||||
unassignedTodoIssues: 0,
|
||||
unassignedReviewIssues: 0,
|
||||
stoppedProjectWorkspaceRuntimes: 0,
|
||||
stoppedExecutionWorkspaceRuntimes: 0,
|
||||
stoppedRuntimeServices: 0,
|
||||
},
|
||||
reboundWorkspaces: [],
|
||||
};
|
||||
return mockVerifiedSeedResult();
|
||||
});
|
||||
|
||||
const results = await Promise.all([
|
||||
@@ -561,7 +643,7 @@ describe("worktree helpers", () => {
|
||||
|
||||
expect(results).toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({ seeded: true, reason: "seeded" }),
|
||||
{ seeded: false, reason: "complete_marker" },
|
||||
{ seeded: false, reason: "verified_manifest" },
|
||||
]));
|
||||
expect(seedDatabase).toHaveBeenCalledTimes(1);
|
||||
expect(fs.existsSync(path.join(targetRoot, ".paperclip", "seed.lock"))).toBe(false);
|
||||
@@ -570,6 +652,58 @@ describe("worktree helpers", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("records an interrupted phase before retrying to a verified terminal state", async () => {
|
||||
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-worktree-interrupted-seed-"));
|
||||
try {
|
||||
const sourceConfigPath = path.join(tempRoot, "source", "config.json");
|
||||
const targetRoot = path.join(tempRoot, "worktree");
|
||||
const targetConfigPath = path.join(targetRoot, ".paperclip", "config.json");
|
||||
const targetPaths = resolveWorktreeLocalPaths({
|
||||
cwd: targetRoot,
|
||||
homeDir: path.join(tempRoot, "worktree-home"),
|
||||
instanceId: "interrupted-seed",
|
||||
});
|
||||
const sourceConfig = buildSourceConfig();
|
||||
const targetConfig = buildWorktreeConfig({
|
||||
sourceConfig,
|
||||
paths: targetPaths,
|
||||
serverPort: 3196,
|
||||
databasePort: 54996,
|
||||
});
|
||||
fs.mkdirSync(path.dirname(sourceConfigPath), { recursive: true });
|
||||
fs.mkdirSync(path.dirname(targetConfigPath), { recursive: true });
|
||||
fs.writeFileSync(sourceConfigPath, `${JSON.stringify(sourceConfig)}\n`);
|
||||
fs.writeFileSync(targetConfigPath, `${JSON.stringify(targetConfig)}\n`);
|
||||
fs.writeFileSync(
|
||||
path.join(targetRoot, ".paperclip", ".env"),
|
||||
`PAPERCLIP_HOME=${targetPaths.homeDir}\nPAPERCLIP_INSTANCE_ID=${targetPaths.instanceId}\n`,
|
||||
);
|
||||
markWorktreeSeedPending({ configPath: targetConfigPath, sourceConfigPath });
|
||||
const interrupted = readWorktreeSeedManifest(targetConfigPath)!;
|
||||
fs.writeFileSync(
|
||||
path.join(targetRoot, ".paperclip", "seed-manifest.json"),
|
||||
`${JSON.stringify({ ...interrupted, state: "running", phase: "restore" }, null, 2)}\n`,
|
||||
);
|
||||
|
||||
await expect(ensureWorktreeSeeded(
|
||||
{ config: targetConfigPath },
|
||||
{ seedDatabase: vi.fn().mockResolvedValue(mockVerifiedSeedResult()) },
|
||||
)).resolves.toMatchObject({ seeded: true, reason: "seeded" });
|
||||
|
||||
const verified = readWorktreeSeedManifest(targetConfigPath)!;
|
||||
expect(verified.state).toBe("verified");
|
||||
expect(verified.diagnostics).toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
phase: "restore",
|
||||
status: "failed",
|
||||
message: "The previous seed attempt ended without a terminal result.",
|
||||
}),
|
||||
]));
|
||||
} finally {
|
||||
fs.rmSync(tempRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("fails closed instead of racing to reclaim a stale seed lock", async () => {
|
||||
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-worktree-ensure-seeded-stale-lock-"));
|
||||
try {
|
||||
@@ -983,15 +1117,7 @@ describe("worktree helpers", () => {
|
||||
const sourceDb = await startEmbeddedPostgresTestDatabase("paperclip-worktree-auth-source-");
|
||||
|
||||
try {
|
||||
const sourceDbClient = createDb(sourceDb.connectionString);
|
||||
await sourceDbClient.insert(authUsers).values({
|
||||
id: "user-existing",
|
||||
email: "existing@paperclip.ing",
|
||||
name: "Existing User",
|
||||
emailVerified: true,
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
});
|
||||
await seedValidWorktreeSource(sourceDb.connectionString);
|
||||
|
||||
fs.mkdirSync(path.dirname(sourceKeyPath), { recursive: true });
|
||||
fs.mkdirSync(worktreeRoot, { recursive: true });
|
||||
@@ -1028,6 +1154,18 @@ describe("worktree helpers", () => {
|
||||
const targetConfig = JSON.parse(
|
||||
fs.readFileSync(path.join(worktreeRoot, ".paperclip", "config.json"), "utf8"),
|
||||
) as PaperclipConfig;
|
||||
const manifestText = fs.readFileSync(
|
||||
path.join(worktreeRoot, ".paperclip", "seed-manifest.json"),
|
||||
"utf8",
|
||||
);
|
||||
expect(JSON.parse(manifestText)).toMatchObject({
|
||||
version: 2,
|
||||
seedMode: "minimal",
|
||||
state: "verified",
|
||||
phase: "complete",
|
||||
});
|
||||
expect(manifestText).not.toContain("fixture-password-hash");
|
||||
expect(manifestText).not.toContain("source-master-key");
|
||||
const { default: EmbeddedPostgres } = await import("embedded-postgres");
|
||||
const targetPg = new EmbeddedPostgres({
|
||||
databaseDir: targetConfig.database.embeddedPostgresDataDir,
|
||||
@@ -1284,9 +1422,8 @@ describe("worktree helpers", () => {
|
||||
const originalCwd = process.cwd();
|
||||
const originalPaperclipConfig = process.env.PAPERCLIP_CONFIG;
|
||||
const currentDatabaseReservation = await reserveTestPort();
|
||||
const sourceDatabaseReservation = await reserveTestPort();
|
||||
const currentDatabasePort = currentDatabaseReservation.port;
|
||||
const sourceDatabasePort = sourceDatabaseReservation.port;
|
||||
const sourceDb = await startEmbeddedPostgresTestDatabase("paperclip-worktree-reseed-source-");
|
||||
|
||||
try {
|
||||
fs.mkdirSync(path.dirname(currentPaths.configPath), { recursive: true });
|
||||
@@ -1301,15 +1438,28 @@ describe("worktree helpers", () => {
|
||||
serverPort: 3114,
|
||||
databasePort: currentDatabasePort,
|
||||
});
|
||||
const sourceConfig = buildWorktreeConfig({
|
||||
sourceConfig: buildSourceConfig(),
|
||||
paths: sourcePaths,
|
||||
serverPort: 3200,
|
||||
databasePort: sourceDatabasePort,
|
||||
});
|
||||
const sourceConfig = buildSourceConfig();
|
||||
sourceConfig.database = {
|
||||
mode: "postgres",
|
||||
embeddedPostgresDataDir: sourcePaths.embeddedPostgresDataDir,
|
||||
embeddedPostgresPort: 54329,
|
||||
backup: {
|
||||
enabled: true,
|
||||
intervalMinutes: 60,
|
||||
retentionDays: 30,
|
||||
dir: sourcePaths.backupDir,
|
||||
},
|
||||
connectionString: sourceDb.connectionString,
|
||||
};
|
||||
sourceConfig.logging.logDir = sourcePaths.logDir;
|
||||
sourceConfig.storage.localDisk.baseDir = sourcePaths.storageDir;
|
||||
sourceConfig.secrets.localEncrypted.keyFilePath = sourcePaths.secretsKeyFilePath;
|
||||
await seedValidWorktreeSource(sourceDb.connectionString);
|
||||
fs.writeFileSync(currentPaths.configPath, JSON.stringify(currentConfig, null, 2), "utf8");
|
||||
fs.writeFileSync(sourcePaths.configPath, JSON.stringify(sourceConfig, null, 2), "utf8");
|
||||
fs.writeFileSync(sourcePaths.secretsKeyFilePath, "source-secret", "utf8");
|
||||
const worktreeSentinelPath = path.join(repoRoot, "user-worktree-file.txt");
|
||||
fs.writeFileSync(worktreeSentinelPath, "preserve me", "utf8");
|
||||
fs.writeFileSync(
|
||||
currentPaths.envPath,
|
||||
[
|
||||
@@ -1325,11 +1475,11 @@ describe("worktree helpers", () => {
|
||||
process.chdir(repoRoot);
|
||||
|
||||
await currentDatabaseReservation.release();
|
||||
await sourceDatabaseReservation.release();
|
||||
|
||||
await worktreeReseedCommand({
|
||||
fromConfig: sourcePaths.configPath,
|
||||
yes: true,
|
||||
backupTarget: true,
|
||||
});
|
||||
|
||||
const rewrittenConfig = JSON.parse(fs.readFileSync(currentPaths.configPath, "utf8"));
|
||||
@@ -1341,9 +1491,13 @@ describe("worktree helpers", () => {
|
||||
expect(rewrittenEnv).toContain(`PAPERCLIP_INSTANCE_ID=${currentInstanceId}`);
|
||||
expect(rewrittenEnv).toContain("PAPERCLIP_WORKTREE_NAME=existing-name");
|
||||
expect(rewrittenEnv).toContain("PAPERCLIP_WORKTREE_COLOR=\"#112233\"");
|
||||
expect(fs.readFileSync(worktreeSentinelPath, "utf8")).toBe("preserve me");
|
||||
expect(
|
||||
fs.readdirSync(path.join(currentPaths.backupDir, "repair")).some((name) => name.endsWith(".sql.gz")),
|
||||
).toBe(true);
|
||||
} finally {
|
||||
await currentDatabaseReservation.release();
|
||||
await sourceDatabaseReservation.release();
|
||||
await sourceDb.cleanup();
|
||||
process.chdir(originalCwd);
|
||||
if (originalPaperclipConfig === undefined) {
|
||||
delete process.env.PAPERCLIP_CONFIG;
|
||||
|
||||
@@ -5,12 +5,52 @@ import { expandHomePrefix } from "../config/home.js";
|
||||
|
||||
export const DEFAULT_WORKTREE_HOME = "~/.paperclip-worktrees";
|
||||
export const WORKTREE_SEED_MODES = ["minimal", "full"] as const;
|
||||
export const WORKTREE_SEED_MANIFEST = "seed-manifest.json";
|
||||
export const WORKTREE_SEED_PENDING_MARKER = "seed-pending";
|
||||
export const WORKTREE_SEED_COMPLETE_MARKER = "seed-complete";
|
||||
export const WORKTREE_SEED_LOCK_MARKER = "seed.lock";
|
||||
|
||||
export type WorktreeSeedMode = (typeof WORKTREE_SEED_MODES)[number];
|
||||
|
||||
export const WORKTREE_SEED_PHASES = [
|
||||
"pending",
|
||||
"source_validation",
|
||||
"snapshot",
|
||||
"restore",
|
||||
"migrations",
|
||||
"execution_quarantine",
|
||||
"routine_pause",
|
||||
"workspace_rebind",
|
||||
"post_restore_validation",
|
||||
"complete",
|
||||
] as const;
|
||||
|
||||
export type WorktreeSeedPhase = (typeof WORKTREE_SEED_PHASES)[number];
|
||||
export type WorktreeSeedState = "pending" | "running" | "verified" | "failed";
|
||||
|
||||
export type WorktreeSeedManifest = {
|
||||
version: 2;
|
||||
source: {
|
||||
instanceId: string;
|
||||
configPath: string;
|
||||
};
|
||||
snapshotAt: string | null;
|
||||
seedMode: WorktreeSeedMode;
|
||||
migrationRevision: string | null;
|
||||
targetInstanceId: string;
|
||||
phase: WorktreeSeedPhase;
|
||||
state: WorktreeSeedState;
|
||||
attemptId: string;
|
||||
startedAt: string | null;
|
||||
finishedAt: string | null;
|
||||
diagnostics: Array<{
|
||||
phase: WorktreeSeedPhase;
|
||||
status: "started" | "succeeded" | "failed";
|
||||
at: string;
|
||||
message?: string;
|
||||
}>;
|
||||
};
|
||||
|
||||
export type WorktreeSeedPlan = {
|
||||
mode: WorktreeSeedMode;
|
||||
excludedTables: string[];
|
||||
@@ -54,6 +94,7 @@ export type WorktreeUiBranding = {
|
||||
};
|
||||
|
||||
export type WorktreeSeedMarkerPaths = {
|
||||
manifest: string;
|
||||
pending: string;
|
||||
complete: string;
|
||||
lock: string;
|
||||
@@ -62,6 +103,7 @@ export type WorktreeSeedMarkerPaths = {
|
||||
export function resolveWorktreeSeedMarkerPaths(configPath: string): WorktreeSeedMarkerPaths {
|
||||
const configDir = path.dirname(path.resolve(configPath));
|
||||
return {
|
||||
manifest: path.resolve(configDir, WORKTREE_SEED_MANIFEST),
|
||||
pending: path.resolve(configDir, WORKTREE_SEED_PENDING_MARKER),
|
||||
complete: path.resolve(configDir, WORKTREE_SEED_COMPLETE_MARKER),
|
||||
lock: path.resolve(configDir, WORKTREE_SEED_LOCK_MARKER),
|
||||
|
||||
+551
-52
@@ -7,6 +7,7 @@ import {
|
||||
readdirSync,
|
||||
readFileSync,
|
||||
readlinkSync,
|
||||
renameSync,
|
||||
rmSync,
|
||||
statSync,
|
||||
symlinkSync,
|
||||
@@ -24,8 +25,11 @@ import { and, eq, inArray, sql } from "drizzle-orm";
|
||||
import {
|
||||
applyPendingMigrations,
|
||||
agents,
|
||||
authAccounts,
|
||||
authUsers,
|
||||
assets,
|
||||
companies,
|
||||
companyMemberships,
|
||||
createDb,
|
||||
documentRevisions,
|
||||
documents,
|
||||
@@ -39,6 +43,7 @@ import {
|
||||
issueComments,
|
||||
issueDocuments,
|
||||
issues,
|
||||
instanceUserRoles,
|
||||
projectWorkspaces,
|
||||
projects,
|
||||
routines,
|
||||
@@ -65,13 +70,16 @@ import {
|
||||
formatShellExports,
|
||||
generateWorktreeColor,
|
||||
isWorktreeSeedMode,
|
||||
WORKTREE_SEED_PHASES,
|
||||
resolveSuggestedWorktreeName,
|
||||
resolveWorktreeSeedPlan,
|
||||
resolveWorktreeSeedMarkerPaths,
|
||||
resolveWorktreeLocalPaths,
|
||||
sanitizeWorktreeInstanceId,
|
||||
type WorktreeSeedPlan,
|
||||
type WorktreeSeedManifest,
|
||||
type WorktreeSeedMode,
|
||||
type WorktreeSeedPhase,
|
||||
type WorktreeLocalPaths,
|
||||
} from "./worktree-lib.js";
|
||||
import {
|
||||
@@ -137,6 +145,7 @@ type WorktreeReseedOptions = {
|
||||
preserveLiveWork?: boolean;
|
||||
yes?: boolean;
|
||||
allowLiveTarget?: boolean;
|
||||
backupTarget?: boolean;
|
||||
};
|
||||
|
||||
type WorktreeRepairOptions = {
|
||||
@@ -197,6 +206,8 @@ type CopiedGitHooksResult = {
|
||||
|
||||
type SeedWorktreeDatabaseResult = {
|
||||
backupSummary: string;
|
||||
snapshotAt: string;
|
||||
migrationRevision: string;
|
||||
pausedScheduledRoutines: number;
|
||||
executionQuarantine: SeededWorktreeExecutionQuarantineSummary;
|
||||
reboundWorkspaces: Array<{
|
||||
@@ -204,28 +215,26 @@ type SeedWorktreeDatabaseResult = {
|
||||
fromCwd: string;
|
||||
toCwd: string;
|
||||
}>;
|
||||
validation: WorktreeSeedValidationSummary;
|
||||
};
|
||||
|
||||
type WorktreeSeedPendingMarker = {
|
||||
version: 1;
|
||||
state: "pending";
|
||||
sourceConfigPath: string;
|
||||
seedMode: "minimal";
|
||||
createdAt: string;
|
||||
};
|
||||
|
||||
type WorktreeSeedCompleteMarker = {
|
||||
version: 1;
|
||||
state: "complete";
|
||||
seedMode: WorktreeSeedMode;
|
||||
completedAt: string;
|
||||
export type WorktreeSeedValidationSummary = {
|
||||
authUserCount: number;
|
||||
credentialAccountCount: number;
|
||||
instanceAdminCount: number;
|
||||
activeMembershipCount: number;
|
||||
companyCount: number;
|
||||
issueCount: number;
|
||||
representativeCompanyId: string;
|
||||
representativeIssueId: string;
|
||||
migrationRevision: string;
|
||||
};
|
||||
|
||||
type SeedWorktreeDatabase = typeof seedWorktreeDatabase;
|
||||
|
||||
export type EnsureWorktreeSeededResult = {
|
||||
seeded: boolean;
|
||||
reason: "seeded" | "complete_marker" | "legacy_unmarked";
|
||||
reason: "seeded" | "verified_manifest" | "complete_marker" | "legacy_unmarked";
|
||||
details?: SeedWorktreeDatabaseResult;
|
||||
};
|
||||
|
||||
@@ -1393,6 +1402,140 @@ export async function quarantineSeededWorktreeExecutionState(
|
||||
}
|
||||
}
|
||||
|
||||
type WorktreeSeedValidationExpectation = {
|
||||
adminUserId: string;
|
||||
representativeCompanyId: string;
|
||||
representativeIssueId: string;
|
||||
};
|
||||
|
||||
async function inspectVerifiedSeedDatabase(
|
||||
connectionString: string,
|
||||
expected?: WorktreeSeedValidationExpectation,
|
||||
): Promise<{ summary: WorktreeSeedValidationSummary; expectation: WorktreeSeedValidationExpectation }> {
|
||||
const migrationState = await inspectMigrations(connectionString);
|
||||
if (migrationState.status !== "upToDate") {
|
||||
throw new Error(
|
||||
`Migration journal is not current (${migrationState.pendingMigrations.length} pending migration(s)).`,
|
||||
);
|
||||
}
|
||||
const migrationRevision = migrationState.appliedMigrations.at(-1);
|
||||
if (!migrationRevision) {
|
||||
throw new Error("Migration journal has no applied revision.");
|
||||
}
|
||||
|
||||
const db = createDb(connectionString);
|
||||
try {
|
||||
const [counts] = await db
|
||||
.select({
|
||||
authUserCount: sql<number>`count(distinct ${authUsers.id})::int`,
|
||||
credentialAccountCount: sql<number>`count(distinct ${authAccounts.id})::int`,
|
||||
instanceAdminCount: sql<number>`count(distinct ${instanceUserRoles.userId})::int`,
|
||||
activeMembershipCount: sql<number>`count(distinct ${companyMemberships.id})::int`,
|
||||
companyCount: sql<number>`count(distinct ${companies.id})::int`,
|
||||
issueCount: sql<number>`count(distinct ${issues.id})::int`,
|
||||
})
|
||||
.from(authUsers)
|
||||
.leftJoin(authAccounts, eq(authAccounts.userId, authUsers.id))
|
||||
.leftJoin(
|
||||
instanceUserRoles,
|
||||
and(eq(instanceUserRoles.userId, authUsers.id), eq(instanceUserRoles.role, "instance_admin")),
|
||||
)
|
||||
.leftJoin(
|
||||
companyMemberships,
|
||||
and(
|
||||
eq(companyMemberships.principalType, "user"),
|
||||
eq(companyMemberships.principalId, authUsers.id),
|
||||
eq(companyMemberships.status, "active"),
|
||||
),
|
||||
)
|
||||
.leftJoin(companies, eq(companies.id, companyMemberships.companyId))
|
||||
.leftJoin(issues, eq(issues.companyId, companies.id));
|
||||
|
||||
const admin = await db
|
||||
.select({ userId: authUsers.id })
|
||||
.from(authUsers)
|
||||
.innerJoin(
|
||||
instanceUserRoles,
|
||||
and(eq(instanceUserRoles.userId, authUsers.id), eq(instanceUserRoles.role, "instance_admin")),
|
||||
)
|
||||
.innerJoin(
|
||||
authAccounts,
|
||||
and(
|
||||
eq(authAccounts.userId, authUsers.id),
|
||||
sql`length(trim(${authAccounts.providerId})) > 0`,
|
||||
sql`length(trim(${authAccounts.accountId})) > 0`,
|
||||
),
|
||||
)
|
||||
.innerJoin(
|
||||
companyMemberships,
|
||||
and(
|
||||
eq(companyMemberships.principalType, "user"),
|
||||
eq(companyMemberships.principalId, authUsers.id),
|
||||
eq(companyMemberships.status, "active"),
|
||||
),
|
||||
)
|
||||
.where(expected ? eq(authUsers.id, expected.adminUserId) : undefined)
|
||||
.limit(1)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
if (!admin) {
|
||||
throw new Error(
|
||||
"No auth user has a non-empty credential account, instance-admin role, and active company membership.",
|
||||
);
|
||||
}
|
||||
|
||||
const representative = await db
|
||||
.select({ companyId: companies.id, issueId: issues.id })
|
||||
.from(companies)
|
||||
.innerJoin(issues, eq(issues.companyId, companies.id))
|
||||
.where(
|
||||
expected
|
||||
? and(
|
||||
eq(companies.id, expected.representativeCompanyId),
|
||||
eq(issues.id, expected.representativeIssueId),
|
||||
)
|
||||
: undefined,
|
||||
)
|
||||
.limit(1)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
if (!representative) {
|
||||
throw new Error("No representative cloned company and issue pair is readable.");
|
||||
}
|
||||
|
||||
const summary: WorktreeSeedValidationSummary = {
|
||||
authUserCount: counts?.authUserCount ?? 0,
|
||||
credentialAccountCount: counts?.credentialAccountCount ?? 0,
|
||||
instanceAdminCount: counts?.instanceAdminCount ?? 0,
|
||||
activeMembershipCount: counts?.activeMembershipCount ?? 0,
|
||||
companyCount: counts?.companyCount ?? 0,
|
||||
issueCount: counts?.issueCount ?? 0,
|
||||
representativeCompanyId: representative.companyId,
|
||||
representativeIssueId: representative.issueId,
|
||||
migrationRevision,
|
||||
};
|
||||
if (
|
||||
summary.authUserCount < 1
|
||||
|| summary.credentialAccountCount < 1
|
||||
|| summary.instanceAdminCount < 1
|
||||
|| summary.activeMembershipCount < 1
|
||||
|| summary.companyCount < 1
|
||||
|| summary.issueCount < 1
|
||||
) {
|
||||
throw new Error("Seed validation found an incomplete auth, membership, company, or issue shape.");
|
||||
}
|
||||
|
||||
return {
|
||||
summary,
|
||||
expectation: {
|
||||
adminUserId: admin.userId,
|
||||
representativeCompanyId: representative.companyId,
|
||||
representativeIssueId: representative.issueId,
|
||||
},
|
||||
};
|
||||
} finally {
|
||||
await db.$client?.end?.({ timeout: 5 }).catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
async function seedWorktreeDatabase(input: {
|
||||
sourceConfigPath: string;
|
||||
sourceConfig: PaperclipConfig;
|
||||
@@ -1401,6 +1544,7 @@ async function seedWorktreeDatabase(input: {
|
||||
instanceId: string;
|
||||
seedMode: WorktreeSeedMode;
|
||||
preserveLiveWork?: boolean;
|
||||
onPhase?: (phase: WorktreeSeedPhase, status: "started" | "succeeded", message?: string) => void;
|
||||
}): Promise<SeedWorktreeDatabaseResult> {
|
||||
const seedPlan = resolveWorktreeSeedPlan(input.seedMode);
|
||||
const sourceEnvFile = resolvePaperclipEnvFile(input.sourceConfigPath);
|
||||
@@ -1428,6 +1572,16 @@ async function seedWorktreeDatabase(input: {
|
||||
sourceEnvEntries,
|
||||
sourceHandle?.port,
|
||||
);
|
||||
input.onPhase?.("source_validation", "started");
|
||||
const sourceValidation = await inspectVerifiedSeedDatabase(sourceConnectionString);
|
||||
input.onPhase?.(
|
||||
"source_validation",
|
||||
"succeeded",
|
||||
`Validated ${sourceValidation.summary.companyCount} company record(s) and ${sourceValidation.summary.issueCount} issue record(s).`,
|
||||
);
|
||||
|
||||
const snapshotAt = new Date().toISOString();
|
||||
input.onPhase?.("snapshot", "started");
|
||||
const backup = await runDatabaseBackup({
|
||||
connectionString: sourceConnectionString,
|
||||
backupDir: path.resolve(input.targetPaths.backupDir, "seed"),
|
||||
@@ -1438,6 +1592,7 @@ async function seedWorktreeDatabase(input: {
|
||||
excludeTables: seedPlan.excludedTables,
|
||||
nullifyColumns: seedPlan.nullifyColumns,
|
||||
});
|
||||
input.onPhase?.("snapshot", "succeeded", `Created ${path.basename(backup.backupFile)}.`);
|
||||
|
||||
targetHandle = await ensureEmbeddedPostgres(
|
||||
input.targetConfig.database.embeddedPostgresDataDir,
|
||||
@@ -1445,27 +1600,56 @@ async function seedWorktreeDatabase(input: {
|
||||
);
|
||||
|
||||
const adminConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${targetHandle.port}/postgres`;
|
||||
input.onPhase?.("restore", "started");
|
||||
await resetPostgresDatabase(adminConnectionString, "paperclip");
|
||||
const targetConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${targetHandle.port}/paperclip`;
|
||||
await runDatabaseRestore({
|
||||
connectionString: targetConnectionString,
|
||||
backupFile: backup.backupFile,
|
||||
});
|
||||
input.onPhase?.("restore", "succeeded");
|
||||
input.onPhase?.("migrations", "started");
|
||||
await applyPendingMigrations(targetConnectionString);
|
||||
input.onPhase?.("migrations", "succeeded");
|
||||
input.onPhase?.("execution_quarantine", "started");
|
||||
const executionQuarantine = input.preserveLiveWork
|
||||
? { ...EMPTY_SEEDED_WORKTREE_EXECUTION_QUARANTINE_SUMMARY }
|
||||
: await quarantineSeededWorktreeExecutionState(targetConnectionString);
|
||||
input.onPhase?.(
|
||||
"execution_quarantine",
|
||||
"succeeded",
|
||||
input.preserveLiveWork
|
||||
? "Preserved copied live work by explicit request."
|
||||
: formatSeededWorktreeExecutionQuarantineSummary(executionQuarantine),
|
||||
);
|
||||
input.onPhase?.("routine_pause", "started");
|
||||
const pausedScheduledRoutines = await pauseSeededScheduledRoutines(targetConnectionString);
|
||||
input.onPhase?.("routine_pause", "succeeded", `Paused ${pausedScheduledRoutines} scheduled routine(s).`);
|
||||
input.onPhase?.("workspace_rebind", "started");
|
||||
const reboundWorkspaces = await rebindSeededProjectWorkspaces({
|
||||
targetConnectionString,
|
||||
currentCwd: input.targetPaths.cwd,
|
||||
});
|
||||
input.onPhase?.("workspace_rebind", "succeeded", `Rebound ${reboundWorkspaces.length} workspace path(s).`);
|
||||
input.onPhase?.("post_restore_validation", "started");
|
||||
const targetValidation = await inspectVerifiedSeedDatabase(
|
||||
targetConnectionString,
|
||||
sourceValidation.expectation,
|
||||
);
|
||||
input.onPhase?.(
|
||||
"post_restore_validation",
|
||||
"succeeded",
|
||||
`Validated migration ${targetValidation.summary.migrationRevision}.`,
|
||||
);
|
||||
|
||||
return {
|
||||
backupSummary: formatDatabaseBackupResult(backup),
|
||||
snapshotAt,
|
||||
migrationRevision: targetValidation.summary.migrationRevision,
|
||||
pausedScheduledRoutines,
|
||||
executionQuarantine,
|
||||
reboundWorkspaces,
|
||||
validation: targetValidation.summary,
|
||||
};
|
||||
} finally {
|
||||
if (targetHandle?.startedByThisProcess) {
|
||||
@@ -1477,46 +1661,184 @@ async function seedWorktreeDatabase(input: {
|
||||
}
|
||||
}
|
||||
|
||||
function writeWorktreeSeedMarker(
|
||||
filePath: string,
|
||||
marker: WorktreeSeedPendingMarker | WorktreeSeedCompleteMarker,
|
||||
): void {
|
||||
const WORKTREE_SEED_DIAGNOSTIC_LIMIT = 32;
|
||||
const WORKTREE_SEED_DIAGNOSTIC_MESSAGE_LIMIT = 512;
|
||||
const activeSeedInterruptHandlers = new Map<string, (signal: NodeJS.Signals) => void>();
|
||||
|
||||
function dispatchSeedInterruption(signal: NodeJS.Signals): void {
|
||||
for (const handler of activeSeedInterruptHandlers.values()) {
|
||||
try {
|
||||
handler(signal);
|
||||
} catch {
|
||||
// Continue terminalizing the other active manifests before exiting.
|
||||
}
|
||||
}
|
||||
process.exit(signal === "SIGINT" ? 130 : 143);
|
||||
}
|
||||
|
||||
const dispatchSeedSigint = () => dispatchSeedInterruption("SIGINT");
|
||||
const dispatchSeedSigterm = () => dispatchSeedInterruption("SIGTERM");
|
||||
|
||||
function registerSeedInterruptHandler(handler: (signal: NodeJS.Signals) => void): () => void {
|
||||
const id = randomUUID();
|
||||
if (activeSeedInterruptHandlers.size === 0) {
|
||||
process.once("SIGINT", dispatchSeedSigint);
|
||||
process.once("SIGTERM", dispatchSeedSigterm);
|
||||
}
|
||||
activeSeedInterruptHandlers.set(id, handler);
|
||||
return () => {
|
||||
activeSeedInterruptHandlers.delete(id);
|
||||
if (activeSeedInterruptHandlers.size === 0) {
|
||||
process.off("SIGINT", dispatchSeedSigint);
|
||||
process.off("SIGTERM", dispatchSeedSigterm);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
type LegacyWorktreeSeedPendingMarker = {
|
||||
version: 1;
|
||||
state: "pending";
|
||||
sourceConfigPath: string;
|
||||
};
|
||||
|
||||
function resolveSeedInstanceId(configPath: string): string {
|
||||
const envEntries = readPaperclipEnvEntries(resolvePaperclipEnvFile(configPath));
|
||||
return nonEmpty(envEntries.PAPERCLIP_INSTANCE_ID)
|
||||
?? sanitizeWorktreeInstanceId(path.basename(path.dirname(path.resolve(configPath))));
|
||||
}
|
||||
|
||||
function writeWorktreeSeedManifest(filePath: string, manifest: WorktreeSeedManifest): void {
|
||||
mkdirSync(path.dirname(filePath), { recursive: true });
|
||||
writeFileSync(filePath, `${JSON.stringify(marker, null, 2)}\n`, { mode: 0o600 });
|
||||
const temporaryPath = `${filePath}.${process.pid}.${randomUUID()}.tmp`;
|
||||
writeFileSync(temporaryPath, `${JSON.stringify(manifest, null, 2)}\n`, { mode: 0o600 });
|
||||
renameSync(temporaryPath, filePath);
|
||||
}
|
||||
|
||||
export function readWorktreeSeedManifest(configPath: string): WorktreeSeedManifest | null {
|
||||
const manifestPath = resolveWorktreeSeedMarkerPaths(configPath).manifest;
|
||||
if (!existsSync(manifestPath)) return null;
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(readFileSync(manifestPath, "utf8"));
|
||||
} catch (error) {
|
||||
throw new Error(
|
||||
`Invalid worktree seed manifest at ${manifestPath}: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
}
|
||||
const value = parsed as Partial<WorktreeSeedManifest>;
|
||||
const diagnosticsValid = Array.isArray(value.diagnostics) && value.diagnostics.every((diagnostic) => (
|
||||
diagnostic
|
||||
&& typeof diagnostic === "object"
|
||||
&& WORKTREE_SEED_PHASES.includes(diagnostic.phase)
|
||||
&& ["started", "succeeded", "failed"].includes(diagnostic.status)
|
||||
&& typeof diagnostic.at === "string"
|
||||
&& (diagnostic.message === undefined || typeof diagnostic.message === "string")
|
||||
));
|
||||
const verifiedTerminalValid = value.state !== "verified" || (
|
||||
value.phase === "complete"
|
||||
&& typeof value.snapshotAt === "string"
|
||||
&& value.snapshotAt.length > 0
|
||||
&& typeof value.migrationRevision === "string"
|
||||
&& value.migrationRevision.length > 0
|
||||
&& typeof value.startedAt === "string"
|
||||
&& typeof value.finishedAt === "string"
|
||||
&& value.diagnostics?.some((diagnostic) => (
|
||||
diagnostic.phase === "complete" && diagnostic.status === "succeeded"
|
||||
)) === true
|
||||
);
|
||||
if (
|
||||
!value
|
||||
|| typeof value !== "object"
|
||||
|| value.version !== 2
|
||||
|| !value.source
|
||||
|| typeof value.source.instanceId !== "string"
|
||||
|| typeof value.source.configPath !== "string"
|
||||
|| typeof value.targetInstanceId !== "string"
|
||||
|| value.targetInstanceId.length === 0
|
||||
|| !isWorktreeSeedMode(String(value.seedMode ?? ""))
|
||||
|| !WORKTREE_SEED_PHASES.includes(value.phase as WorktreeSeedPhase)
|
||||
|| !["pending", "running", "verified", "failed"].includes(String(value.state ?? ""))
|
||||
|| typeof value.attemptId !== "string"
|
||||
|| value.attemptId.length === 0
|
||||
|| !diagnosticsValid
|
||||
|| !verifiedTerminalValid
|
||||
) {
|
||||
throw new Error(`Invalid worktree seed manifest at ${manifestPath}.`);
|
||||
}
|
||||
return value as WorktreeSeedManifest;
|
||||
}
|
||||
|
||||
export function markWorktreeSeedPending(input: {
|
||||
configPath: string;
|
||||
sourceConfigPath: string;
|
||||
now?: Date;
|
||||
}): void {
|
||||
const markers = resolveWorktreeSeedMarkerPaths(input.configPath);
|
||||
rmSync(markers.complete, { force: true });
|
||||
writeWorktreeSeedMarker(markers.pending, {
|
||||
version: 1,
|
||||
state: "pending",
|
||||
sourceConfigPath: path.resolve(input.sourceConfigPath),
|
||||
seedMode: "minimal",
|
||||
createdAt: (input.now ?? new Date()).toISOString(),
|
||||
});
|
||||
}
|
||||
|
||||
export function markWorktreeSeedComplete(input: {
|
||||
configPath: string;
|
||||
targetInstanceId?: string;
|
||||
seedMode?: WorktreeSeedMode;
|
||||
now?: Date;
|
||||
}): void {
|
||||
const markers = resolveWorktreeSeedMarkerPaths(input.configPath);
|
||||
writeWorktreeSeedMarker(markers.complete, {
|
||||
version: 1,
|
||||
state: "complete",
|
||||
const at = (input.now ?? new Date()).toISOString();
|
||||
writeWorktreeSeedManifest(markers.manifest, {
|
||||
version: 2,
|
||||
source: {
|
||||
instanceId: resolveSeedInstanceId(input.sourceConfigPath),
|
||||
configPath: path.resolve(input.sourceConfigPath),
|
||||
},
|
||||
snapshotAt: null,
|
||||
seedMode: input.seedMode ?? "minimal",
|
||||
completedAt: (input.now ?? new Date()).toISOString(),
|
||||
migrationRevision: null,
|
||||
targetInstanceId: input.targetInstanceId ?? resolveSeedInstanceId(input.configPath),
|
||||
phase: "pending",
|
||||
state: "pending",
|
||||
attemptId: randomUUID(),
|
||||
startedAt: null,
|
||||
finishedAt: null,
|
||||
diagnostics: [{ phase: "pending", status: "succeeded", at }],
|
||||
});
|
||||
// New manifests are authoritative. Legacy files are removed so no caller can
|
||||
// mistake a stale binary marker for current verified seed state.
|
||||
rmSync(markers.complete, { force: true });
|
||||
rmSync(markers.pending, { force: true });
|
||||
}
|
||||
|
||||
function readWorktreeSeedPendingMarker(filePath: string): WorktreeSeedPendingMarker {
|
||||
function updateWorktreeSeedManifest(input: {
|
||||
configPath: string;
|
||||
phase: WorktreeSeedPhase;
|
||||
status: "started" | "succeeded" | "failed";
|
||||
state?: WorktreeSeedManifest["state"];
|
||||
message?: string;
|
||||
snapshotAt?: string | null;
|
||||
migrationRevision?: string | null;
|
||||
now?: Date;
|
||||
}): WorktreeSeedManifest {
|
||||
const markers = resolveWorktreeSeedMarkerPaths(input.configPath);
|
||||
const current = readWorktreeSeedManifest(input.configPath);
|
||||
if (!current) throw new Error(`Worktree seed manifest does not exist at ${markers.manifest}.`);
|
||||
const at = (input.now ?? new Date()).toISOString();
|
||||
const nextState = input.state ?? current.state;
|
||||
const diagnostic = {
|
||||
phase: input.phase,
|
||||
status: input.status,
|
||||
at,
|
||||
...(input.message
|
||||
? { message: input.message.slice(0, WORKTREE_SEED_DIAGNOSTIC_MESSAGE_LIMIT) }
|
||||
: {}),
|
||||
};
|
||||
const next: WorktreeSeedManifest = {
|
||||
...current,
|
||||
phase: input.phase,
|
||||
state: nextState,
|
||||
snapshotAt: input.snapshotAt === undefined ? current.snapshotAt : input.snapshotAt,
|
||||
migrationRevision:
|
||||
input.migrationRevision === undefined ? current.migrationRevision : input.migrationRevision,
|
||||
startedAt: current.startedAt ?? (input.status === "started" ? at : null),
|
||||
finishedAt: nextState === "verified" || nextState === "failed" ? at : null,
|
||||
diagnostics: [...current.diagnostics, diagnostic].slice(-WORKTREE_SEED_DIAGNOSTIC_LIMIT),
|
||||
};
|
||||
writeWorktreeSeedManifest(markers.manifest, next);
|
||||
return next;
|
||||
}
|
||||
|
||||
function readLegacyWorktreeSeedPendingMarker(filePath: string): LegacyWorktreeSeedPendingMarker {
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(readFileSync(filePath, "utf8"));
|
||||
@@ -1537,7 +1859,7 @@ function readWorktreeSeedPendingMarker(filePath: string): WorktreeSeedPendingMar
|
||||
throw new Error(`Invalid worktree seed-pending marker at ${filePath}.`);
|
||||
}
|
||||
|
||||
return parsed as WorktreeSeedPendingMarker;
|
||||
return parsed as LegacyWorktreeSeedPendingMarker;
|
||||
}
|
||||
|
||||
const WORKTREE_SEED_LOCK_POLL_MS = 50;
|
||||
@@ -1631,6 +1953,117 @@ async function acquireWorktreeSeedLock(lockPath: string): Promise<() => Promise<
|
||||
}
|
||||
}
|
||||
|
||||
function startWorktreeSeedAttempt(configPath: string, now = new Date()): WorktreeSeedManifest {
|
||||
const markers = resolveWorktreeSeedMarkerPaths(configPath);
|
||||
const current = readWorktreeSeedManifest(configPath);
|
||||
if (!current) throw new Error(`Worktree seed manifest does not exist at ${markers.manifest}.`);
|
||||
const at = now.toISOString();
|
||||
const next: WorktreeSeedManifest = {
|
||||
...current,
|
||||
state: "running",
|
||||
phase: "pending",
|
||||
attemptId: randomUUID(),
|
||||
snapshotAt: null,
|
||||
migrationRevision: null,
|
||||
startedAt: at,
|
||||
finishedAt: null,
|
||||
diagnostics: [
|
||||
...current.diagnostics,
|
||||
{ phase: "pending" as const, status: "started" as const, at },
|
||||
].slice(-WORKTREE_SEED_DIAGNOSTIC_LIMIT),
|
||||
};
|
||||
writeWorktreeSeedManifest(markers.manifest, next);
|
||||
return next;
|
||||
}
|
||||
|
||||
async function runVerifiedWorktreeSeed(input: {
|
||||
configPath: string;
|
||||
sourceConfigPath: string;
|
||||
sourceConfig: PaperclipConfig;
|
||||
targetConfig: PaperclipConfig;
|
||||
targetPaths: WorktreeLocalPaths;
|
||||
instanceId: string;
|
||||
seedMode: WorktreeSeedMode;
|
||||
preserveLiveWork?: boolean;
|
||||
seedDatabase: SeedWorktreeDatabase;
|
||||
}): Promise<SeedWorktreeDatabaseResult> {
|
||||
let activePhase: WorktreeSeedPhase = "pending";
|
||||
const previous = readWorktreeSeedManifest(input.configPath);
|
||||
if (previous?.state === "running") {
|
||||
updateWorktreeSeedManifest({
|
||||
configPath: input.configPath,
|
||||
phase: previous.phase,
|
||||
status: "failed",
|
||||
state: "failed",
|
||||
message: "The previous seed attempt ended without a terminal result.",
|
||||
});
|
||||
}
|
||||
startWorktreeSeedAttempt(input.configPath);
|
||||
|
||||
const unregisterInterruption = registerSeedInterruptHandler((signal) => {
|
||||
updateWorktreeSeedManifest({
|
||||
configPath: input.configPath,
|
||||
phase: activePhase,
|
||||
status: "failed",
|
||||
state: "failed",
|
||||
message: `Seed interrupted by ${signal} during ${activePhase}.`,
|
||||
});
|
||||
});
|
||||
|
||||
try {
|
||||
const details = await input.seedDatabase({
|
||||
sourceConfigPath: input.sourceConfigPath,
|
||||
sourceConfig: input.sourceConfig,
|
||||
targetConfig: input.targetConfig,
|
||||
targetPaths: input.targetPaths,
|
||||
instanceId: input.instanceId,
|
||||
seedMode: input.seedMode,
|
||||
preserveLiveWork: input.preserveLiveWork,
|
||||
onPhase: (phase, status, message) => {
|
||||
activePhase = phase;
|
||||
updateWorktreeSeedManifest({
|
||||
configPath: input.configPath,
|
||||
phase,
|
||||
status,
|
||||
state: "running",
|
||||
message,
|
||||
...(phase === "snapshot" && status === "started"
|
||||
? { snapshotAt: new Date().toISOString() }
|
||||
: {}),
|
||||
});
|
||||
},
|
||||
});
|
||||
if (!details.snapshotAt || !details.migrationRevision || !details.validation) {
|
||||
throw new Error("Seed implementation returned without required validation evidence.");
|
||||
}
|
||||
updateWorktreeSeedManifest({
|
||||
configPath: input.configPath,
|
||||
phase: "complete",
|
||||
status: "succeeded",
|
||||
state: "verified",
|
||||
snapshotAt: details.snapshotAt,
|
||||
migrationRevision: details.migrationRevision,
|
||||
message:
|
||||
`Verified ${details.validation.companyCount} company record(s), `
|
||||
+ `${details.validation.issueCount} issue record(s), auth, admin, membership, and migration state.`,
|
||||
});
|
||||
return details;
|
||||
} catch (error) {
|
||||
updateWorktreeSeedManifest({
|
||||
configPath: input.configPath,
|
||||
phase: activePhase,
|
||||
status: "failed",
|
||||
state: "failed",
|
||||
// Do not persist the underlying error: database/driver errors may contain
|
||||
// connection credentials. The CLI still returns the exact error to its caller.
|
||||
message: `Seed failed during ${activePhase}.`,
|
||||
});
|
||||
throw error;
|
||||
} finally {
|
||||
unregisterInterruption();
|
||||
}
|
||||
}
|
||||
|
||||
export async function ensureWorktreeSeeded(
|
||||
opts: WorktreeEnsureSeededOptions = {},
|
||||
dependencies: { seedDatabase?: SeedWorktreeDatabase } = {},
|
||||
@@ -1642,24 +2075,36 @@ export async function ensureWorktreeSeeded(
|
||||
try {
|
||||
// These checks deliberately happen under the cross-process lock. A second
|
||||
// service process waits for the first seed transaction, then observes the
|
||||
// complete marker instead of cloning the same database concurrently.
|
||||
if (existsSync(markers.complete)) {
|
||||
// verified manifest instead of cloning the same database concurrently.
|
||||
let manifest = readWorktreeSeedManifest(configPath);
|
||||
if (manifest?.state === "verified") {
|
||||
return { seeded: false, reason: "verified_manifest" };
|
||||
}
|
||||
if (!manifest && existsSync(markers.complete)) {
|
||||
return { seeded: false, reason: "complete_marker" };
|
||||
}
|
||||
if (!existsSync(markers.pending)) {
|
||||
if (!manifest && existsSync(markers.pending)) {
|
||||
const legacyPending = readLegacyWorktreeSeedPendingMarker(markers.pending);
|
||||
markWorktreeSeedPending({
|
||||
configPath,
|
||||
sourceConfigPath: legacyPending.sourceConfigPath,
|
||||
seedMode: "minimal",
|
||||
});
|
||||
manifest = readWorktreeSeedManifest(configPath);
|
||||
}
|
||||
if (!manifest) {
|
||||
// Worktrees created before lazy seeding shipped were seeded eagerly and
|
||||
// have neither marker. Preserve that compatibility without re-cloning.
|
||||
return { seeded: false, reason: "legacy_unmarked" };
|
||||
}
|
||||
|
||||
const pending = readWorktreeSeedPendingMarker(markers.pending);
|
||||
const sourceConfigPath = opts.fromConfig || opts.fromDataDir || opts.fromInstance
|
||||
? resolveSourceConfigPath({
|
||||
fromConfig: opts.fromConfig,
|
||||
fromDataDir: opts.fromDataDir,
|
||||
fromInstance: opts.fromInstance,
|
||||
})
|
||||
: path.resolve(pending.sourceConfigPath);
|
||||
: path.resolve(manifest.source.configPath);
|
||||
|
||||
if (path.resolve(sourceConfigPath) === path.resolve(configPath)) {
|
||||
throw new Error(
|
||||
@@ -1679,16 +2124,17 @@ export async function ensureWorktreeSeeded(
|
||||
const targetRoot = path.dirname(path.dirname(configPath));
|
||||
const targetPaths = resolveWorktreeReseedTargetPaths({ configPath, rootPath: targetRoot });
|
||||
const seedDatabase = dependencies.seedDatabase ?? seedWorktreeDatabase;
|
||||
const details = await seedDatabase({
|
||||
const details = await runVerifiedWorktreeSeed({
|
||||
configPath,
|
||||
sourceConfigPath,
|
||||
sourceConfig,
|
||||
targetConfig,
|
||||
targetPaths,
|
||||
instanceId: targetPaths.instanceId,
|
||||
seedMode: "minimal",
|
||||
seedMode: manifest.seedMode,
|
||||
preserveLiveWork: opts.preserveLiveWork,
|
||||
seedDatabase,
|
||||
});
|
||||
markWorktreeSeedComplete({ configPath });
|
||||
return { seeded: true, reason: "seeded", details };
|
||||
} finally {
|
||||
await releaseLock();
|
||||
@@ -1762,6 +2208,8 @@ async function runWorktreeInit(opts: WorktreeInitOptions): Promise<void> {
|
||||
markWorktreeSeedPending({
|
||||
configPath: paths.configPath,
|
||||
sourceConfigPath,
|
||||
targetInstanceId: instanceId,
|
||||
seedMode,
|
||||
});
|
||||
const sourceEnvEntries = readPaperclipEnvEntries(resolvePaperclipEnvFile(sourceConfigPath));
|
||||
const existingAgentJwtSecret =
|
||||
@@ -1790,8 +2238,11 @@ async function runWorktreeInit(opts: WorktreeInitOptions): Promise<void> {
|
||||
}
|
||||
const spinner = p.spinner();
|
||||
spinner.start(`Seeding isolated worktree database from source instance (${seedMode})...`);
|
||||
const markers = resolveWorktreeSeedMarkerPaths(paths.configPath);
|
||||
const releaseSeedLock = await acquireWorktreeSeedLock(markers.lock);
|
||||
try {
|
||||
const seeded = await seedWorktreeDatabase({
|
||||
const seeded = await runVerifiedWorktreeSeed({
|
||||
configPath: paths.configPath,
|
||||
sourceConfigPath,
|
||||
sourceConfig,
|
||||
targetConfig,
|
||||
@@ -1799,16 +2250,18 @@ async function runWorktreeInit(opts: WorktreeInitOptions): Promise<void> {
|
||||
instanceId,
|
||||
seedMode,
|
||||
preserveLiveWork: opts.preserveLiveWork,
|
||||
seedDatabase: seedWorktreeDatabase,
|
||||
});
|
||||
seedSummary = seeded.backupSummary;
|
||||
seedExecutionQuarantineSummary = seeded.executionQuarantine;
|
||||
pausedScheduledRoutineCount = seeded.pausedScheduledRoutines;
|
||||
reboundWorkspaceSummary = seeded.reboundWorkspaces;
|
||||
markWorktreeSeedComplete({ configPath: paths.configPath, seedMode });
|
||||
spinner.stop(`Seeded isolated worktree database (${seedMode}).`);
|
||||
} catch (error) {
|
||||
spinner.stop(pc.red("Failed to seed worktree database."));
|
||||
throw error;
|
||||
} finally {
|
||||
await releaseSeedLock();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3480,6 +3933,34 @@ export async function worktreeMergeHistoryCommand(sourceArg: string | undefined,
|
||||
}
|
||||
}
|
||||
|
||||
async function backupWorktreeReseedTarget(input: {
|
||||
targetConfig: PaperclipConfig;
|
||||
targetPaths: WorktreeLocalPaths;
|
||||
}): Promise<string> {
|
||||
if (input.targetConfig.database.mode !== "embedded-postgres") {
|
||||
throw new Error("Managed worktree repair requires an embedded PostgreSQL target.");
|
||||
}
|
||||
const targetHandle = await ensureEmbeddedPostgres(
|
||||
input.targetConfig.database.embeddedPostgresDataDir,
|
||||
input.targetConfig.database.embeddedPostgresPort,
|
||||
);
|
||||
try {
|
||||
const adminConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${targetHandle.port}/postgres`;
|
||||
await ensurePostgresDatabase(adminConnectionString, "paperclip");
|
||||
const result = await runDatabaseBackup({
|
||||
connectionString: `postgres://paperclip:paperclip@127.0.0.1:${targetHandle.port}/paperclip`,
|
||||
backupDir: path.resolve(input.targetPaths.backupDir, "repair"),
|
||||
retention: { dailyDays: 30, weeklyWeeks: 12, monthlyMonths: 12 },
|
||||
filenamePrefix: `${input.targetPaths.instanceId}-pre-repair`,
|
||||
backupEngine: "auto",
|
||||
includeMigrationJournal: true,
|
||||
});
|
||||
return formatDatabaseBackupResult(result);
|
||||
} finally {
|
||||
if (targetHandle.startedByThisProcess) await targetHandle.stop();
|
||||
}
|
||||
}
|
||||
|
||||
async function runWorktreeReseed(opts: WorktreeReseedOptions): Promise<void> {
|
||||
const seedMode = opts.seedMode ?? "full";
|
||||
if (!isWorktreeSeedMode(seedMode)) {
|
||||
@@ -3535,8 +4016,23 @@ async function runWorktreeReseed(opts: WorktreeReseedOptions): Promise<void> {
|
||||
|
||||
const spinner = p.spinner();
|
||||
spinner.start(`Reseeding ${targetEndpoint.label} from ${source.label} (${seedMode})...`);
|
||||
const markers = resolveWorktreeSeedMarkerPaths(targetEndpoint.configPath);
|
||||
mkdirSync(path.dirname(markers.lock), { recursive: true });
|
||||
const releaseSeedLock = await acquireWorktreeSeedLock(markers.lock);
|
||||
try {
|
||||
const seeded = await seedWorktreeDatabase({
|
||||
let targetBackupSummary: string | null = null;
|
||||
if (opts.backupTarget) {
|
||||
targetBackupSummary = await backupWorktreeReseedTarget({ targetConfig, targetPaths });
|
||||
p.log.message(pc.dim(`Recoverable pre-repair backup: ${targetBackupSummary}`));
|
||||
}
|
||||
markWorktreeSeedPending({
|
||||
configPath: targetEndpoint.configPath,
|
||||
sourceConfigPath: source.configPath,
|
||||
targetInstanceId: targetPaths.instanceId,
|
||||
seedMode,
|
||||
});
|
||||
const seeded = await runVerifiedWorktreeSeed({
|
||||
configPath: targetEndpoint.configPath,
|
||||
sourceConfigPath: source.configPath,
|
||||
sourceConfig,
|
||||
targetConfig,
|
||||
@@ -3544,8 +4040,8 @@ async function runWorktreeReseed(opts: WorktreeReseedOptions): Promise<void> {
|
||||
instanceId: targetPaths.instanceId,
|
||||
seedMode,
|
||||
preserveLiveWork: opts.preserveLiveWork,
|
||||
seedDatabase: seedWorktreeDatabase,
|
||||
});
|
||||
markWorktreeSeedComplete({ configPath: targetEndpoint.configPath, seedMode });
|
||||
spinner.stop(`Reseeded ${targetEndpoint.label} (${seedMode}).`);
|
||||
p.log.message(pc.dim(`Source: ${source.configPath}`));
|
||||
p.log.message(pc.dim(`Target: ${targetEndpoint.configPath}`));
|
||||
@@ -3567,6 +4063,8 @@ async function runWorktreeReseed(opts: WorktreeReseedOptions): Promise<void> {
|
||||
} catch (error) {
|
||||
spinner.stop(pc.red("Failed to reseed worktree database."));
|
||||
throw error;
|
||||
} finally {
|
||||
await releaseSeedLock();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3748,6 +4246,7 @@ export function registerWorktreeCommands(program: Command): void {
|
||||
.option("--preserve-live-work", "Do not quarantine copied agent work or workspace runtime services in the seeded worktree", false)
|
||||
.option("--yes", "Skip the destructive confirmation prompt", false)
|
||||
.option("--allow-live-target", "Override the guard that requires the target worktree DB to be stopped first", false)
|
||||
.option("--backup-target", "Retain a recoverable full backup of the isolated target DB before reseeding", false)
|
||||
.action(worktreeReseedCommand);
|
||||
|
||||
worktree
|
||||
|
||||
+11
-9
@@ -479,25 +479,24 @@ After `worktree init`, both the server and the CLI auto-load the repo-local `.pa
|
||||
|
||||
Seeding a worktree database is the heaviest part of `worktree init`. That work can be deferred so a worktree is cheap to create and only pays the seed cost the first time it is actually used — the CLI/dev-time analog of the server's lazy runtime provisioning (see the board-operator guide's "Lazy runtime provisioning" section).
|
||||
|
||||
Seeding state is tracked with two marker files under the worktree's `.paperclip/` directory:
|
||||
Seeding state is tracked in `.paperclip/seed-manifest.json`. The versioned manifest records only non-secret evidence: source instance id/config path, target instance id, seed mode, snapshot time, migration revision, attempt timestamps, current phase, terminal state, and a bounded phase diagnostic history. It never stores database credentials or auth credential material. The legacy `seed-pending` and `seed-complete` files remain read-only compatibility signals for worktrees created before the manifest shipped.
|
||||
|
||||
- `seed-pending` — the isolated database has not been seeded yet (a **lean** worktree). Written by `worktree init` before any seed runs.
|
||||
- `seed-complete` — the database was seeded; the pending marker is removed.
|
||||
The default `worktree init` still seeds eagerly. A lean worktree (created without an eager seed) has a `pending` manifest until something seeds it on demand:
|
||||
|
||||
The default `worktree init` still seeds eagerly and writes `seed-complete` immediately. A lean worktree (created without an eager seed) keeps its `seed-pending` marker until something seeds it on demand:
|
||||
|
||||
- `pnpm paperclipai worktree ensure-seeded` performs the deferred seed **exactly once**. It is lock-guarded and idempotent: a present `seed-complete` marker or a missing `seed-pending` marker short-circuits it, so it is safe to call repeatedly and from concurrent processes. It reads the source instance from the `seed-pending` marker unless you pass `--from-config`.
|
||||
- `pnpm paperclipai worktree ensure-seeded` performs the deferred seed **exactly once**. It is lock-guarded and idempotent: only a complete `verified` manifest short-circuits it, so it is safe to call repeatedly and from concurrent processes. It reads the source identity from the manifest unless you pass `--from-config`.
|
||||
- `paperclipai run` calls `ensureWorktreeSeeded` automatically before doctor/boot, so `run` transparently seeds a lean worktree on first launch.
|
||||
- Managed git-worktree runtime startup also runs `scripts/provision-worktree-runtime.sh` automatically when a legacy workspace policy has no explicit runtime provision command and the worktree is still `seed-pending`. An explicitly configured runtime provision command always takes precedence.
|
||||
- Managed git-worktree runtime startup also runs `scripts/provision-worktree-runtime.sh` automatically when a legacy workspace policy has no explicit runtime provision command and the manifest is not verified. An explicitly configured runtime provision command always takes precedence.
|
||||
- Worktrees created before lazy seeding shipped have neither marker; they are treated as already-seeded for backward compatibility (never re-cloned).
|
||||
|
||||
**Seed-pending guard.** `pnpm dev` (the dev-runner) refuses to boot a worktree whose database is still `seed-pending` and points you at the fix:
|
||||
Both `minimal` and `full` modes use the same terminal validation contract. Before a manifest can become `verified`, Paperclip proves the migration journal is current and reads a credential-backed auth user, instance administrator role, active company membership, and representative cloned company/issue pair in both source and target. Restore, migrations, execution quarantine, routine pausing, workspace rebinding, and post-restore validation all run under the seed lock. An interruption leaves the exact active phase in terminal `failed` state; it cannot produce readiness evidence.
|
||||
|
||||
**Unverified-seed guard.** `pnpm dev` (the dev-runner) refuses to boot a worktree whose manifest is pending, running, failed, malformed, or missing required verification evidence and points you at the fix:
|
||||
|
||||
```
|
||||
[paperclip] this worktree database is seed-pending. Run `pnpm paperclipai worktree ensure-seeded` before `pnpm dev`.
|
||||
```
|
||||
|
||||
This guard (`isWorktreeSeedPending` in `server/src/dev-runner-worktree.ts`) prevents `pnpm dev` from starting the app against an empty, unseeded database — run `worktree ensure-seeded` once and re-run `pnpm dev`.
|
||||
This guard (`isWorktreeSeedPending` in `server/src/dev-runner-worktree.ts`) prevents `pnpm dev` from starting the app against an empty or partially restored database — run `worktree ensure-seeded` once and re-run `pnpm dev`.
|
||||
|
||||
Provisioned git worktrees also pause seeded routines that still have enabled schedule triggers in the isolated worktree database by default. This prevents copied daily/cron routines from firing unexpectedly inside the new workspace instance during development without disabling webhook/API-only routines.
|
||||
|
||||
@@ -603,6 +602,7 @@ For an already-created worktree where you want to keep the existing repo-local c
|
||||
| `--seed-mode <mode>` | Seed profile: `minimal` or `full` (default: `full`) |
|
||||
| `--yes` | Skip the destructive confirmation prompt |
|
||||
| `--allow-live-target` | Override the guard that requires the target worktree DB to be stopped first |
|
||||
| `--backup-target` | Retain a recoverable full target-DB backup before reseeding |
|
||||
|
||||
Examples:
|
||||
|
||||
@@ -622,6 +622,8 @@ npx paperclipai worktree reseed \
|
||||
--seed-mode full
|
||||
```
|
||||
|
||||
Managed workspace repair uses this same verified full-reseed contract through `POST /api/execution-workspaces/:id/runtime-commands/repair`. The exclusive, audited operation stops managed services, writes a recoverable pre-repair database backup under the isolated instance's backup directory, performs the full seed/migration/quarantine/rebinding sequence, and restarts only after terminal manifest and service-health validation. It preserves the worktree filesystem. On failure, services remain stopped while the database backup, seed manifest, bounded phase diagnostics, and operation log are retained for inspection; repair never retries itself in a loop.
|
||||
|
||||
**`npx paperclipai worktree:make <name> [options]`** — Create `~/NAME` as a git worktree, then initialize an isolated Paperclip instance inside it. This combines `git worktree add` with `worktree init` in a single step.
|
||||
|
||||
| Option | Description |
|
||||
|
||||
@@ -3,6 +3,7 @@ export type WorkspaceOperationPhase =
|
||||
| "workspace_config_freshness"
|
||||
| "workspace_provision"
|
||||
| "workspace_runtime_provision"
|
||||
| "workspace_repair"
|
||||
| "workspace_teardown"
|
||||
| "worktree_cleanup"
|
||||
| "workspace_finalize";
|
||||
|
||||
@@ -137,7 +137,10 @@ test("uses the base CLI when its import graph boots", () => {
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
const config = readWorktreeConfig(worktreeCwd);
|
||||
assert.equal(config.$meta.source, "fake-cli");
|
||||
assert.ok(fs.existsSync(path.join(worktreeCwd, ".paperclip", "seed-pending")));
|
||||
assert.equal(
|
||||
JSON.parse(fs.readFileSync(path.join(worktreeCwd, ".paperclip", "seed-manifest.json"), "utf8")).state,
|
||||
"pending",
|
||||
);
|
||||
const initInvocation = readCliInvocations(baseCwd).find(
|
||||
(args) => args[0] === "worktree" && args[1] === "init",
|
||||
);
|
||||
@@ -166,7 +169,10 @@ test("falls back to an isolated config when the base CLI cannot boot", () => {
|
||||
);
|
||||
const env = fs.readFileSync(path.join(worktreeCwd, ".paperclip", ".env"), "utf8");
|
||||
assert.match(env, /PAPERCLIP_IN_WORKTREE=true/);
|
||||
assert.ok(fs.existsSync(path.join(worktreeCwd, ".paperclip", "seed-pending")));
|
||||
assert.equal(
|
||||
JSON.parse(fs.readFileSync(path.join(worktreeCwd, ".paperclip", "seed-manifest.json"), "utf8")).state,
|
||||
"pending",
|
||||
);
|
||||
});
|
||||
|
||||
test("repairs an unhealthy base install under the lock and then uses the CLI", (t) => {
|
||||
@@ -271,7 +277,7 @@ test("runtime provisioning invokes ensure-seeded once and fast-exits after succe
|
||||
|
||||
const second = runRuntimeProvision(baseCwd, worktreeCwd);
|
||||
assert.equal(second.status, 0, second.stderr);
|
||||
assert.match(second.stderr, /already seeded; skipping/);
|
||||
assert.match(second.stderr, /already seeded.*skipping/);
|
||||
const ensureCallsAfterSecond = readCliInvocations(baseCwd)
|
||||
.filter((args) => args[0] === "worktree" && args[1] === "ensure-seeded");
|
||||
assert.equal(ensureCallsAfterSecond.length, 1);
|
||||
@@ -290,3 +296,22 @@ test("runtime provisioning leaves seed-pending in place when ensure-seeded fails
|
||||
assert.ok(fs.existsSync(path.join(worktreeCwd, ".paperclip", "seed-pending")));
|
||||
assert.ok(!fs.existsSync(path.join(worktreeCwd, ".paperclip", "seed-complete")));
|
||||
});
|
||||
|
||||
test("runtime provisioning does not trust a truncated verified manifest", () => {
|
||||
const baseCwd = makeBaseWorkspace({ helpExit: 0, initExit: 0, ensureExit: 4 });
|
||||
const worktreeCwd = makeTempDir("paperclip-provision-runtime-truncated-");
|
||||
fs.mkdirSync(path.join(worktreeCwd, ".paperclip"), { recursive: true });
|
||||
fs.writeFileSync(path.join(worktreeCwd, ".paperclip", "config.json"), "{}\n");
|
||||
fs.writeFileSync(
|
||||
path.join(worktreeCwd, ".paperclip", "seed-manifest.json"),
|
||||
JSON.stringify({ version: 2, state: "verified" }),
|
||||
);
|
||||
|
||||
const result = runRuntimeProvision(baseCwd, worktreeCwd);
|
||||
assert.equal(result.status, 4, result.stderr);
|
||||
assert.equal(
|
||||
readCliInvocations(baseCwd)
|
||||
.filter((args) => args[0] === "worktree" && args[1] === "ensure-seeded").length,
|
||||
1,
|
||||
);
|
||||
});
|
||||
@@ -7,6 +7,7 @@ paperclip_home="${PAPERCLIP_HOME:-$HOME/.paperclip}"
|
||||
paperclip_instance_id="${PAPERCLIP_INSTANCE_ID:-default}"
|
||||
paperclip_dir="$worktree_cwd/.paperclip"
|
||||
worktree_config_path="$paperclip_dir/config.json"
|
||||
seed_manifest_path="$paperclip_dir/seed-manifest.json"
|
||||
seed_pending_marker_path="$paperclip_dir/seed-pending"
|
||||
seed_complete_marker_path="$paperclip_dir/seed-complete"
|
||||
|
||||
@@ -20,8 +21,37 @@ if [[ ! -d "$worktree_cwd" ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -e "$seed_complete_marker_path" || ! -e "$seed_pending_marker_path" ]]; then
|
||||
echo "Worktree database is already seeded; skipping runtime provisioning." >&2
|
||||
if [[ -e "$seed_manifest_path" ]]; then
|
||||
seed_manifest_state="$(SEED_MANIFEST_PATH="$seed_manifest_path" node <<'EOF'
|
||||
const fs = require("node:fs");
|
||||
try {
|
||||
const value = JSON.parse(fs.readFileSync(process.env.SEED_MANIFEST_PATH, "utf8"));
|
||||
const complete = value?.version === 2
|
||||
&& value?.state === "verified"
|
||||
&& value?.phase === "complete"
|
||||
&& typeof value?.source?.instanceId === "string" && value.source.instanceId.length > 0
|
||||
&& typeof value?.source?.configPath === "string" && value.source.configPath.length > 0
|
||||
&& (value?.seedMode === "minimal" || value?.seedMode === "full")
|
||||
&& typeof value?.snapshotAt === "string" && value.snapshotAt.length > 0
|
||||
&& typeof value?.migrationRevision === "string" && value.migrationRevision.length > 0
|
||||
&& typeof value?.targetInstanceId === "string" && value.targetInstanceId.length > 0
|
||||
&& typeof value?.attemptId === "string" && value.attemptId.length > 0
|
||||
&& typeof value?.startedAt === "string"
|
||||
&& typeof value?.finishedAt === "string"
|
||||
&& Array.isArray(value?.diagnostics)
|
||||
&& value.diagnostics.some((entry) => entry?.phase === "complete" && entry?.status === "succeeded" && typeof entry?.at === "string");
|
||||
process.stdout.write(complete ? "verified" : "incomplete");
|
||||
} catch {
|
||||
process.stdout.write("invalid");
|
||||
}
|
||||
EOF
|
||||
)"
|
||||
if [[ "$seed_manifest_state" == "verified" ]]; then
|
||||
echo "Worktree database has a verified seed manifest; skipping runtime provisioning." >&2
|
||||
exit 0
|
||||
fi
|
||||
elif [[ -e "$seed_complete_marker_path" || ! -e "$seed_pending_marker_path" ]]; then
|
||||
echo "Worktree database is already seeded by a legacy marker; skipping runtime provisioning." >&2
|
||||
exit 0
|
||||
fi
|
||||
|
||||
@@ -31,6 +61,17 @@ if [[ ! -f "$worktree_config_path" ]]; then
|
||||
fi
|
||||
|
||||
source_config_path="${PAPERCLIP_CONFIG:-}"
|
||||
if [[ -e "$seed_manifest_path" ]]; then
|
||||
manifest_source_config_path="$(SEED_MANIFEST_PATH="$seed_manifest_path" node <<'EOF'
|
||||
const fs = require("node:fs");
|
||||
const value = JSON.parse(fs.readFileSync(process.env.SEED_MANIFEST_PATH, "utf8"));
|
||||
process.stdout.write(typeof value?.source?.configPath === "string" ? value.source.configPath : "");
|
||||
EOF
|
||||
)"
|
||||
if [[ -n "$manifest_source_config_path" ]]; then
|
||||
source_config_path="$manifest_source_config_path"
|
||||
fi
|
||||
fi
|
||||
if [[ -z "$source_config_path" && ( -e "$base_cwd/.paperclip/config.json" || -L "$base_cwd/.paperclip/config.json" ) ]]; then
|
||||
source_config_path="$base_cwd/.paperclip/config.json"
|
||||
fi
|
||||
@@ -41,7 +82,7 @@ source_config_args=(--from-config "$source_config_path")
|
||||
if [[ "$source_config_path" == "$worktree_config_path" ]]; then
|
||||
# A human may invoke this after sourcing `worktree env`, which points
|
||||
# PAPERCLIP_CONFIG at the target. In that case the CLI reads the original
|
||||
# source config from the seed-pending marker instead.
|
||||
# source config from the seed manifest or legacy pending marker instead.
|
||||
source_config_args=()
|
||||
fi
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ paperclip_instance_id="${PAPERCLIP_INSTANCE_ID:-default}"
|
||||
paperclip_dir="$worktree_cwd/.paperclip"
|
||||
worktree_config_path="$paperclip_dir/config.json"
|
||||
worktree_env_path="$paperclip_dir/.env"
|
||||
seed_manifest_path="$paperclip_dir/seed-manifest.json"
|
||||
seed_pending_marker_path="$paperclip_dir/seed-pending"
|
||||
seed_complete_marker_path="$paperclip_dir/seed-complete"
|
||||
worktree_name="${PAPERCLIP_WORKSPACE_BRANCH:-$(basename "$worktree_cwd")}"
|
||||
@@ -237,25 +238,48 @@ for (const rawValue of runtimePaths) {
|
||||
EOF
|
||||
}
|
||||
|
||||
write_seed_pending_marker() {
|
||||
write_seed_pending_manifest() {
|
||||
SEED_MANIFEST_PATH="$seed_manifest_path" \
|
||||
SEED_PENDING_MARKER_PATH="$seed_pending_marker_path" \
|
||||
SEED_COMPLETE_MARKER_PATH="$seed_complete_marker_path" \
|
||||
SOURCE_CONFIG_PATH="$source_config_path" \
|
||||
TARGET_INSTANCE_ID="$worktree_instance_id" \
|
||||
node <<'EOF'
|
||||
const crypto = require("node:crypto");
|
||||
const fs = require("node:fs");
|
||||
const path = require("node:path");
|
||||
|
||||
const manifestPath = process.env.SEED_MANIFEST_PATH;
|
||||
const pendingPath = process.env.SEED_PENDING_MARKER_PATH;
|
||||
const completePath = process.env.SEED_COMPLETE_MARKER_PATH;
|
||||
const sourceConfigPath = path.resolve(process.env.SOURCE_CONFIG_PATH);
|
||||
const sourceEnvPath = path.join(path.dirname(sourceConfigPath), ".env");
|
||||
let sourceInstanceId = path.basename(path.dirname(sourceConfigPath));
|
||||
if (fs.existsSync(sourceEnvPath)) {
|
||||
const match = fs.readFileSync(sourceEnvPath, "utf8").match(/^\s*(?:export\s+)?PAPERCLIP_INSTANCE_ID\s*=\s*["']?([^\s"'#]+)["']?/m);
|
||||
if (match?.[1]) sourceInstanceId = match[1];
|
||||
}
|
||||
fs.rmSync(completePath, { force: true });
|
||||
fs.rmSync(pendingPath, { force: true });
|
||||
const at = new Date().toISOString();
|
||||
fs.writeFileSync(
|
||||
pendingPath,
|
||||
manifestPath,
|
||||
`${JSON.stringify({
|
||||
version: 1,
|
||||
state: "pending",
|
||||
sourceConfigPath: path.resolve(process.env.SOURCE_CONFIG_PATH),
|
||||
version: 2,
|
||||
source: {
|
||||
instanceId: sourceInstanceId,
|
||||
configPath: sourceConfigPath,
|
||||
},
|
||||
snapshotAt: null,
|
||||
seedMode: "minimal",
|
||||
createdAt: new Date().toISOString(),
|
||||
migrationRevision: null,
|
||||
targetInstanceId: process.env.TARGET_INSTANCE_ID,
|
||||
phase: "pending",
|
||||
state: "pending",
|
||||
attemptId: crypto.randomUUID(),
|
||||
startedAt: null,
|
||||
finishedAt: null,
|
||||
diagnostics: [{ phase: "pending", status: "succeeded", at }],
|
||||
}, null, 2)}\n`,
|
||||
{ mode: 0o600 },
|
||||
);
|
||||
@@ -551,8 +575,8 @@ else
|
||||
created_worktree_config=1
|
||||
fi
|
||||
|
||||
if [[ "$created_worktree_config" -eq 1 && ! -e "$seed_pending_marker_path" && ! -e "$seed_complete_marker_path" ]]; then
|
||||
write_seed_pending_marker
|
||||
if [[ "$created_worktree_config" -eq 1 && ! -e "$seed_manifest_path" && ! -e "$seed_pending_marker_path" && ! -e "$seed_complete_marker_path" ]]; then
|
||||
write_seed_pending_manifest
|
||||
fi
|
||||
|
||||
list_base_node_modules_paths() {
|
||||
|
||||
@@ -36,6 +36,36 @@ describe("dev-runner worktree env bootstrap", () => {
|
||||
expect(isWorktreeSeedPending(root)).toBe(false);
|
||||
});
|
||||
|
||||
it("guards every manifest state except a complete verified manifest", () => {
|
||||
const root = createTempRoot("paperclip-dev-runner-seed-manifest-");
|
||||
const manifestPath = path.join(root, ".paperclip", "seed-manifest.json");
|
||||
fs.mkdirSync(path.dirname(manifestPath), { recursive: true });
|
||||
fs.writeFileSync(manifestPath, JSON.stringify({ version: 2, state: "failed" }), "utf8");
|
||||
expect(isWorktreeSeedPending(root)).toBe(true);
|
||||
|
||||
fs.writeFileSync(manifestPath, JSON.stringify({ version: 2, state: "verified" }), "utf8");
|
||||
expect(isWorktreeSeedPending(root)).toBe(true);
|
||||
|
||||
fs.writeFileSync(manifestPath, JSON.stringify({
|
||||
version: 2,
|
||||
source: { instanceId: "source", configPath: "/source/config.json" },
|
||||
snapshotAt: "2026-08-18T00:00:00.000Z",
|
||||
seedMode: "minimal",
|
||||
migrationRevision: "0001",
|
||||
targetInstanceId: "target",
|
||||
phase: "complete",
|
||||
state: "verified",
|
||||
attemptId: "attempt",
|
||||
startedAt: "2026-08-18T00:00:00.000Z",
|
||||
finishedAt: "2026-08-18T00:01:00.000Z",
|
||||
diagnostics: [{ phase: "complete", status: "succeeded", at: "2026-08-18T00:01:00.000Z" }],
|
||||
}), "utf8");
|
||||
expect(isWorktreeSeedPending(root)).toBe(false);
|
||||
|
||||
fs.writeFileSync(manifestPath, "not-json", "utf8");
|
||||
expect(isWorktreeSeedPending(root)).toBe(true);
|
||||
});
|
||||
|
||||
it("detects linked git worktrees from .git files", () => {
|
||||
const root = createTempRoot("paperclip-dev-runner-worktree-");
|
||||
fs.writeFileSync(path.join(root, ".git"), "gitdir: /tmp/paperclip/.git/worktrees/feature\n", "utf8");
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
import { EventEmitter } from "node:events";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { PassThrough } from "node:stream";
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
@@ -29,6 +34,12 @@ const mockBuildWorkspaceRuntimeDesiredStatePatch = vi.hoisted(() => vi.fn());
|
||||
// `execution-workspace-runtime-lease-route.test.ts` exercises the lease itself against a real db.
|
||||
const mockClaimRuntimeLease = vi.hoisted(() => vi.fn(async () => null));
|
||||
const mockReleaseRuntimeLease = vi.hoisted(() => vi.fn(async () => undefined));
|
||||
const mockSpawn = vi.hoisted(() => vi.fn());
|
||||
|
||||
vi.mock("node:child_process", async () => ({
|
||||
...(await vi.importActual<typeof import("node:child_process")>("node:child_process")),
|
||||
spawn: mockSpawn,
|
||||
}));
|
||||
|
||||
vi.mock("../telemetry.js", () => ({ getTelemetryClient: mockGetTelemetryClient }));
|
||||
|
||||
@@ -45,7 +56,7 @@ vi.mock("../services/index.js", () => ({
|
||||
claim: mockClaimRuntimeLease,
|
||||
release: mockReleaseRuntimeLease,
|
||||
}),
|
||||
LEASED_WORKSPACE_RUNTIME_ACTIONS: ["start", "stop", "restart"],
|
||||
LEASED_WORKSPACE_RUNTIME_ACTIONS: ["start", "stop", "restart", "repair"],
|
||||
}));
|
||||
|
||||
vi.mock("../services/workspace-runtime.js", () => ({
|
||||
@@ -135,6 +146,7 @@ async function createApp() {
|
||||
describe.sequential("execution workspace runtime control conflict and failure reconciliation", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
mockSpawn.mockReset();
|
||||
mockAccessService.decide.mockResolvedValue({
|
||||
allowed: true,
|
||||
action: "runtime:manage",
|
||||
@@ -165,7 +177,7 @@ describe.sequential("execution workspace runtime control conflict and failure re
|
||||
// Mirror the real recorder: a throwing `run` becomes a terminal failed operation and
|
||||
// the error still propagates to the caller.
|
||||
try {
|
||||
await input.run();
|
||||
await input.run(async () => undefined);
|
||||
} catch (error) {
|
||||
(error as any).recordedOperationStatus = "failed";
|
||||
throw error;
|
||||
@@ -247,6 +259,156 @@ describe.sequential("execution workspace runtime control conflict and failure re
|
||||
);
|
||||
});
|
||||
|
||||
it("runs an audited full repair through terminal seed and service readiness validation", async () => {
|
||||
const workspaceCwd = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-route-repair-success-"));
|
||||
try {
|
||||
const configDir = path.join(workspaceCwd, ".paperclip");
|
||||
const sourceConfigPath = path.join(workspaceCwd, "source", "config.json");
|
||||
const cliRunner = path.join(workspaceCwd, "cli", "node_modules", "tsx", "dist", "cli.mjs");
|
||||
const cliEntry = path.join(workspaceCwd, "cli", "src", "index.ts");
|
||||
fs.mkdirSync(configDir, { recursive: true });
|
||||
fs.mkdirSync(path.dirname(sourceConfigPath), { recursive: true });
|
||||
fs.mkdirSync(path.dirname(cliRunner), { recursive: true });
|
||||
fs.mkdirSync(path.dirname(cliEntry), { recursive: true });
|
||||
fs.writeFileSync(path.join(configDir, "config.json"), "{}\n");
|
||||
fs.writeFileSync(sourceConfigPath, "{}\n");
|
||||
fs.writeFileSync(cliRunner, "// test runner\n");
|
||||
fs.writeFileSync(cliEntry, "// test entry\n");
|
||||
fs.writeFileSync(path.join(configDir, "seed-manifest.json"), JSON.stringify({
|
||||
version: 2,
|
||||
source: { instanceId: "source", configPath: sourceConfigPath },
|
||||
state: "failed",
|
||||
attemptId: "previous",
|
||||
}));
|
||||
|
||||
const { deriveWorktreeInstanceId } = await import("../services/workspace-instance-cleanup.js");
|
||||
mockExecutionWorkspaceService.getById.mockResolvedValue(buildExecutionWorkspace({ cwd: workspaceCwd }));
|
||||
mockEnsurePersistedExecutionWorkspaceAvailable.mockResolvedValue({ cwd: workspaceCwd });
|
||||
mockStartRuntimeServices.mockResolvedValue([{
|
||||
id: "service-1",
|
||||
status: "running",
|
||||
healthStatus: "healthy",
|
||||
}]);
|
||||
mockSpawn.mockImplementation((_command: string, args: string[]) => {
|
||||
expect(args).toEqual(expect.arrayContaining([
|
||||
"worktree", "reseed", "--seed-mode", "full", "--yes", "--backup-target",
|
||||
]));
|
||||
const child = new EventEmitter() as EventEmitter & {
|
||||
stdout: PassThrough;
|
||||
stderr: PassThrough;
|
||||
kill: ReturnType<typeof vi.fn>;
|
||||
};
|
||||
child.stdout = new PassThrough();
|
||||
child.stderr = new PassThrough();
|
||||
child.kill = vi.fn();
|
||||
queueMicrotask(() => {
|
||||
fs.writeFileSync(path.join(configDir, "seed-manifest.json"), JSON.stringify({
|
||||
version: 2,
|
||||
source: { instanceId: "source", configPath: sourceConfigPath },
|
||||
snapshotAt: "2026-08-18T00:00:00.000Z",
|
||||
seedMode: "full",
|
||||
migrationRevision: "0142_test.sql",
|
||||
targetInstanceId: deriveWorktreeInstanceId(workspaceCwd),
|
||||
phase: "complete",
|
||||
state: "verified",
|
||||
attemptId: "repair-attempt",
|
||||
startedAt: "2026-08-18T00:00:00.000Z",
|
||||
finishedAt: "2026-08-18T00:01:00.000Z",
|
||||
diagnostics: [{ phase: "complete", status: "succeeded", at: "2026-08-18T00:01:00.000Z" }],
|
||||
}));
|
||||
child.emit("exit", 0);
|
||||
});
|
||||
return child;
|
||||
});
|
||||
|
||||
const app = await createApp();
|
||||
const res = await request(app)
|
||||
.post(`/api/execution-workspaces/${executionWorkspaceId}/runtime-commands/repair`)
|
||||
.send({});
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(mockStopRuntimeServicesForExecutionWorkspace).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ executionWorkspaceId, workspaceCwd }),
|
||||
);
|
||||
expect(mockStartRuntimeServices).toHaveBeenCalledTimes(1);
|
||||
expect(mockLogActivity).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({ action: "execution_workspace.runtime_repair" }),
|
||||
);
|
||||
} finally {
|
||||
fs.rmSync(workspaceCwd, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("fails repair at the exact seed phase and leaves managed services stopped", async () => {
|
||||
const workspaceCwd = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-route-repair-failure-"));
|
||||
try {
|
||||
const configDir = path.join(workspaceCwd, ".paperclip");
|
||||
const sourceConfigPath = path.join(workspaceCwd, "source", "config.json");
|
||||
const cliRunner = path.join(workspaceCwd, "cli", "node_modules", "tsx", "dist", "cli.mjs");
|
||||
const cliEntry = path.join(workspaceCwd, "cli", "src", "index.ts");
|
||||
for (const dir of [configDir, path.dirname(sourceConfigPath), path.dirname(cliRunner), path.dirname(cliEntry)]) {
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
fs.writeFileSync(path.join(configDir, "config.json"), "{}\n");
|
||||
fs.writeFileSync(sourceConfigPath, "{}\n");
|
||||
fs.writeFileSync(cliRunner, "// test runner\n");
|
||||
fs.writeFileSync(cliEntry, "// test entry\n");
|
||||
fs.writeFileSync(path.join(configDir, "seed-manifest.json"), JSON.stringify({
|
||||
version: 2,
|
||||
source: { instanceId: "source", configPath: sourceConfigPath },
|
||||
state: "failed",
|
||||
attemptId: "previous",
|
||||
}));
|
||||
mockExecutionWorkspaceService.getById.mockResolvedValue(buildExecutionWorkspace({ cwd: workspaceCwd }));
|
||||
mockEnsurePersistedExecutionWorkspaceAvailable.mockResolvedValue({ cwd: workspaceCwd });
|
||||
mockSpawn.mockImplementation(() => {
|
||||
const child = new EventEmitter() as EventEmitter & {
|
||||
stdout: PassThrough;
|
||||
stderr: PassThrough;
|
||||
kill: ReturnType<typeof vi.fn>;
|
||||
};
|
||||
child.stdout = new PassThrough();
|
||||
child.stderr = new PassThrough();
|
||||
child.kill = vi.fn();
|
||||
queueMicrotask(() => {
|
||||
fs.writeFileSync(path.join(configDir, "seed-manifest.json"), JSON.stringify({
|
||||
version: 2,
|
||||
source: { instanceId: "source", configPath: sourceConfigPath },
|
||||
state: "failed",
|
||||
phase: "restore",
|
||||
attemptId: "repair-attempt",
|
||||
}));
|
||||
child.emit("exit", 4);
|
||||
});
|
||||
return child;
|
||||
});
|
||||
|
||||
const progress: Array<Record<string, unknown>> = [];
|
||||
mockWorkspaceOperationService.createRecorder.mockReturnValue({
|
||||
attachExecutionWorkspaceId: vi.fn(),
|
||||
recordOperation: async (input: any) => {
|
||||
await input.run(async (entry: Record<string, unknown>) => {
|
||||
progress.push(entry);
|
||||
});
|
||||
},
|
||||
});
|
||||
const app = await createApp();
|
||||
const res = await request(app)
|
||||
.post(`/api/execution-workspaces/${executionWorkspaceId}/runtime-commands/repair`)
|
||||
.send({});
|
||||
|
||||
expect(res.status).toBeGreaterThanOrEqual(400);
|
||||
expect(progress).toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({ metadata: { seedFailurePhase: "restore" } }),
|
||||
]));
|
||||
expect(mockStartRuntimeServices).not.toHaveBeenCalled();
|
||||
expect(mockStopRuntimeServicesForExecutionWorkspace).toHaveBeenCalledTimes(2);
|
||||
} finally {
|
||||
fs.rmSync(workspaceCwd, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("reconciles once when the operation's own time budget fails a start that never settles", async () => {
|
||||
const { WorkspaceOperationTimeoutError } = await import("../services/workspace-operations.js");
|
||||
// The recorder's ceiling fires outside `run`, so only the outer handler can reconcile.
|
||||
|
||||
@@ -19,6 +19,7 @@ const mockExecutionWorkspaceService = vi.hoisted(() => ({
|
||||
const mockWorkspaceOperationService = vi.hoisted(() => ({
|
||||
listForExecutionWorkspace: vi.fn(),
|
||||
createRecorder: vi.fn(),
|
||||
assertRuntimeControlAvailable: vi.fn(async () => undefined),
|
||||
}));
|
||||
|
||||
const mockWorkspaceRuntimeLeaseService = vi.hoisted(() => ({
|
||||
@@ -47,7 +48,7 @@ vi.mock("../services/index.js", () => ({
|
||||
logActivity: mockLogActivity,
|
||||
workspaceOperationService: () => mockWorkspaceOperationService,
|
||||
workspaceRuntimeLeaseService: () => mockWorkspaceRuntimeLeaseService,
|
||||
LEASED_WORKSPACE_RUNTIME_ACTIONS: ["start", "stop", "restart"],
|
||||
LEASED_WORKSPACE_RUNTIME_ACTIONS: ["start", "stop", "restart", "repair"],
|
||||
}));
|
||||
|
||||
vi.mock("../services/environment-runtime.js", () => ({
|
||||
|
||||
@@ -44,14 +44,14 @@ describe("workspace runtime control serialization", () => {
|
||||
|
||||
await expect(runExclusiveWorkspaceRuntimeControl({
|
||||
executionWorkspaceId: "workspace-1",
|
||||
action: "restart",
|
||||
run: async () => "restarted",
|
||||
action: "repair",
|
||||
run: async () => "repaired",
|
||||
})).rejects.toMatchObject({
|
||||
status: 409,
|
||||
details: {
|
||||
code: "workspace_runtime_control_in_progress",
|
||||
activeAction: "start",
|
||||
requestedAction: "restart",
|
||||
requestedAction: "repair",
|
||||
},
|
||||
});
|
||||
|
||||
@@ -204,4 +204,71 @@ describeEmbeddedPostgres("workspace operation reconciliation", () => {
|
||||
.then((rows) => rows[0]);
|
||||
expect(unrelated?.status).toBe("running");
|
||||
});
|
||||
|
||||
it("persists bounded repair phase progress on the operation row and log", async () => {
|
||||
const companyId = randomUUID();
|
||||
const projectId = randomUUID();
|
||||
const executionWorkspaceId = randomUUID();
|
||||
await db.insert(companies).values({
|
||||
id: companyId,
|
||||
name: "Workspace repair diagnostics",
|
||||
issuePrefix: `R${companyId.replace(/-/g, "").slice(0, 7).toUpperCase()}`,
|
||||
requireBoardApprovalForNewAgents: false,
|
||||
});
|
||||
await db.insert(projects).values({
|
||||
id: projectId,
|
||||
companyId,
|
||||
name: "Repair diagnostics",
|
||||
status: "in_progress",
|
||||
});
|
||||
await db.insert(executionWorkspaces).values({
|
||||
id: executionWorkspaceId,
|
||||
companyId,
|
||||
projectId,
|
||||
mode: "isolated_workspace",
|
||||
strategyType: "git_worktree",
|
||||
name: "Repair diagnostics workspace",
|
||||
status: "active",
|
||||
cwd: "/tmp/repair-diagnostics-workspace",
|
||||
});
|
||||
|
||||
const operation = await workspaceOperationService(db)
|
||||
.createRecorder({ companyId, executionWorkspaceId })
|
||||
.recordOperation({
|
||||
phase: "workspace_repair",
|
||||
command: "workspace command repair",
|
||||
cwd: "/tmp/repair-diagnostics-workspace",
|
||||
metadata: { action: "repair" },
|
||||
run: async (reportProgress) => {
|
||||
await reportProgress({
|
||||
metadata: {
|
||||
repairPhase: "target_backup",
|
||||
repairDiagnostics: [{
|
||||
phase: "target_backup",
|
||||
status: "succeeded",
|
||||
at: "2026-08-18T00:00:00.000Z",
|
||||
}],
|
||||
databaseOnly: true,
|
||||
worktreePreserved: true,
|
||||
},
|
||||
system: "Workspace repair target_backup: succeeded.\n",
|
||||
});
|
||||
return { status: "succeeded", metadata: { backupRetained: true } };
|
||||
},
|
||||
});
|
||||
|
||||
expect(operation).toMatchObject({
|
||||
status: "succeeded",
|
||||
metadata: {
|
||||
action: "repair",
|
||||
repairPhase: "target_backup",
|
||||
databaseOnly: true,
|
||||
worktreePreserved: true,
|
||||
backupRetained: true,
|
||||
},
|
||||
});
|
||||
expect((await workspaceOperationService(db).readLog(operation.id)).content).toContain(
|
||||
"Workspace repair target_backup: succeeded.",
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -54,7 +54,7 @@ vi.mock("../services/index.js", () => ({
|
||||
secretService: () => mockSecretService,
|
||||
workspaceOperationService: () => mockWorkspaceOperationService,
|
||||
workspaceRuntimeLeaseService: () => mockWorkspaceRuntimeLeaseService,
|
||||
LEASED_WORKSPACE_RUNTIME_ACTIONS: ["start", "stop", "restart"],
|
||||
LEASED_WORKSPACE_RUNTIME_ACTIONS: ["start", "stop", "restart", "repair"],
|
||||
}));
|
||||
|
||||
vi.mock("../services/workspace-runtime.js", () => ({
|
||||
@@ -84,7 +84,7 @@ function registerWorkspaceRouteMocks() {
|
||||
secretService: () => mockSecretService,
|
||||
workspaceOperationService: () => mockWorkspaceOperationService,
|
||||
workspaceRuntimeLeaseService: () => mockWorkspaceRuntimeLeaseService,
|
||||
LEASED_WORKSPACE_RUNTIME_ACTIONS: ["start", "stop", "restart"],
|
||||
LEASED_WORKSPACE_RUNTIME_ACTIONS: ["start", "stop", "restart", "repair"],
|
||||
}));
|
||||
|
||||
vi.doMock("../services/workspace-runtime.js", () => ({
|
||||
|
||||
@@ -456,6 +456,39 @@ describe("resolveRuntimeProvisionCommand", () => {
|
||||
|
||||
await fs.writeFile(path.join(cwd, ".paperclip", "seed-complete"), "{}\n");
|
||||
expect(resolveRuntimeProvisionCommand({ config: {}, workspace })).toBe("");
|
||||
|
||||
await fs.writeFile(
|
||||
path.join(cwd, ".paperclip", "seed-manifest.json"),
|
||||
JSON.stringify({ version: 2, state: "failed" }),
|
||||
);
|
||||
expect(resolveRuntimeProvisionCommand({ config: {}, workspace })).toBe(
|
||||
"bash ./scripts/provision-worktree-runtime.sh",
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(cwd, ".paperclip", "seed-manifest.json"),
|
||||
JSON.stringify({ version: 2, state: "verified" }),
|
||||
);
|
||||
expect(resolveRuntimeProvisionCommand({ config: {}, workspace })).toBe(
|
||||
"bash ./scripts/provision-worktree-runtime.sh",
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(cwd, ".paperclip", "seed-manifest.json"),
|
||||
JSON.stringify({
|
||||
version: 2,
|
||||
source: { instanceId: "source", configPath: "/source/config.json" },
|
||||
snapshotAt: "2026-08-18T00:00:00.000Z",
|
||||
seedMode: "full",
|
||||
migrationRevision: "0001",
|
||||
targetInstanceId: "target",
|
||||
phase: "complete",
|
||||
state: "verified",
|
||||
attemptId: "attempt",
|
||||
startedAt: "2026-08-18T00:00:00.000Z",
|
||||
finishedAt: "2026-08-18T00:01:00.000Z",
|
||||
diagnostics: [{ phase: "complete", status: "succeeded", at: "2026-08-18T00:01:00.000Z" }],
|
||||
}),
|
||||
);
|
||||
expect(resolveRuntimeProvisionCommand({ config: {}, workspace })).toBe("");
|
||||
} finally {
|
||||
await fs.rm(baseCwd, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { existsSync, lstatSync, readFileSync } from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { hasVerifiedWorktreeSeedManifest } from "./worktree-seed-manifest.js";
|
||||
|
||||
function parseEnvFile(contents: string): Record<string, string> {
|
||||
const entries: Record<string, string> = {};
|
||||
@@ -58,6 +59,10 @@ export function resolveWorktreeEnvFilePath(rootDir: string): string {
|
||||
|
||||
export function isWorktreeSeedPending(rootDir: string): boolean {
|
||||
const markerDir = path.resolve(rootDir, ".paperclip");
|
||||
const manifestPath = path.resolve(markerDir, "seed-manifest.json");
|
||||
if (existsSync(manifestPath)) {
|
||||
return !hasVerifiedWorktreeSeedManifest(manifestPath);
|
||||
}
|
||||
return existsSync(path.resolve(markerDir, "seed-pending"))
|
||||
&& !existsSync(path.resolve(markerDir, "seed-complete"));
|
||||
}
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { Router, type Request, type Response } from "express";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
@@ -45,6 +48,8 @@ import { appendWithCap } from "../adapters/utils.js";
|
||||
import { environmentRuntimeService } from "../services/environment-runtime.js";
|
||||
import type { PluginWorkerManager } from "../services/plugin-worker-manager.js";
|
||||
import { runExclusiveWorkspaceRuntimeControl } from "../services/workspace-operations.js";
|
||||
import { deriveWorktreeInstanceId } from "../services/workspace-instance-cleanup.js";
|
||||
import { isVerifiedWorktreeSeedManifest } from "../worktree-seed-manifest.js";
|
||||
|
||||
const WORKSPACE_CONTROL_OUTPUT_MAX_CHARS = 256 * 1024;
|
||||
|
||||
@@ -151,7 +156,7 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
|
||||
async function handleExecutionWorkspaceRuntimeCommand(req: Request, res: Response) {
|
||||
const id = req.params.id as string;
|
||||
const action = String(req.params.action ?? "").trim().toLowerCase();
|
||||
if (action !== "start" && action !== "stop" && action !== "restart" && action !== "run") {
|
||||
if (action !== "start" && action !== "stop" && action !== "restart" && action !== "repair" && action !== "run") {
|
||||
res.status(404).json({ error: "Workspace command action not found" });
|
||||
return;
|
||||
}
|
||||
@@ -262,7 +267,7 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
|
||||
return;
|
||||
}
|
||||
|
||||
if ((action === "start" || action === "restart") && !effectiveRuntimeConfig) {
|
||||
if ((action === "start" || action === "restart" || action === "repair") && !effectiveRuntimeConfig) {
|
||||
res.status(422).json({ error: "Execution workspace has no workspace command configuration or inherited project workspace default" });
|
||||
return;
|
||||
}
|
||||
@@ -327,7 +332,11 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
|
||||
}
|
||||
|
||||
const recordRuntimeControlOperation = () => recorder.recordOperation({
|
||||
phase: action === "stop" ? "workspace_teardown" : "workspace_provision",
|
||||
phase: action === "stop"
|
||||
? "workspace_teardown"
|
||||
: action === "repair"
|
||||
? "workspace_repair"
|
||||
: "workspace_provision",
|
||||
command: workspaceCommand?.command ?? `workspace command ${action}`,
|
||||
cwd: existing.cwd,
|
||||
metadata: {
|
||||
@@ -339,7 +348,7 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
|
||||
runtimeServiceId: selectedRuntimeServiceId,
|
||||
serviceIndex: selectedServiceIndex,
|
||||
},
|
||||
run: async () => {
|
||||
run: async (reportProgress) => {
|
||||
const ensureWorkspaceAvailable = async () =>
|
||||
await ensurePersistedExecutionWorkspaceAvailable({
|
||||
base: {
|
||||
@@ -429,7 +438,234 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
|
||||
else stderr = appendWithCap(stderr, chunk, WORKSPACE_CONTROL_OUTPUT_MAX_CHARS);
|
||||
};
|
||||
|
||||
if (action === "stop" || action === "restart") {
|
||||
if (action === "repair") {
|
||||
type RepairPhase =
|
||||
| "managed_stop"
|
||||
| "target_backup"
|
||||
| "full_reseed"
|
||||
| "managed_restart"
|
||||
| "readiness_validation";
|
||||
const repairDiagnostics: Array<{
|
||||
phase: RepairPhase;
|
||||
status: "started" | "succeeded" | "failed";
|
||||
at: string;
|
||||
}> = [];
|
||||
let repairPhase: RepairPhase = "managed_stop";
|
||||
const reportRepairPhase = async (
|
||||
phase: RepairPhase,
|
||||
status: "started" | "succeeded" | "failed",
|
||||
) => {
|
||||
repairPhase = phase;
|
||||
repairDiagnostics.push({ phase, status, at: new Date().toISOString() });
|
||||
if (repairDiagnostics.length > 32) repairDiagnostics.splice(0, repairDiagnostics.length - 32);
|
||||
await reportProgress({
|
||||
metadata: {
|
||||
repairPhase,
|
||||
repairDiagnostics: [...repairDiagnostics],
|
||||
databaseOnly: true,
|
||||
worktreePreserved: true,
|
||||
},
|
||||
system: `Workspace repair ${phase}: ${status}.\n`,
|
||||
});
|
||||
};
|
||||
|
||||
try {
|
||||
await reportRepairPhase("managed_stop", "started");
|
||||
await stopRuntimeServicesForExecutionWorkspace({
|
||||
db,
|
||||
executionWorkspaceId: existing.id,
|
||||
workspaceCwd,
|
||||
});
|
||||
await reportRepairPhase("managed_stop", "succeeded");
|
||||
|
||||
const manifestPath = path.join(workspaceCwd, ".paperclip", "seed-manifest.json");
|
||||
const targetConfigPath = path.join(workspaceCwd, ".paperclip", "config.json");
|
||||
let sourceConfigPath: string | null = null;
|
||||
let previousAttemptId: string | null = null;
|
||||
if (existsSync(manifestPath)) {
|
||||
try {
|
||||
const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as {
|
||||
attemptId?: unknown;
|
||||
source?: { configPath?: unknown };
|
||||
};
|
||||
previousAttemptId = typeof manifest.attemptId === "string" ? manifest.attemptId : null;
|
||||
sourceConfigPath = typeof manifest.source?.configPath === "string"
|
||||
? path.resolve(manifest.source.configPath)
|
||||
: null;
|
||||
} catch {
|
||||
throw new Error("Workspace seed manifest is malformed; repair source identity cannot be trusted.");
|
||||
}
|
||||
}
|
||||
const baseWorkspaceCwd = projectWorkspace?.cwd ?? workspaceCwd;
|
||||
const baseConfigPath = path.join(baseWorkspaceCwd, ".paperclip", "config.json");
|
||||
if (!sourceConfigPath && existsSync(baseConfigPath) && path.resolve(baseConfigPath) !== path.resolve(targetConfigPath)) {
|
||||
sourceConfigPath = baseConfigPath;
|
||||
}
|
||||
if (!sourceConfigPath || !existsSync(sourceConfigPath)) {
|
||||
throw new Error("Workspace repair has no readable configured source instance.");
|
||||
}
|
||||
|
||||
const cliRunner = path.join(baseWorkspaceCwd, "cli", "node_modules", "tsx", "dist", "cli.mjs");
|
||||
const cliEntry = path.join(baseWorkspaceCwd, "cli", "src", "index.ts");
|
||||
const cliDist = path.join(baseWorkspaceCwd, "cli", "dist", "index.js");
|
||||
const cliArgs = existsSync(cliRunner) && existsSync(cliEntry)
|
||||
? [cliRunner, cliEntry]
|
||||
: existsSync(cliDist)
|
||||
? [cliDist]
|
||||
: null;
|
||||
if (!cliArgs) {
|
||||
throw new Error("Workspace repair cannot find a runnable Paperclip CLI in the base workspace.");
|
||||
}
|
||||
|
||||
await reportRepairPhase("target_backup", "started");
|
||||
const child = spawn(process.execPath, [
|
||||
...cliArgs,
|
||||
"worktree",
|
||||
"reseed",
|
||||
"--from-config",
|
||||
sourceConfigPath,
|
||||
"--to",
|
||||
workspaceCwd,
|
||||
"--seed-mode",
|
||||
"full",
|
||||
"--yes",
|
||||
"--backup-target",
|
||||
], {
|
||||
cwd: baseWorkspaceCwd,
|
||||
env: process.env,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
child.stdout?.on("data", (chunk: Buffer) => {
|
||||
void onLog("stdout", chunk.toString("utf8"));
|
||||
});
|
||||
child.stderr?.on("data", (chunk: Buffer) => {
|
||||
void onLog("stderr", chunk.toString("utf8"));
|
||||
});
|
||||
let repairCommandTimedOut = false;
|
||||
let repairCommandForceTimeout: NodeJS.Timeout | null = null;
|
||||
const repairCommandTimeout = setTimeout(() => {
|
||||
repairCommandTimedOut = true;
|
||||
child.kill("SIGTERM");
|
||||
repairCommandForceTimeout = setTimeout(() => child.kill("SIGKILL"), 5_000);
|
||||
repairCommandForceTimeout.unref?.();
|
||||
}, 20 * 60_000);
|
||||
repairCommandTimeout.unref?.();
|
||||
|
||||
let reseedObserved = false;
|
||||
const manifestPoll = setInterval(() => {
|
||||
if (reseedObserved || !existsSync(manifestPath)) return;
|
||||
try {
|
||||
const current = JSON.parse(readFileSync(manifestPath, "utf8")) as {
|
||||
attemptId?: unknown;
|
||||
state?: unknown;
|
||||
};
|
||||
if (
|
||||
typeof current.attemptId === "string"
|
||||
&& current.attemptId !== previousAttemptId
|
||||
&& (current.state === "pending" || current.state === "running" || current.state === "failed")
|
||||
) {
|
||||
reseedObserved = true;
|
||||
}
|
||||
} catch {
|
||||
// The atomic manifest writer makes this unlikely; the terminal
|
||||
// read below remains authoritative.
|
||||
}
|
||||
}, 100);
|
||||
manifestPoll.unref?.();
|
||||
const exitCode = await new Promise<number>((resolve, reject) => {
|
||||
child.once("error", reject);
|
||||
child.once("exit", (code) => resolve(code ?? 1));
|
||||
}).finally(() => {
|
||||
clearInterval(manifestPoll);
|
||||
clearTimeout(repairCommandTimeout);
|
||||
if (repairCommandForceTimeout) clearTimeout(repairCommandForceTimeout);
|
||||
});
|
||||
if (reseedObserved) {
|
||||
await reportRepairPhase("target_backup", "succeeded");
|
||||
await reportRepairPhase("full_reseed", "started");
|
||||
}
|
||||
if (exitCode !== 0) {
|
||||
let seedFailurePhase: string | null = null;
|
||||
try {
|
||||
const failedManifest = JSON.parse(readFileSync(manifestPath, "utf8")) as {
|
||||
phase?: unknown;
|
||||
state?: unknown;
|
||||
};
|
||||
seedFailurePhase = failedManifest.state === "failed" && typeof failedManifest.phase === "string"
|
||||
? failedManifest.phase
|
||||
: null;
|
||||
} catch {
|
||||
// The outer repair phase remains exact when no seed phase was persisted.
|
||||
}
|
||||
await reportProgress({
|
||||
metadata: { seedFailurePhase },
|
||||
system: seedFailurePhase ? `Workspace seed failed during ${seedFailurePhase}.\n` : null,
|
||||
});
|
||||
throw new Error(
|
||||
repairCommandTimedOut
|
||||
? `Workspace database repair command timed out during ${repairPhase}.`
|
||||
: `Workspace database repair command failed during ${repairPhase}.`,
|
||||
);
|
||||
}
|
||||
if (!reseedObserved) {
|
||||
await reportRepairPhase("target_backup", "succeeded");
|
||||
await reportRepairPhase("full_reseed", "started");
|
||||
}
|
||||
|
||||
const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as Record<string, unknown>;
|
||||
if (!isVerifiedWorktreeSeedManifest(manifest)) {
|
||||
throw new Error("Workspace reseed returned without a verified terminal manifest.");
|
||||
}
|
||||
const expectedInstanceId = deriveWorktreeInstanceId(workspaceCwd);
|
||||
if (manifest.targetInstanceId !== expectedInstanceId) {
|
||||
throw new Error("Verified seed manifest belongs to a different workspace instance.");
|
||||
}
|
||||
await reportRepairPhase("full_reseed", "succeeded");
|
||||
|
||||
await reportRepairPhase("managed_restart", "started");
|
||||
const availableWorkspace = await ensureWorkspaceAvailable();
|
||||
if (!availableWorkspace) {
|
||||
throw new Error("Execution workspace needs a local path before Paperclip can restart it.");
|
||||
}
|
||||
const startedServices = await startRuntimeServicesForWorkspaceControl({
|
||||
db,
|
||||
actor: {
|
||||
id: actor.agentId ?? null,
|
||||
name: actor.actorType === "user" ? "Board" : "Agent",
|
||||
companyId: existing.companyId,
|
||||
},
|
||||
issue: existing.sourceIssueId
|
||||
? { id: existing.sourceIssueId, identifier: null, title: existing.name }
|
||||
: null,
|
||||
workspace: availableWorkspace,
|
||||
executionWorkspaceId: existing.id,
|
||||
config: {
|
||||
workspaceRuntime: effectiveRuntimeConfig,
|
||||
runtimeProvisionCommand:
|
||||
existing.config?.runtimeProvisionCommand
|
||||
?? projectPolicy?.workspaceStrategy?.runtimeProvisionCommand
|
||||
?? null,
|
||||
},
|
||||
adapterEnv: {},
|
||||
onLog,
|
||||
recorder,
|
||||
});
|
||||
runtimeServiceCount = startedServices.length;
|
||||
await reportRepairPhase("managed_restart", "succeeded");
|
||||
|
||||
await reportRepairPhase("readiness_validation", "started");
|
||||
if (
|
||||
startedServices.length === 0
|
||||
|| startedServices.some((service) => service.status !== "running" || service.healthStatus !== "healthy")
|
||||
) {
|
||||
throw new Error("Managed restart did not produce healthy runtime services.");
|
||||
}
|
||||
await reportRepairPhase("readiness_validation", "succeeded");
|
||||
} catch (error) {
|
||||
await reportRepairPhase(repairPhase, "failed");
|
||||
throw error;
|
||||
}
|
||||
} else if (action === "stop" || action === "restart") {
|
||||
await stopRuntimeServicesForExecutionWorkspace({
|
||||
db,
|
||||
executionWorkspaceId: existing.id,
|
||||
@@ -481,7 +717,7 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
|
||||
throw error;
|
||||
}
|
||||
runtimeServiceCount = startedServices.length;
|
||||
} else {
|
||||
} else if (action !== "repair") {
|
||||
runtimeServiceCount = selectedRuntimeServiceId ? Math.max(0, (existing.runtimeServices?.length ?? 1) - 1) : 0;
|
||||
}
|
||||
|
||||
@@ -504,7 +740,7 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
|
||||
config: { workspaceRuntime: effectiveRuntimeConfig },
|
||||
currentDesiredState,
|
||||
currentServiceStates: existing.config?.serviceStates ?? null,
|
||||
action,
|
||||
action: action === "repair" ? "start" : action,
|
||||
serviceIndex: selectedServiceIndex,
|
||||
});
|
||||
const metadata = mergeExecutionWorkspaceConfig(existing.metadata as Record<string, unknown> | null, {
|
||||
@@ -522,6 +758,8 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
|
||||
? "Stopped execution workspace runtime services.\n"
|
||||
: action === "restart"
|
||||
? "Restarted execution workspace runtime services.\n"
|
||||
: action === "repair"
|
||||
? "Repaired the isolated workspace database and restarted healthy runtime services.\n"
|
||||
: "Started execution workspace runtime services.\n",
|
||||
metadata: {
|
||||
runtimeServiceCount,
|
||||
@@ -565,7 +803,9 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
|
||||
// The operation is already terminal here. This also catches the recorder's own time
|
||||
// budget expiring on a start that never settles, which is the one failure the inner
|
||||
// handler cannot see — reconcile there too so no listener or desired state is left over.
|
||||
if (action === "start" || action === "restart") await reconcileFailedRuntimeStart(error);
|
||||
if (action === "start" || action === "restart" || action === "repair") {
|
||||
await reconcileFailedRuntimeStart(error);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
|
||||
@@ -14,7 +14,7 @@ type WorkspaceOperationRow = typeof workspaceOperations.$inferSelect;
|
||||
* Managed runtime control actions. Every one of these mutates the runtime rows and
|
||||
* local listeners of a single execution workspace, so only one may be live at a time.
|
||||
*/
|
||||
const RUNTIME_CONTROL_ACTIONS = new Set(["start", "stop", "restart", "run"]);
|
||||
const RUNTIME_CONTROL_ACTIONS = new Set(["start", "stop", "restart", "repair", "run"]);
|
||||
|
||||
/**
|
||||
* Identity of this server process. A `running` runtime-control operation stamped with a
|
||||
@@ -234,7 +234,10 @@ export interface WorkspaceOperationRecorder {
|
||||
* timeout error is rethrown, so a hung provider can never leave an active operation.
|
||||
*/
|
||||
timeoutMs?: number | null;
|
||||
run: () => Promise<{
|
||||
run: (reportProgress: (input: {
|
||||
metadata?: Record<string, unknown> | null;
|
||||
system?: string | null;
|
||||
}) => Promise<void>) => Promise<{
|
||||
status?: WorkspaceOperationStatus;
|
||||
exitCode?: number | null;
|
||||
stdout?: string | null;
|
||||
@@ -484,7 +487,7 @@ export function workspaceOperationService(db: Db) {
|
||||
// Managed runtime controls get an ownership stamp so bounded recovery can tell a
|
||||
// slow-but-live operation from one abandoned by a dead request or server process.
|
||||
const runtimeControlAction = readRuntimeControlAction(recordInput.metadata);
|
||||
const insertedMetadata = runtimeControlAction
|
||||
let currentMetadata = runtimeControlAction
|
||||
? {
|
||||
...(recordInput.metadata ?? {}),
|
||||
runtimeControlOwner: {
|
||||
@@ -515,7 +518,7 @@ export function workspaceOperationService(db: Db) {
|
||||
logStore: handle.store,
|
||||
logRef: handle.logRef,
|
||||
metadata: redactCurrentUserValue(
|
||||
insertedMetadata,
|
||||
currentMetadata,
|
||||
currentUserRedactionOptions,
|
||||
) as Record<string, unknown> | null,
|
||||
startedAt,
|
||||
@@ -532,17 +535,21 @@ export function workspaceOperationService(db: Db) {
|
||||
if (runtimeControlAction) {
|
||||
heartbeatTimer = setInterval(() => {
|
||||
const heartbeatAt = new Date();
|
||||
currentMetadata = combineMetadata(currentMetadata, {
|
||||
runtimeControlOwner: {
|
||||
ownerId: RUNTIME_CONTROL_OWNER_ID,
|
||||
pid: process.pid,
|
||||
action: runtimeControlAction,
|
||||
heartbeatAt: heartbeatAt.toISOString(),
|
||||
} satisfies RuntimeControlOwnerStamp,
|
||||
});
|
||||
void db
|
||||
.update(workspaceOperations)
|
||||
.set({
|
||||
metadata: combineMetadata(insertedMetadata, {
|
||||
runtimeControlOwner: {
|
||||
ownerId: RUNTIME_CONTROL_OWNER_ID,
|
||||
pid: process.pid,
|
||||
action: runtimeControlAction,
|
||||
heartbeatAt: heartbeatAt.toISOString(),
|
||||
} satisfies RuntimeControlOwnerStamp,
|
||||
}),
|
||||
metadata: redactCurrentUserValue(
|
||||
currentMetadata,
|
||||
currentUserRedactionOptions,
|
||||
) as Record<string, unknown> | null,
|
||||
updatedAt: heartbeatAt,
|
||||
})
|
||||
.where(and(eq(workspaceOperations.id, id), eq(workspaceOperations.status, "running")))
|
||||
@@ -551,9 +558,29 @@ export function workspaceOperationService(db: Db) {
|
||||
heartbeatTimer.unref?.();
|
||||
}
|
||||
|
||||
const reportProgress = async (progress: {
|
||||
metadata?: Record<string, unknown> | null;
|
||||
system?: string | null;
|
||||
}) => {
|
||||
await append("system", progress.system ?? null);
|
||||
currentMetadata = combineMetadata(currentMetadata, progress.metadata);
|
||||
await db
|
||||
.update(workspaceOperations)
|
||||
.set({
|
||||
metadata: redactCurrentUserValue(
|
||||
currentMetadata,
|
||||
currentUserRedactionOptions,
|
||||
) as Record<string, unknown> | null,
|
||||
stdoutExcerpt: stdoutExcerpt || null,
|
||||
stderrExcerpt: stderrExcerpt || null,
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(and(eq(workspaceOperations.id, id), eq(workspaceOperations.status, "running")));
|
||||
};
|
||||
|
||||
const timeoutMs = recordInput.timeoutMs ?? defaultRuntimeControlTimeoutMs(runtimeControlAction);
|
||||
const settle = async () => {
|
||||
if (!timeoutMs || timeoutMs <= 0) return await recordInput.run();
|
||||
if (!timeoutMs || timeoutMs <= 0) return await recordInput.run(reportProgress);
|
||||
const timeout = new Promise<never>((_resolve, reject) => {
|
||||
timeoutTimer = setTimeout(
|
||||
() => reject(new WorkspaceOperationTimeoutError(timeoutMs, runtimeControlAction)),
|
||||
@@ -561,7 +588,7 @@ export function workspaceOperationService(db: Db) {
|
||||
);
|
||||
timeoutTimer.unref?.();
|
||||
});
|
||||
return await Promise.race([recordInput.run(), timeout]);
|
||||
return await Promise.race([recordInput.run(reportProgress), timeout]);
|
||||
};
|
||||
|
||||
try {
|
||||
@@ -583,7 +610,7 @@ export function workspaceOperationService(db: Db) {
|
||||
logSha256: finalized.sha256,
|
||||
logCompressed: finalized.compressed,
|
||||
metadata: redactCurrentUserValue(
|
||||
combineMetadata(insertedMetadata, result.metadata),
|
||||
combineMetadata(currentMetadata, result.metadata),
|
||||
currentUserRedactionOptions,
|
||||
) as Record<string, unknown> | null,
|
||||
finishedAt,
|
||||
@@ -613,7 +640,7 @@ export function workspaceOperationService(db: Db) {
|
||||
...(runtimeControlAction
|
||||
? {
|
||||
metadata: redactCurrentUserValue(
|
||||
combineMetadata(insertedMetadata, {
|
||||
combineMetadata(currentMetadata, {
|
||||
failureReason: error instanceof WorkspaceOperationTimeoutError
|
||||
? "runtime_control_timeout"
|
||||
: "runtime_control_error",
|
||||
|
||||
@@ -28,7 +28,7 @@ const TERMINAL_HEARTBEAT_RUN_STATUSES = new Set([
|
||||
]);
|
||||
|
||||
/** Runtime control actions that mutate the workspace and therefore need the lease. */
|
||||
export const LEASED_WORKSPACE_RUNTIME_ACTIONS: readonly string[] = ["start", "stop", "restart"];
|
||||
export const LEASED_WORKSPACE_RUNTIME_ACTIONS: readonly string[] = ["start", "stop", "restart", "repair"];
|
||||
|
||||
/**
|
||||
* Upper bound on how long a lease survives without the owner touching it. Recovery
|
||||
|
||||
@@ -36,6 +36,7 @@ import { and, desc, eq, gte, inArray, isNull, lte, ne, or } from "drizzle-orm";
|
||||
import { asNumber, asString, parseObject, renderTemplate } from "../adapters/utils.js";
|
||||
import { conflict } from "../errors.js";
|
||||
import { resolveHomeAwarePath } from "../home-paths.js";
|
||||
import { hasVerifiedWorktreeSeedManifest } from "../worktree-seed-manifest.js";
|
||||
import {
|
||||
createLocalServiceKey,
|
||||
findLocalServiceRegistryRecordByRuntimeServiceId,
|
||||
@@ -5015,6 +5016,7 @@ export function resolveRuntimeProvisionCommand(input: {
|
||||
if (input.workspace.strategy !== "git_worktree") return "";
|
||||
|
||||
const stateDir = path.join(input.workspace.cwd, ".paperclip");
|
||||
const manifestPath = path.join(stateDir, "seed-manifest.json");
|
||||
const pendingMarker = path.join(stateDir, "seed-pending");
|
||||
const completeMarker = path.join(stateDir, "seed-complete");
|
||||
const provisionScript = path.join(
|
||||
@@ -5022,11 +5024,11 @@ export function resolveRuntimeProvisionCommand(input: {
|
||||
"scripts",
|
||||
"provision-worktree-runtime.sh",
|
||||
);
|
||||
if (
|
||||
!existsSync(pendingMarker)
|
||||
|| existsSync(completeMarker)
|
||||
|| !existsSync(provisionScript)
|
||||
) {
|
||||
let needsSeed = existsSync(pendingMarker) && !existsSync(completeMarker);
|
||||
if (existsSync(manifestPath)) {
|
||||
needsSeed = !hasVerifiedWorktreeSeedManifest(manifestPath);
|
||||
}
|
||||
if (!needsSeed || !existsSync(provisionScript)) {
|
||||
return "";
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
|
||||
const WORKTREE_SEED_MODES = new Set(["minimal", "full"]);
|
||||
|
||||
type SeedDiagnostic = {
|
||||
phase?: unknown;
|
||||
status?: unknown;
|
||||
at?: unknown;
|
||||
};
|
||||
|
||||
/**
|
||||
* A manifest is readiness evidence only when the complete validation phase was
|
||||
* durably recorded. A bare `state: verified` is deliberately insufficient so
|
||||
* truncated or hand-edited files cannot start a partially restored workspace.
|
||||
*/
|
||||
export function isVerifiedWorktreeSeedManifest(value: unknown): boolean {
|
||||
if (!value || typeof value !== "object") return false;
|
||||
const manifest = value as Record<string, unknown>;
|
||||
const source = manifest.source as Record<string, unknown> | null;
|
||||
const diagnostics = manifest.diagnostics as SeedDiagnostic[] | null;
|
||||
return manifest.version === 2
|
||||
&& manifest.state === "verified"
|
||||
&& manifest.phase === "complete"
|
||||
&& typeof source?.instanceId === "string"
|
||||
&& source.instanceId.length > 0
|
||||
&& typeof source.configPath === "string"
|
||||
&& source.configPath.length > 0
|
||||
&& WORKTREE_SEED_MODES.has(String(manifest.seedMode ?? ""))
|
||||
&& typeof manifest.snapshotAt === "string"
|
||||
&& manifest.snapshotAt.length > 0
|
||||
&& typeof manifest.migrationRevision === "string"
|
||||
&& manifest.migrationRevision.length > 0
|
||||
&& typeof manifest.targetInstanceId === "string"
|
||||
&& manifest.targetInstanceId.length > 0
|
||||
&& typeof manifest.attemptId === "string"
|
||||
&& manifest.attemptId.length > 0
|
||||
&& typeof manifest.startedAt === "string"
|
||||
&& typeof manifest.finishedAt === "string"
|
||||
&& Array.isArray(diagnostics)
|
||||
&& diagnostics.some((diagnostic) => (
|
||||
diagnostic?.phase === "complete"
|
||||
&& diagnostic.status === "succeeded"
|
||||
&& typeof diagnostic.at === "string"
|
||||
));
|
||||
}
|
||||
|
||||
export function hasVerifiedWorktreeSeedManifest(manifestPath: string): boolean {
|
||||
if (!existsSync(manifestPath)) return false;
|
||||
try {
|
||||
return isVerifiedWorktreeSeedManifest(JSON.parse(readFileSync(manifestPath, "utf8")));
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -114,6 +114,11 @@ export const executionWorkspacesApi = {
|
||||
`/execution-workspaces/${id}/runtime-commands/${action}`,
|
||||
sanitizeWorkspaceRuntimeControlTarget(target),
|
||||
),
|
||||
repair: (id: string) =>
|
||||
api.post<{ workspace: ExecutionWorkspace; operation: WorkspaceOperation }>(
|
||||
`/execution-workspaces/${id}/runtime-commands/repair`,
|
||||
{},
|
||||
),
|
||||
update: (id: string, data: Record<string, unknown>) => api.patch<ExecutionWorkspace>(`/execution-workspaces/${id}`, data),
|
||||
/**
|
||||
* Reconcile a git-worktree branch divergence via the S4 (`PAP-1586`) op.
|
||||
|
||||
Reference in new issue
Block a user