mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-01 23:43:16 +01:00
Merge branch 'codex/boat-environment-core' into codex/boat-environment
* codex/boat-environment-core: fix: verify original workspace lineage after warm retirement test: synchronize remote upload concurrency assertions Keep remote instructions usable in large persistent folders fix(computers): prune retired desktop viewers during reconciliation fix(computers): serialize shared project checkout setup
This commit is contained in:
commit
029fde0ac1
7 files changed
+243
-69
No files matched your search
@@ -13,7 +13,7 @@ import {
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import { remoteProgram } from "./remote-program.js";
|
||||
import { afterEach, describe, it, expect, vi } from "vitest";
|
||||
import { boatBackend, createBoatTransportAdmission, desktopReadinessProgram, processProgram } from "./boat.js";
|
||||
@@ -350,6 +350,67 @@ process.stdout.write(result.stdout||"");process.stderr.write(result.stderr||"");
|
||||
expect(config).not.toContain("credential");
|
||||
expect(spawnSync(realGit, ["rev-parse", "HEAD"], { cwd: result.remoteCwd }).status).toBe(0);
|
||||
});
|
||||
it.each(["shared", "worktree"])("serializes concurrent first checkout creation for %s workspaces", async mode => {
|
||||
const f = fixture();
|
||||
const source = join(f.temp, "source"), bin = join(f.temp, "bin");
|
||||
mkdirSync(source); mkdirSync(bin);
|
||||
const git = spawnSync("which", ["git"], { encoding: "utf8" }).stdout.trim();
|
||||
for (const args of [["init"], ["-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "--allow-empty", "-m", "base"]]) {
|
||||
expect(spawnSync(git, args, { cwd: source, encoding: "utf8" }).status).toBe(0);
|
||||
}
|
||||
const started = join(f.temp, "clone-started"), release = join(f.temp, "clone-release"), clones = join(f.temp, "clones");
|
||||
// Pause real Git cloning after its destination exists. The second actual
|
||||
// remote program must wait instead of inspecting this unfinished checkout.
|
||||
writeFileSync(join(bin, "git"), `#!${process.execPath}
|
||||
const fs=require("node:fs"),{spawnSync}=require("node:child_process");
|
||||
const args=process.argv.slice(2),clone=args.indexOf("clone");
|
||||
if(clone>=0){
|
||||
fs.appendFileSync(process.env.FIXTURE_CLONES,"clone\\n");
|
||||
fs.mkdirSync(args.at(-1),{recursive:true});fs.writeFileSync(process.env.FIXTURE_STARTED,"ready");
|
||||
const deadline=Date.now()+5000;
|
||||
while(!fs.existsSync(process.env.FIXTURE_RELEASE)){if(Date.now()>deadline)process.exit(90);Atomics.wait(new Int32Array(new SharedArrayBuffer(4)),0,0,10);}
|
||||
const url=args[clone+2];args[clone+2]=process.env.FIXTURE_SOURCE;
|
||||
const result=spawnSync(process.env.FIXTURE_GIT,args,{encoding:"utf8"});
|
||||
if(result.status!==0){process.stderr.write(result.stderr);process.exit(result.status??1);}
|
||||
process.exit(spawnSync(process.env.FIXTURE_GIT,["-C",args.at(-1),"remote","set-url","origin",url]).status??1);
|
||||
}
|
||||
const result=spawnSync(process.env.FIXTURE_GIT,args,{encoding:"utf8"});
|
||||
process.stdout.write(result.stdout||"");process.stderr.write(result.stderr||"");process.exit(result.status??1);
|
||||
`, { mode: 0o700 });
|
||||
const env = { ...process.env, PATH: `${bin}:${process.env.PATH}`, FIXTURE_GIT: git, FIXTURE_SOURCE: source,
|
||||
FIXTURE_STARTED: started, FIXTURE_RELEASE: release, FIXTURE_CLONES: clones };
|
||||
const call = (taskId: string) => new Promise<{ remoteCwd: string }>((resolve, reject) => {
|
||||
const child = spawn("python3", ["-c", remoteProgram.replaceAll("/home/user/paperclip/", `${f.temp}/`)], { env });
|
||||
let stdout = "", stderr = "";
|
||||
child.stdout.on("data", value => { stdout += value; }); child.stderr.on("data", value => { stderr += value; });
|
||||
child.on("error", reject); child.on("close", code => {
|
||||
if (code !== 0) reject(new Error(stderr));
|
||||
else { try { resolve(JSON.parse(stdout)); } catch (error) { reject(error); } }
|
||||
});
|
||||
child.stdin.end(JSON.stringify({ root: f.root, action: "workspace", mode, taskId, repositoryUrl: "https://github.com/company/shared.git" }));
|
||||
});
|
||||
const first = call("first");
|
||||
await vi.waitFor(() => expect(existsSync(started)).toBe(true));
|
||||
const second = call("second");
|
||||
let status: unknown;
|
||||
try {
|
||||
status = await Promise.race([second.then(() => "completed"), new Promise(resolve => setTimeout(() => resolve("waiting"), 100))]);
|
||||
} finally { writeFileSync(release, "continue"); }
|
||||
const results = await Promise.all([first, second]);
|
||||
expect(status).toBe("waiting");
|
||||
expect(readFileSync(clones, "utf8").trim().split("\n")).toHaveLength(1);
|
||||
for (const [index, result] of results.entries()) {
|
||||
expect(result.remoteCwd).toBe(mode === "shared" ? join(f.root, "checkout") : join(f.root, "tasks", index === 0 ? "first" : "second"));
|
||||
expect(spawnSync(git, ["rev-parse", "--show-toplevel"], { cwd: result.remoteCwd, encoding: "utf8" }).stdout.trim()).toBe(result.remoteCwd);
|
||||
}
|
||||
});
|
||||
it("rejects substituted project locks without touching their target", () => {
|
||||
const f = fixture(); mkdirSync(f.root);
|
||||
const outside = join(f.temp, "outside"); writeFileSync(outside, "keep");
|
||||
symlinkSync(outside, join(f.root, ".paperclip-workspace.lock"));
|
||||
expect(f.call({ action: "workspace", mode: "shared" })).toEqual({ error: "invalid" });
|
||||
expect(readFileSync(outside, "utf8")).toBe("keep");
|
||||
});
|
||||
it.each([false, true])("uses the requested worktree branch separately from its start ref (explicit base: %s)", explicitBase => {
|
||||
const f = fixture();
|
||||
const checkout = join(f.root, "checkout");
|
||||
|
||||
@@ -60,36 +60,47 @@ if act=='seed':
|
||||
sys.exit(0)
|
||||
if act=='workspace':
|
||||
os.makedirs(root,exist_ok=True)
|
||||
repo=p.get('repositoryUrl');branch=p.get('branch');task=p.get('taskId');base_ref=p.get('baseRef')
|
||||
if base_ref and (base_ref.startswith('-') or '\n' in base_ref):fail('invalid')
|
||||
checkout=os.path.join(root,'checkout')
|
||||
if repo:
|
||||
if not (repo.startswith('https://') or repo.startswith('ssh://') or repo.startswith('git@')) or '\n' in repo:fail('invalid')
|
||||
if not os.path.exists(checkout):
|
||||
# Resolve credentials on the controller for this operation. Only the clone
|
||||
# child receives the helper environment; no credential file or Git config persists.
|
||||
auth=p.get('gitAuth') or {};git_env=dict(os.environ)
|
||||
git_env.update({'GIT_TERMINAL_PROMPT':'0','GIT_CONFIG_GLOBAL':'/dev/null','GIT_CONFIG_NOSYSTEM':'1'})
|
||||
git_env.update(auth.get('env',{}))
|
||||
run(['git']+auth.get('configArgs',[])+['clone','--',repo,checkout],env=git_env)
|
||||
elif run(['git','remote','get-url','origin'],checkout)!=repo:fail('conflict')
|
||||
else:
|
||||
os.makedirs(checkout,exist_ok=True)
|
||||
if not os.path.isdir(os.path.join(checkout,'.git')):run(['git','init',checkout])
|
||||
if branch:
|
||||
if branch.startswith('-') or '\n' in branch:fail('invalid')
|
||||
run(['git','check-ref-format','--branch',branch])
|
||||
if p.get('mode')=='worktree':
|
||||
if not task:fail('invalid')
|
||||
target=os.path.join(root,'tasks',task);os.makedirs(os.path.dirname(target),exist_ok=True)
|
||||
task_branch=branch or 'paperclip/task-'+task
|
||||
if not os.path.exists(target):run(['git','worktree','add','-b',task_branch,target,base_ref or 'HEAD'],checkout)
|
||||
run(['git','rev-parse','--show-toplevel'],target)
|
||||
if run(['git','symbolic-ref','--quiet','--short','HEAD'],target)!=task_branch:fail('conflict')
|
||||
else:
|
||||
target=checkout
|
||||
if branch:run(['git','checkout',branch],checkout)
|
||||
print(json.dumps({'remoteCwd':target}));sys.exit(0)
|
||||
# The root is the company/project placement on this physical computer. Keep
|
||||
# clone validation, shared checkout, and worktree creation under one flock.
|
||||
rootfd=os.open(root,os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW)
|
||||
try:
|
||||
lock=os.open('.paperclip-workspace.lock',os.O_CREAT|os.O_RDWR|os.O_NOFOLLOW,0o600,dir_fd=rootfd)
|
||||
except OSError:fail('invalid')
|
||||
finally:os.close(rootfd)
|
||||
with os.fdopen(lock,'r+') as workspace_lock:
|
||||
info=os.fstat(workspace_lock.fileno())
|
||||
if not stat.S_ISREG(info.st_mode) or info.st_nlink!=1:fail('invalid')
|
||||
fcntl.flock(workspace_lock,fcntl.LOCK_EX)
|
||||
repo=p.get('repositoryUrl');branch=p.get('branch');task=p.get('taskId');base_ref=p.get('baseRef')
|
||||
if base_ref and (base_ref.startswith('-') or '\n' in base_ref):fail('invalid')
|
||||
checkout=os.path.join(root,'checkout')
|
||||
if repo:
|
||||
if not (repo.startswith('https://') or repo.startswith('ssh://') or repo.startswith('git@')) or '\n' in repo:fail('invalid')
|
||||
if not os.path.exists(checkout):
|
||||
# Resolve credentials on the controller for this operation. Only the clone
|
||||
# child receives the helper environment; no credential file or Git config persists.
|
||||
auth=p.get('gitAuth') or {};git_env=dict(os.environ)
|
||||
git_env.update({'GIT_TERMINAL_PROMPT':'0','GIT_CONFIG_GLOBAL':'/dev/null','GIT_CONFIG_NOSYSTEM':'1'})
|
||||
git_env.update(auth.get('env',{}))
|
||||
run(['git']+auth.get('configArgs',[])+['clone','--',repo,checkout],env=git_env)
|
||||
elif run(['git','remote','get-url','origin'],checkout)!=repo:fail('conflict')
|
||||
else:
|
||||
os.makedirs(checkout,exist_ok=True)
|
||||
if not os.path.isdir(os.path.join(checkout,'.git')):run(['git','init',checkout])
|
||||
if branch:
|
||||
if branch.startswith('-') or '\n' in branch:fail('invalid')
|
||||
run(['git','check-ref-format','--branch',branch])
|
||||
if p.get('mode')=='worktree':
|
||||
if not task:fail('invalid')
|
||||
target=os.path.join(root,'tasks',task);os.makedirs(os.path.dirname(target),exist_ok=True)
|
||||
task_branch=branch or 'paperclip/task-'+task
|
||||
if not os.path.exists(target):run(['git','worktree','add','-b',task_branch,target,base_ref or 'HEAD'],checkout)
|
||||
run(['git','rev-parse','--show-toplevel'],target)
|
||||
if run(['git','symbolic-ref','--quiet','--short','HEAD'],target)!=task_branch:fail('conflict')
|
||||
else:
|
||||
target=checkout
|
||||
if branch:run(['git','checkout',branch],checkout)
|
||||
print(json.dumps({'remoteCwd':target}));sys.exit(0)
|
||||
# Hold directory descriptors throughout each operation. Ancestor symlink swaps cannot
|
||||
# redirect a checked pathname outside the selected placement.
|
||||
def directory(parts,create=False,start=None):
|
||||
|
||||
@@ -259,6 +259,24 @@ describe("computer ownership", () => {
|
||||
await f.service.reconcile();
|
||||
expect(f.backend.stop).toHaveBeenCalledOnce();
|
||||
});
|
||||
it("prunes retired viewers while preserving active viewers and runner stop proof", async () => {
|
||||
const f = fixture();
|
||||
await f.attach();
|
||||
const runner = await f.admit();
|
||||
await f.service.retire({ ...f.scope, owner: runner.owner });
|
||||
const expired = await f.service.connect({ ...f.scope, userId: "alice", idleTimeoutMs: 1000 });
|
||||
const active = await f.service.connect({ ...f.scope, userId: "bob", idleTimeoutMs: 60_000 });
|
||||
const disconnected = await f.service.connect({ ...f.scope, userId: "alice", idleTimeoutMs: 60_000 });
|
||||
await f.service.disconnectViewer({ ...f.scope, owner: disconnected.owner, userId: "alice" });
|
||||
f.advance(1001);
|
||||
await f.service.reconcile();
|
||||
const owners = (await f.repository.get(f.scope)).ledger.owners;
|
||||
expect(owners.map(owner => owner.id).sort()).toEqual([runner.owner.ownerId, active.owner.ownerId].sort());
|
||||
expect(owners.some(owner => [expired.owner.ownerId, disconnected.owner.ownerId].includes(owner.id))).toBe(false);
|
||||
expect(await f.service.isRetired({ ...f.scope, owner: runner.owner, agentId: "agent", runId: "run" })).toBe(true);
|
||||
await expect(f.service.renewViewer({ ...f.scope, owner: active.owner, userId: "bob" })).resolves.toHaveProperty("viewerUrl");
|
||||
expect(f.backend.stop).not.toHaveBeenCalled();
|
||||
});
|
||||
it("a failed exact-process retirement keeps the machine held", async () => {
|
||||
const f = fixture();
|
||||
await f.attach();
|
||||
|
||||
@@ -942,7 +942,8 @@ finally:
|
||||
await retire({ ...record, owner: ref(record, owner) });
|
||||
await repository.update(record, (current) => {
|
||||
current.ledger.owners = current.ledger.owners.filter(
|
||||
(owner) => owner.kind !== "file-operation" || owner.phase !== "retired",
|
||||
// Only runner tombstones are needed for durable process-stop proof.
|
||||
(owner) => owner.kind === "runner" || owner.phase !== "retired",
|
||||
);
|
||||
});
|
||||
record = await repository.get(record);
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { persistentAgentFiles, readPersistentAgentFile, seedPersistentAgentHome } from "./persistent-agent-files.js";
|
||||
import { persistentAgentFiles, readPersistentAgentFile, seedPersistentAgentHome, isMissingRemoteFile } from "./persistent-agent-files.js";
|
||||
import { adoptAgentFiles, inspectAgentFile, fileHash, agentFilePath, MAX_AGENT_FILE_BYTES } from "./agent-file-store.js";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
@@ -509,20 +509,41 @@ export function agentInstructionsService(db?: Db) {
|
||||
}
|
||||
return remote;
|
||||
}
|
||||
async function remotePaths(remote: NonNullable<Awaited<ReturnType<typeof remoteFiles>>>, relative = ""): Promise<Array<{ path: string; size: number }>> {
|
||||
const result: Array<{ path: string; size: number }> = [];
|
||||
for (const entry of await remote.list(relative || undefined)) {
|
||||
const name = relative ? `${relative}/${entry.name}` : entry.name;
|
||||
// Only the home-level directory belongs to the runtime; nested names can
|
||||
// be ordinary user content and retain their existing listing behavior.
|
||||
if (!relative && entry.name === ".paperclip-runtime") continue;
|
||||
instructionPath(name);
|
||||
if (entry.kind === "directory") {
|
||||
if (!IGNORED_INSTRUCTIONS_DIRECTORY_NAMES.has(entry.name)) result.push(...await remotePaths(remote, name));
|
||||
} else if (!IGNORED_INSTRUCTIONS_FILE_NAMES.has(entry.name)) result.push({ path: name, size: entry.size });
|
||||
if (result.length > 100_000) throw unprocessable("Agent folder exceeds its entry limit");
|
||||
async function remotePaths(remote: NonNullable<Awaited<ReturnType<typeof remoteFiles>>>, entryFile: string) {
|
||||
const files = new Map<string, { path: string; size: number }>();
|
||||
let remaining = 1000;
|
||||
let truncated = false;
|
||||
// Reserve the configured entry before browsing. It can be beyond the first
|
||||
// page, or below a directory whose siblings consume the metadata budget.
|
||||
if (entryFile.split("/")[0] !== ".paperclip-runtime") {
|
||||
try {
|
||||
const entry = await remote.stat(instructionPath(entryFile));
|
||||
if (entry.kind === "file") files.set(entryFile, { path: entryFile, size: entry.size });
|
||||
} catch (error) { if (!isMissingRemoteFile(error)) throw error; }
|
||||
remaining -= 1;
|
||||
}
|
||||
return result;
|
||||
const directories = [""];
|
||||
while (directories.length > 0 && remaining > 0) {
|
||||
const relative = directories.shift()!;
|
||||
const page = await remote.listPage(relative, { limit: remaining });
|
||||
remaining -= page.entries.length;
|
||||
truncated ||= page.truncated;
|
||||
for (const entry of page.entries) {
|
||||
const name = relative ? `${relative}/${entry.name}` : entry.name;
|
||||
// Nested names can be user content; only the home-level runtime tree
|
||||
// contains provider packs and mutable session state owned by Paperclip.
|
||||
if (!relative && entry.name === ".paperclip-runtime") continue;
|
||||
instructionPath(name);
|
||||
if (entry.kind === "directory") {
|
||||
if (!IGNORED_INSTRUCTIONS_DIRECTORY_NAMES.has(entry.name)) directories.push(name);
|
||||
} else if (!IGNORED_INSTRUCTIONS_FILE_NAMES.has(entry.name)) files.set(name, { path: name, size: entry.size });
|
||||
}
|
||||
}
|
||||
truncated ||= directories.length > 0;
|
||||
return {
|
||||
files: [...files.values()],
|
||||
warnings: truncated ? ["The agent folder listing and export are partial because they exceed the 1,000-entry limit. The configured instructions entry is included when available."] : [],
|
||||
};
|
||||
}
|
||||
async function assertUnversionedEntry(agent: AgentLike, entryFile: string, connection: Db | Parameters<Parameters<Db["transaction"]>[0]>[0] | undefined = db) {
|
||||
if (!connection) throw unprocessable("Bundle initialization requires the database-backed instructions service");
|
||||
@@ -535,11 +556,12 @@ export function agentInstructionsService(db?: Db) {
|
||||
const remote = await remoteFiles(agent);
|
||||
if (remote) {
|
||||
const summaries: AgentInstructionsFileSummary[] = [];
|
||||
for (const { path: relative, size } of await remotePaths(remote)) {
|
||||
const listing = await remotePaths(remote, state.entryFile);
|
||||
for (const { path: relative, size } of listing.files) {
|
||||
const file = size <= MAX_INSTRUCTION_BYTES ? await readPersistentAgentFile(remote, relative) : null;
|
||||
summaries.push(summarizeFile(relative, state.entryFile, size, file?.bytes ?? null, file?.sha256));
|
||||
}
|
||||
return toBundle(agent, { ...state, rootPath: remote.root, resolvedEntryPath: `${remote.root}/${state.entryFile}` }, summaries);
|
||||
return toBundle(agent, { ...state, rootPath: remote.root, resolvedEntryPath: `${remote.root}/${state.entryFile}`, warnings: [...state.warnings, ...listing.warnings] }, summaries);
|
||||
}
|
||||
if (!state.rootPath) return toBundle(agent, state, []);
|
||||
await assertInstructionPathSafe(state.rootPath, state.entryFile);
|
||||
@@ -792,12 +814,13 @@ export function agentInstructionsService(db?: Db) {
|
||||
const remote = await remoteFiles(agent);
|
||||
if (remote) {
|
||||
const files: Record<string, string> = {};
|
||||
for (const { path: relative, size } of await remotePaths(remote)) {
|
||||
const listing = await remotePaths(remote, state.entryFile);
|
||||
for (const { path: relative, size } of listing.files) {
|
||||
if (size > MAX_INSTRUCTION_BYTES) continue;
|
||||
const file = await readPersistentAgentFile(remote, relative);
|
||||
if (file) files[relative] = file.bytes.toString("utf8");
|
||||
}
|
||||
return { files, entryFile: state.entryFile, warnings: state.warnings };
|
||||
return { files, entryFile: state.entryFile, warnings: [...state.warnings, ...listing.warnings] };
|
||||
}
|
||||
if (state.rootPath) {
|
||||
const stat = await statIfExists(state.rootPath);
|
||||
|
||||
@@ -2635,16 +2635,21 @@ describe("remote provider checkpoint restores", () => {
|
||||
await writeFile(targetPath, "previous verified binary");
|
||||
let inFlight = 0;
|
||||
let maxInFlight = 0;
|
||||
let releaseFirstBatch!: () => void;
|
||||
const firstBatch = new Promise<void>((resolve) => { releaseFirstBatch = resolve; });
|
||||
const execute = vi.fn(async (request: { command: string; args: string[]; stdin?: string }) => {
|
||||
expect(request.stdin?.length ?? 0).toBeLessThanOrEqual(Math.ceil(4 * 1024 * 1024 / 3) * 4);
|
||||
inFlight++;
|
||||
maxInFlight = Math.max(maxInFlight, inFlight);
|
||||
if (request.stdin) {
|
||||
inFlight++;
|
||||
maxInFlight = Math.max(maxInFlight, inFlight);
|
||||
if (inFlight === 3) releaseFirstBatch();
|
||||
}
|
||||
try {
|
||||
await new Promise((resolve) => setTimeout(resolve, 5));
|
||||
if (request.stdin) await firstBatch;
|
||||
const stdin = corrupt && request.stdin ? Buffer.from("corrupt").toString("base64") : request.stdin;
|
||||
execFileSync(request.command, request.args, { input: stdin });
|
||||
return { exitCode: 0, timedOut: false, stdout: "", stderr: "" };
|
||||
} finally { inFlight--; }
|
||||
} finally { if (request.stdin) inFlight--; }
|
||||
});
|
||||
try {
|
||||
const stage = stageRemoteRunnerFile({
|
||||
@@ -2670,10 +2675,16 @@ describe("remote provider checkpoint restores", () => {
|
||||
await writeFile(sourcePath, randomBytes(20 * 1024 * 1024));
|
||||
let active = 0;
|
||||
let peak = 0;
|
||||
let releaseFirstBatch!: () => void;
|
||||
const firstBatch = new Promise<void>((resolve) => { releaseFirstBatch = resolve; });
|
||||
const execute = vi.fn(async (request: { command: string; args: string[]; stdin?: string }) => {
|
||||
if (request.stdin) { active++; peak = Math.max(peak, active); }
|
||||
if (request.stdin) {
|
||||
active++;
|
||||
peak = Math.max(peak, active);
|
||||
if (active === 8) releaseFirstBatch();
|
||||
}
|
||||
try {
|
||||
if (request.stdin) await new Promise((resolve) => setTimeout(resolve, 25));
|
||||
if (request.stdin) await firstBatch;
|
||||
execFileSync(request.command, request.args, { input: request.stdin });
|
||||
return { exitCode: 0, timedOut: false, stdout: "", stderr: "" };
|
||||
} finally { if (request.stdin) active--; }
|
||||
@@ -13446,6 +13457,13 @@ describe("runnerd provider runtime wiring", () => {
|
||||
it.each([
|
||||
{ condition: "retired", expected: "verified" },
|
||||
{ condition: "workspace-coordinate", expected: "verified" },
|
||||
{ condition: "warm-origin", expected: "verified" },
|
||||
{ condition: "origin-other-session", expected: "scope_mismatch" },
|
||||
{ condition: "origin-other-owner", expected: "scope_mismatch" },
|
||||
{ condition: "origin-newer-generation", expected: "scope_mismatch" },
|
||||
{ condition: "origin-active", expected: "scope_mismatch" },
|
||||
{ condition: "origin-ambiguous", expected: "scope_mismatch" },
|
||||
{ condition: "origin-wrong-lease-run", expected: "scope_mismatch" },
|
||||
{ condition: "wrong-workspace-run", expected: "scope_mismatch" },
|
||||
{ condition: "uncertain", expected: "terminal_state_indeterminate" },
|
||||
{ condition: "stale-generation", expected: "terminal_state_indeterminate" },
|
||||
@@ -13456,20 +13474,28 @@ describe("runnerd provider runtime wiring", () => {
|
||||
{ condition: "corrupt-local", expected: "terminal_state_indeterminate" },
|
||||
])("verifies remote idle retirement before a fresh same-session turn ($condition)", async ({ condition, expected }) => {
|
||||
const root = await mkdtemp(join(tmpdir(), "paperclip-computer-idle-retirement-"));
|
||||
const prior = execution;
|
||||
const current = { ...execution, binding: { ...execution.binding, runId: "after-idle" } };
|
||||
const originCase = condition === "warm-origin" || condition.startsWith("origin-");
|
||||
const prior = originCase ? { ...execution, binding: { ...execution.binding, executionWorkspaceId: execution.binding.runId } } : execution;
|
||||
const current = { ...prior, binding: { ...prior.binding, runId: "after-idle", ...(originCase ? { executionWorkspaceId: "after-idle" } : {}) } };
|
||||
const owner = { computerId: "computer", ownerId: "retired-owner", generation: 2, listenerPort: 43127 };
|
||||
const descriptor = { kind: "remote-persistent", leaseId: "prior-lease", providerLeaseId: owner.ownerId,
|
||||
remoteCwd: "/remote/agent", computerId: owner.computerId, ownerGeneration: owner.generation,
|
||||
listenerPort: owner.listenerPort, placementId: "placement" };
|
||||
let leaseQuery: SQL | undefined;
|
||||
const originExecution = { ...prior, binding: { ...prior.binding, runId: "origin-workspace", executionWorkspaceId: "origin-workspace" },
|
||||
session: { ...prior.session, normalizedSessionId: condition === "origin-other-session" ? "other-session" : prior.session.normalizedSessionId } };
|
||||
const originDescriptor = { ...descriptor, leaseId: "origin-lease",
|
||||
providerLeaseId: condition === "origin-other-owner" ? "other-owner" : owner.ownerId,
|
||||
ownerGeneration: condition === "origin-newer-generation" ? 3 : 1 };
|
||||
const originRow = { status: condition === "origin-active" ? "running" : "succeeded",
|
||||
runnerProfileJson: { nativeExecutionInput: originExecution, nativeComputerWorkspace: originDescriptor } };
|
||||
const workspaceCoordinate = ["workspace-coordinate", "wrong-workspace-run"].includes(condition);
|
||||
const db = { select: () => ({ from: (table: unknown) => ({ where: (query: SQL) => { if (table === environmentLeases) leaseQuery = query; return ({ limit: async () =>
|
||||
const db = { select: () => ({ from: (table: unknown) => ({ where: (query: SQL) => { if (table === environmentLeases) leaseQuery = query; return ({ limit: async (limit: number) =>
|
||||
table === environmentLeases ? condition === "wrong-lease" ? [] : [{
|
||||
providerLeaseId: owner.ownerId,
|
||||
heartbeatRunId: condition === "wrong-workspace-run" ? "unrelated-run" : prior.binding.runId,
|
||||
heartbeatRunId: condition === "wrong-workspace-run" || condition === "origin-wrong-lease-run" ? "unrelated-run" : originCase ? "origin-workspace" : prior.binding.runId,
|
||||
metadata: { agentId: prior.binding.agentId, computerOwner: { ...owner, generation: 1 } },
|
||||
}] : [{ status: condition === "active-run" ? "running" : "succeeded",
|
||||
}] : limit === 2 ? (originCase ? condition === "origin-ambiguous" ? [originRow, originRow] : [originRow] : []) : [{ status: condition === "active-run" ? "running" : "succeeded",
|
||||
runnerProfileJson: { nativeExecutionInput: prior, nativeComputerWorkspace: descriptor } }],
|
||||
}); } }) }) } as unknown as Db;
|
||||
computerRetirement.isRetired.mockReset().mockImplementation(async ({ owner: requested }) => {
|
||||
@@ -13478,7 +13504,7 @@ describe("runnerd provider runtime wiring", () => {
|
||||
});
|
||||
const input = { db, root, execution: current, identity: { runId: prior.binding.runId,
|
||||
normalizedSessionId: prior.session.normalizedSessionId!, runnerInstanceId: "prior-runner",
|
||||
environmentLeaseId: workspaceCoordinate ? prior.binding.executionWorkspaceId : condition === "wrong-lease" ? "other-lease" : "initial-physical-lease" },
|
||||
environmentLeaseId: originCase ? "origin-workspace" : workspaceCoordinate ? prior.binding.executionWorkspaceId : condition === "wrong-lease" ? "other-lease" : "initial-physical-lease" },
|
||||
allowVerifiedBackup: false, remoteRunnerState: true,
|
||||
runnerExecutionTarget: { kind: "remote", transport: "computer", environmentId: "environment",
|
||||
remoteCwd: "/remote/agent", resourceAuthority: { ...owner, ownerId: "successor-owner", generation: 1 },
|
||||
@@ -13498,12 +13524,17 @@ describe("runnerd provider runtime wiring", () => {
|
||||
expect(computerRetirement.isRetired).not.toHaveBeenCalled();
|
||||
}
|
||||
expect(await verifyPriorRunnerdStateForSessionScope({ ...input, allowRetainedWarmRunner: false })).toBe(expected);
|
||||
if (["retired", "workspace-coordinate", "uncertain", "stale-generation", "unavailable"].includes(condition)) {
|
||||
if (["retired", "workspace-coordinate", "warm-origin", "uncertain", "stale-generation", "unavailable"].includes(condition)) {
|
||||
expect(computerRetirement.isRetired).toHaveBeenCalledExactlyOnceWith({
|
||||
companyId: current.binding.companyId, environmentId: "environment", agentId: current.binding.agentId,
|
||||
runId: prior.binding.runId, owner,
|
||||
});
|
||||
} else expect(computerRetirement.isRetired).not.toHaveBeenCalled();
|
||||
if (condition === "warm-origin") {
|
||||
const query = new PgDialect().sqlToQuery(leaseQuery!);
|
||||
expect(query.params).toContain("origin-lease");
|
||||
expect(query.params).not.toContain("origin-workspace");
|
||||
}
|
||||
if (workspaceCoordinate) {
|
||||
const query = new PgDialect().sqlToQuery(leaseQuery!);
|
||||
expect(query.params).toContain(descriptor.leaseId);
|
||||
|
||||
@@ -4389,11 +4389,40 @@ export async function verifyPriorRunnerdStateForSessionScope(input: {
|
||||
// environmentLeaseId. Computer placement records the actual lease. Only
|
||||
// this exact prior workspace coordinate may resolve through that record.
|
||||
// Explicit physical lease identities still preserve their warm lineage.
|
||||
const workspaceCoordinate = input.identity.environmentLeaseId ===
|
||||
priorExecution.binding.executionWorkspaceId;
|
||||
const physicalLeaseId = workspaceCoordinate
|
||||
? priorWorkspace.leaseId
|
||||
: input.identity.environmentLeaseId;
|
||||
let workspaceRunId: string | null = input.identity.environmentLeaseId ===
|
||||
priorExecution.binding.executionWorkspaceId ? input.identity.runId : null;
|
||||
let physicalLeaseId = workspaceRunId ? priorWorkspace.leaseId : input.identity.environmentLeaseId;
|
||||
if (!workspaceRunId) {
|
||||
// Warm attachment advances runId while retaining the process's original
|
||||
// workspace coordinate. Resolve that recorded origin, never an arbitrary
|
||||
// run ID, before checking the physical lease and latest retirement.
|
||||
const origins = await input.db.select({
|
||||
status: heartbeatRuns.status,
|
||||
runnerProfileJson: heartbeatRuns.runnerProfileJson,
|
||||
}).from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.companyId, input.execution.binding.companyId),
|
||||
eq(heartbeatRuns.agentId, input.execution.binding.agentId),
|
||||
eq(sql<string>`${heartbeatRuns.runnerProfileJson}->'nativeExecutionInput'->'binding'->>'executionWorkspaceId'`,
|
||||
input.identity.environmentLeaseId),
|
||||
)).limit(2);
|
||||
if (origins.length > 1) return "scope_mismatch";
|
||||
if (origins.length === 1) {
|
||||
const origin = origins[0]!;
|
||||
const originProfile = record(origin.runnerProfileJson);
|
||||
const originExecution = parseNativeExecutionInput(originProfile.nativeExecutionInput);
|
||||
const originWorkspace = readNativeComputerWorkspaceReference(originProfile.nativeComputerWorkspace);
|
||||
if (!TERMINAL_HEARTBEAT_RUN_STATUSES.has(origin.status) ||
|
||||
originExecution.binding.executionWorkspaceId !== input.identity.environmentLeaseId ||
|
||||
nativeSessionScopeKey(originExecution) !== nativeSessionScopeKey(priorExecution) ||
|
||||
!originWorkspace || originWorkspace.remoteCwd !== priorWorkspace.remoteCwd ||
|
||||
record(originProfile.nativeComputerWorkspace).placementId !== record(descriptor).placementId ||
|
||||
originWorkspace.computerOwner.computerId !== priorWorkspace.computerOwner.computerId ||
|
||||
originWorkspace.computerOwner.ownerId !== priorWorkspace.computerOwner.ownerId ||
|
||||
originWorkspace.computerOwner.generation > priorWorkspace.computerOwner.generation) return "scope_mismatch";
|
||||
workspaceRunId = originExecution.binding.runId;
|
||||
physicalLeaseId = originWorkspace.leaseId;
|
||||
}
|
||||
}
|
||||
const [physicalLease] = await input.db.select({
|
||||
providerLeaseId: environmentLeases.providerLeaseId,
|
||||
heartbeatRunId: environmentLeases.heartbeatRunId,
|
||||
@@ -4405,7 +4434,7 @@ export async function verifyPriorRunnerdStateForSessionScope(input: {
|
||||
)).limit(1);
|
||||
const leaseMetadata = record(physicalLease?.metadata);
|
||||
const leaseOwner = record(leaseMetadata.computerOwner);
|
||||
if ((workspaceCoordinate && physicalLease?.heartbeatRunId !== input.identity.runId) ||
|
||||
if ((workspaceRunId && physicalLease?.heartbeatRunId !== workspaceRunId) ||
|
||||
physicalLease?.providerLeaseId !== priorWorkspace.computerOwner.ownerId ||
|
||||
leaseMetadata.agentId !== input.execution.binding.agentId ||
|
||||
leaseOwner.computerId !== priorWorkspace.computerOwner.computerId ||
|
||||
|
||||
Reference in new issue
Block a user