Jannik/PaperClipAI
1
0 Fork 0
mirror of https://github.com/paperclipai/paperclip.git synced 2026-11-05 03:51:54 +01:00
Code Issues Packages Projects Releases Wiki Activity

test(evals): allow authorized AWS fleet allocation fallback

Browse Source
This commit is contained in:
Dotta committed 2026-10-27 10:23:57 -05:00
1 parent 9414380536
commit 3084702f49
2 files changed
+12 -7
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/runner-e2e/chat-remote-host.test.ts
+8 -3
View File
@@ -37,9 +37,14 @@ describe("remote-only chat host qualification without native/provider calls", ()
expect(f.fetch.mock.calls.every(([url]) => String(url).startsWith("http://169.254.169.254/latest/"))).toBe(true);
expect(f.fetch.mock.invocationCallOrder.at(-1)!).toBeLessThan(f.docker.mock.invocationCallOrder[0]!);
});
it("refuses On-Demand fallback before any Docker/native/provider/DB action", async () => {
const f = await fixture("on-demand"); await expect(prepareChatRemoteHost(f)).rejects.toThrow("on-demand-or-unknown-host-refused");
expect(f.docker).not.toHaveBeenCalled(); expect(await f.receipt()).toMatchObject({ lifecycle: "on-demand", status: "blocked", evaluationAdmitted: false });
it("accepts the user-authorized fleet fallback while truthfully recording On-Demand", async () => {
const f = await fixture("on-demand"); await prepareChatRemoteHost(f);
expect(await f.receipt()).toMatchObject({ lifecycle: "on-demand", status: "qualified-host-only", spotVerified: false, hostPolicy: "aws-fleet-spot-or-on-demand", evaluationAdmitted: false });
expect(f.docker).toHaveBeenCalled();
});
it.each(["", "unknown", "preemptible"])("refuses unproven host lifecycle %s before Docker", async lifecycle => {
const f = await fixture(lifecycle); await expect(prepareChatRemoteHost(f)).rejects.toThrow("unknown-host-lifecycle-refused");
expect(f.docker).not.toHaveBeenCalled(); expect(await f.receipt()).toMatchObject({ lifecycle, status: "blocked", evaluationAdmitted: false });
});
it.each(["local", "branch-workflow", "wrong-source", "wrong-model", "automatic-retry", "foreign-repository"])("fails closed on %s before metadata or Docker", async kind => {
const f = await fixture();
tests/runner-e2e/chat-remote-host.ts
+4 -4
View File
@@ -27,7 +27,7 @@ export async function prepareChatRemoteHost(input: { executions: readonly Select
if (!selectsRemoteChat(input.executions)) return undefined;
const deps = { ...defaults, ...input.dependencies }, env = input.env;
const receipt: Record<string, unknown> = { schema: "paperclip.chat-remote-host/v1", status: "blocked", scope: "before-evaluation", evaluationAdmitted: false,
allocationSpotOnlyClaimed: false, databaseLane: "owned-docker-postgres", automaticRetries: 0, maximumParallel: 2,
allocationSpotOnlyClaimed: false, hostPolicy: "aws-fleet-spot-or-on-demand", databaseLane: "owned-docker-postgres", automaticRetries: 0, maximumParallel: 2,
executionIds: input.executions.map(e => e.id) };
const filename = path.join(input.outputDirectory, "chat-remote-host.json");
await mkdir(input.outputDirectory, { recursive: true });
@@ -52,8 +52,8 @@ export async function prepareChatRemoteHost(input: { executions: readonly Select
const identity = JSON.parse(await metadata("http://169.254.169.254/latest/dynamic/instance-identity/document"));
if (!/^i-[a-f0-9]{8,17}$/.test(identity.instanceId ?? "") || !/^ami-[a-f0-9]{8,17}$/.test(identity.imageId ?? "") || identity.region !== "us-east-1" || identity.architecture !== "x86_64") throw new Error(reason);
Object.assign(receipt, { instanceId: identity.instanceId, imageId: identity.imageId, region: identity.region, architecture: identity.architecture, lifecycle });
reason = "on-demand-or-unknown-host-refused-before-evaluation";
if (lifecycle !== "spot") throw new Error(reason);
reason = "unknown-host-lifecycle-refused-before-evaluation";
if (lifecycle !== "spot" && lifecycle !== "on-demand") throw new Error(reason);
reason = "owned-linux-amd64-postgres-image-unqualified";
receipt.qualificationStage = "docker-engine";
const engine = await currentDockerEngine(deps.docker, env);
@@ -69,7 +69,7 @@ export async function prepareChatRemoteHost(input: { executions: readonly Select
os: ["linux", "windows"].includes(image?.Os) ? image.Os : null, architecture: ["amd64", "arm64"].includes(image?.Architecture) ? image.Architecture : null,
version: typeof pgVersion === "string" && /^[0-9][a-zA-Z0-9.+~-]{0,79}$/.test(pgVersion) ? pgVersion : null };
if (Object.values(checks).some(value => !value)) throw new Error(reason);
Object.assign(receipt, { status: "qualified-host-only", spotVerified: true, postgres: { image: POSTGRES_AMD64_IMAGE, imageId: image.Id, architecture: image.Architecture, version: pgVersion }, dockerEngineSha256: hash(engine.engineId) });
Object.assign(receipt, { status: "qualified-host-only", spotVerified: lifecycle === "spot", postgres: { image: POSTGRES_AMD64_IMAGE, imageId: image.Id, architecture: image.Architecture, version: pgVersion }, dockerEngineSha256: hash(engine.engineId) });
await persist();
return { receiptPath: filename, databaseLane: "owned-docker-postgres" as const, maxAutomaticRetries: 0, maxParallel: 2, postgresArtifact: "linux-amd64" as const, postgresImageId: image.Id as typeof POSTGRES_AMD64_IMAGE_IDS[number] };
} catch (error) {