mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-04 19:28:46 +01:00
228 lines
8.1 KiB
TypeScript
228 lines
8.1 KiB
TypeScript
import { afterEach, expect, it } from "vitest";
|
|
import { mkdtemp, mkdir, writeFile, rm, chmod } from "node:fs/promises";
|
|
import { tmpdir } from "node:os";
|
|
import path from "node:path";
|
|
import { CHAT_PRIVATE_INPUTS_ENV } from "./chat-private-inputs.js";
|
|
import {
|
|
buildPaperclipServerEnvironment,
|
|
buildRunnerE2EProcessEnvironment,
|
|
} from "./harness-env.js";
|
|
import {
|
|
readSelectedVaultServerEnvironment,
|
|
privateVaultMetadataMatches,
|
|
validVaultBinding,
|
|
vaultPrivateFields,
|
|
vaultExecutionId,
|
|
vaultExecutionIds,
|
|
} from "./chat-vault-private-server.js";
|
|
import type { MatrixExecution } from "./types.js";
|
|
|
|
const roots: string[] = [];
|
|
afterEach(async () => {
|
|
for (const directory of roots.splice(0))
|
|
await rm(directory, { recursive: true, force: true });
|
|
});
|
|
function binding() {
|
|
return {
|
|
disposable: true,
|
|
resourceIds: [
|
|
"arn:aws:secretsmanager:us-east-1:123456789012:secret:disposable/deployment-AbCdEf",
|
|
"arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012",
|
|
],
|
|
credentialEnv: [
|
|
"PAPERCLIP_CHAT_EVAL_AWS_ACCESS",
|
|
"PAPERCLIP_CHAT_EVAL_AWS_SECRET",
|
|
"PAPERCLIP_CHAT_EVAL_AWS_SESSION",
|
|
"PAPERCLIP_CHAT_EVAL_VAULT_EXPECTED",
|
|
],
|
|
prerequisites: {
|
|
provider: "aws_secrets_manager",
|
|
region: "us-east-1",
|
|
deploymentId: "disposable-catalog",
|
|
kmsKeyId:
|
|
"arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012",
|
|
secretReference:
|
|
"arn:aws:secretsmanager:us-east-1:123456789012:secret:disposable/deployment-AbCdEf",
|
|
accessKeyIdEnv: "PAPERCLIP_CHAT_EVAL_AWS_ACCESS",
|
|
secretAccessKeyEnv: "PAPERCLIP_CHAT_EVAL_AWS_SECRET",
|
|
sessionTokenEnv: "PAPERCLIP_CHAT_EVAL_AWS_SESSION",
|
|
expectedValueEnv: "PAPERCLIP_CHAT_EVAL_VAULT_EXPECTED",
|
|
},
|
|
};
|
|
}
|
|
async function fixture() {
|
|
const root = await mkdtemp(
|
|
path.join(tmpdir(), "paperclip-runner-e2e-vault-control-"),
|
|
);
|
|
roots.push(root);
|
|
const directory = path.join(root, "selected-private-inputs");
|
|
await mkdir(directory, { mode: 0o700 });
|
|
const filename = path.join(directory, "chat-selected-inputs.json"),
|
|
b = binding();
|
|
const envelope = {
|
|
version: 1,
|
|
executionId: vaultExecutionId,
|
|
references: {
|
|
PAPERCLIP_CHAT_EVAL_AWS_ACCESS: "ASIA1234567890123456",
|
|
PAPERCLIP_CHAT_EVAL_AWS_SECRET: "synthetic-selected-secret-key",
|
|
PAPERCLIP_CHAT_EVAL_AWS_SESSION: "synthetic-selected-session-token",
|
|
PAPERCLIP_CHAT_EVAL_VAULT_EXPECTED: "synthetic-vault-oracle-only",
|
|
},
|
|
external: { schemaVersion: 1, providers: { vault: b } },
|
|
declaredReferenceNames: b.credentialEnv,
|
|
declaredExternalRequirements: [
|
|
{ provider: "vault", fields: [...vaultPrivateFields] },
|
|
],
|
|
setupBlockers: [] as string[],
|
|
};
|
|
const save = () =>
|
|
writeFile(filename, JSON.stringify(envelope), { mode: 0o600 });
|
|
await save();
|
|
return {
|
|
envelope,
|
|
save,
|
|
filename,
|
|
input: {
|
|
executionIds: [vaultExecutionId],
|
|
temporaryRoot: root,
|
|
backendOrigin: "http://127.0.0.1:3399",
|
|
env: {
|
|
[CHAT_PRIVATE_INPUTS_ENV]: filename,
|
|
AWS_ACCESS_KEY_ID: "AMBIENT-DO-NOT-USE",
|
|
AWS_PROFILE: "unrelated-personal-profile",
|
|
PAPERCLIP_SECRETS_AWS_ENDPOINT: "http://unrelated.test",
|
|
},
|
|
},
|
|
};
|
|
}
|
|
it("supplies only explicitly selected server credentials/config and excludes oracle values/default profiles", async () => {
|
|
const f = await fixture(),
|
|
env = await readSelectedVaultServerEnvironment(f.input);
|
|
expect(env.AWS_ACCESS_KEY_ID).toBe(
|
|
f.envelope.references.PAPERCLIP_CHAT_EVAL_AWS_ACCESS,
|
|
);
|
|
expect(env.AWS_SECRET_ACCESS_KEY).toBe(
|
|
f.envelope.references.PAPERCLIP_CHAT_EVAL_AWS_SECRET,
|
|
);
|
|
expect(env.AWS_SESSION_TOKEN).toBe(
|
|
f.envelope.references.PAPERCLIP_CHAT_EVAL_AWS_SESSION,
|
|
);
|
|
expect(env.AWS_EC2_METADATA_DISABLED).toBe("true");
|
|
expect(env.AWS_PROFILE).toBeUndefined();
|
|
expect(env.PAPERCLIP_SECRETS_AWS_ENDPOINT).toBeUndefined();
|
|
expect(JSON.stringify(env)).not.toContain(
|
|
f.envelope.references.PAPERCLIP_CHAT_EVAL_VAULT_EXPECTED,
|
|
);
|
|
expect(
|
|
Object.keys(env).some((name) => name.startsWith("PAPERCLIP_CHAT_EVAL_")),
|
|
).toBe(false);
|
|
expect(env.AWS_CONFIG_FILE).toBe(env.AWS_SHARED_CREDENTIALS_FILE);
|
|
});
|
|
it("keeps missing selected inputs as reserved setup prerequisites without using ambient AWS", async () => {
|
|
const f = await fixture();
|
|
expect(
|
|
await readSelectedVaultServerEnvironment({
|
|
...f.input,
|
|
env: { AWS_ACCESS_KEY_ID: "ambient" },
|
|
}),
|
|
).toEqual({});
|
|
f.envelope.setupBlockers = [
|
|
"missing-external-reference:PAPERCLIP_CHAT_EVAL_AWS_ACCESS",
|
|
];
|
|
await f.save();
|
|
expect(await readSelectedVaultServerEnvironment(f.input)).toEqual({});
|
|
expect(
|
|
await readSelectedVaultServerEnvironment({
|
|
...f.input,
|
|
executionIds: ["unrelated"],
|
|
}),
|
|
).toEqual({});
|
|
});
|
|
it("rejects another execution, unsafe private files, undeclared references and cross-account resources", async () => {
|
|
const f = await fixture();
|
|
await expect(
|
|
readSelectedVaultServerEnvironment({
|
|
...f.input,
|
|
executionIds: [vaultExecutionId, "unrelated"],
|
|
}),
|
|
).rejects.toThrow(/single selected/);
|
|
f.envelope.executionId = "another-execution";
|
|
await f.save();
|
|
await expect(readSelectedVaultServerEnvironment(f.input)).rejects.toThrow(
|
|
/another execution/,
|
|
);
|
|
f.envelope.executionId = vaultExecutionId;
|
|
f.envelope.declaredReferenceNames = [];
|
|
await f.save();
|
|
await expect(readSelectedVaultServerEnvironment(f.input)).rejects.toThrow(
|
|
/selected reference/,
|
|
);
|
|
f.envelope.declaredReferenceNames = binding().credentialEnv;
|
|
await f.save();
|
|
await chmod(f.filename, 0o644);
|
|
await expect(readSelectedVaultServerEnvironment(f.input)).rejects.toThrow(
|
|
/private and owned/,
|
|
);
|
|
expect(privateVaultMetadataMatches({ uid: 101, mode: 0o600 }, 101)).toBe(
|
|
true,
|
|
);
|
|
expect(privateVaultMetadataMatches({ uid: 102, mode: 0o600 }, 101)).toBe(
|
|
false,
|
|
);
|
|
const b = binding();
|
|
expect(validVaultBinding(b)).toBe(true);
|
|
b.prerequisites.kmsKeyId = b.prerequisites.kmsKeyId.replace(
|
|
"123456789012",
|
|
"987654321012",
|
|
);
|
|
expect(validVaultBinding(b)).toBe(false);
|
|
const other = binding();
|
|
other.resourceIds = [];
|
|
expect(validVaultBinding(other)).toBe(false);
|
|
});
|
|
it("scrubs ambient AWS and vault SDK config while preserving explicit unrelated suite mechanisms", () => {
|
|
const ambient = {
|
|
AWS_ACCESS_KEY_ID: "ambient",
|
|
AWS_SECRET_ACCESS_KEY: "ambient-secret",
|
|
AWS_PROFILE: "personal",
|
|
AWS_CONTAINER_CREDENTIALS_FULL_URI: "https://unrelated.test",
|
|
AWS_WEB_IDENTITY_TOKEN_FILE: "/private/personal-token",
|
|
PAPERCLIP_SECRETS_AWS_ENDPOINT: "https://unrelated.test",
|
|
KEEP: "yes",
|
|
};
|
|
expect(buildPaperclipServerEnvironment(ambient)).toEqual({ KEEP: "yes" });
|
|
expect(
|
|
buildPaperclipServerEnvironment(ambient, {
|
|
AWS_ACCESS_KEY_ID: "explicit-selected",
|
|
}).AWS_ACCESS_KEY_ID,
|
|
).toBe("explicit-selected");
|
|
const execution = {
|
|
suite: { id: "chat-features" },
|
|
task: { id: "secrets.vault-reference" },
|
|
profile: { generation: "native", provider: "codex" },
|
|
environment: { id: "local" },
|
|
} as MatrixExecution;
|
|
const chat = buildRunnerE2EProcessEnvironment(ambient, [execution]);
|
|
expect(
|
|
Object.keys(chat).some(
|
|
(name) =>
|
|
name.startsWith("AWS_") || name.startsWith("PAPERCLIP_SECRETS_AWS_"),
|
|
),
|
|
).toBe(false);
|
|
const other = buildRunnerE2EProcessEnvironment(ambient, [
|
|
{
|
|
...execution,
|
|
suite: { ...execution.suite, id: "explicit-unrelated-suite" },
|
|
},
|
|
]);
|
|
expect(other.AWS_ACCESS_KEY_ID).toBe("ambient");
|
|
});
|
|
|
|
it("binds the OpenCode vault envelope to the exact selected new execution, never the prior campaign", async () => {
|
|
const f = await fixture();
|
|
f.input.executionIds = [vaultExecutionIds[1]!]; f.envelope.executionId = vaultExecutionIds[1]!; await f.save();
|
|
expect(await readSelectedVaultServerEnvironment(f.input)).toHaveProperty("AWS_ACCESS_KEY_ID", f.envelope.references.PAPERCLIP_CHAT_EVAL_AWS_ACCESS);
|
|
f.envelope.executionId = vaultExecutionId; await f.save();
|
|
await expect(readSelectedVaultServerEnvironment(f.input)).rejects.toThrow();
|
|
});
|