mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-26 18:31:34 +01:00
fix(runner): close phase 3 security integrity gaps
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed
1 parent
486069e369
commit
313c40cd9b
7 files changed
+2141
-319
No files matched your search
+230
@@ -2,24 +2,200 @@
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "aead"
|
||||
version = "0.5.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0"
|
||||
dependencies = [
|
||||
"crypto-common",
|
||||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aes"
|
||||
version = "0.8.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cipher",
|
||||
"cpufeatures",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aes-gcm"
|
||||
version = "0.10.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1"
|
||||
dependencies = [
|
||||
"aead",
|
||||
"aes",
|
||||
"cipher",
|
||||
"ctr",
|
||||
"ghash",
|
||||
"subtle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "block-buffer"
|
||||
version = "0.10.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
|
||||
|
||||
[[package]]
|
||||
name = "cipher"
|
||||
version = "0.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
|
||||
dependencies = [
|
||||
"crypto-common",
|
||||
"inout",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cpufeatures"
|
||||
version = "0.2.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
"rand_core",
|
||||
"typenum",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ctr"
|
||||
version = "0.9.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835"
|
||||
dependencies = [
|
||||
"cipher",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "digest"
|
||||
version = "0.10.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
||||
dependencies = [
|
||||
"block-buffer",
|
||||
"crypto-common",
|
||||
"subtle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "0.14.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
|
||||
dependencies = [
|
||||
"typenum",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.2.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"wasi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ghash"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1"
|
||||
dependencies = [
|
||||
"opaque-debug",
|
||||
"polyval",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hmac"
|
||||
version = "0.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e"
|
||||
dependencies = [
|
||||
"digest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "inout"
|
||||
version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "1.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.189"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
||||
|
||||
[[package]]
|
||||
name = "opaque-debug"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"
|
||||
|
||||
[[package]]
|
||||
name = "paperclip-runner-core"
|
||||
version = "0.0.0"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"hmac",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "polyval"
|
||||
version = "0.6.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cpufeatures",
|
||||
"opaque-debug",
|
||||
"universal-hash",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -40,6 +216,15 @@ dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_core"
|
||||
version = "0.6.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
|
||||
dependencies = [
|
||||
"getrandom",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.229"
|
||||
@@ -83,6 +268,23 @@ dependencies = [
|
||||
"zmij",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha2"
|
||||
version = "0.10.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cpufeatures",
|
||||
"digest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "subtle"
|
||||
version = "2.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "3.0.3"
|
||||
@@ -94,12 +296,40 @@ dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "typenum"
|
||||
version = "1.20.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
|
||||
|
||||
[[package]]
|
||||
name = "universal-hash"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea"
|
||||
dependencies = [
|
||||
"crypto-common",
|
||||
"subtle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "version_check"
|
||||
version = "0.9.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
version = "0.11.1+wasi-snapshot-preview1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
|
||||
|
||||
[[package]]
|
||||
name = "zmij"
|
||||
version = "1.0.23"
|
||||
|
||||
@@ -9,5 +9,8 @@ license = "MIT"
|
||||
publish = false
|
||||
|
||||
[workspace.dependencies]
|
||||
aes-gcm = "0.10"
|
||||
hmac = "0.12"
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_json = "1.0"
|
||||
sha2 = "0.10"
|
||||
@@ -6,8 +6,11 @@ license.workspace = true
|
||||
publish.workspace = true
|
||||
|
||||
[dependencies]
|
||||
aes-gcm.workspace = true
|
||||
hmac.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
sha2.workspace = true
|
||||
|
||||
[[bin]]
|
||||
name = "phase0-tracer"
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -62,6 +62,8 @@ export interface Phase3RunnerState extends Phase3Identity {
|
||||
peakOutboxBytes: number;
|
||||
outbox: Phase3StoredEvent[];
|
||||
processedCommands: Record<string, Phase3ProcessedCommand>;
|
||||
compactedCommandFilter: string;
|
||||
compactedCommandCount: number;
|
||||
diagnostics: string[];
|
||||
backpressure: boolean;
|
||||
recoverableFailure: string | null;
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import {
|
||||
chmodSync,
|
||||
lstatSync,
|
||||
@@ -20,38 +21,7 @@ import {
|
||||
runPhase3Recovery,
|
||||
} from "./phase3-recovery.js";
|
||||
|
||||
async function upgradeStatus(url: string, ticket: string): Promise<number> {
|
||||
const parsed = new URL(url);
|
||||
return new Promise((resolveStatus, rejectStatus) => {
|
||||
const socket = connect(Number(parsed.port), parsed.hostname);
|
||||
let response = "";
|
||||
socket.once("error", rejectStatus);
|
||||
socket.once("connect", () => {
|
||||
socket.write(
|
||||
[
|
||||
`GET ${parsed.pathname} HTTP/1.1`,
|
||||
`Host: ${parsed.host}`,
|
||||
"Upgrade: websocket",
|
||||
"Connection: Upgrade",
|
||||
"Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==",
|
||||
"Sec-WebSocket-Version: 13",
|
||||
`Authorization: Bearer ${ticket}`,
|
||||
"\r\n",
|
||||
].join("\r\n"),
|
||||
);
|
||||
});
|
||||
socket.on("data", (chunk) => {
|
||||
response += chunk.toString("utf8");
|
||||
if (response.includes("\r\n\r\n")) {
|
||||
const status = Number(response.match(/^HTTP\/1\.1 (\d{3})/)?.[1]);
|
||||
socket.destroy();
|
||||
resolveStatus(status);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function upgradeSocket(url: string, ticket: string): Promise<Socket> {
|
||||
async function upgradeSocket(url: string): Promise<Socket> {
|
||||
const parsed = new URL(url);
|
||||
return new Promise((resolveSocket, rejectSocket) => {
|
||||
const socket = connect(Number(parsed.port), parsed.hostname);
|
||||
@@ -81,7 +51,6 @@ async function upgradeSocket(url: string, ticket: string): Promise<Socket> {
|
||||
"Connection: Upgrade",
|
||||
"Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==",
|
||||
"Sec-WebSocket-Version: 13",
|
||||
`Authorization: Bearer ${ticket}`,
|
||||
"\r\n",
|
||||
].join("\r\n"),
|
||||
);
|
||||
@@ -90,6 +59,39 @@ async function upgradeSocket(url: string, ticket: string): Promise<Socket> {
|
||||
});
|
||||
}
|
||||
|
||||
async function receiveServerJson(socket: Socket): Promise<Record<string, unknown>> {
|
||||
return new Promise((resolveValue, rejectValue) => {
|
||||
let buffer = Buffer.alloc(0);
|
||||
const timer = setTimeout(() => rejectValue(new Error("Server frame timed out.")), 2_000);
|
||||
const onData = (chunk: Buffer): void => {
|
||||
buffer = Buffer.concat([buffer, chunk]);
|
||||
if (buffer.length < 2) return;
|
||||
let length = buffer[1]! & 0x7f;
|
||||
let cursor = 2;
|
||||
if (length === 126) {
|
||||
if (buffer.length < 4) return;
|
||||
length = buffer.readUInt16BE(2);
|
||||
cursor = 4;
|
||||
} else if (length === 127) {
|
||||
if (buffer.length < 10) return;
|
||||
length = Number(buffer.readBigUInt64BE(2));
|
||||
cursor = 10;
|
||||
}
|
||||
if (buffer.length < cursor + length) return;
|
||||
clearTimeout(timer);
|
||||
socket.off("data", onData);
|
||||
resolveValue(
|
||||
JSON.parse(buffer.subarray(cursor, cursor + length).toString("utf8")) as Record<
|
||||
string,
|
||||
unknown
|
||||
>,
|
||||
);
|
||||
};
|
||||
socket.on("data", onData);
|
||||
socket.once("error", rejectValue);
|
||||
});
|
||||
}
|
||||
|
||||
function sendMaskedJson(socket: Socket, value: unknown): void {
|
||||
const payload = Buffer.from(JSON.stringify(value));
|
||||
const mask = Buffer.from([0x11, 0x22, 0x33, 0x44]);
|
||||
@@ -116,23 +118,93 @@ async function expectSocketClosed(socket: Socket): Promise<void> {
|
||||
});
|
||||
}
|
||||
|
||||
function hello(identity: ReturnType<typeof phase3Internals.phase3Identity>): Record<string, unknown> {
|
||||
function authHello(
|
||||
identity: ReturnType<typeof phase3Internals.phase3Identity>,
|
||||
credentialId: string,
|
||||
): Record<string, unknown> {
|
||||
return {
|
||||
protocol: "paperclip.runner",
|
||||
version: 1,
|
||||
envelopeId: "hello_auth_regression",
|
||||
kind: "hello",
|
||||
runnerInstanceId: identity.runnerInstanceId,
|
||||
kind: "auth_hello",
|
||||
payload: {
|
||||
credentialId,
|
||||
clientNonce: "client_nonce_auth_regression",
|
||||
protocolMin: 1,
|
||||
protocolMax: 1,
|
||||
runnerInstanceId: identity.runnerInstanceId,
|
||||
runnerVersion: "0.3.0",
|
||||
runnerDigest: "sha256:phase3-approved",
|
||||
environmentLeaseId: identity.environmentLeaseId,
|
||||
runId: identity.runId,
|
||||
normalizedSessionId: identity.normalizedSessionId,
|
||||
turnId: identity.turnId,
|
||||
itemId: identity.itemId,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function runRunnerProcess(options: {
|
||||
connectUrl: string;
|
||||
stateDirectory: string;
|
||||
identity: ReturnType<typeof phase3Internals.phase3Identity>;
|
||||
ticket: string;
|
||||
maxRuntimeMs: number;
|
||||
}): Promise<{ code: number | null; stderr: string }> {
|
||||
const child = spawn(
|
||||
phase3Internals.runnerBinary,
|
||||
[
|
||||
"--connect-url",
|
||||
options.connectUrl,
|
||||
"--state-dir",
|
||||
options.stateDirectory,
|
||||
"--runner-id",
|
||||
options.identity.runnerInstanceId,
|
||||
"--environment-lease-id",
|
||||
options.identity.environmentLeaseId,
|
||||
"--run-id",
|
||||
options.identity.runId,
|
||||
"--session-id",
|
||||
options.identity.normalizedSessionId,
|
||||
"--turn-id",
|
||||
options.identity.turnId,
|
||||
"--item-id",
|
||||
options.identity.itemId,
|
||||
"--runner-version",
|
||||
"0.3.0",
|
||||
"--runner-digest",
|
||||
"sha256:phase3-approved",
|
||||
"--max-outbox-bytes",
|
||||
String(64 * 1024),
|
||||
"--p0-reserve-bytes",
|
||||
String(32 * 1024),
|
||||
"--reconnect-delay-ms",
|
||||
"10",
|
||||
"--max-runtime-ms",
|
||||
String(options.maxRuntimeMs),
|
||||
],
|
||||
{
|
||||
env: phase3Internals.runnerEnvironment(options.ticket),
|
||||
stdio: "pipe",
|
||||
},
|
||||
);
|
||||
let stderr = "";
|
||||
child.stderr.setEncoding("utf8").on("data", (chunk: string) => {
|
||||
stderr += chunk;
|
||||
});
|
||||
const code = await new Promise<number | null>((resolveExit, rejectExit) => {
|
||||
const timer = setTimeout(() => {
|
||||
child.kill("SIGKILL");
|
||||
rejectExit(new Error("Phase 3 runner process did not terminate."));
|
||||
}, options.maxRuntimeMs + 3_000);
|
||||
child.once("error", rejectExit);
|
||||
child.once("exit", (exitCode) => {
|
||||
clearTimeout(timer);
|
||||
resolveExit(exitCode);
|
||||
});
|
||||
});
|
||||
return { code, stderr };
|
||||
}
|
||||
|
||||
describe.sequential("Phase 3 durable transport and recovery", () => {
|
||||
it("accepts a short-lived bootstrap ticket once and rejects its reuse", async () => {
|
||||
const scratch =
|
||||
@@ -147,19 +219,50 @@ describe.sequential("Phase 3 durable transport and recovery", () => {
|
||||
});
|
||||
try {
|
||||
await core.start();
|
||||
core.queueCommand("runner.shutdown", {});
|
||||
const ticket = core.issueBootstrapTicket();
|
||||
const stateDirectory = resolve(root, "runner");
|
||||
|
||||
const first = await runRunnerProcess({
|
||||
connectUrl: core.connectUrl,
|
||||
stateDirectory,
|
||||
identity: core.identity,
|
||||
ticket,
|
||||
maxRuntimeMs: 2_000,
|
||||
});
|
||||
expect(first.code, first.stderr).toBe(0);
|
||||
expect(core.store.state.connectionCount).toBe(1);
|
||||
|
||||
const reused = await runRunnerProcess({
|
||||
connectUrl: core.connectUrl,
|
||||
stateDirectory,
|
||||
identity: core.identity,
|
||||
ticket,
|
||||
maxRuntimeMs: 100,
|
||||
});
|
||||
expect(reused.code).not.toBe(0);
|
||||
expect(core.store.state.connectionCount).toBe(1);
|
||||
|
||||
await expect(upgradeStatus(core.connectUrl, ticket)).resolves.toBe(101);
|
||||
await expect(upgradeStatus(core.connectUrl, ticket)).resolves.toBe(401);
|
||||
const expiredTicket = core.issueBootstrapTicket(-1);
|
||||
await expect(upgradeStatus(core.connectUrl, expiredTicket)).resolves.toBe(401);
|
||||
const expired = await runRunnerProcess({
|
||||
connectUrl: core.connectUrl,
|
||||
stateDirectory: resolve(root, "expired-runner"),
|
||||
identity: core.identity,
|
||||
ticket: expiredTicket,
|
||||
maxRuntimeMs: 100,
|
||||
});
|
||||
expect(expired.code).not.toBe(0);
|
||||
expect(core.store.state.connectionCount).toBe(1);
|
||||
expect(
|
||||
Object.values(core.store.state.tickets).filter((record) => record.usedAt !== null),
|
||||
).toHaveLength(1);
|
||||
} finally {
|
||||
await core.stop();
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects bootstrap and resumed lease credentials bound to a different runner identity", async () => {
|
||||
it("requires proof before consuming a ticket and rejects mismatched session identity", async () => {
|
||||
const scratch =
|
||||
process.env.PAPERCLIP_RUN_SCRATCH_DIR ??
|
||||
process.env.PAPERCLIP_SCRATCH_DIR ??
|
||||
@@ -175,26 +278,31 @@ describe.sequential("Phase 3 durable transport and recovery", () => {
|
||||
await core.start();
|
||||
|
||||
const bootstrap = core.issueBootstrapTicket();
|
||||
const bootstrapSocket = await upgradeSocket(core.connectUrl, bootstrap);
|
||||
sendMaskedJson(bootstrapSocket, {
|
||||
...hello(identity),
|
||||
runnerInstanceId: "runner_wrong_identity",
|
||||
});
|
||||
await expectSocketClosed(bootstrapSocket);
|
||||
const credentialId = phase3Internals.credentialMaterial(bootstrap).credentialId;
|
||||
const wrongIdentitySocket = await upgradeSocket(core.connectUrl);
|
||||
const wrongHello = authHello(identity, credentialId);
|
||||
(wrongHello.payload as Record<string, unknown>).runId = "run_wrong_identity";
|
||||
sendMaskedJson(wrongIdentitySocket, wrongHello);
|
||||
await expectSocketClosed(wrongIdentitySocket);
|
||||
expect(core.store.state.tickets[credentialId]?.usedAt).toBeNull();
|
||||
|
||||
const leaseToken = "lease_wrong_identity_regression";
|
||||
const digest = phase3Internals.tokenDigest(leaseToken);
|
||||
core.store.state.leases[digest] = {
|
||||
digest,
|
||||
leaseId: "lease_wrong_identity",
|
||||
runnerInstanceId: "runner_other_identity",
|
||||
expiresAt: new Date(Date.now() + 30_000).toISOString(),
|
||||
revokedAt: null,
|
||||
};
|
||||
core.store.save();
|
||||
const leaseSocket = await upgradeSocket(core.connectUrl, leaseToken);
|
||||
sendMaskedJson(leaseSocket, hello(identity));
|
||||
await expectSocketClosed(leaseSocket);
|
||||
const forgedProofSocket = await upgradeSocket(core.connectUrl);
|
||||
sendMaskedJson(forgedProofSocket, authHello(identity, credentialId));
|
||||
const challenge = await receiveServerJson(forgedProofSocket);
|
||||
const challengePayload = challenge.payload as Record<string, unknown>;
|
||||
sendMaskedJson(forgedProofSocket, {
|
||||
protocol: "paperclip.runner",
|
||||
version: 1,
|
||||
kind: "auth_response",
|
||||
payload: {
|
||||
credentialId,
|
||||
clientNonce: challengePayload.clientNonce,
|
||||
serverNonce: challengePayload.serverNonce,
|
||||
clientProof: "00".repeat(32),
|
||||
},
|
||||
});
|
||||
await expectSocketClosed(forgedProofSocket);
|
||||
expect(core.store.state.tickets[credentialId]?.usedAt).toBeNull();
|
||||
} finally {
|
||||
await core.stop();
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
@@ -295,6 +403,100 @@ describe.sequential("Phase 3 durable transport and recovery", () => {
|
||||
expect(trace.assertions.sourceCursorContinuous).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps a restarted revoked runner network-silent and byte-stable with a fresh ticket", async () => {
|
||||
const scratch =
|
||||
process.env.PAPERCLIP_RUN_SCRATCH_DIR ??
|
||||
process.env.PAPERCLIP_SCRATCH_DIR ??
|
||||
tmpdir();
|
||||
const root = mkdtempSync(resolve(scratch, "phase3-revoked-restart-test-"));
|
||||
const runnerStateDirectory = resolve(root, "runner");
|
||||
const identity = phase3Internals.phase3Identity();
|
||||
mkdirSync(runnerStateDirectory, { recursive: true, mode: 0o700 });
|
||||
const envelope = {
|
||||
protocol: "paperclip.runner",
|
||||
version: 1,
|
||||
envelopeId: "envelope_event_000004",
|
||||
kind: "event",
|
||||
runnerInstanceId: identity.runnerInstanceId,
|
||||
payload: {
|
||||
sourceEventId: `event_${identity.runnerInstanceId}_000004`,
|
||||
sourceSeq: 4,
|
||||
eventType: "run.terminal",
|
||||
priority: 0,
|
||||
payload: { status: "succeeded" },
|
||||
},
|
||||
};
|
||||
const byteSize = Buffer.byteLength(JSON.stringify(envelope));
|
||||
const statePath = resolve(runnerStateDirectory, "runner-state.json");
|
||||
writeFileSync(
|
||||
statePath,
|
||||
JSON.stringify(
|
||||
{
|
||||
schema: "paperclip.runner.phase3.state.v1",
|
||||
...identity,
|
||||
lifecycle: "revoked",
|
||||
nextSourceSeq: 5,
|
||||
ackedSourceSeq: 3,
|
||||
lastControllerCommandSeq: 2,
|
||||
reconnectCount: 7,
|
||||
maxOutboxBytes: 64 * 1024,
|
||||
peakOutboxBytes: byteSize,
|
||||
outbox: [
|
||||
{
|
||||
sourceSeq: 4,
|
||||
sourceEventId: `event_${identity.runnerInstanceId}_000004`,
|
||||
priority: 0,
|
||||
eventType: "run.terminal",
|
||||
itemId: null,
|
||||
envelope,
|
||||
byteSize,
|
||||
},
|
||||
],
|
||||
processedCommands: {},
|
||||
compactedCommandFilter: "00".repeat(4_096),
|
||||
compactedCommandCount: 0,
|
||||
diagnostics: ["connection lease revoked; preserving durable events"],
|
||||
backpressure: false,
|
||||
recoverableFailure: null,
|
||||
unrecoverableOutcome: null,
|
||||
harnessGeneration: 1,
|
||||
stopAfterFlush: true,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
{ mode: 0o600 },
|
||||
);
|
||||
const before = readFileSync(statePath);
|
||||
const core = new Phase3MockCore({
|
||||
stateDirectory: resolve(root, "mock-core"),
|
||||
identity,
|
||||
fault: "none",
|
||||
});
|
||||
try {
|
||||
await core.start();
|
||||
const ticket = core.issueBootstrapTicket();
|
||||
const connectionsBefore = core.store.state.connectionCount;
|
||||
const result = await runRunnerProcess({
|
||||
connectUrl: core.connectUrl,
|
||||
stateDirectory: runnerStateDirectory,
|
||||
identity,
|
||||
ticket,
|
||||
maxRuntimeMs: 1_000,
|
||||
});
|
||||
|
||||
expect(result.code, result.stderr).toBe(0);
|
||||
expect(core.store.state.connectionCount).toBe(connectionsBefore);
|
||||
expect(Object.values(core.store.state.tickets).map((record) => record.usedAt)).toEqual([
|
||||
null,
|
||||
]);
|
||||
expect(readFileSync(statePath)).toEqual(before);
|
||||
} finally {
|
||||
await core.stop();
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("reconciles a restarted harness without replacing session, turn, or item IDs", async () => {
|
||||
const trace = await runPhase3Recovery({ fault: "harness-restart" });
|
||||
const reconciled = trace.diagnostics.committedEvents.find(
|
||||
|
||||
@@ -1,5 +1,12 @@
|
||||
import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import {
|
||||
createCipheriv,
|
||||
createDecipheriv,
|
||||
createHash,
|
||||
createHmac,
|
||||
randomUUID,
|
||||
timingSafeEqual,
|
||||
} from "node:crypto";
|
||||
import {
|
||||
chmodSync,
|
||||
closeSync,
|
||||
@@ -51,22 +58,31 @@ const fakeHarnessScript = resolve(
|
||||
);
|
||||
const protocol = "paperclip.runner";
|
||||
const protocolVersion = 1;
|
||||
const secureFrameSchema = "paperclip.runner.secure-frame.v1";
|
||||
const websocketGuid = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11";
|
||||
const coreStateSchema = "paperclip.runner.phase3.mock-core-state.v1";
|
||||
const maxFrameBytes = 1024 * 1024;
|
||||
|
||||
interface BootstrapTicketRecord {
|
||||
digest: string;
|
||||
runnerInstanceId: string;
|
||||
credentialId: string;
|
||||
authKeyDigest: string;
|
||||
identity: Phase3Identity;
|
||||
runnerVersion: string;
|
||||
runnerDigest: string;
|
||||
expiresAt: string;
|
||||
expiresAtUnixMs: number;
|
||||
usedAt: string | null;
|
||||
}
|
||||
|
||||
interface ConnectionLeaseRecord {
|
||||
digest: string;
|
||||
credentialId: string;
|
||||
authKeyDigest: string;
|
||||
leaseId: string;
|
||||
runnerInstanceId: string;
|
||||
identity: Phase3Identity;
|
||||
protocolVersion: number;
|
||||
expiresAt: string;
|
||||
expiresAtUnixMs: number;
|
||||
revocationEpoch: number;
|
||||
revokedAt: string | null;
|
||||
}
|
||||
|
||||
@@ -88,20 +104,34 @@ interface StoredCoreState {
|
||||
lastLeaseExpiresAt: string | null;
|
||||
}
|
||||
|
||||
type Authorization =
|
||||
type PendingAuthorization =
|
||||
| {
|
||||
kind: "bootstrap";
|
||||
digest: string;
|
||||
rawToken: string;
|
||||
authKey: Buffer;
|
||||
ticket: BootstrapTicketRecord;
|
||||
}
|
||||
| {
|
||||
kind: "lease";
|
||||
digest: string;
|
||||
rawToken: string;
|
||||
authKey: Buffer;
|
||||
lease: ConnectionLeaseRecord;
|
||||
};
|
||||
|
||||
interface PendingChallenge {
|
||||
authorization: PendingAuthorization;
|
||||
canonicalChallenge: string;
|
||||
serverProof: string;
|
||||
clientNonce: string;
|
||||
serverNonce: string;
|
||||
}
|
||||
|
||||
interface SecureChannel {
|
||||
sendKey: Buffer;
|
||||
receiveKey: Buffer;
|
||||
sendCounter: bigint;
|
||||
receiveCounter: bigint;
|
||||
sessionId: string;
|
||||
}
|
||||
|
||||
interface MockCoreOptions {
|
||||
stateDirectory: string;
|
||||
identity: Phase3Identity;
|
||||
@@ -122,8 +152,36 @@ interface RunnerProcessHandle {
|
||||
completion: Promise<RunnerProcessResult>;
|
||||
}
|
||||
|
||||
function domainDigest(domain: string, parts: readonly Buffer[]): Buffer {
|
||||
const digest = createHash("sha256").update(domain).update(Buffer.from([0]));
|
||||
for (const part of parts) {
|
||||
const length = Buffer.alloc(8);
|
||||
length.writeBigUInt64BE(BigInt(part.length));
|
||||
digest.update(length).update(part);
|
||||
}
|
||||
return digest.digest();
|
||||
}
|
||||
|
||||
function domainHmac(key: Buffer, domain: string, parts: readonly Buffer[]): Buffer {
|
||||
const digest = createHmac("sha256", key).update(domain).update(Buffer.from([0]));
|
||||
for (const part of parts) {
|
||||
const length = Buffer.alloc(8);
|
||||
length.writeBigUInt64BE(BigInt(part.length));
|
||||
digest.update(length).update(part);
|
||||
}
|
||||
return digest.digest();
|
||||
}
|
||||
|
||||
function credentialMaterial(token: string): { credentialId: string; authKey: Buffer } {
|
||||
const bytes = Buffer.from(token);
|
||||
return {
|
||||
credentialId: `sha256:${domainDigest("paperclip-runner-credential-id-v1", [bytes]).toString("hex")}`,
|
||||
authKey: domainDigest("paperclip-runner-auth-key-v1", [bytes]),
|
||||
};
|
||||
}
|
||||
|
||||
function tokenDigest(token: string): string {
|
||||
return `sha256:${createHash("sha256").update(token).digest("hex")}`;
|
||||
return `sha256:${credentialMaterial(token).authKey.toString("hex")}`;
|
||||
}
|
||||
|
||||
function canonicalJson(value: unknown): string {
|
||||
@@ -144,6 +202,103 @@ function safeDate(offsetMs: number): string {
|
||||
return new Date(Date.now() + offsetMs).toISOString();
|
||||
}
|
||||
|
||||
function authKeyFromDigest(digest: string): Buffer {
|
||||
const hex = digest.match(/^sha256:([0-9a-f]{64})$/)?.[1];
|
||||
if (hex === undefined) throw new Error("Stored transport authentication key is malformed.");
|
||||
return Buffer.from(hex, "hex");
|
||||
}
|
||||
|
||||
function proofMatches(expected: Buffer, supplied: unknown): boolean {
|
||||
if (typeof supplied !== "string" || !/^[0-9a-f]{64}$/.test(supplied)) return false;
|
||||
return timingSafeEqual(expected, Buffer.from(supplied, "hex"));
|
||||
}
|
||||
|
||||
function createSecureChannel(
|
||||
authKey: Buffer,
|
||||
canonicalChallenge: string,
|
||||
serverProof: string,
|
||||
clientProof: string,
|
||||
): SecureChannel {
|
||||
const parts = [
|
||||
Buffer.from(canonicalChallenge),
|
||||
Buffer.from(serverProof),
|
||||
Buffer.from(clientProof),
|
||||
];
|
||||
const binding = domainDigest("paperclip-runner-session-binding-v1", parts);
|
||||
return {
|
||||
sendKey: domainHmac(authKey, "paperclip-runner-core-to-client-key-v1", [binding]),
|
||||
receiveKey: domainHmac(authKey, "paperclip-runner-client-to-core-key-v1", [binding]),
|
||||
sendCounter: 0n,
|
||||
receiveCounter: 0n,
|
||||
sessionId: `sha256:${binding.toString("hex")}`,
|
||||
};
|
||||
}
|
||||
|
||||
function secureNonce(prefix: "P3C1" | "P3S1", counter: bigint): Buffer {
|
||||
const nonce = Buffer.alloc(12);
|
||||
nonce.write(prefix, 0, "ascii");
|
||||
nonce.writeBigUInt64BE(counter, 4);
|
||||
return nonce;
|
||||
}
|
||||
|
||||
function secureAad(
|
||||
channel: SecureChannel,
|
||||
direction: "client_to_core" | "core_to_client",
|
||||
counter: bigint,
|
||||
): Buffer {
|
||||
return Buffer.from(`${secureFrameSchema}\0${channel.sessionId}\0${direction}\0${counter}`);
|
||||
}
|
||||
|
||||
function encryptSecureJson(channel: SecureChannel, value: unknown): Record<string, unknown> {
|
||||
const counter = channel.sendCounter;
|
||||
const cipher = createCipheriv("aes-256-gcm", channel.sendKey, secureNonce("P3S1", counter));
|
||||
cipher.setAAD(secureAad(channel, "core_to_client", counter));
|
||||
const ciphertext = Buffer.concat([
|
||||
cipher.update(Buffer.from(JSON.stringify(value))),
|
||||
cipher.final(),
|
||||
cipher.getAuthTag(),
|
||||
]);
|
||||
channel.sendCounter += 1n;
|
||||
return {
|
||||
schema: secureFrameSchema,
|
||||
counter: Number(counter),
|
||||
ciphertext: ciphertext.toString("hex"),
|
||||
};
|
||||
}
|
||||
|
||||
function decryptSecureJson(channel: SecureChannel, value: unknown): Record<string, unknown> {
|
||||
if (typeof value !== "object" || value === null) {
|
||||
throw new Error("Secure frame must be an object.");
|
||||
}
|
||||
const frame = value as Record<string, unknown>;
|
||||
if (
|
||||
frame.schema !== secureFrameSchema ||
|
||||
typeof frame.counter !== "number" ||
|
||||
!Number.isSafeInteger(frame.counter) ||
|
||||
BigInt(frame.counter) !== channel.receiveCounter ||
|
||||
typeof frame.ciphertext !== "string" ||
|
||||
!/^[0-9a-f]+$/.test(frame.ciphertext) ||
|
||||
frame.ciphertext.length % 2 !== 0
|
||||
) {
|
||||
throw new Error("Secure frame metadata or counter is invalid.");
|
||||
}
|
||||
const sealed = Buffer.from(frame.ciphertext, "hex");
|
||||
if (sealed.length < 16) throw new Error("Secure frame authentication tag is missing.");
|
||||
const ciphertext = sealed.subarray(0, -16);
|
||||
const tag = sealed.subarray(-16);
|
||||
const counter = channel.receiveCounter;
|
||||
const decipher = createDecipheriv(
|
||||
"aes-256-gcm",
|
||||
channel.receiveKey,
|
||||
secureNonce("P3C1", counter),
|
||||
);
|
||||
decipher.setAAD(secureAad(channel, "client_to_core", counter));
|
||||
decipher.setAuthTag(tag);
|
||||
const plaintext = Buffer.concat([decipher.update(ciphertext), decipher.final()]);
|
||||
channel.receiveCounter += 1n;
|
||||
return JSON.parse(plaintext.toString("utf8")) as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function initialCoreState(identity: Phase3Identity): StoredCoreState {
|
||||
return {
|
||||
schema: coreStateSchema,
|
||||
@@ -304,8 +459,10 @@ class DurableCoreStore {
|
||||
|
||||
class MockWebSocketConnection {
|
||||
readonly socket: Duplex;
|
||||
readonly authorization: Authorization;
|
||||
pendingChallenge: PendingChallenge | null = null;
|
||||
secureChannel: SecureChannel | null = null;
|
||||
lease: ConnectionLeaseRecord | null = null;
|
||||
connectionId: string | null = null;
|
||||
#buffer = Buffer.alloc(0);
|
||||
#closed = false;
|
||||
#onText: (text: string) => void;
|
||||
@@ -313,12 +470,10 @@ class MockWebSocketConnection {
|
||||
|
||||
constructor(
|
||||
socket: Duplex,
|
||||
authorization: Authorization,
|
||||
onText: (text: string) => void,
|
||||
onClose: () => void,
|
||||
) {
|
||||
this.socket = socket;
|
||||
this.authorization = authorization;
|
||||
this.#onText = onText;
|
||||
this.#onClose = onClose;
|
||||
socket.on("data", (chunk: Buffer) => this.#consume(chunk));
|
||||
@@ -332,7 +487,8 @@ class MockWebSocketConnection {
|
||||
}
|
||||
|
||||
sendJson(value: unknown): void {
|
||||
this.sendText(JSON.stringify(value));
|
||||
const wire = this.secureChannel === null ? value : encryptSecureJson(this.secureChannel, value);
|
||||
this.sendText(JSON.stringify(wire));
|
||||
}
|
||||
|
||||
sendText(text: string): void {
|
||||
@@ -492,11 +648,16 @@ export class Phase3MockCore {
|
||||
|
||||
issueBootstrapTicket(ttlMs = 5_000): string {
|
||||
const ticket = `bootstrap_${randomUUID()}`;
|
||||
const digest = tokenDigest(ticket);
|
||||
this.store.state.tickets[digest] = {
|
||||
digest,
|
||||
runnerInstanceId: this.identity.runnerInstanceId,
|
||||
expiresAt: safeDate(ttlMs),
|
||||
const material = credentialMaterial(ticket);
|
||||
const expiresAtUnixMs = Date.now() + ttlMs;
|
||||
this.store.state.tickets[material.credentialId] = {
|
||||
credentialId: material.credentialId,
|
||||
authKeyDigest: `sha256:${material.authKey.toString("hex")}`,
|
||||
identity: structuredClone(this.identity),
|
||||
runnerVersion: this.#expectedRunnerVersion,
|
||||
runnerDigest: this.#expectedRunnerDigest,
|
||||
expiresAt: new Date(expiresAtUnixMs).toISOString(),
|
||||
expiresAtUnixMs,
|
||||
usedAt: null,
|
||||
};
|
||||
this.store.state.freshBootstraps += 1;
|
||||
@@ -542,12 +703,6 @@ export class Phase3MockCore {
|
||||
socket.destroy();
|
||||
return;
|
||||
}
|
||||
const authorization = this.#authorize(request.headers.authorization);
|
||||
if (authorization === null) {
|
||||
socket.write("HTTP/1.1 401 Unauthorized\r\nConnection: close\r\n\r\n");
|
||||
socket.destroy();
|
||||
return;
|
||||
}
|
||||
const websocketKey = request.headers["sec-websocket-key"];
|
||||
if (typeof websocketKey !== "string") {
|
||||
socket.write("HTTP/1.1 400 Bad Request\r\nConnection: close\r\n\r\n");
|
||||
@@ -568,38 +723,20 @@ export class Phase3MockCore {
|
||||
);
|
||||
const connection = new MockWebSocketConnection(
|
||||
socket,
|
||||
authorization,
|
||||
(text) => this.#handleText(connection, text),
|
||||
() => this.#connections.delete(connection),
|
||||
);
|
||||
this.#connections.add(connection);
|
||||
}
|
||||
|
||||
#authorize(header: string | undefined): Authorization | null {
|
||||
const rawToken = header?.match(/^Bearer (.+)$/i)?.[1];
|
||||
if (rawToken === undefined) return null;
|
||||
const digest = tokenDigest(rawToken);
|
||||
const ticket = this.store.state.tickets[digest];
|
||||
if (ticket !== undefined && ticket.usedAt === null && Date.parse(ticket.expiresAt) > Date.now()) {
|
||||
ticket.usedAt = new Date().toISOString();
|
||||
this.store.save();
|
||||
return { kind: "bootstrap", digest, rawToken, ticket };
|
||||
}
|
||||
const lease = this.store.state.leases[digest];
|
||||
if (
|
||||
lease !== undefined &&
|
||||
lease.revokedAt === null &&
|
||||
Date.parse(lease.expiresAt) > Date.now()
|
||||
) {
|
||||
return { kind: "lease", digest, rawToken, lease };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
#handleText(connection: MockWebSocketConnection, text: string): void {
|
||||
let envelope: Record<string, unknown>;
|
||||
try {
|
||||
envelope = JSON.parse(text) as Record<string, unknown>;
|
||||
const wire = JSON.parse(text) as unknown;
|
||||
envelope =
|
||||
connection.secureChannel === null
|
||||
? (wire as Record<string, unknown>)
|
||||
: decryptSecureJson(connection.secureChannel, wire);
|
||||
} catch {
|
||||
this.store.state.malformedFrames += 1;
|
||||
this.store.save();
|
||||
@@ -611,11 +748,15 @@ export class Phase3MockCore {
|
||||
return;
|
||||
}
|
||||
const kind = envelope.kind;
|
||||
if (kind === "hello") {
|
||||
this.#hello(connection, envelope);
|
||||
if (connection.secureChannel === null && kind === "auth_hello") {
|
||||
this.#authHello(connection, envelope);
|
||||
return;
|
||||
}
|
||||
if (connection.lease === null) {
|
||||
if (connection.secureChannel === null && kind === "auth_response") {
|
||||
this.#authResponse(connection, envelope);
|
||||
return;
|
||||
}
|
||||
if (connection.secureChannel === null || connection.lease === null) {
|
||||
connection.close();
|
||||
return;
|
||||
}
|
||||
@@ -632,42 +773,165 @@ export class Phase3MockCore {
|
||||
}
|
||||
}
|
||||
|
||||
#hello(connection: MockWebSocketConnection, envelope: Record<string, unknown>): void {
|
||||
const payload = envelope.payload as Record<string, unknown> | undefined;
|
||||
const authorizedRunnerInstanceId =
|
||||
connection.authorization.kind === "bootstrap"
|
||||
? connection.authorization.ticket.runnerInstanceId
|
||||
: connection.authorization.lease.runnerInstanceId;
|
||||
#authorizeHello(payload: Record<string, unknown>): PendingAuthorization | null {
|
||||
const credentialId = payload.credentialId;
|
||||
if (typeof credentialId !== "string") return null;
|
||||
const ticket = this.store.state.tickets[credentialId];
|
||||
const lease = this.store.state.leases[credentialId];
|
||||
const authorization: PendingAuthorization | null =
|
||||
ticket !== undefined && ticket.usedAt === null && ticket.expiresAtUnixMs > Date.now()
|
||||
? { kind: "bootstrap", authKey: authKeyFromDigest(ticket.authKeyDigest), ticket }
|
||||
: lease !== undefined &&
|
||||
lease.revokedAt === null &&
|
||||
lease.expiresAtUnixMs > Date.now()
|
||||
? { kind: "lease", authKey: authKeyFromDigest(lease.authKeyDigest), lease }
|
||||
: null;
|
||||
if (authorization === null) return null;
|
||||
const binding = authorization.kind === "bootstrap" ? authorization.ticket : authorization.lease;
|
||||
const identity = binding.identity;
|
||||
if (
|
||||
envelope.runnerInstanceId !== this.identity.runnerInstanceId ||
|
||||
envelope.runnerInstanceId !== authorizedRunnerInstanceId ||
|
||||
payload?.environmentLeaseId !== this.identity.environmentLeaseId ||
|
||||
payload?.runnerVersion !== this.#expectedRunnerVersion ||
|
||||
payload?.runnerDigest !== this.#expectedRunnerDigest ||
|
||||
payload?.protocolMin !== 1 ||
|
||||
payload.protocolMax !== 1
|
||||
payload.runnerInstanceId !== identity.runnerInstanceId ||
|
||||
payload.environmentLeaseId !== identity.environmentLeaseId ||
|
||||
payload.runId !== identity.runId ||
|
||||
payload.normalizedSessionId !== identity.normalizedSessionId ||
|
||||
payload.turnId !== identity.turnId ||
|
||||
payload.itemId !== identity.itemId ||
|
||||
payload.runnerVersion !== this.#expectedRunnerVersion ||
|
||||
payload.runnerDigest !== this.#expectedRunnerDigest ||
|
||||
payload.protocolMin !== 1 ||
|
||||
payload.protocolMax !== 1 ||
|
||||
(authorization.kind === "lease" && authorization.lease.protocolVersion !== protocolVersion)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return authorization;
|
||||
}
|
||||
|
||||
#authHello(connection: MockWebSocketConnection, envelope: Record<string, unknown>): void {
|
||||
if (connection.pendingChallenge !== null) {
|
||||
connection.close();
|
||||
return;
|
||||
}
|
||||
const payload = envelope.payload as Record<string, unknown> | undefined;
|
||||
if (payload === undefined || typeof payload.clientNonce !== "string") {
|
||||
connection.close();
|
||||
return;
|
||||
}
|
||||
const authorization = this.#authorizeHello(payload);
|
||||
if (authorization === null) {
|
||||
connection.close();
|
||||
return;
|
||||
}
|
||||
const credential = authorization.kind === "bootstrap" ? authorization.ticket : authorization.lease;
|
||||
const serverNonce = randomUUID();
|
||||
const challengePayload: Record<string, unknown> = {
|
||||
credentialId: credential.credentialId,
|
||||
credentialKind: authorization.kind,
|
||||
clientNonce: payload.clientNonce,
|
||||
serverNonce,
|
||||
runnerInstanceId: payload.runnerInstanceId,
|
||||
environmentLeaseId: payload.environmentLeaseId,
|
||||
runId: payload.runId,
|
||||
normalizedSessionId: payload.normalizedSessionId,
|
||||
turnId: payload.turnId,
|
||||
itemId: payload.itemId,
|
||||
runnerVersion: payload.runnerVersion,
|
||||
runnerDigest: payload.runnerDigest,
|
||||
selectedVersion: protocolVersion,
|
||||
credentialLeaseId: authorization.kind === "lease" ? authorization.lease.leaseId : null,
|
||||
credentialExpiresAt: credential.expiresAt,
|
||||
credentialExpiresAtUnixMs: credential.expiresAtUnixMs,
|
||||
revocationEpoch: authorization.kind === "lease" ? authorization.lease.revocationEpoch : 0,
|
||||
};
|
||||
const canonicalChallenge = canonicalJson(challengePayload);
|
||||
const serverProof = domainHmac(
|
||||
authorization.authKey,
|
||||
"paperclip-runner-server-proof-v1",
|
||||
[Buffer.from(canonicalChallenge)],
|
||||
).toString("hex");
|
||||
connection.pendingChallenge = {
|
||||
authorization,
|
||||
canonicalChallenge,
|
||||
serverProof,
|
||||
clientNonce: payload.clientNonce,
|
||||
serverNonce,
|
||||
};
|
||||
connection.sendJson({
|
||||
protocol,
|
||||
version: protocolVersion,
|
||||
kind: "auth_challenge",
|
||||
payload: { ...challengePayload, serverProof },
|
||||
});
|
||||
}
|
||||
|
||||
#authResponse(connection: MockWebSocketConnection, envelope: Record<string, unknown>): void {
|
||||
const pending = connection.pendingChallenge;
|
||||
const payload = envelope.payload as Record<string, unknown> | undefined;
|
||||
if (
|
||||
pending === null ||
|
||||
payload === undefined ||
|
||||
payload.credentialId !==
|
||||
(pending.authorization.kind === "bootstrap"
|
||||
? pending.authorization.ticket.credentialId
|
||||
: pending.authorization.lease.credentialId) ||
|
||||
payload.clientNonce !== pending.clientNonce ||
|
||||
payload.serverNonce !== pending.serverNonce
|
||||
) {
|
||||
connection.close();
|
||||
return;
|
||||
}
|
||||
|
||||
let leaseToken = connection.authorization.rawToken;
|
||||
const expectedClientProof = domainHmac(
|
||||
pending.authorization.authKey,
|
||||
"paperclip-runner-client-proof-v1",
|
||||
[Buffer.from(pending.canonicalChallenge), Buffer.from(pending.serverProof)],
|
||||
);
|
||||
if (!proofMatches(expectedClientProof, payload.clientProof)) {
|
||||
connection.close();
|
||||
return;
|
||||
}
|
||||
const clientProof = expectedClientProof.toString("hex");
|
||||
let leaseToken: string | null = null;
|
||||
let lease: ConnectionLeaseRecord;
|
||||
if (connection.authorization.kind === "bootstrap") {
|
||||
if (pending.authorization.kind === "bootstrap") {
|
||||
pending.authorization.ticket.usedAt = new Date().toISOString();
|
||||
leaseToken = `lease_${randomUUID()}`;
|
||||
const digest = tokenDigest(leaseToken);
|
||||
const material = credentialMaterial(leaseToken);
|
||||
const ttlMs = this.fault === "lease-expiry" && !this.#faultTriggered ? 50 : 30_000;
|
||||
const expiresAtUnixMs = Date.now() + ttlMs;
|
||||
lease = {
|
||||
digest,
|
||||
credentialId: material.credentialId,
|
||||
authKeyDigest: `sha256:${material.authKey.toString("hex")}`,
|
||||
leaseId: `connection_lease_${randomUUID()}`,
|
||||
runnerInstanceId: this.identity.runnerInstanceId,
|
||||
expiresAt: safeDate(this.fault === "lease-expiry" && !this.#faultTriggered ? 50 : 30_000),
|
||||
identity: structuredClone(this.identity),
|
||||
protocolVersion,
|
||||
expiresAt: new Date(expiresAtUnixMs).toISOString(),
|
||||
expiresAtUnixMs,
|
||||
revocationEpoch: 0,
|
||||
revokedAt: null,
|
||||
};
|
||||
this.store.state.leases[digest] = lease;
|
||||
this.store.state.leases[material.credentialId] = lease;
|
||||
} else {
|
||||
lease = connection.authorization.lease;
|
||||
lease = pending.authorization.lease;
|
||||
}
|
||||
connection.pendingChallenge = null;
|
||||
connection.lease = lease;
|
||||
connection.connectionId = `connection_${this.store.state.connectionCount + 1}`;
|
||||
connection.secureChannel = createSecureChannel(
|
||||
pending.authorization.authKey,
|
||||
pending.canonicalChallenge,
|
||||
pending.serverProof,
|
||||
clientProof,
|
||||
);
|
||||
this.#welcome(connection, leaseToken);
|
||||
}
|
||||
|
||||
#welcome(connection: MockWebSocketConnection, leaseToken: string | null): void {
|
||||
const lease = connection.lease;
|
||||
if (lease === null || connection.connectionId === null) {
|
||||
connection.close();
|
||||
return;
|
||||
}
|
||||
|
||||
this.store.state.connectionCount += 1;
|
||||
this.store.state.lastLeaseId = lease.leaseId;
|
||||
this.store.state.lastLeaseExpiresAt = lease.expiresAt;
|
||||
@@ -691,14 +955,29 @@ export class Phase3MockCore {
|
||||
envelopeId: `welcome_${this.store.state.connectionCount}`,
|
||||
kind: "welcome",
|
||||
runnerInstanceId: this.identity.runnerInstanceId,
|
||||
connectionId: `connection_${this.store.state.connectionCount}`,
|
||||
environmentLeaseId: this.identity.environmentLeaseId,
|
||||
runId: this.identity.runId,
|
||||
normalizedSessionId: this.identity.normalizedSessionId,
|
||||
turnId: this.identity.turnId,
|
||||
itemId: this.identity.itemId,
|
||||
connectionId: connection.connectionId,
|
||||
connectionLeaseId: lease.leaseId,
|
||||
sentAt: new Date().toISOString(),
|
||||
payload: {
|
||||
selectedVersion: 1,
|
||||
heartbeatIntervalMs: 250,
|
||||
connectionLeaseId: lease.leaseId,
|
||||
connectionLeaseToken: leaseToken,
|
||||
...(leaseToken === null ? {} : { connectionLeaseToken: leaseToken }),
|
||||
connectionLeaseExpiresAt: lease.expiresAt,
|
||||
connectionLeaseExpiresAtUnixMs: lease.expiresAtUnixMs,
|
||||
connectionLeaseRevocationEpoch: lease.revocationEpoch,
|
||||
leaseBinding: {
|
||||
runnerInstanceId: this.identity.runnerInstanceId,
|
||||
environmentLeaseId: this.identity.environmentLeaseId,
|
||||
runId: this.identity.runId,
|
||||
normalizedSessionId: this.identity.normalizedSessionId,
|
||||
protocolVersion,
|
||||
},
|
||||
maxFrameBytes,
|
||||
maxBatchEvents: 100,
|
||||
ackedSourceSeq: reportedAck,
|
||||
@@ -715,6 +994,7 @@ export class Phase3MockCore {
|
||||
}
|
||||
if (this.fault === "lease-expiry") {
|
||||
lease.expiresAt = safeDate(-1);
|
||||
lease.expiresAtUnixMs = Date.now() - 1;
|
||||
this.store.state.lastLeaseExpiresAt = lease.expiresAt;
|
||||
this.store.save();
|
||||
}
|
||||
@@ -732,23 +1012,47 @@ export class Phase3MockCore {
|
||||
return command === undefined ? [] : [command];
|
||||
}
|
||||
|
||||
#controlEnvelope(
|
||||
connection: MockWebSocketConnection,
|
||||
envelopeId: string,
|
||||
kind: string,
|
||||
payload: Record<string, unknown>,
|
||||
): Record<string, unknown> {
|
||||
if (connection.lease === null || connection.connectionId === null) {
|
||||
throw new Error("Cannot send control data before transport authentication.");
|
||||
}
|
||||
return {
|
||||
protocol,
|
||||
version: protocolVersion,
|
||||
envelopeId,
|
||||
kind,
|
||||
runnerInstanceId: this.identity.runnerInstanceId,
|
||||
environmentLeaseId: this.identity.environmentLeaseId,
|
||||
runId: this.identity.runId,
|
||||
normalizedSessionId: this.identity.normalizedSessionId,
|
||||
turnId: this.identity.turnId,
|
||||
itemId: this.identity.itemId,
|
||||
connectionId: connection.connectionId,
|
||||
connectionLeaseId: connection.lease.leaseId,
|
||||
sentAt: new Date().toISOString(),
|
||||
payload,
|
||||
};
|
||||
}
|
||||
|
||||
#sendNextCommand(connection: MockWebSocketConnection): void {
|
||||
const [command] = this.#nextPendingCommand();
|
||||
if (command === undefined) return;
|
||||
this.store.state.commandDeliveryCounts[command.commandId] =
|
||||
(this.store.state.commandDeliveryCounts[command.commandId] ?? 0) + 1;
|
||||
this.store.save();
|
||||
connection.sendJson({
|
||||
protocol,
|
||||
version: protocolVersion,
|
||||
envelopeId: `command_${command.commandId}_${this.store.state.commandDeliveryCounts[command.commandId]}`,
|
||||
kind: "command",
|
||||
runnerInstanceId: this.identity.runnerInstanceId,
|
||||
runId: this.identity.runId,
|
||||
normalizedSessionId: this.identity.normalizedSessionId,
|
||||
sentAt: new Date().toISOString(),
|
||||
payload: this.#wireCommand(command),
|
||||
});
|
||||
connection.sendJson(
|
||||
this.#controlEnvelope(
|
||||
connection,
|
||||
`command_${command.commandId}_${this.store.state.commandDeliveryCounts[command.commandId]}`,
|
||||
"command",
|
||||
this.#wireCommand(command),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
#commandResult(
|
||||
@@ -834,6 +1138,7 @@ export class Phase3MockCore {
|
||||
if (this.fault === "revoke" && eventType === "run.terminal" && !this.#faultTriggered) {
|
||||
if (connection.lease !== null) {
|
||||
connection.lease.revokedAt = new Date().toISOString();
|
||||
connection.lease.revocationEpoch += 1;
|
||||
}
|
||||
this.queueCommand(
|
||||
"turn.start",
|
||||
@@ -843,15 +1148,13 @@ export class Phase3MockCore {
|
||||
this.#triggerFault();
|
||||
this.store.save();
|
||||
// Revoke before ACK so the runner must flush a genuinely non-empty durable outbox.
|
||||
connection.sendJson({
|
||||
protocol,
|
||||
version: protocolVersion,
|
||||
envelopeId: "revoke_phase3",
|
||||
kind: "revoke",
|
||||
runnerInstanceId: this.identity.runnerInstanceId,
|
||||
sentAt: new Date().toISOString(),
|
||||
payload: { reason: "fault_injection_revoke", drain: true },
|
||||
});
|
||||
connection.sendJson(
|
||||
this.#controlEnvelope(connection, "revoke_phase3", "revoke", {
|
||||
reason: "fault_injection_revoke",
|
||||
drain: true,
|
||||
revocationEpoch: connection.lease?.revocationEpoch,
|
||||
}),
|
||||
);
|
||||
this.#sendNextCommand(connection);
|
||||
return;
|
||||
}
|
||||
@@ -867,15 +1170,11 @@ export class Phase3MockCore {
|
||||
return;
|
||||
}
|
||||
|
||||
connection.sendJson({
|
||||
protocol,
|
||||
version: protocolVersion,
|
||||
envelopeId: `ack_${this.store.state.ackedSourceSeq}`,
|
||||
kind: "ack",
|
||||
runnerInstanceId: this.identity.runnerInstanceId,
|
||||
sentAt: new Date().toISOString(),
|
||||
payload: { ackedSourceSeq: this.store.state.ackedSourceSeq },
|
||||
});
|
||||
connection.sendJson(
|
||||
this.#controlEnvelope(connection, `ack_${this.store.state.ackedSourceSeq}`, "ack", {
|
||||
ackedSourceSeq: this.store.state.ackedSourceSeq,
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1249,6 +1548,7 @@ export async function runPhase3Recovery(
|
||||
export const phase3Internals = {
|
||||
runnerBinary,
|
||||
runnerEnvironment,
|
||||
credentialMaterial,
|
||||
tokenDigest,
|
||||
canonicalJson,
|
||||
phase3Identity,
|
||||
|
||||
Reference in new issue
Block a user