mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-10 11:01:23 +01:00
Merge master and deduplicate workflow target fixture inputs
Two upstream fixes added IMAGE_ONLY and TESTOPS_REQUESTED to the same fixture. GitHub's merge ref therefore failed test-project typecheck with TS1117 even though the change branch contained one copy. Retain one copy of each input after merging current master. Verification: Runner test-project typecheck, all nine workflow-target tests, and all 483 relevant UI tests pass. Preserve the fixture's existing false input values and workflow target behavior. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
commit
0e77123ba1
75 files changed
+5096
-430
No files matched your search
@@ -0,0 +1,177 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const repository = "paperclipai/paperclip";
|
||||
const shaPattern = /^[a-f0-9]{40}$/;
|
||||
const minimumChecks = ["ci / verify", "ci / e2e"];
|
||||
const gh = (args) => JSON.parse(execFileSync("gh", args, {
|
||||
encoding: "utf8", maxBuffer: 16 * 1024 * 1024, timeout: 30_000,
|
||||
// Never allow command diagnostics or a response to reach the Actions log.
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
}));
|
||||
const api = (endpoint) => gh(["api", endpoint]);
|
||||
const pages = (endpoint) => gh(["api", "--paginate", "--slurp", endpoint]).flat();
|
||||
const query = `query($number:Int!, $endCursor:String) {
|
||||
repository(owner:"paperclipai", name:"paperclip") {
|
||||
pullRequest(number:$number) {
|
||||
number url state isDraft headRefOid baseRefOid baseRefName changedFiles
|
||||
statusCheckRollup { contexts(first:100, after:$endCursor) {
|
||||
totalCount pageInfo { hasNextPage endCursor }
|
||||
nodes {
|
||||
__typename
|
||||
... on CheckRun { name status conclusion checkSuite { databaseId app { databaseId }
|
||||
workflowRun { databaseId } } }
|
||||
... on StatusContext { context state }
|
||||
}
|
||||
} }
|
||||
}
|
||||
}
|
||||
}`;
|
||||
|
||||
export function snapshot(number, request = gh) {
|
||||
let candidate;
|
||||
let cursor;
|
||||
const checks = [];
|
||||
do {
|
||||
const args = ["api", "graphql", "-f", `query=${query}`, "-F", `number=${number}`];
|
||||
if (cursor) args.push("-f", `endCursor=${cursor}`);
|
||||
const result = request(args);
|
||||
if (result.errors?.length) throw new Error("Incomplete check listing");
|
||||
const pr = result.data.repository.pullRequest;
|
||||
if (candidate && (pr.headRefOid !== candidate.headRefOid || pr.baseRefOid !== candidate.baseRefOid)) {
|
||||
throw new Error("Candidate changed during check pagination");
|
||||
}
|
||||
candidate = pr;
|
||||
const contexts = pr.statusCheckRollup.contexts;
|
||||
checks.push(...contexts.nodes);
|
||||
if (!contexts.pageInfo.hasNextPage) {
|
||||
if (checks.length !== contexts.totalCount) throw new Error("Incomplete check listing");
|
||||
break;
|
||||
}
|
||||
const next = contexts.pageInfo.endCursor;
|
||||
if (!next || next === cursor || checks.length >= contexts.totalCount) throw new Error("Incomplete check listing");
|
||||
cursor = next;
|
||||
} while (checks.length < 10_000);
|
||||
if (candidate.statusCheckRollup.contexts.pageInfo.hasNextPage) throw new Error("Check listing limit exceeded");
|
||||
return { ...candidate, statusCheckRollup: checks };
|
||||
}
|
||||
|
||||
export function green(pr, requiredNames = minimumChecks) {
|
||||
if (pr.state !== "OPEN" || pr.isDraft || pr.baseRefName !== "master" ||
|
||||
!shaPattern.test(pr.headRefOid) || !shaPattern.test(pr.baseRefOid)) return false;
|
||||
const checks = pr.statusCheckRollup;
|
||||
if (!Array.isArray(checks) || checks.length === 0) return false;
|
||||
const relevant = checks;
|
||||
const passed = (check) => check.__typename === "StatusContext"
|
||||
? check.state === "SUCCESS"
|
||||
: check.__typename === "CheckRun" && check.status === "COMPLETED" && check.conclusion === "SUCCESS";
|
||||
// Optional skipped/neutral checks can mean no work or unavailable coverage.
|
||||
// They permit assessment, not approval. Required checks must really succeed.
|
||||
return relevant.every((check) => passed(check) || (check.__typename === "CheckRun" &&
|
||||
check.status === "COMPLETED" && ["SKIPPED", "NEUTRAL"].includes(check.conclusion))) &&
|
||||
[...new Set([...minimumChecks, ...requiredNames])].every((name) =>
|
||||
relevant.some((check) => (check.name ?? check.context) === name && passed(check)));
|
||||
}
|
||||
|
||||
export function trustedVerification(pr, { read = api, list = pages } = {}) {
|
||||
// Names alone are forgeable in pull_request YAML. Require the real Actions
|
||||
// suite for the unmodified PR caller and its master-owned reusable workflow.
|
||||
const actionsChecks = pr.statusCheckRollup.filter((check) =>
|
||||
check.__typename === "CheckRun" && check.status === "COMPLETED" && check.conclusion === "SUCCESS" &&
|
||||
check.checkSuite?.app?.databaseId === 15368 &&
|
||||
Number.isSafeInteger(check.checkSuite?.workflowRun?.databaseId));
|
||||
const runIds = new Set(actionsChecks.filter((check) => minimumChecks.includes(check.name))
|
||||
.map((check) => check.checkSuite.workflowRun.databaseId));
|
||||
let trusted = false;
|
||||
for (const id of runIds) {
|
||||
const run = read(`/repos/${repository}/actions/runs/${id}`);
|
||||
if (run.repository?.full_name !== repository || run.path !== ".github/workflows/pr.yml" ||
|
||||
run.event !== "pull_request" || run.head_sha !== pr.headRefOid ||
|
||||
run.status !== "completed" || run.conclusion !== "success") continue;
|
||||
if (!minimumChecks.every((name) => actionsChecks.some((check) => check.name === name &&
|
||||
check.checkSuite.workflowRun.databaseId === id && check.checkSuite.databaseId === run.check_suite_id))) continue;
|
||||
const reusable = run.referenced_workflows?.find((entry) =>
|
||||
entry.path === `${repository}/.github/workflows/pr-trusted.yml@master` &&
|
||||
entry.ref === "refs/heads/master" && shaPattern.test(entry.sha));
|
||||
if (reusable) { trusted = true; break; }
|
||||
}
|
||||
if (!trusted || !Number.isSafeInteger(pr.changedFiles) || pr.changedFiles < 0) return false;
|
||||
const files = list(`/repos/${repository}/pulls/${pr.number}/files?per_page=100`);
|
||||
// GitHub caps this API at 3,000 files. A partial listing cannot prove that
|
||||
// the caller is unmodified. Do not execute or download any of these files.
|
||||
return files.length === pr.changedFiles && files.every((file) =>
|
||||
typeof file.filename === "string" && ![file.filename, file.previous_filename].includes(".github/workflows/pr.yml"));
|
||||
}
|
||||
|
||||
export async function deliver(pr, { url, token, fetcher = fetch } = {}) {
|
||||
if (!url || !token) return "unconfigured";
|
||||
try {
|
||||
const target = new URL(url);
|
||||
if (target.protocol !== "https:" || target.username || target.password || target.search || target.hash ||
|
||||
!/^\/api\/routine-triggers\/public\/[a-f0-9]+\/fire$/.test(target.pathname)) return "invalid_configuration";
|
||||
const response = await fetcher(target, {
|
||||
method: "POST", redirect: "error", signal: AbortSignal.timeout(15_000),
|
||||
headers: {
|
||||
"Content-Type": "application/json", Authorization: `Bearer ${token}`,
|
||||
"Idempotency-Key": `pr-review:${repository}:${pr.number}:${pr.headRefOid}:${pr.baseRefOid}`,
|
||||
},
|
||||
// No PR title, body, comments, or executable instructions cross this boundary.
|
||||
body: JSON.stringify({ variables: {
|
||||
pr_url: `https://github.com/${repository}/pull/${pr.number}`,
|
||||
pr_number: String(pr.number), head_sha: pr.headRefOid, base_sha: pr.baseRefOid,
|
||||
} }),
|
||||
});
|
||||
// Do not read the response body: a proxy can echo the URL or credentials.
|
||||
await response.body?.cancel();
|
||||
return response.ok ? "delivered" : "not_delivered";
|
||||
} catch {
|
||||
// Fetch errors can contain the private URL. Keep the result fixed and quiet.
|
||||
return "not_delivered";
|
||||
}
|
||||
}
|
||||
|
||||
export async function notify(event, { read = api, list = pages, view = snapshot, send = deliver } = {}) {
|
||||
if (event.repository?.full_name !== repository) return [];
|
||||
const run = event.workflow_run;
|
||||
if (run && (run.status !== "completed" || !["pull_request", "pull_request_target"].includes(run.event))) return [];
|
||||
const sha = run?.head_sha ?? event.check_run?.head_sha ?? event.sha;
|
||||
if (!shaPattern.test(sha)) return [];
|
||||
// Resolve forks through the commit API too: workflow_run.pull_requests can be empty.
|
||||
const associated = run?.pull_requests?.length ? run.pull_requests
|
||||
: list(`/repos/${repository}/commits/${sha}/pulls?per_page=100`);
|
||||
const required = read(`/repos/${repository}/rules/branches/master`)
|
||||
.filter((rule) => rule.type === "required_status_checks")
|
||||
.flatMap((rule) => rule.parameters.required_status_checks.map((check) => check.context));
|
||||
const outcomes = [];
|
||||
for (const number of new Set(associated.map((pr) => pr.number))) {
|
||||
if (!Number.isSafeInteger(number) || number <= 0) continue;
|
||||
const pr = view(number);
|
||||
// Target-context workflows use the base SHA. All other events must still
|
||||
// describe this PR's current head; an old successful run cannot trigger it.
|
||||
if ((run?.event !== "pull_request_target" && pr.headRefOid !== sha) || !green(pr, required)) continue;
|
||||
const fresh = view(number);
|
||||
if (fresh.headRefOid !== pr.headRefOid || fresh.baseRefOid !== pr.baseRefOid ||
|
||||
!green(fresh, required) || !trustedVerification(fresh, { read, list })) continue;
|
||||
outcomes.push(await send(fresh));
|
||||
}
|
||||
return outcomes;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
try {
|
||||
const url = process.env.PAPERCLIP_PR_REVIEW_WEBHOOK_URL;
|
||||
const token = process.env.PAPERCLIP_PR_REVIEW_WEBHOOK_TOKEN;
|
||||
if (!url || !token) {
|
||||
console.log("PR review notification is not configured; skipped.");
|
||||
return;
|
||||
}
|
||||
const event = JSON.parse(readFileSync(process.env.GITHUB_EVENT_PATH, "utf8"));
|
||||
const outcomes = await notify(event, { send: (pr) => deliver(pr, { url, token }) });
|
||||
console.log(outcomes.length ? `PR review notification: ${outcomes.join(", ")}.` : "No current green PR to notify.");
|
||||
} catch {
|
||||
console.log("PR review notification was unavailable; verification results are unchanged.");
|
||||
}
|
||||
}
|
||||
|
||||
if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) await main();
|
||||
@@ -0,0 +1,263 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import { readFileSync, mkdtempSync, writeFileSync, rmSync } from "node:fs";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { green, deliver, notify, snapshot, trustedVerification } from "../notify-pr-merge-review.mjs";
|
||||
|
||||
const head = "a".repeat(40);
|
||||
const base = "b".repeat(40);
|
||||
const check = (name, conclusion = "SUCCESS", extra = {}) => ({
|
||||
__typename: "CheckRun", name, status: "COMPLETED", conclusion, workflowName: "PR",
|
||||
checkSuite: { databaseId: 8, app: { databaseId: 15368 }, workflowRun: { databaseId: 7 } },
|
||||
...extra,
|
||||
});
|
||||
const candidate = (extra = {}) => ({
|
||||
number: 42, state: "OPEN", isDraft: false, headRefOid: head, baseRefOid: base,
|
||||
baseRefName: "master", changedFiles: 1, statusCheckRollup: [check("ci / verify"), check("ci / e2e")], ...extra,
|
||||
});
|
||||
const event = (extra = {}) => ({ repository: { full_name: "paperclipai/paperclip" }, sha: head, ...extra });
|
||||
const trustedRun = (extra = {}) => ({
|
||||
id: 7, repository: { full_name: "paperclipai/paperclip" }, path: ".github/workflows/pr.yml",
|
||||
event: "pull_request", head_sha: head, status: "completed", conclusion: "success", check_suite_id: 8,
|
||||
referenced_workflows: [{ path: "paperclipai/paperclip/.github/workflows/pr-trusted.yml@master",
|
||||
ref: "refs/heads/master", sha: base }], ...extra,
|
||||
});
|
||||
const files = [{ filename: "README.md" }];
|
||||
const read = (endpoint) => endpoint.includes("/rules/") ? [] : trustedRun();
|
||||
const list = (endpoint) => endpoint.includes("/files?") ? files : [{ number: 42 }];
|
||||
const deps = (extra = {}) => ({
|
||||
read, list, view: () => candidate(),
|
||||
send: async () => "delivered", ...extra,
|
||||
});
|
||||
|
||||
test("reads every check page and fails closed on incomplete or changing listings", () => {
|
||||
const page = (nodes, hasNextPage, extra = {}) => ({ data: { repository: { pullRequest: candidate({
|
||||
statusCheckRollup: { contexts: { totalCount: 2, nodes,
|
||||
pageInfo: { hasNextPage, endCursor: "next" } } }, ...extra,
|
||||
}) } } });
|
||||
let calls = 0;
|
||||
const pr = snapshot(42, (args) => {
|
||||
if (++calls === 1) return page([check("ci / verify")], true);
|
||||
assert.ok(args.includes("endCursor=next"));
|
||||
return page([check("ci / e2e")], false);
|
||||
});
|
||||
assert.equal(green(pr), true);
|
||||
const own = check("notify", "", { status: "IN_PROGRESS", workflowName: undefined,
|
||||
checkSuite: { workflowRun: { workflow: { name: "PR merge review notification" } } } });
|
||||
const withOwn = snapshot(42, () => ({ data: { repository: { pullRequest: candidate({
|
||||
statusCheckRollup: { contexts: { totalCount: 3, nodes: [...candidate().statusCheckRollup, own],
|
||||
pageInfo: { hasNextPage: false } } },
|
||||
}) } } }));
|
||||
assert.equal(green(withOwn), false);
|
||||
assert.throws(() => snapshot(42, () => page([check("ci / verify")], false)), /Incomplete/);
|
||||
calls = 0;
|
||||
assert.throws(() => snapshot(42, () => ++calls === 1 ? page([check("ci / verify")], true)
|
||||
: page([check("ci / e2e")], false, { headRefOid: "d".repeat(40) })), /Candidate changed/);
|
||||
assert.throws(() => snapshot(42, () => ({ errors: [{ message: "unavailable" }] })), /Incomplete/);
|
||||
});
|
||||
|
||||
test("requires real success from both aggregate checks and additional repository rules", () => {
|
||||
assert.equal(green(candidate()), true);
|
||||
for (const state of ["FAILURE", "CANCELLED", "TIMED_OUT", "ACTION_REQUIRED", "NEUTRAL", "SKIPPED", ""]) {
|
||||
assert.equal(green(candidate({ statusCheckRollup: [check("ci / verify", state), check("ci / e2e")] })), false, state);
|
||||
}
|
||||
assert.equal(green(candidate({ statusCheckRollup: [check("ci / verify")] })), false);
|
||||
assert.equal(green(candidate(), ["extra required"]), false);
|
||||
assert.equal(green(candidate({ statusCheckRollup: [...candidate().statusCheckRollup, check("extra required")] }), ["extra required"]), true);
|
||||
assert.equal(green(candidate({ statusCheckRollup: [] })), false);
|
||||
});
|
||||
|
||||
test("waits for pending checks and statuses, permits skipped optional routing lanes", () => {
|
||||
for (const extra of [check("tests", "", { status: "IN_PROGRESS" }), check("tests", "FAILURE"),
|
||||
{ __typename: "StatusContext", context: "security", state: "PENDING" },
|
||||
{ __typename: "StatusContext", context: "security", state: "ERROR" }]) {
|
||||
assert.equal(green(candidate({ statusCheckRollup: [...candidate().statusCheckRollup, extra] })), false);
|
||||
}
|
||||
assert.equal(green(candidate({ statusCheckRollup: [...candidate().statusCheckRollup,
|
||||
check("not selected", "SKIPPED"), { __typename: "StatusContext", context: "security", state: "SUCCESS" }] })), true);
|
||||
assert.equal(green(candidate({ statusCheckRollup: [...candidate().statusCheckRollup,
|
||||
check("optional scanner coverage", "NEUTRAL")] })), true);
|
||||
});
|
||||
|
||||
test("workflow names cannot hide pending or failing checks", () => {
|
||||
const extras = ["PR merge review notification", "CI profile"].map((workflowName) =>
|
||||
check("notify", "", { status: "IN_PROGRESS", workflowName }));
|
||||
assert.equal(green(candidate({ statusCheckRollup: [...candidate().statusCheckRollup, ...extras] })), false);
|
||||
assert.equal(green(candidate({ statusCheckRollup: [...candidate().statusCheckRollup,
|
||||
check("another", "", { status: "QUEUED", workflowName: "Other verification" })] })), false);
|
||||
});
|
||||
|
||||
test("rejects closed, draft, wrong-base, and malformed candidates", () => {
|
||||
for (const extra of [{ state: "MERGED" }, { isDraft: true }, { baseRefName: "release" },
|
||||
{ headRefOid: "invalid" }, { baseRefOid: "invalid" }]) assert.equal(green(candidate(extra)), false);
|
||||
});
|
||||
|
||||
test("resolves fork PRs from the head commit when workflow payload has no PRs", async () => {
|
||||
const seen = [];
|
||||
const outcomes = await notify(event({ workflow_run: {
|
||||
status: "completed", event: "pull_request", head_sha: head, pull_requests: [],
|
||||
} }), deps({ list: (endpoint) => { seen.push(endpoint); return list(endpoint); } }));
|
||||
assert.deepEqual(outcomes, ["delivered"]);
|
||||
assert.match(seen[0], new RegExp(`/commits/${head}/pulls\\?per_page=100$`));
|
||||
});
|
||||
|
||||
test("reads additional required rules and holds until their check succeeds", async () => {
|
||||
const outcomes = await notify(event(), deps({ read: () => [{ type: "required_status_checks",
|
||||
parameters: { required_status_checks: [{ context: "new gate" }] } }] }));
|
||||
assert.deepEqual(outcomes, []);
|
||||
});
|
||||
|
||||
test("forged successful check names from a PR-controlled workflow cannot notify", async () => {
|
||||
const input = deps({ read: (endpoint) => endpoint.includes("/rules/") ? [] : {
|
||||
id: 7, path: ".github/workflows/attacker.yml", event: "pull_request",
|
||||
head_sha: head, status: "completed", conclusion: "success",
|
||||
} });
|
||||
assert.deepEqual(await notify(event(), input), []);
|
||||
});
|
||||
|
||||
test("requires genuine suite, caller, revision, and master reusable workflow provenance", () => {
|
||||
assert.equal(trustedVerification(candidate(), deps()), true);
|
||||
for (const changes of [
|
||||
{ path: ".github/workflows/attacker.yml" }, { event: "pull_request_target" },
|
||||
{ head_sha: "c".repeat(40) }, { status: "in_progress" }, { conclusion: "failure" },
|
||||
{ repository: { full_name: "attacker/repo" } }, { check_suite_id: 99 },
|
||||
{ referenced_workflows: [] },
|
||||
{ referenced_workflows: [{ path: "paperclipai/paperclip/.github/workflows/pr-trusted.yml@master",
|
||||
ref: "refs/pull/42/merge", sha: base }] },
|
||||
]) {
|
||||
assert.equal(trustedVerification(candidate(), deps({ read: () => trustedRun(changes) })), false);
|
||||
}
|
||||
const wrongSuite = check("ci / verify", "SUCCESS", {
|
||||
checkSuite: { databaseId: 99, app: { databaseId: 1 }, workflowRun: { databaseId: 7 } },
|
||||
});
|
||||
assert.equal(trustedVerification(candidate({
|
||||
statusCheckRollup: [wrongSuite, check("ci / e2e")],
|
||||
}), deps()), false);
|
||||
});
|
||||
|
||||
test("modified or renamed caller and truncated file listings cannot notify", async () => {
|
||||
for (const fileList of [[], [{ filename: ".github/workflows/pr.yml" }],
|
||||
[{ filename: "moved.yml", previous_filename: ".github/workflows/pr.yml" }]]) {
|
||||
const dependencies = deps({ list: (endpoint) => endpoint.includes("/files?")
|
||||
? fileList : [{ number: 42 }] });
|
||||
assert.deepEqual(await notify(event(), dependencies), []);
|
||||
}
|
||||
assert.equal(trustedVerification(candidate({ changedFiles: undefined }), deps()), false);
|
||||
});
|
||||
|
||||
test("rejects stale head events and a head, base, or check change during refresh", async () => {
|
||||
assert.deepEqual(await notify(event({ sha: "c".repeat(40) }), deps()), []);
|
||||
for (const extra of [{ headRefOid: "c".repeat(40) }, { baseRefOid: "d".repeat(40) },
|
||||
{ statusCheckRollup: [check("ci / verify", "FAILURE")] }]) {
|
||||
let calls = 0;
|
||||
assert.deepEqual(await notify(event(), deps({ view: () => candidate(++calls === 1 ? {} : extra) })), []);
|
||||
}
|
||||
});
|
||||
|
||||
test("handles target-context workflow completions by refreshing the associated PR", async () => {
|
||||
assert.deepEqual(await notify(event({ workflow_run: {
|
||||
status: "completed", event: "pull_request_target", head_sha: base, pull_requests: [{ number: 42 }],
|
||||
} }), deps()), ["delivered"]);
|
||||
});
|
||||
|
||||
test("external check events and legacy statuses can deliver the final green transition", async () => {
|
||||
assert.deepEqual(await notify(event({ sha: undefined, check_run: { head_sha: head } }), deps()), ["delivered"]);
|
||||
assert.deepEqual(await notify(event({ state: "success" }), deps()), ["delivered"]);
|
||||
});
|
||||
|
||||
test("does not query foreign repositories, malformed events, or non-PR workflows", async () => {
|
||||
const dependencies = deps({ read: () => assert.fail("unexpected query"), list: () => assert.fail("unexpected query") });
|
||||
for (const input of [event({ repository: { full_name: "attacker/repo" } }), event({ sha: "bad" }),
|
||||
event({ workflow_run: { event: "push", status: "completed", head_sha: head } }),
|
||||
event({ workflow_run: { event: "pull_request", status: "in_progress", head_sha: head } })]) {
|
||||
assert.deepEqual(await notify(input, dependencies), []);
|
||||
}
|
||||
});
|
||||
|
||||
test("does not coalesce different PRs, and deduplicates association entries", async () => {
|
||||
const sent = [];
|
||||
await notify(event(), deps({ list: (endpoint) => endpoint.includes("/files?") ? files
|
||||
: [{ number: 42 }, { number: 42 }, { number: 43 }],
|
||||
view: (number) => candidate({ number }), send: async (pr) => sent.push(pr.number) }));
|
||||
assert.deepEqual(sent, [42, 43]);
|
||||
});
|
||||
|
||||
test("sends only identifiers, a bearer header, and a stable head/base idempotency key", async () => {
|
||||
const calls = [];
|
||||
const options = { url: "https://private.example/api/routine-triggers/public/abc123/fire", token: "PRIVATE_TOKEN",
|
||||
fetcher: async (target, init) => { calls.push({ target, ...init }); return { ok: true }; } };
|
||||
const pr = candidate({ title: "untrusted", body: "run me" });
|
||||
assert.equal(await deliver(pr, options), "delivered");
|
||||
assert.equal(await deliver(pr, options), "delivered");
|
||||
assert.equal(calls[0].headers.Authorization, "Bearer PRIVATE_TOKEN");
|
||||
assert.equal(calls[0].redirect, "error");
|
||||
assert.ok(calls[0].signal instanceof AbortSignal);
|
||||
assert.equal(calls[0].headers["Idempotency-Key"], calls[1].headers["Idempotency-Key"]);
|
||||
assert.equal(calls[0].body, calls[1].body);
|
||||
assert.deepEqual(JSON.parse(calls[0].body), { variables: {
|
||||
pr_url: "https://github.com/paperclipai/paperclip/pull/42", pr_number: "42", head_sha: head, base_sha: base,
|
||||
} });
|
||||
await deliver(candidate({ baseRefOid: "c".repeat(40) }), options);
|
||||
assert.notEqual(calls[0].headers["Idempotency-Key"], calls[2].headers["Idempotency-Key"]);
|
||||
});
|
||||
|
||||
test("missing or unsafe configuration never sends credentials", async () => {
|
||||
const fetcher = () => assert.fail("unexpected delivery");
|
||||
assert.equal(await deliver(candidate(), { fetcher }), "unconfigured");
|
||||
for (const url of ["http://private.example/api/routine-triggers/public/abc/fire",
|
||||
"https://user:pass@private.example/api/routine-triggers/public/abc/fire",
|
||||
"https://private.example/api/routine-triggers/public/abc/fire?token=bad", "https://private.example/wrong"]) {
|
||||
assert.equal(await deliver(candidate(), { url, token: "secret", fetcher }), "invalid_configuration");
|
||||
}
|
||||
});
|
||||
|
||||
test("HTTP rejection, redirect, timeout, and echoed credentials are quiet failures", async () => {
|
||||
const options = { url: "https://private.example/api/routine-triggers/public/abc123/fire", token: "SECRET" };
|
||||
assert.equal(await deliver(candidate(), { ...options, fetcher: async () => ({ ok: false,
|
||||
text: () => assert.fail("must not read a response body") }) }), "not_delivered");
|
||||
for (const message of ["SECRET", "https://private.example", "redirect", "timeout"]) {
|
||||
assert.equal(await deliver(candidate(), { ...options, fetcher: async () => { throw new Error(message); } }), "not_delivered");
|
||||
}
|
||||
});
|
||||
|
||||
test("command failures cannot print a private URL, token, or event payload", () => {
|
||||
const directory = mkdtempSync(path.join(tmpdir(), "pr-notifier-test-"));
|
||||
try {
|
||||
const file = path.join(directory, "event.json");
|
||||
writeFileSync(file, "PRIVATE_EVENT_INVALID_JSON");
|
||||
const run = spawnSync(process.execPath, [".github/scripts/notify-pr-merge-review.mjs"], {
|
||||
encoding: "utf8", env: { ...process.env, GITHUB_EVENT_PATH: file,
|
||||
PAPERCLIP_PR_REVIEW_WEBHOOK_URL: "PRIVATE_URL_SENTINEL", PAPERCLIP_PR_REVIEW_WEBHOOK_TOKEN: "PRIVATE_TOKEN_SENTINEL" },
|
||||
});
|
||||
assert.equal(run.status, 0);
|
||||
assert.doesNotMatch(run.stdout + run.stderr, /PRIVATE_/);
|
||||
assert.match(run.stdout, /verification results are unchanged/);
|
||||
} finally { rmSync(directory, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test("workflow uses read permissions, a protected environment, trusted checkout, and optional delivery", () => {
|
||||
const workflow = readFileSync(".github/workflows/pr-merge-review.yml", "utf8");
|
||||
assert.match(workflow, /environment: pr-merge-review/);
|
||||
assert.match(workflow, /ref: \$\{\{ github.sha \}\}/);
|
||||
assert.match(workflow, /persist-credentials: false/);
|
||||
assert.match(workflow, /continue-on-error: true/);
|
||||
for (const permission of ["actions", "checks", "contents", "pull-requests", "statuses"]) {
|
||||
assert.ok(workflow.includes(` ${permission}: read`), `Missing ${permission}: read`);
|
||||
}
|
||||
assert.doesNotMatch(workflow, /: write|download-artifact|restore@|pull_request_target:/);
|
||||
for (const name of ["PAPERCLIP_PR_REVIEW_WEBHOOK_URL", "PAPERCLIP_PR_REVIEW_WEBHOOK_TOKEN"]) {
|
||||
assert.ok(workflow.includes(`secrets.${name}`));
|
||||
assert.ok(!workflow.includes(`vars.${name}`));
|
||||
}
|
||||
});
|
||||
|
||||
test("API probe uses the same read permissions without private webhook access", () => {
|
||||
const probe = readFileSync(".github/workflows/pr-review-api-access.yml", "utf8");
|
||||
for (const permission of ["actions", "checks", "contents", "pull-requests", "statuses"]) {
|
||||
assert.ok(probe.includes(` ${permission}: read`));
|
||||
}
|
||||
assert.match(probe, /snapshot\(Number\(process.env.PR_NUMBER\)\)/);
|
||||
assert.doesNotMatch(probe, /\$\{\{ secrets\.|environment:|: write|deliver\(/);
|
||||
});
|
||||
@@ -0,0 +1,57 @@
|
||||
name: PR merge review notification
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows:
|
||||
- PR
|
||||
- Runner Windows
|
||||
- Sentry SDK contract
|
||||
- Docker Runner check
|
||||
- commitperclip PR Review
|
||||
- Storybook Visual
|
||||
- Storybook Deploy
|
||||
- CI work benchmark
|
||||
- Incremental CI qualification
|
||||
- Backup checkpoint qualification
|
||||
- PR review API access
|
||||
types: [completed]
|
||||
# External check providers and legacy commit statuses can finish after CI.
|
||||
check_run:
|
||||
types: [completed]
|
||||
status:
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
checks: read
|
||||
contents: read
|
||||
pull-requests: read
|
||||
statuses: read
|
||||
|
||||
jobs:
|
||||
notify:
|
||||
if: >-
|
||||
github.repository == 'paperclipai/paperclip' &&
|
||||
(github.event_name != 'workflow_run' ||
|
||||
github.event.workflow_run.event == 'pull_request' ||
|
||||
github.event.workflow_run.event == 'pull_request_target')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
# Restrict this environment to master. Repository secrets would also be
|
||||
# available to workflows changed by an in-repository PR.
|
||||
environment: pr-merge-review
|
||||
continue-on-error: true
|
||||
steps:
|
||||
# Only the trusted default-branch commit. Never use the PR SHA, caches,
|
||||
# artifacts, dependencies, or executable files from a pull request.
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
ref: ${{ github.sha }}
|
||||
persist-credentials: false
|
||||
sparse-checkout: .github/scripts
|
||||
- name: Notify the advisory review routine
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
PAPERCLIP_PR_REVIEW_WEBHOOK_URL: ${{ secrets.PAPERCLIP_PR_REVIEW_WEBHOOK_URL }}
|
||||
PAPERCLIP_PR_REVIEW_WEBHOOK_TOKEN: ${{ secrets.PAPERCLIP_PR_REVIEW_WEBHOOK_TOKEN }}
|
||||
run: node .github/scripts/notify-pr-merge-review.mjs
|
||||
@@ -0,0 +1,43 @@
|
||||
name: PR review API access
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- .github/scripts/notify-pr-merge-review.mjs
|
||||
- .github/scripts/tests/notify-pr-merge-review.test.mjs
|
||||
- .github/workflows/pr-merge-review.yml
|
||||
- .github/workflows/pr-review-api-access.yml
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
checks: read
|
||||
contents: read
|
||||
pull-requests: read
|
||||
statuses: read
|
||||
|
||||
jobs:
|
||||
probe:
|
||||
if: github.repository == 'paperclipai/paperclip'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
# This job has no environment or private webhook secrets. It proves that
|
||||
# the notifier's status query works with the actual read-only Actions token.
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
sparse-checkout: .github/scripts
|
||||
- name: Read the PR status rollup with the Actions token
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
run: |
|
||||
node --input-type=module <<'JS'
|
||||
import assert from 'node:assert/strict';
|
||||
import { snapshot } from './.github/scripts/notify-pr-merge-review.mjs';
|
||||
const pr = snapshot(Number(process.env.PR_NUMBER));
|
||||
assert.equal(pr.headRefOid, process.env.PR_HEAD_SHA);
|
||||
assert.ok(pr.statusCheckRollup.length > 0);
|
||||
console.log('Actions token can read the complete PR status rollup.');
|
||||
JS
|
||||
@@ -522,6 +522,16 @@ pnpm build-storybook
|
||||
|
||||
These run the `@paperclipai/ui` Storybook on port `6006` and build the static output to `ui/storybook-static/`.
|
||||
|
||||
**Inline links** mounts the shared Markdown renderer with GitHub repository,
|
||||
pull request, and issue references, custom labels, connected status metadata,
|
||||
generic URL fallbacks, and streaming playback. Pasted GitHub resource URLs use compact labels;
|
||||
the exact destination, query, and fragment stay intact. The presentation makes
|
||||
no remote metadata requests. Run its light/dark desktop/mobile checks with:
|
||||
|
||||
```sh
|
||||
STORYBOOK_TEST_BASE_URL=http://localhost:6006 pnpm exec playwright test inline-links.spec.ts --config=tests/storybook-visual/playwright.config.ts --project=stories
|
||||
```
|
||||
|
||||
**Task chat → Failure notices** covers compact run errors and adjacent recovery
|
||||
notices in both presentation modes, with expanded details, fixture retry
|
||||
callbacks (including the legacy run retry when recovery retry is unavailable),
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
# Advisory PR review notifications
|
||||
|
||||
The optional `PR merge review notification` workflow sends a private webhook
|
||||
after a current, open, non-draft PR targeting `master` passes verification.
|
||||
It does not approve or merge a PR. Webhook failures do not change PR CI results.
|
||||
|
||||
## Private configuration
|
||||
|
||||
Create a GitHub Actions environment named `pr-merge-review`. Restrict its
|
||||
deployment branch policy to the `master` branch only, with no tag policy.
|
||||
Keep these two values in **environment secrets**, not repository variables,
|
||||
repository secrets, committed files, task comments, or workflow outputs:
|
||||
|
||||
- `PAPERCLIP_PR_REVIEW_WEBHOOK_URL`: a Paperclip routine webhook URL.
|
||||
- `PAPERCLIP_PR_REVIEW_WEBHOOK_TOKEN`: its separate bearer signing secret.
|
||||
|
||||
Use a webhook trigger with `signingMode: bearer`. The routine should use
|
||||
`always_enqueue` concurrency so separate PRs cannot be coalesced into one task.
|
||||
Define required text variables `pr_url`, `pr_number`, `head_sha`, and `base_sha`.
|
||||
The notifier sends only those identifiers. The receiving agent must treat them
|
||||
as untrusted data and verify the repository, PR identity, and current revision.
|
||||
|
||||
Give the routine instructions to run `pr-merge-review` and publish its MERGE,
|
||||
HOLD, or DO NOT MERGE recommendation on the routine execution's Paperclip task.
|
||||
Require a fresh check of the head, base, reviews, required checks, and task gates.
|
||||
Keep GitHub approval, merge, auto-merge, and public-comment actions disabled in
|
||||
the instructions for this advisory phase. Configure the routine in Paperclip;
|
||||
do not put instance-specific URLs or credentials in this repository.
|
||||
|
||||
## Delivery and verification
|
||||
|
||||
The notification workflow runs trusted default-branch code. It never checks out
|
||||
PR code or downloads PR artifacts or caches. It reacts to the repository's PR
|
||||
workflow completions, external check completions, and legacy commit statuses.
|
||||
When adding another PR verification workflow, add its name to `workflow_run`.
|
||||
|
||||
The separate `PR review API access` workflow checks notifier changes using
|
||||
the real read-only Actions token. It has no private environment or webhook
|
||||
credentials and sends no webhook. Both workflows explicitly grant
|
||||
`actions`, `checks`, `contents`, `pull-requests`, and `statuses` read access.
|
||||
The GraphQL status rollup needs both check and commit-status permissions.
|
||||
|
||||
The notifier reads the current PR's status rollup and branch rules. Both
|
||||
`ci / verify` and `ci / e2e`, plus any additional required checks, must succeed.
|
||||
Other checks must succeed or finish as skipped or neutral. Optional skipped or
|
||||
neutral checks permit an assessment; they do not prove successful verification.
|
||||
For example, a neutral scanner can mean coverage is unavailable. The advisory
|
||||
routine evaluates these evidence gaps and reviews before recommending merge.
|
||||
Workflow names do not exclude checks. The notifier and CI profile exporter run
|
||||
on the default branch, so their own checks do not belong to the PR revision.
|
||||
|
||||
Check names alone are not trusted. Both core aggregates must belong to the same
|
||||
GitHub Actions suite and a successful run of the repository's actual
|
||||
`.github/workflows/pr.yml`. Its run metadata must reference
|
||||
`pr-trusted.yml@master`. The complete PR file list must prove that the caller
|
||||
`pr.yml` was not changed or renamed. A missing or truncated list stops delivery.
|
||||
PRs that change this caller need a manual advisory review until their caller
|
||||
change reaches the trusted branch. Other workflow changes can still be reviewed.
|
||||
|
||||
A final refresh rejects head or base changes during evaluation. Paperclip's
|
||||
idempotency receipt deduplicates events for the same PR, head, and base. A new
|
||||
base can produce a new review. The routine must refresh again before its final
|
||||
recommendation. Event delivery and host check visibility can race; notifications
|
||||
are best effort, with no polling or delivery guarantee.
|
||||
|
||||
The HTTP request uses TLS, a bearer header, no redirects, and a 15 second timeout.
|
||||
Logs contain fixed delivery states only. Request and response bodies, URLs,
|
||||
tokens, and raw exceptions are never logged. Missing secrets skip delivery.
|
||||
|
||||
Run the focused regression checks with:
|
||||
|
||||
```sh
|
||||
node --test .github/scripts/tests/notify-pr-merge-review.test.mjs
|
||||
```
|
||||
|
||||
For setup verification, use Paperclip's webhook `setupPending` test mode before
|
||||
activation. This verifies authentication and ingress without starting a review.
|
||||
After the workflow is merged to `master`, inspect a real successful workflow
|
||||
delivery and its linked routine task to prove end-to-end operation. A saved
|
||||
configuration or a setup test alone does not prove that journey.
|
||||
|
||||
To pause reviews, pause the routine in Paperclip or disable its trigger. To stop
|
||||
notifications, disable this workflow or remove the two environment secrets.
|
||||
Rotate the trigger signing secret in Paperclip and replace the token environment
|
||||
secret together. Keep the URL secret even though it has no embedded bearer token.
|
||||
@@ -179,3 +179,101 @@ An explicit managed reconnect reinstalls the saved app and verifies the replacem
|
||||
bot credentials. Its request ID is saved before dispatch, so a duplicate completed
|
||||
request does not reinstall again. Credential-configuration recovery still resumes
|
||||
the saved token after a downstream failure. No reconnect creates another Slack app.
|
||||
|
||||
## Durable Cloud Events API delivery
|
||||
|
||||
A broker that advertises `slackEvents.version=1` can accept a signed Events API
|
||||
registration after ordinary configuration settles. The producer uses the saved
|
||||
child app ID, current workspace, signing key, endpoint/connection and runtime
|
||||
generation. It retains the existing public callback URL. Missing verified app
|
||||
identity or broker support leaves the existing setup in place; it does not guess
|
||||
an identity or substitute the Slack manager's signing key.
|
||||
|
||||
Cloud stores the child signing key through its existing encrypted stack vault.
|
||||
Registration metadata and receipts contain no plaintext credentials. An unchanged
|
||||
credential/endpoint identity reuses its immutable vault ref across generations;
|
||||
rotation and current registration authority remain separately fenced. Removal
|
||||
retires exact queue membership and prevents an old generation from reenrolling.
|
||||
The local configuration transaction first saves nonsecret enrollment or
|
||||
retirement intent and signals the existing connection-events owner. Broker work
|
||||
runs through that owner after commit; configuration APIs do not await an optional
|
||||
Cloud request. Configure, pause and removal can complete locally while Cloud is unavailable. The owner
|
||||
retries only retained due intent and reconstructs it after restart, including
|
||||
removal for archived endpoints after credential cleanup. Confirmation replaces
|
||||
only the same current intent and generation; a completed remote call with a
|
||||
lost response remains pending. Current authorization errors retain visible
|
||||
attention debt instead of claiming registration succeeded. Event draining and
|
||||
acknowledgments run before enrollment reconciliation. The owner then admits one
|
||||
due enrollment operation and schedules the earliest retained responsibility.
|
||||
A busy credential lease or enrollment failure retains a bounded retry without
|
||||
aborting that event drain. Rollback leaves no committed intent to run.
|
||||
|
||||
Retirement sends the original responsible subject and the exact instance,
|
||||
company, endpoint, public ID, connection and monotonic generation. Cloud can
|
||||
publish a nonreceiving generation floor without a signing key, live descriptor
|
||||
or previously returned binding ID. This fences delayed enrollment after local
|
||||
removal. Normal current broker authority still applies; the subject alone does
|
||||
not authorize an operation. An instance that never enrolled with Cloud creates
|
||||
no new intent or background retry when Cloud is absent. Previously configured
|
||||
retirement responsibility retains its original instance identity as visible
|
||||
attention if that identity becomes unavailable. Unsupported capability or disabled
|
||||
new enrollment retains bounded retry responsibility; a closed generation or
|
||||
identity conflict requires attention. Current authorized confirmation is required
|
||||
before clearing the exact pending intent.
|
||||
|
||||
Ordinary redeploys retain the same stack identity. To move enrollment to another
|
||||
stack, first confirm retirement under the old stack's authority, with no pending
|
||||
old intent. Then explicitly reconfigure under the new authorized stack at a newer
|
||||
runtime generation. The old stored identity remains until the new broker receipt
|
||||
matches the current intent and generation. An environment change cannot transfer
|
||||
live responsibility. Unavailable old authority remains visible attention.
|
||||
|
||||
A missing or disabled signing credential also requires attention. Enrollment does
|
||||
not retry it as a broker outage. Incoming receipts retain typed attention before
|
||||
SDK admission; uncertain authentication does not acknowledge the receipt.
|
||||
|
||||
Only authenticated `event_callback` messages use this durable lane. The sealed
|
||||
queue retains original bytes, signature, timestamp and acceptance time. The app
|
||||
checks current endpoint/generation authority before SDK handling. It verifies
|
||||
original freshness at Cloud acceptance; ordinary external requests still use
|
||||
current time. The credential snapshot lease releases before normal lifecycle
|
||||
callbacks reacquire their own leases.
|
||||
|
||||
Private replay provenance prevents internal replay URLs from replacing saved
|
||||
external callback verification evidence. The normal transactional chat delivery
|
||||
ledger remains the message admission authority. An ambiguous callback or ACK
|
||||
failure retains its receipt; malformed receipts rejected before useful admission
|
||||
can be explicitly ignored with bounded diagnostics. Credential-authentication
|
||||
uncertainty requires attention and is not silently acknowledged. The broker's
|
||||
existing retention horizon remains the disclosed recovery boundary.
|
||||
|
||||
This does not qualify a complete Slack endpoint for sleep. Slash commands,
|
||||
interactive actions, modal validation and challenges still need their full
|
||||
synchronous response semantics. The endpoint blocker remains. Qualification
|
||||
must cover normal signed enrollment, durable retry after removal, sealed drain,
|
||||
the actual event worker and SDK, lost-ACK redelivery and one domain effect,
|
||||
generation changes, and callback evidence. Synthetic protocol coverage does not
|
||||
prove live Slack, complete configuration UI or a deployed stop/restart/wake journey.
|
||||
|
||||
### Reproduce the paired Cloud acceptance
|
||||
|
||||
Use the repository's supported Node version and installed dependencies. Compile
|
||||
the companion Cloud checkout first. Set `PAPERCLIP_SLACK_TEST_DATABASE_URL` and
|
||||
`SLACK_EVENT_TEST_DATABASE_URL` to two distinct, owned loopback PostgreSQL
|
||||
`*_test` databases. Set `SLACK_TEST_CLOUD_DIST` to its absolute compiled `dist`
|
||||
directory. This test applies actual migrations and uses only those two databases.
|
||||
It does not create a PostgreSQL cluster or use a live Slack provider.
|
||||
|
||||
From the app repository root, run:
|
||||
|
||||
```sh
|
||||
node server/scripts/test-slack-cloud-postgres.mjs
|
||||
```
|
||||
|
||||
The launcher uses the current Node executable and one Vitest worker. It loads the
|
||||
checked-in prewire guard in that actual worker before client construction. The
|
||||
controlled process child explicitly loads the same guard before its script.
|
||||
The guard rejects off-host and unowned socket targets before transport. Caught
|
||||
unexpected attempts still fail the fixture. No private preload path or manual
|
||||
`SLACK_GATE_GUARD_PID` setting is needed. The fixture remains opt-in during the
|
||||
normal test suite; its default owner and guard regressions run without databases.
|
||||
@@ -1647,6 +1647,14 @@ On startup and on the periodic recovery loop, Paperclip performs the following r
|
||||
3. reconcile stranded assigned work
|
||||
4. scan silent active runs only for source-aware terminal folding and legacy cleanup; API reads classify ordinary output silence for the board UI
|
||||
|
||||
The orphan reap normally waits until a `running` run has been unchanged for five
|
||||
minutes. A legacy run whose controller lease belongs to an earlier server boot
|
||||
and has expired, which was still preparing (before adapter dispatch), and which
|
||||
has no recorded process is recovered when that lease expires instead. The
|
||||
expired lease proves that the earlier owner is gone, and no provider work
|
||||
started. A live lease, a lease of the current boot, a dispatched adapter, or a
|
||||
recorded process keeps the five-minute guard.
|
||||
|
||||
The stranded-work pass closes the gap where issue state survives a crash but the wake/run path does not. The silent-run scan covers the separate case where a live process exists but has stopped producing observable output.
|
||||
|
||||
Native finalization reconciliation (part of native recovery and the orphan
|
||||
@@ -2379,6 +2387,22 @@ All of these must be true:
|
||||
- Each quarantined root for the scope passes the same journal and local
|
||||
Runner evidence checks for the same owner.
|
||||
|
||||
A local per-turn Runner can end the same way: the result was accepted, but its
|
||||
`runner.suspend` command did not finish, so its own receipt is still
|
||||
unsuspended. The next local run retires that owner with the same checks and
|
||||
these differences:
|
||||
|
||||
- The Runner's own receipt may be unsuspended. It may not be unreadable or
|
||||
name another owner.
|
||||
- Only the Runner's own lifecycle commands (`runner.suspend`, `runner.drain`,
|
||||
`runner.shutdown`) may be unfinished. All other commands must be complete.
|
||||
- The prior run's recorded Runner process is proven stopped by exact PID and
|
||||
start time, as in restart recovery. A PID that now belongs to another
|
||||
process counts as stopped. A live recorded process, a live process group, a
|
||||
live recorded provider process, a missing identity, or a start time that
|
||||
cannot be compared keeps the rejection, for example
|
||||
`retirement_refused=runner_process_alive`. A later Retry checks again.
|
||||
|
||||
The provider session then follows the normal resume-or-replace policy. A
|
||||
resumed sandbox, a warm session, a plugin task, a restart claim, or any
|
||||
missing proof keeps the existing `prior_owner_terminal_state_indeterminate`
|
||||
@@ -3317,6 +3341,13 @@ A graceful shutdown can also suspend a restart-recovery claim whose execution
|
||||
has not yet reached provider dispatch. That execution has lost its authority:
|
||||
it leaves the run running, its budget hold held and its workspace unmarked, and
|
||||
the next controller adopts the same run. It does not settle the run as failed.
|
||||
Shutdown closes the runtime API listener before it drains runs. A local Runner
|
||||
execution that then needs the listener (for example, to give a new runner its
|
||||
callback origin) detaches the same way instead of failing. It keeps its claim,
|
||||
the drain suspends it, and the next controller adopts the run. A remote Runner
|
||||
execution has no process evidence on the controller host, so it keeps the
|
||||
same-run bootstrap retry. A listener that never started is still a startup
|
||||
failure.
|
||||
Remote Runner processes use their authenticated connection lease as the bound
|
||||
on disconnected lifetime. The controller's shorter connection-recovery budget
|
||||
limits its own attempt; it must not terminate the remote owner while another
|
||||
|
||||
@@ -1589,12 +1589,11 @@ it.each(["before", "after"] as const)(
|
||||
JSON.stringify(controlPlaneState),
|
||||
);
|
||||
await expect(
|
||||
runnerdRecoveryInternals.rotateExternalAuthorityEpoch(
|
||||
runnerdRecoveryInternals.rotateAuthorityEpoch(
|
||||
root,
|
||||
controlPlaneState,
|
||||
desiredIdentity,
|
||||
readRunnerState,
|
||||
archiveRunnerState,
|
||||
{ read: readRunnerState, archive: archiveRunnerState },
|
||||
),
|
||||
).rejects.toThrow(`crashed ${crashPoint} remote archive`);
|
||||
await expect(stat(join(root, "control-plane"))).rejects.toMatchObject({
|
||||
@@ -1602,12 +1601,11 @@ it.each(["before", "after"] as const)(
|
||||
});
|
||||
|
||||
await expect(
|
||||
runnerdRecoveryInternals.rotateExternalAuthorityEpoch(
|
||||
runnerdRecoveryInternals.rotateAuthorityEpoch(
|
||||
root,
|
||||
controlPlaneState,
|
||||
desiredIdentity,
|
||||
readRunnerState,
|
||||
archiveRunnerState,
|
||||
{ read: readRunnerState, archive: archiveRunnerState },
|
||||
),
|
||||
).resolves.toEqual(controlPlaneState);
|
||||
expect(readCount).toBe(1);
|
||||
@@ -1622,6 +1620,261 @@ it.each(["before", "after"] as const)(
|
||||
},
|
||||
);
|
||||
|
||||
describe("authority rotation after the prior controller state is restored", () => {
|
||||
const priorIdentity = {
|
||||
runnerInstanceId: "runner-restored-rotation",
|
||||
environmentLeaseId: "lease-restored-rotation",
|
||||
runId: "run-restored-rotation-prior",
|
||||
normalizedSessionId: "session-restored-rotation",
|
||||
turnId: "turn-restored-rotation-prior",
|
||||
itemId: "item-restored-rotation-prior",
|
||||
};
|
||||
const desiredIdentity = {
|
||||
...priorIdentity,
|
||||
runId: "run-restored-rotation-next",
|
||||
turnId: "turn-restored-rotation-next",
|
||||
itemId: "item-restored-rotation-next",
|
||||
};
|
||||
// The live controller state carries fields that a server checkpoint
|
||||
// restore does not reproduce, so a restored copy can differ in bytes.
|
||||
const controlPlaneState = {
|
||||
schema: "paperclip.runner.durable.control-plane-state.v1",
|
||||
identity: priorIdentity,
|
||||
ackedSourceSeq: 7,
|
||||
};
|
||||
const suspendedRunnerState = {
|
||||
schema: "paperclip.runner.durable.state.v1",
|
||||
...priorIdentity,
|
||||
lifecycle: "suspended",
|
||||
};
|
||||
const restoredCopies = {
|
||||
"byte-identical": JSON.stringify(controlPlaneState),
|
||||
"re-serialized": JSON.stringify(
|
||||
{ schema: controlPlaneState.schema, identity: priorIdentity },
|
||||
null,
|
||||
2,
|
||||
),
|
||||
"other-authority": JSON.stringify({
|
||||
...controlPlaneState,
|
||||
identity: { ...priorIdentity, turnId: "turn-restored-rotation-other" },
|
||||
}),
|
||||
} as const;
|
||||
type RestoredCopy = keyof typeof restoredCopies;
|
||||
const cases = Object.keys(restoredCopies) as RestoredCopy[];
|
||||
|
||||
async function restoreControlPlane(root: string, copy: RestoredCopy) {
|
||||
await mkdir(join(root, "control-plane"), { recursive: true });
|
||||
await writeFile(
|
||||
join(root, "control-plane", "control-plane-state.json"),
|
||||
restoredCopies[copy],
|
||||
);
|
||||
}
|
||||
|
||||
it.each(cases)("external rotation: %s restored copy", async (copy) => {
|
||||
const root = await mkdtemp(join(tmpdir(), "runner-external-rotation-restore-"));
|
||||
let activeRunnerState: Record<string, unknown> | null = suspendedRunnerState;
|
||||
let archivedRunnerState: Record<string, unknown> | null = null;
|
||||
let readCount = 0;
|
||||
const readRunnerState = async () => {
|
||||
readCount += 1;
|
||||
if (activeRunnerState === null) throw new Error("runner state moved");
|
||||
return activeRunnerState;
|
||||
};
|
||||
// The remote archive is idempotent: a retry returns the saved snapshot.
|
||||
const archiveRunnerState = async () => {
|
||||
if (activeRunnerState !== null) {
|
||||
archivedRunnerState = activeRunnerState;
|
||||
activeRunnerState = null;
|
||||
}
|
||||
if (archivedRunnerState === null) {
|
||||
throw new Error("archived runner state unavailable");
|
||||
}
|
||||
return archivedRunnerState;
|
||||
};
|
||||
const rotate = () =>
|
||||
runnerdRecoveryInternals.rotateAuthorityEpoch(
|
||||
root,
|
||||
controlPlaneState,
|
||||
desiredIdentity,
|
||||
{ read: readRunnerState, archive: archiveRunnerState },
|
||||
);
|
||||
try {
|
||||
await mkdir(join(root, "control-plane"), { recursive: true });
|
||||
await writeFile(
|
||||
join(root, "control-plane", "control-plane-state.json"),
|
||||
JSON.stringify(controlPlaneState),
|
||||
);
|
||||
// The first attempt completes both archives, then fails before the
|
||||
// new Runner starts.
|
||||
await expect(rotate()).resolves.toEqual(controlPlaneState);
|
||||
await expect(stat(join(root, "control-plane"))).rejects.toMatchObject({
|
||||
code: "ENOENT",
|
||||
});
|
||||
|
||||
// Retry preparation restores the prior run's controller checkpoint.
|
||||
await restoreControlPlane(root, copy);
|
||||
if (copy === "other-authority") {
|
||||
await expect(rotate()).rejects.toThrow(
|
||||
"native_runner_authority_archive_conflict",
|
||||
);
|
||||
// Contradictory state stays in place for operator recovery.
|
||||
await expect(
|
||||
stat(join(root, "control-plane", "control-plane-state.json")),
|
||||
).resolves.toBeDefined();
|
||||
return;
|
||||
}
|
||||
// The retry resumes from the archived snapshot, not the restored copy.
|
||||
await expect(rotate()).resolves.toEqual(controlPlaneState);
|
||||
await expect(stat(join(root, "control-plane"))).rejects.toMatchObject({
|
||||
code: "ENOENT",
|
||||
});
|
||||
expect(readCount).toBe(1);
|
||||
expect(archivedRunnerState).toEqual(suspendedRunnerState);
|
||||
} finally {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it.each(cases)("local rotation: %s restored copy", async (copy) => {
|
||||
const root = await mkdtemp(join(tmpdir(), "runner-local-rotation-restore-"));
|
||||
const rotate = () =>
|
||||
runnerdRecoveryInternals.rotateAuthorityEpoch(
|
||||
root,
|
||||
controlPlaneState,
|
||||
desiredIdentity,
|
||||
runnerdRecoveryInternals.localRunnerStateArchive(root),
|
||||
);
|
||||
try {
|
||||
await mkdir(join(root, "control-plane"), { recursive: true });
|
||||
await mkdir(join(root, "runner"), { recursive: true });
|
||||
await writeFile(
|
||||
join(root, "control-plane", "control-plane-state.json"),
|
||||
JSON.stringify(controlPlaneState),
|
||||
);
|
||||
await writeFile(
|
||||
join(root, "runner", "runner-state.json"),
|
||||
JSON.stringify(suspendedRunnerState),
|
||||
);
|
||||
await expect(rotate()).resolves.toEqual(controlPlaneState);
|
||||
await expect(stat(join(root, "control-plane"))).rejects.toMatchObject({
|
||||
code: "ENOENT",
|
||||
});
|
||||
await expect(
|
||||
stat(join(root, "runner", "runner-state.json")),
|
||||
).rejects.toMatchObject({ code: "ENOENT" });
|
||||
|
||||
await restoreControlPlane(root, copy);
|
||||
if (copy === "other-authority") {
|
||||
await expect(rotate()).rejects.toThrow("native_runner_authority_archive_conflict");
|
||||
await expect(
|
||||
stat(join(root, "control-plane", "control-plane-state.json")),
|
||||
).resolves.toBeDefined();
|
||||
return;
|
||||
}
|
||||
await expect(rotate()).resolves.toEqual(controlPlaneState);
|
||||
await expect(stat(join(root, "control-plane"))).rejects.toMatchObject({
|
||||
code: "ENOENT",
|
||||
});
|
||||
} finally {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it.each(["identical", "different"] as const)(
|
||||
"local rotation: %s duplicate runner state",
|
||||
async (duplicate) => {
|
||||
const root = await mkdtemp(join(tmpdir(), "runner-local-rotation-runner-state-"));
|
||||
const runnerStatePath = join(root, "runner", "runner-state.json");
|
||||
const rotate = () =>
|
||||
runnerdRecoveryInternals.rotateAuthorityEpoch(
|
||||
root,
|
||||
controlPlaneState,
|
||||
desiredIdentity,
|
||||
runnerdRecoveryInternals.localRunnerStateArchive(root),
|
||||
);
|
||||
try {
|
||||
await mkdir(join(root, "control-plane"), { recursive: true });
|
||||
await mkdir(join(root, "runner"), { recursive: true });
|
||||
await writeFile(
|
||||
join(root, "control-plane", "control-plane-state.json"),
|
||||
JSON.stringify(controlPlaneState),
|
||||
);
|
||||
await writeFile(runnerStatePath, JSON.stringify(suspendedRunnerState));
|
||||
await expect(rotate()).resolves.toEqual(controlPlaneState);
|
||||
|
||||
// A settled Runner state reappears beside its archived snapshot.
|
||||
await writeFile(
|
||||
runnerStatePath,
|
||||
duplicate === "identical"
|
||||
? JSON.stringify(suspendedRunnerState)
|
||||
: JSON.stringify({ ...suspendedRunnerState, outbox: [] }),
|
||||
);
|
||||
if (duplicate === "different") {
|
||||
await expect(rotate()).rejects.toThrow("native_runner_authority_archive_conflict");
|
||||
await expect(stat(runnerStatePath)).resolves.toBeDefined();
|
||||
return;
|
||||
}
|
||||
await expect(rotate()).resolves.toEqual(controlPlaneState);
|
||||
await expect(stat(runnerStatePath)).rejects.toMatchObject({
|
||||
code: "ENOENT",
|
||||
});
|
||||
} finally {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
// Drives the transport's real start path. The missing runnerd binary stops
|
||||
// it right after the rotation, so its ENOENT proves the retry passed every
|
||||
// earlier check without reading the retired live Runner state.
|
||||
it.each([
|
||||
["a completed warm handoff", { completedWarmTransition: { receipt: { commandId: "warm-command" } } }, "resumes"],
|
||||
["a source attachment hook", {}, "resumes"],
|
||||
["an unfinished warm transition", { warmTransition: { receipt: { commandId: "warm-command" } } }, "conflicts"],
|
||||
["a pending attachment", { commands: [{ commandId: "attach", type: "run.attach", status: "pending", payload: {} }] }, "conflicts"],
|
||||
] as const)("transport retry after a committed rotation with %s", async (_name, restoredFields, outcome) => {
|
||||
const root = await mkdtemp(join(tmpdir(), "runner-committed-rotation-transport-"));
|
||||
const archiveKey = createHash("sha256").update(JSON.stringify(priorIdentity)).digest("hex").slice(0, 24);
|
||||
const archivedControlPlane = join(root, "authority-epochs", `epoch-${archiveKey}`, "control-plane");
|
||||
const restoredState = { ...controlPlaneState, ...restoredFields };
|
||||
const steps: string[] = [];
|
||||
const { transport } = createCapabilityRunnerdCodexTransport({
|
||||
provider: "codex",
|
||||
stateDirectory: root,
|
||||
runnerBinary: join(root, "missing-paperclip-runnerd"),
|
||||
prpIdentity: desiredIdentity,
|
||||
...(_name === "a source attachment hook" ? { onRecoveryAccountingEvent: () => undefined } : {}),
|
||||
// The previous attempt already moved the live Runner state.
|
||||
prepareExternalRunnerState: async () => { steps.push("prepared"); },
|
||||
readRunnerState: async () => { steps.push("read"); throw new Error("live runner state was archived"); },
|
||||
archiveExternalRunnerState: async (input) => {
|
||||
steps.push(`archived:${input.archiveKey}`);
|
||||
return suspendedRunnerState;
|
||||
},
|
||||
} as Parameters<typeof createCapabilityRunnerdCodexTransport>[0]);
|
||||
try {
|
||||
await mkdir(archivedControlPlane, { recursive: true });
|
||||
await writeFile(join(archivedControlPlane, "control-plane-state.json"), JSON.stringify(controlPlaneState));
|
||||
// Retry preparation restored the prior run's controller checkpoint.
|
||||
await mkdir(join(root, "control-plane"), { recursive: true });
|
||||
await writeFile(join(root, "control-plane", "control-plane-state.json"), JSON.stringify(restoredState));
|
||||
const started = transport.request("thread/read", {});
|
||||
if (outcome === "conflicts") {
|
||||
await expect(started).rejects.toThrow("native_runner_authority_archive_conflict");
|
||||
expect(steps).toEqual([]);
|
||||
await expect(stat(join(root, "control-plane", "control-plane-state.json"))).resolves.toBeDefined();
|
||||
return;
|
||||
}
|
||||
await expect(started).rejects.toThrow(/ENOENT|missing-paperclip-runnerd/);
|
||||
expect(steps).toEqual([`archived:${archiveKey}`]);
|
||||
await expect(stat(join(root, "control-plane"))).rejects.toMatchObject({ code: "ENOENT" });
|
||||
} finally {
|
||||
await transport.close();
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
it("quiesces the control route before checkpoint and containment regardless of process completion", async () => {
|
||||
const settledSteps: string[] = [];
|
||||
await runnerdRecoveryInternals.releaseRunnerProcessOwnership({
|
||||
|
||||
@@ -229,6 +229,100 @@ function assertRealDirectory(path: string): void {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the archived controller state of a rotation that already started.
|
||||
*
|
||||
* The archive directory is the durable rotation marker for exactly
|
||||
* `priorIdentity`. Retry preparation can restore that run's controller
|
||||
* checkpoint into the root after the archive was written. A restored copy
|
||||
* with the same schema and the exact archived identity belongs to authority
|
||||
* that this rotation already retired, so it is removed and the rotation
|
||||
* resumes from the archived snapshot. The restore can serialize the state
|
||||
* differently, so this compares identity, not bytes. Any other active state
|
||||
* fails closed.
|
||||
*/
|
||||
function resumeArchivedControlPlane(
|
||||
activeControlPlane: string,
|
||||
archivedControlPlane: string,
|
||||
priorIdentity: DurableRecoveryIdentity,
|
||||
): Record<string, unknown> {
|
||||
assertRealDirectory(archivedControlPlane);
|
||||
const archivedState = readControlPlaneState(archivedControlPlane);
|
||||
if (!recoveryIdentityMatches(controlPlaneIdentity(archivedState), priorIdentity)) {
|
||||
throw new Error("native_runner_authority_archive_conflict");
|
||||
}
|
||||
if (existsSync(activeControlPlane)) {
|
||||
assertRealDirectory(activeControlPlane);
|
||||
if (!existsSync(resolve(activeControlPlane, "control-plane-state.json"))) {
|
||||
throw new Error("native_runner_authority_archive_conflict");
|
||||
}
|
||||
const activeState = readControlPlaneState(activeControlPlane);
|
||||
if (
|
||||
activeState.schema !== archivedState.schema ||
|
||||
!recoveryIdentityMatches(controlPlaneIdentity(activeState), priorIdentity)
|
||||
) {
|
||||
throw new Error("native_runner_authority_archive_conflict");
|
||||
}
|
||||
rmSync(activeControlPlane, { recursive: true });
|
||||
}
|
||||
return archivedState;
|
||||
}
|
||||
|
||||
/**
|
||||
* Recognize a rotation that an earlier attempt already committed.
|
||||
*
|
||||
* The epoch archive for the active control plane's exact identity means a
|
||||
* rotation away from that authority already started. A retry can restore the
|
||||
* prior run's controller checkpoint into the root after that, which makes the
|
||||
* root look as if the rotation never happened while its Runner state is
|
||||
* already archived. Returns the archived snapshot (and removes the restored
|
||||
* copy) so every caller continues from the same archive-only state, or null
|
||||
* when no rotation was committed. A copy that still holds an unfinished warm
|
||||
* transition, staged attachment, or pending attachment is not a plain restore
|
||||
* and fails closed.
|
||||
*/
|
||||
function takeCommittedRotation(
|
||||
root: string,
|
||||
activeState: Record<string, unknown>,
|
||||
): Record<string, unknown> | null {
|
||||
const identity = controlPlaneIdentity(activeState);
|
||||
const archivedControlPlane = resolve(
|
||||
authorityArchiveDirectory(root, identity),
|
||||
"control-plane",
|
||||
);
|
||||
if (!existsSync(archivedControlPlane)) return null;
|
||||
const pendingAttachment =
|
||||
Array.isArray(activeState.commands) &&
|
||||
activeState.commands.some((entry) => {
|
||||
const command = record(entry);
|
||||
return command.type === "run.attach" && command.status === "pending";
|
||||
});
|
||||
if (
|
||||
activeState.warmTransition !== undefined ||
|
||||
activeState.stagedRunAttachment !== undefined ||
|
||||
pendingAttachment
|
||||
) {
|
||||
throw new Error("native_runner_authority_archive_conflict");
|
||||
}
|
||||
return resumeArchivedControlPlane(
|
||||
resolve(root, "control-plane"),
|
||||
archivedControlPlane,
|
||||
identity,
|
||||
);
|
||||
}
|
||||
|
||||
/** True only for two regular files with the same bytes. */
|
||||
function sameRegularFile(left: string, right: string): boolean {
|
||||
const leftMetadata = lstatSync(left);
|
||||
const rightMetadata = lstatSync(right);
|
||||
return (
|
||||
leftMetadata.isFile() &&
|
||||
rightMetadata.isFile() &&
|
||||
leftMetadata.size === rightMetadata.size &&
|
||||
readFileSync(left).equals(readFileSync(right))
|
||||
);
|
||||
}
|
||||
|
||||
function quarantineLocalRuntimeState(root: string, reason: unknown): never {
|
||||
assertRealDirectory(root);
|
||||
const quarantine = resolve(
|
||||
@@ -288,87 +382,74 @@ function latestArchivedControlPlaneState(
|
||||
return null;
|
||||
}
|
||||
|
||||
function rotateLocalAuthorityEpoch(
|
||||
root: string,
|
||||
controlPlaneState: Record<string, unknown>,
|
||||
desired: DurableRecoveryIdentity,
|
||||
): Record<string, unknown> {
|
||||
const priorIdentity = controlPlaneIdentity(controlPlaneState);
|
||||
if (
|
||||
priorIdentity.runnerInstanceId !== desired.runnerInstanceId ||
|
||||
priorIdentity.environmentLeaseId !== desired.environmentLeaseId ||
|
||||
priorIdentity.normalizedSessionId !== desired.normalizedSessionId ||
|
||||
priorIdentity.runId === desired.runId
|
||||
) {
|
||||
throw new Error(
|
||||
"PRP recovery identity does not match the durable session binding",
|
||||
);
|
||||
}
|
||||
const runnerDirectory = resolve(root, "runner");
|
||||
const runnerStatePath = resolve(runnerDirectory, "runner-state.json");
|
||||
const archive = authorityArchiveDirectory(root, priorIdentity);
|
||||
const archivedControlPlane = resolve(archive, "control-plane");
|
||||
const archivedRunnerState = resolve(archive, "runner-state.json");
|
||||
const runnerStateSource = existsSync(runnerStatePath)
|
||||
? runnerStatePath
|
||||
: archivedRunnerState;
|
||||
if (!existsSync(runnerStateSource)) {
|
||||
throw new Error("native_runner_authority_rotation_state_unavailable");
|
||||
}
|
||||
assertRealDirectory(runnerDirectory);
|
||||
const runnerState = readRunnerState(runnerStateSource);
|
||||
if (
|
||||
runnerState.runnerInstanceId !== priorIdentity.runnerInstanceId ||
|
||||
runnerState.environmentLeaseId !== priorIdentity.environmentLeaseId ||
|
||||
runnerState.runId !== priorIdentity.runId ||
|
||||
runnerState.normalizedSessionId !== priorIdentity.normalizedSessionId ||
|
||||
runnerState.turnId !== priorIdentity.turnId ||
|
||||
runnerState.itemId !== priorIdentity.itemId ||
|
||||
runnerState.lifecycle !== "suspended"
|
||||
) {
|
||||
throw new Error("native_runner_authority_rotation_requires_settled_state");
|
||||
}
|
||||
const archivesRoot = resolve(root, "authority-epochs");
|
||||
if (existsSync(archivesRoot)) {
|
||||
assertRealDirectory(archivesRoot);
|
||||
} else {
|
||||
mkdirSync(archivesRoot, { mode: 0o700 });
|
||||
}
|
||||
if (existsSync(archive)) {
|
||||
assertRealDirectory(archive);
|
||||
} else {
|
||||
mkdirSync(archive, { mode: 0o700 });
|
||||
}
|
||||
assertRealDirectory(archive);
|
||||
const activeControlPlane = resolve(root, "control-plane");
|
||||
if (existsSync(activeControlPlane)) {
|
||||
assertRealDirectory(activeControlPlane);
|
||||
if (existsSync(archivedControlPlane)) {
|
||||
throw new Error("native_runner_authority_archive_conflict");
|
||||
}
|
||||
renameSync(activeControlPlane, archivedControlPlane);
|
||||
}
|
||||
if (existsSync(runnerStatePath)) {
|
||||
if (existsSync(archivedRunnerState)) {
|
||||
throw new Error("native_runner_authority_archive_conflict");
|
||||
}
|
||||
renameSync(runnerStatePath, archivedRunnerState);
|
||||
}
|
||||
if (!existsSync(archivedControlPlane) || !existsSync(archivedRunnerState)) {
|
||||
throw new Error("native_runner_authority_archive_incomplete");
|
||||
}
|
||||
return controlPlaneState;
|
||||
}
|
||||
|
||||
async function rotateExternalAuthorityEpoch(
|
||||
root: string,
|
||||
controlPlaneState: Record<string, unknown>,
|
||||
desired: DurableRecoveryIdentity,
|
||||
readRunnerState: () => Promise<Record<string, unknown>>,
|
||||
archiveRunnerState: (input: {
|
||||
/**
|
||||
* Where an authority rotation reads and archives the prior run's Runner
|
||||
* state. Local and external Runners differ only here, so every provider and
|
||||
* placement shares one rotation path.
|
||||
*/
|
||||
interface RunnerStateArchive {
|
||||
/** Read the live Runner state before its control plane is archived. */
|
||||
read: () => Promise<Record<string, unknown>>;
|
||||
/**
|
||||
* Move the live Runner state into the epoch archive and return the archived
|
||||
* snapshot. A retry must succeed when the state is already archived, and
|
||||
* when a byte-identical live copy sits beside the archive.
|
||||
*/
|
||||
archive: (input: {
|
||||
archiveKey: string;
|
||||
priorIdentity: DurableRecoveryIdentity;
|
||||
}) => Promise<Record<string, unknown>>,
|
||||
}) => Promise<Record<string, unknown>>;
|
||||
}
|
||||
|
||||
/** Runner state that this controller owns under `<root>/runner`. */
|
||||
function localRunnerStateArchive(root: string): RunnerStateArchive {
|
||||
const runnerDirectory = resolve(root, "runner");
|
||||
const live = resolve(runnerDirectory, "runner-state.json");
|
||||
const readLive = () => {
|
||||
if (!existsSync(live)) {
|
||||
throw new Error("native_runner_authority_rotation_state_unavailable");
|
||||
}
|
||||
assertRealDirectory(runnerDirectory);
|
||||
return readRunnerState(live);
|
||||
};
|
||||
return {
|
||||
read: async () => readLive(),
|
||||
archive: async ({ archiveKey, priorIdentity }) => {
|
||||
const archived = resolve(
|
||||
root,
|
||||
"authority-epochs",
|
||||
`epoch-${archiveKey}`,
|
||||
"runner-state.json",
|
||||
);
|
||||
if (existsSync(live)) {
|
||||
assertSuspendedRunnerState(readLive(), priorIdentity);
|
||||
if (!existsSync(archived)) {
|
||||
renameSync(live, archived);
|
||||
} else if (sameRegularFile(live, archived)) {
|
||||
rmSync(live);
|
||||
} else {
|
||||
throw new Error("native_runner_authority_archive_conflict");
|
||||
}
|
||||
}
|
||||
if (!existsSync(archived)) {
|
||||
throw new Error("native_runner_authority_archive_incomplete");
|
||||
}
|
||||
return readRunnerState(archived);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Retire the prior run's PRP authority so a new run can attach to the same
|
||||
* durable session. The epoch archive directory is the durable transaction
|
||||
* marker: once it holds the control plane, every retry resumes the same
|
||||
* idempotent archive instead of reading live state again.
|
||||
*/
|
||||
async function rotateAuthorityEpoch(
|
||||
root: string,
|
||||
controlPlaneState: Record<string, unknown>,
|
||||
desired: DurableRecoveryIdentity,
|
||||
runnerState: RunnerStateArchive,
|
||||
): Promise<Record<string, unknown>> {
|
||||
const priorIdentity = controlPlaneIdentity(controlPlaneState);
|
||||
if (
|
||||
@@ -396,32 +477,30 @@ async function rotateExternalAuthorityEpoch(
|
||||
mkdirSync(archive, { mode: 0o700 });
|
||||
}
|
||||
assertRealDirectory(archive);
|
||||
let resumedState: Record<string, unknown> | null = null;
|
||||
if (existsSync(archivedControlPlane)) {
|
||||
// This directory is the durable transaction marker. A prior controller
|
||||
// may have stopped before or after the remote move, so resume the same
|
||||
// idempotent archive instead of starting a new external runner.
|
||||
assertRealDirectory(archivedControlPlane);
|
||||
if (existsSync(activeControlPlane)) {
|
||||
throw new Error("native_runner_authority_archive_conflict");
|
||||
}
|
||||
const archivedIdentity = controlPlaneIdentity(
|
||||
readControlPlaneState(archivedControlPlane),
|
||||
// A prior attempt may have stopped before or after the Runner state
|
||||
// moved. Resume from the archived snapshot, and drop a restored copy of
|
||||
// the same retired authority (see resumeArchivedControlPlane).
|
||||
resumedState = resumeArchivedControlPlane(
|
||||
activeControlPlane,
|
||||
archivedControlPlane,
|
||||
priorIdentity,
|
||||
);
|
||||
if (!recoveryIdentityMatches(archivedIdentity, priorIdentity)) {
|
||||
throw new Error("native_runner_authority_archive_conflict");
|
||||
}
|
||||
} else {
|
||||
const runnerState = await readRunnerState();
|
||||
assertSuspendedRunnerState(runnerState, priorIdentity);
|
||||
assertSuspendedRunnerState(await runnerState.read(), priorIdentity);
|
||||
if (!existsSync(activeControlPlane)) {
|
||||
throw new Error("native_runner_authority_archive_incomplete");
|
||||
}
|
||||
assertRealDirectory(activeControlPlane);
|
||||
renameSync(activeControlPlane, archivedControlPlane);
|
||||
}
|
||||
const archivedRunnerState = await archiveRunnerState({
|
||||
const archivedRunnerState = await runnerState.archive({
|
||||
archiveKey: basename(archive).replace(/^epoch-/, ""),
|
||||
priorIdentity,
|
||||
});
|
||||
assertSuspendedRunnerState(archivedRunnerState, priorIdentity);
|
||||
return controlPlaneState;
|
||||
return resumedState ?? controlPlaneState;
|
||||
}
|
||||
|
||||
function projectCopilotRunGrant(
|
||||
@@ -5421,6 +5500,23 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
|
||||
return this.options.measureOperation ? this.options.measureOperation("runner.transport.cold_start", () => this.#startWork(params)) : this.#startWork(params);
|
||||
}
|
||||
|
||||
/** Runner state held by an external Runner host (a plugin task or a sandbox). */
|
||||
#externalRunnerStateArchive(): RunnerStateArchive {
|
||||
const { readRunnerState, prepareExternalRunnerState, archiveExternalRunnerState } = this.options;
|
||||
if (readRunnerState === undefined || archiveExternalRunnerState === undefined) {
|
||||
throw new Error("native_runner_prp_run_rotation_unavailable");
|
||||
}
|
||||
return {
|
||||
// Prepare the host only to read live state. A resumed rotation finds
|
||||
// that state already archived, and preparing again could re-stage it.
|
||||
read: async () => {
|
||||
await prepareExternalRunnerState?.();
|
||||
return readRunnerState();
|
||||
},
|
||||
archive: archiveExternalRunnerState,
|
||||
};
|
||||
}
|
||||
|
||||
async #startWork(
|
||||
params: Record<string, unknown>,
|
||||
): Promise<Record<string, unknown>> {
|
||||
@@ -5646,10 +5742,19 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
|
||||
candidateRunnerState?.schema ===
|
||||
"paperclip.runner.durable.state.warm-transition.v1";
|
||||
let controlPlaneState: Record<string, unknown> | null;
|
||||
// Set when the active control plane was a restored copy of authority that
|
||||
// an earlier attempt already rotated away from (see takeCommittedRotation).
|
||||
let committedRotationSource: Record<string, unknown> | null = null;
|
||||
try {
|
||||
controlPlaneState = existsSync(controlPlaneStatePath)
|
||||
? readControlPlaneState(controlPlaneDirectory)
|
||||
: null;
|
||||
// Normalize before any check trusts the active copy. Every later lane
|
||||
// then sees the same archive-only state that the rotation committed.
|
||||
if (controlPlaneState && supportsColdRotation) {
|
||||
committedRotationSource = takeCommittedRotation(this.#root, controlPlaneState);
|
||||
if (committedRotationSource) controlPlaneState = null;
|
||||
}
|
||||
} catch (error) {
|
||||
if (supportsColdRotation && localStateOwner && !hasRunnerWarmBoundary) {
|
||||
quarantineLocalRuntimeState(this.#root, error);
|
||||
@@ -5827,77 +5932,26 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
|
||||
identity.turnId === desiredIdentity.turnId &&
|
||||
identity.itemId === desiredIdentity.itemId;
|
||||
let rotatedAuthority = false;
|
||||
// One rotation path serves every provider and Runner placement. Only the
|
||||
// Runner state store differs between a local and an external Runner.
|
||||
let rotationSource: Record<string, unknown> | null = null;
|
||||
if (controlPlaneState === null) {
|
||||
if (!supportsColdRotation) {
|
||||
throw new Error("PRP provider resume state is unavailable");
|
||||
}
|
||||
const archivedState = latestArchivedControlPlaneState(
|
||||
// An interrupted rotation leaves only its archive marker.
|
||||
rotationSource = committedRotationSource ?? latestArchivedControlPlaneState(
|
||||
this.#root,
|
||||
desiredIdentity,
|
||||
);
|
||||
if (!archivedState) {
|
||||
if (!rotationSource) {
|
||||
throw new Error("PRP provider resume state is unavailable");
|
||||
}
|
||||
if (localStateOwner) {
|
||||
try {
|
||||
controlPlaneState = rotateLocalAuthorityEpoch(
|
||||
this.#root,
|
||||
archivedState,
|
||||
desiredIdentity,
|
||||
);
|
||||
} catch (error) {
|
||||
quarantineLocalRuntimeState(this.#root, error);
|
||||
}
|
||||
} else {
|
||||
if (
|
||||
this.options.readRunnerState === undefined ||
|
||||
this.options.archiveExternalRunnerState === undefined
|
||||
) {
|
||||
throw new Error("native_runner_prp_run_rotation_unavailable");
|
||||
}
|
||||
controlPlaneState = await rotateExternalAuthorityEpoch(
|
||||
this.#root,
|
||||
archivedState,
|
||||
desiredIdentity,
|
||||
this.options.readRunnerState,
|
||||
this.options.archiveExternalRunnerState,
|
||||
);
|
||||
}
|
||||
identity = desiredIdentity;
|
||||
rotatedAuthority = true;
|
||||
} else if (!exactAuthority && warmRecovery === null) {
|
||||
if (!supportsColdRotation || controlPlaneState === null) {
|
||||
if (!supportsColdRotation) {
|
||||
throw new Error("native_runner_prp_run_rotation_unavailable");
|
||||
}
|
||||
if (localStateOwner) {
|
||||
try {
|
||||
controlPlaneState = rotateLocalAuthorityEpoch(
|
||||
this.#root,
|
||||
controlPlaneState,
|
||||
desiredIdentity,
|
||||
);
|
||||
} catch (error) {
|
||||
quarantineLocalRuntimeState(this.#root, error);
|
||||
}
|
||||
} else {
|
||||
if (
|
||||
this.options.readRunnerState === undefined ||
|
||||
this.options.prepareExternalRunnerState === undefined ||
|
||||
this.options.archiveExternalRunnerState === undefined
|
||||
) {
|
||||
throw new Error("native_runner_prp_run_rotation_unavailable");
|
||||
}
|
||||
await this.options.prepareExternalRunnerState();
|
||||
controlPlaneState = await rotateExternalAuthorityEpoch(
|
||||
this.#root,
|
||||
controlPlaneState,
|
||||
desiredIdentity,
|
||||
this.options.readRunnerState,
|
||||
this.options.archiveExternalRunnerState,
|
||||
);
|
||||
}
|
||||
identity = desiredIdentity;
|
||||
rotatedAuthority = true;
|
||||
rotationSource = controlPlaneState;
|
||||
} else if (
|
||||
localStateOwner &&
|
||||
!existsSync(resolve(this.#root, "runner", "runner-state.json"))
|
||||
@@ -5907,6 +5961,26 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
|
||||
new Error("PRP provider resume state is unavailable"),
|
||||
);
|
||||
}
|
||||
if (rotationSource) {
|
||||
try {
|
||||
controlPlaneState = await rotateAuthorityEpoch(
|
||||
this.#root,
|
||||
rotationSource,
|
||||
desiredIdentity,
|
||||
localStateOwner ? localRunnerStateArchive(this.#root) : this.#externalRunnerStateArchive(),
|
||||
);
|
||||
} catch (error) {
|
||||
// Locally owned state is preserved aside for operator recovery.
|
||||
if (localStateOwner) quarantineLocalRuntimeState(this.#root, error);
|
||||
throw error;
|
||||
}
|
||||
identity = desiredIdentity;
|
||||
rotatedAuthority = true;
|
||||
}
|
||||
// A missing control plane either rotated from its archive above or threw.
|
||||
if (controlPlaneState === null) {
|
||||
throw new Error("PRP provider resume state is unavailable");
|
||||
}
|
||||
const runnerBinaryPath =
|
||||
this.options.runnerBinary ?? (this.options.prepareLaunch ? "paperclip-runnerd" : defaultCapabilityRunnerdBinary());
|
||||
const runnerArtifact = this.options.runnerArtifact ?? approvedRunnerArtifact(runnerBinaryPath);
|
||||
@@ -8284,7 +8358,8 @@ export const runnerdRecoveryInternals = Object.freeze({
|
||||
runnerCloseDeadlines,
|
||||
runnerCloseGraceMs,
|
||||
rotatedRunAttachPayload,
|
||||
rotateExternalAuthorityEpoch,
|
||||
localRunnerStateArchive,
|
||||
rotateAuthorityEpoch,
|
||||
runnerdTerminalProviderTurnId,
|
||||
terminalMayBelongToPendingTurnStart,
|
||||
turnStartCommandResultValid,
|
||||
|
||||
+3
-1
@@ -483,7 +483,9 @@ packages:
|
||||
'@smithy/core@3.33.3':
|
||||
resolution: {integrity: sha512-CsOeKq/9kA3y6VJHt+/+VTCtBaxJ4OTFpgrjIUhPpDIKxBci1k2bJaQASF2h/ELWrulGp+t97DZ0mevfAD8idg==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
deprecated: Deprecated due to a bug in custom logger data redaction. See https://github.com/aws/aws-sdk-js-v3/issues/8393.
|
||||
deprecated: |-
|
||||
Deprecated due to a bug in custom logger data redaction. Upgrade to the latest version. If you used a custom logger with this
|
||||
version, review your logs and rotate any credentials or other secrets. See https://github.com/aws/aws-sdk-js-v3/issues/8393.
|
||||
|
||||
'@smithy/credential-provider-imds@4.5.2':
|
||||
resolution: {integrity: sha512-A9uSdn72ozbRUSit0eib0TW7nXuNPlaeM0zcGkJ+nE6tFcSDbnmtwoxbTCFBukVQcszDAyvsd7+rTduPTXpygg==}
|
||||
|
||||
Generated
+3
-1
@@ -5048,7 +5048,9 @@ packages:
|
||||
'@smithy/core@3.35.1':
|
||||
resolution: {integrity: sha512-i4YPS4B6ts7bjn7UwLnGjiZdprOvHvgGobFZsYK3GIY3E5hIqtj0rReU69BcTpGp+fvtraSNXeG1l+jtJvF55w==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
deprecated: Deprecated due to a bug in custom logger data redaction. See https://github.com/aws/aws-sdk-js-v3/issues/8393.
|
||||
deprecated: |-
|
||||
Deprecated due to a bug in custom logger data redaction. Upgrade to the latest version. If you used a custom logger with this
|
||||
version, review your logs and rotate any credentials or other secrets. See https://github.com/aws/aws-sdk-js-v3/issues/8393.
|
||||
|
||||
'@smithy/credential-provider-imds@4.5.2':
|
||||
resolution: {integrity: sha512-A9uSdn72ozbRUSit0eib0TW7nXuNPlaeM0zcGkJ+nE6tFcSDbnmtwoxbTCFBukVQcszDAyvsd7+rTduPTXpygg==}
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
# Paperclip v2026.1103.0
|
||||
# Paperclip v2026.1106.0
|
||||
|
||||
> Released: 2026-11-03
|
||||
> Released: 2026-11-06
|
||||
|
||||
Paperclip v2026.1103.0 carries 515 commits, promoted from `2026.1023.0-beta.0`. The headline is that the Paperclip Runner is now the default for new agents: new Claude, Codex, OpenCode, and Pi agents start on the native Runner on qualified macOS and Linux hosts, existing agents get a one-click "Upgrade runner" action, and GitHub Copilot joins as a native harness. Around that: project repositories that automatically become skill sources, model-backed prompt injection prevention, agent observability to Braintrust or any OTLP endpoint, a unified Google Workspace connection, bulk edits on the Agents roster, workspaces that compact and expire themselves with encrypted restore points, and Grok Bot and Muse as experimental Runner providers. Underneath, server restarts report ready sooner, the Docker image boots from a precompiled bundle, task pages load faster, and more background services stop polling the database.
|
||||
Paperclip v2026.1106.0 carries 515 commits, promoted from `2026.1023.0-beta.0`. The headline is that the Paperclip Runner is now the default for new agents: new Claude, Codex, OpenCode, and Pi agents start on the native Runner on qualified macOS and Linux hosts, existing agents get a one-click "Upgrade runner" action, and GitHub Copilot joins as a native harness. Around that: project repositories that automatically become skill sources, model-backed prompt injection prevention, agent observability to Braintrust or any OTLP endpoint, a unified Google Workspace connection, bulk edits on the Agents roster, workspaces that compact and expire themselves with encrypted restore points, and Grok Bot and Muse as experimental Runner providers. Underneath, server restarts report ready sooner, the Docker image boots from a precompiled bundle, task pages load faster, and more background services stop polling the database.
|
||||
|
||||
## Breaking Changes
|
||||
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { createRequire } from "node:module";
|
||||
import { existsSync, mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const patterns = { "--tool-removal-only": "public tool removal retains Slack retirement", "--fairness-only": "two due registrations", "--malformed-only": "malformed sealed Slack receipt",
|
||||
"--wiring-only": "normal configure stages SQL intent", "--cas-only": "real SQL refuses an enrollment response" };
|
||||
const args = process.argv.slice(2);
|
||||
if (!process.version.startsWith("v24.")) throw new Error("Slack owner fixture requires the current Node24 executable");
|
||||
if (args.length > 1 || args.length && !Object.hasOwn(patterns, args[0])) throw new Error("Unknown Slack owner fixture selector");
|
||||
const value = process.env.PAPERCLIP_SLACK_OWNER_TEST_DATABASE_URL;
|
||||
if (!value) throw new Error("Set PAPERCLIP_SLACK_OWNER_TEST_DATABASE_URL to the exact owned external test database");
|
||||
const fixture = new URL(value), admin = new URL(process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL ?? "invalid:");
|
||||
if (!["postgres:", "postgresql:"].includes(fixture.protocol) || fixture.hostname !== "127.0.0.1" || fixture.port !== "55448"
|
||||
|| !/^[a-zA-Z0-9_]+_test$/.test(fixture.pathname.slice(1)) || fixture.search || fixture.hash
|
||||
|| admin.protocol !== fixture.protocol || admin.hostname !== fixture.hostname || admin.port !== fixture.port
|
||||
|| admin.pathname !== "/postgres" || admin.search || admin.hash) throw new Error("Slack owner fixture requires its explicit owned PG17 URL/admin grant");
|
||||
const log = process.env.SLACK_GATE_PROCESS_LOG;
|
||||
if (!log || !path.isAbsolute(log) || existsSync(log)) throw new Error("Slack owner fixture requires a fresh absolute guard log");
|
||||
const server = fileURLToPath(new URL("../", import.meta.url));
|
||||
const require = createRequire(new URL("../package.json", import.meta.url));
|
||||
const vitest = path.join(path.dirname(require.resolve("vitest/package.json")), "vitest.mjs");
|
||||
const guard = path.join(server, "src/__tests__/helpers/slack-cloud/prewire.cjs");
|
||||
const home = mkdtempSync(path.join(tmpdir(), "slack-owner-fixture-"));
|
||||
const env = { PATH: process.env.PATH, HOME: process.env.HOME, CODEX_HOME: process.env.CODEX_HOME, LANG: process.env.LANG,
|
||||
TMPDIR: home, PAPERCLIP_HOME: home, NODE_ENV: "test", NODE_OPTIONS: "", SLACK_GATE_EXPECT_PRELOAD: "1",
|
||||
PAPERCLIP_SLACK_OWNER_TEST_DATABASE_URL: value, PAPERCLIP_SLACK_TEST_DATABASE_URL: value,
|
||||
PAPERCLIP_TEST_POSTGRES_ADMIN_URL: process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL, SLACK_GATE_PROCESS_LOG: log };
|
||||
const child = spawn(process.execPath, ["--require", guard, vitest, "run", "src/__tests__/slack-cloud-owner-work.test.ts",
|
||||
"--maxWorkers=1", "--no-file-parallelism", `--execArgv=--require=${guard}`,
|
||||
...(args.length ? [`--testNamePattern=${patterns[args[0]]}`] : [])], { cwd: server, env, stdio: "inherit" });
|
||||
for (const signal of ["SIGINT", "SIGTERM"]) process.on(signal, () => child.kill(signal));
|
||||
child.on("error", error => { console.error(error); process.exitCode = 1; });
|
||||
child.on("close", (code, signal) => { rmSync(home, { recursive: true, force: true }); process.exitCode = signal ? 1 : code ?? 1; });
|
||||
@@ -0,0 +1,47 @@
|
||||
import { spawn } from 'node:child_process';
|
||||
import { createRequire } from 'node:module';
|
||||
import { existsSync, mkdtempSync, rmSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
if (process.argv.slice(2).length > 1 || process.argv.slice(2).some(arg => !['--instance-transition-only', '--normal-owner-only'].includes(arg))) throw new Error('Unknown Slack fixture selector');
|
||||
const transitionOnly = process.argv.includes('--instance-transition-only');
|
||||
const normalOwnerOnly = process.argv.includes('--normal-owner-only');
|
||||
const server = fileURLToPath(new URL('../', import.meta.url));
|
||||
const require = createRequire(new URL('../package.json', import.meta.url));
|
||||
const guard = path.join(server, 'src/__tests__/helpers/slack-cloud/prewire.cjs');
|
||||
const names = ['PAPERCLIP_SLACK_TEST_DATABASE_URL', 'SLACK_EVENT_TEST_DATABASE_URL'];
|
||||
const urls = names.map(name => {
|
||||
if (!process.env[name]) throw new Error(`Set ${name} to an owned dedicated PostgreSQL test database`);
|
||||
let url;
|
||||
try { url = new URL(process.env[name]); } catch { throw new Error(`${name} must name an explicit loopback *_test database`); }
|
||||
if (!['postgres:', 'postgresql:'].includes(url.protocol) || !['127.0.0.1', 'localhost', '::1', '[::1]'].includes(url.hostname)
|
||||
|| !/^[a-zA-Z0-9_]+_test$/.test(decodeURIComponent(url.pathname.slice(1)))
|
||||
|| ['options', 'database', 'dbname'].some(k => url.searchParams.has(k))) throw new Error(`${name} must name an explicit loopback *_test database`);
|
||||
return url;
|
||||
});
|
||||
if (urls[0].pathname === urls[1].pathname) throw new Error('Slack app and Cloud fixtures require distinct databases');
|
||||
if (!process.env.SLACK_TEST_CLOUD_DIST || !path.isAbsolute(process.env.SLACK_TEST_CLOUD_DIST)
|
||||
|| !existsSync(path.join(process.env.SLACK_TEST_CLOUD_DIST, 'connections/broker.js'))) throw new Error('SLACK_TEST_CLOUD_DIST must name the compiled companion Cloud dist directory');
|
||||
const vitest = path.join(path.dirname(require.resolve('vitest/package.json')), 'vitest.mjs');
|
||||
const ownedHome = mkdtempSync(path.join(tmpdir(), 'paperclip-slack-paired-'));
|
||||
const env = {
|
||||
PATH: process.env.PATH, HOME: process.env.HOME, CODEX_HOME: process.env.CODEX_HOME,
|
||||
LANG: process.env.LANG, TMPDIR: ownedHome, PAPERCLIP_HOME: ownedHome,
|
||||
NODE_ENV: 'test', NODE_OPTIONS: '', SLACK_GATE_EXPECT_PRELOAD: '1',
|
||||
PAPERCLIP_SLACK_TEST_DATABASE_URL: process.env.PAPERCLIP_SLACK_TEST_DATABASE_URL,
|
||||
SLACK_EVENT_TEST_DATABASE_URL: process.env.SLACK_EVENT_TEST_DATABASE_URL,
|
||||
SLACK_TEST_CLOUD_DIST: process.env.SLACK_TEST_CLOUD_DIST,
|
||||
SLACK_GATE_PROCESS_LOG: process.env.SLACK_GATE_PROCESS_LOG,
|
||||
};
|
||||
const child = spawn(process.execPath, ['--require', guard, vitest, 'run',
|
||||
...(transitionOnly ? ['src/services/chat-channels/slack/cloud-instance-transition.postgres.test.ts'] : normalOwnerOnly
|
||||
? ['src/services/chat-channels/slack/cloud-enrollment.postgres.test.ts'] : ['src/services/chat-channels/slack/cloud-enrollment.postgres.test.ts', 'src/services/chat-channels/slack/cloud-instance-transition.postgres.test.ts']), '--maxWorkers=1', '--no-file-parallelism', `--execArgv=--require=${guard}`],
|
||||
{ cwd: server, env, stdio: 'inherit' });
|
||||
for (const signal of ['SIGINT', 'SIGTERM']) process.on(signal, () => child.kill(signal));
|
||||
child.on('error', error => { console.error(error); process.exitCode = 1; });
|
||||
child.on('close', (code, signal) => {
|
||||
rmSync(ownedHome, { recursive: true, force: true });
|
||||
process.exitCode = signal ? 1 : code ?? 1;
|
||||
});
|
||||
@@ -17,7 +17,7 @@ import { mkdtemp, realpath, rm, access, readFile, writeFile, stat } from "node:f
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { createDb, companies, agents, agentTaskSessions, heartbeatRuns, companyMemberships, connectionGrants, toolApplications, connectionGrantDelegations, connectionGrantMembers, toolConnections, toolConnectionInstalls, aiConnectionDefaults, aiProviderDefaults, adapterAuthSessions, environments, plugins, issues, issueThreadInteractions, issueRecoveryActions, connectionIntentDeliveries, agentWakeupRequests, companySecrets, companySecretVersions, userSecretDefinitions, principalPermissionGrants } from "@paperclipai/db";
|
||||
import { createDb, type Db, companies, agents, agentTaskSessions, heartbeatRuns, companyMemberships, connectionGrants, toolApplications, connectionGrantDelegations, connectionGrantMembers, toolConnections, toolConnectionInstalls, aiConnectionDefaults, aiProviderDefaults, adapterAuthSessions, environments, plugins, issues, issueThreadInteractions, issueRecoveryActions, connectionIntentDeliveries, agentWakeupRequests, companySecrets, companySecretVersions, userSecretDefinitions, principalPermissionGrants } from "@paperclipai/db";
|
||||
import { startEmbeddedPostgresTestDatabase } from "@paperclipai/db/test-embedded-postgres";
|
||||
import { fetchCompanyQuotaWindows } from "../services/quota-windows.js";
|
||||
import { quotaCredentialRecoveryPath, quotaCredentialRecovery, quotaCredentialHash } from "../services/quota-credential-recovery.js";
|
||||
@@ -2213,6 +2213,33 @@ console.log(JSON.stringify({ type: "result", subtype: "success", result: "hello"
|
||||
const [agent] = await db.select().from(agents).where(eq(agents.id, agentId));
|
||||
expect(agent.runtimeConfig.aiConnection).toBeUndefined();
|
||||
});
|
||||
it("keeps account summaries ordered across sequential scans and repeat migration", async () => {
|
||||
const orderedCompanyId = randomUUID(), userId = "stable-list-owner";
|
||||
await db.insert(companies).values({ id: orderedCompanyId, name: "Stable account list", issuePrefix: `O${orderedCompanyId.slice(0, 5)}` });
|
||||
await db.insert(companyMemberships).values({ companyId: orderedCompanyId, principalId: userId, principalType: "user", status: "active", membershipRole: "member" });
|
||||
const accounts = [];
|
||||
for (const name of ["First account", "Second account", "Third account"]) {
|
||||
accounts.push(await service.save(orderedCompanyId, userId, { provider: "anthropic", method: "api_key", ownership: "personal", name, apiKey: "fixture-stable-list", agentIds: [], allAgents: false }, "fixture-stable-list"));
|
||||
}
|
||||
const expected = accounts.map(({ connectionId: id, grantId }) => ({ id, grantId }))
|
||||
.sort((a, b) => a.id.localeCompare(b.id) || a.grantId.localeCompare(b.grantId));
|
||||
await db.transaction(async tx => {
|
||||
await tx.execute(sql`SET LOCAL enable_indexscan = off`);
|
||||
await tx.execute(sql`SET LOCAL enable_bitmapscan = off`);
|
||||
await tx.execute(sql`SET LOCAL enable_hashjoin = off`);
|
||||
await tx.execute(sql`SET LOCAL enable_mergejoin = off`);
|
||||
for (const account of [...expected].reverse()) {
|
||||
await tx.update(toolConnections).set({ name: sql`${toolConnections.name}` }).where(eq(toolConnections.id, account.id));
|
||||
await tx.update(connectionGrants).set({ status: sql`${connectionGrants.status}` }).where(eq(connectionGrants.id, account.grantId));
|
||||
}
|
||||
const transactionalService = aiConnectionService(tx as unknown as Db);
|
||||
const before = await transactionalService.list(orderedCompanyId, userId);
|
||||
expect(before.map(({ id, grantId }) => ({ id, grantId }))).toEqual(expected);
|
||||
const migration = await readFile(new URL("../../../packages/db/src/migrations/0276_hard_mandroid.sql", import.meta.url), "utf8");
|
||||
for (const statement of migration.split("--> statement-breakpoint").filter(value => value.trim())) await tx.execute(sql.raw(statement));
|
||||
expect(await transactionalService.list(orderedCompanyId, userId)).toEqual(before);
|
||||
});
|
||||
});
|
||||
it("runs two Claude subscription executions for the same grant at the same time", async () => {
|
||||
// Claude writes no auth file back to the grant, so two runs share no
|
||||
// mutable state and must not wait for each other.
|
||||
|
||||
@@ -90,6 +90,13 @@ const support = await getEmbeddedPostgresTestSupport();
|
||||
grantId = grant!.id;
|
||||
const env = { PAPERCLIP_CLOUD_CONNECTOR_BASE_URL: origin, PAPERCLIP_CLOUD_CONNECTOR_ENVIRONMENT: "test" };
|
||||
const service = githubConnectionEventService(db, { env });
|
||||
const reserved: Awaited<ReturnType<typeof db.$client.reserve>>[] = [];
|
||||
try {
|
||||
for (let slot = 0; slot < 4; slot++) reserved.push(await db.$client.reserve());
|
||||
await Promise.all(reserved.map(connection => connection`SELECT 1`));
|
||||
} finally {
|
||||
for (const connection of reserved) connection.release();
|
||||
}
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: e => errors.push(e) });
|
||||
const selects = vi.spyOn(db, "select"), transactions = vi.spyOn(db, "transaction"), executes = vi.spyOn(db, "execute");
|
||||
|
||||
@@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { authUsers, budgetReservations, companyMemberships, completionContracts, heartbeatRunEvents, heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults, workspaceOperations } from "@paperclipai/db";
|
||||
import { expect, it, vi } from "vitest";
|
||||
import { resolveLocalRuntimeApiUrl } from "../runtime-api.js";
|
||||
import { heartbeatService } from "../services/heartbeat.js";
|
||||
import { instanceSettingsService } from "../services/instance-settings.js";
|
||||
import { currentNativeControllerIdentity } from "../services/native-runtime/native-restart-recovery.js";
|
||||
@@ -77,6 +78,56 @@ describeEmbeddedPostgres("heartbeat native recovery interrupted by graceful shut
|
||||
90_000,
|
||||
);
|
||||
|
||||
it("hands a recovery to the next controller when shutdown closed the listener before backend setup", async () => {
|
||||
const f = await seedRecoverableRun("LISTENER-CLOSED");
|
||||
const listener = fixture.current.server!;
|
||||
const address = listener.address();
|
||||
if (!address || typeof address === "string") throw new Error("Expected a TCP runtime API listener");
|
||||
let listenerClosed: Promise<void> | undefined;
|
||||
// Graceful shutdown closes HTTP admission before it drains heartbeat runs.
|
||||
// Backend setup then asks for the callback origin, as runnerd setup does.
|
||||
const dying = vi.fn(() => ({
|
||||
descriptor: async () => { resolveLocalRuntimeApiUrl(); throw new Error("unreachable"); },
|
||||
openSession: async () => { resolveLocalRuntimeApiUrl(); throw new Error("unreachable"); },
|
||||
}));
|
||||
const heartbeat = heartbeatService(f.db, {
|
||||
nativeSessionBackendFactory: dying as never,
|
||||
beforeNativeRuntimeSelection: async () => {
|
||||
listener.closeIdleConnections();
|
||||
listenerClosed = new Promise<void>(resolve => listener.close(() => resolve()));
|
||||
},
|
||||
});
|
||||
let drained: unknown;
|
||||
try {
|
||||
try {
|
||||
const { claims } = await heartbeat.recoverNativeRunsAfterRestart(new Date(), [f.runId]);
|
||||
expect(claims).toEqual([expect.objectContaining({ runId: f.runId })]);
|
||||
} finally { await heartbeat.drainActiveRunExecutions(); }
|
||||
expect(dying).toHaveBeenCalled();
|
||||
drained = await heartbeat.drainRunningRunsForShutdown("SIGTERM", new Date(), [f.runId]);
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
} finally {
|
||||
await listenerClosed;
|
||||
await new Promise<void>(resolve => listener.listen(address.port, "127.0.0.1", resolve));
|
||||
}
|
||||
expect(drained).toMatchObject({ restartSuspendedRunIds: [f.runId] });
|
||||
const [run] = await f.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId));
|
||||
const [owner] = await f.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId));
|
||||
expect({ status: run.status, errorCode: run.errorCode, finishedAt: run.finishedAt }).toEqual({
|
||||
status: "running", errorCode: null, finishedAt: null });
|
||||
expect(owner).toMatchObject({ phase: "observed", resultId: null, recoveryState: "awaiting_runner_reattach" });
|
||||
|
||||
await f.db.update(nativeRunFinalizations).set({ controllerPid: 2_000_000_001 })
|
||||
.where(eq(nativeRunFinalizations.runId, f.runId));
|
||||
const resumed = vi.fn(() => { throw new Error("next boot reached provider dispatch"); });
|
||||
const next = heartbeatService(f.db, { nativeSessionBackendFactory: resumed });
|
||||
try {
|
||||
const { claims } = await next.recoverNativeRunsAfterRestart(new Date(), [f.runId]);
|
||||
expect(claims).toEqual([expect.objectContaining({ runId: f.runId })]);
|
||||
} finally { await next.drainActiveRunExecutions(); }
|
||||
expect(resumed).toHaveBeenCalledOnce();
|
||||
}, 90_000);
|
||||
|
||||
it("still settles a recovery whose claim merely expired without a shutdown suspension", async () => {
|
||||
const f = await seedRecoverableRun("EXPIRED-NOT-SUSPENDED");
|
||||
const dying = vi.fn(() => { throw new Error("An expired claim must not open a provider session"); });
|
||||
|
||||
@@ -244,7 +244,7 @@ export function nativeRestartFixture() {
|
||||
};
|
||||
}
|
||||
|
||||
return { seedRun, transportOptions, get current() { return { temporary, runtimeRoot, paperclipHome, apiUrl, companyId, agentId, successor }; } };
|
||||
return { seedRun, transportOptions, get current() { return { temporary, runtimeRoot, paperclipHome, apiUrl, companyId, agentId, successor, server }; } };
|
||||
}
|
||||
|
||||
export function remotePreparedWorkspace() {
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
import { createCipheriv, createHash, createHmac, diffieHellman, generateKeyPairSync, hkdfSync, randomBytes, verify } from "node:crypto";
|
||||
import { expect, vi } from "vitest";
|
||||
import { createPaperclipCloudConnector, type PaperclipCloudConnectorConfig } from "../../services/paperclip-cloud-connector.js";
|
||||
|
||||
export function createSlackCloudConnectorFixture(instanceId: string, companyId: string, subject: string) {
|
||||
const signing = generateKeyPairSync("ed25519"), sealing = generateKeyPairSync("x25519");
|
||||
const config: PaperclipCloudConnectorConfig = { baseUrl: "http://127.0.0.1:1", instanceId, environment: "test",
|
||||
signPrivateKey: signing.privateKey.export({ format: "jwk" }).d!, sealPrivateKey: sealing.privateKey.export({ format: "jwk" }).d! };
|
||||
const calls: { operation: string; enrollment?: Record<string, unknown> }[] = [];
|
||||
const acknowledged: string[] = [];
|
||||
const events: Record<string, unknown>[] = [];
|
||||
let unavailable = false;
|
||||
let beforeEnrollmentReceipt: (() => Promise<void>) | undefined;
|
||||
function seal(payload: unknown, profile: "github.code" | "slack.events.v1") {
|
||||
const provider = profile === "github.code" ? "github" : "slack";
|
||||
const ephemeral = generateKeyPairSync("x25519");
|
||||
const epk = ephemeral.publicKey.export({ format: "jwk" }).x!;
|
||||
const recipient = sealing.publicKey.export({ format: "jwk" }).x!;
|
||||
const aad = Buffer.from([1, "X25519-HKDF-SHA256-A256GCM", "events", instanceId, "test", provider, profile, ""].join("\n"));
|
||||
const key = Buffer.from(hkdfSync("sha256", diffieHellman({ privateKey: ephemeral.privateKey, publicKey: sealing.publicKey }),
|
||||
Buffer.concat([Buffer.from(epk, "base64url"), Buffer.from(recipient, "base64url")]), aad, 32));
|
||||
const iv = randomBytes(12), cipher = createCipheriv("aes-256-gcm", key, iv); cipher.setAAD(aad);
|
||||
const ct = Buffer.concat([cipher.update(JSON.stringify(payload), "utf8"), cipher.final(), cipher.getAuthTag()]);
|
||||
return { v: 1, alg: "X25519-HKDF-SHA256-A256GCM", purpose: "events", provider, profile, epk,
|
||||
iv: iv.toString("base64url"), ct: ct.toString("base64url") };
|
||||
}
|
||||
const requestErrors: unknown[] = [];
|
||||
const request: typeof fetch = vi.fn(async (input: RequestInfo | URL, init?: RequestInit): Promise<Response> => {
|
||||
try {
|
||||
const url = new URL(input instanceof Request ? input.url : String(input));
|
||||
expect(url.origin).toBe(config.baseUrl);
|
||||
expect(init?.method).toBe("POST");
|
||||
const body = JSON.parse(String(init?.body)) as { request: string; enrollment?: string; acknowledgement?: string };
|
||||
const [header, payload, signature] = body.request.split(".");
|
||||
expect(verify(null, Buffer.from(`${header}.${payload}`), signing.publicKey, Buffer.from(signature!, "base64url"))).toBe(true);
|
||||
const claims = JSON.parse(Buffer.from(payload!, "base64url").toString("utf8")) as Record<string, unknown>;
|
||||
expect(claims).toMatchObject({ iss: instanceId, env: "test", aud: url.toString() });
|
||||
const operation = String(claims.op);
|
||||
if (operation !== "status") {
|
||||
const expectedSubject = operation === "slack-events" ? subject
|
||||
: ["event-lease", "event-ack"].includes(operation) ? "github-bot-events" : undefined;
|
||||
if (!expectedSubject) throw new Error(`Unexpected synthetic connector authority ${operation}`);
|
||||
expect(claims).toMatchObject({ sub: expectedSubject, cid: companyId });
|
||||
}
|
||||
if (unavailable) return Response.json({ error: "fixture_broker_unavailable" }, { status: 503 });
|
||||
if (operation === "status") return Response.json({ active: true, status: "active", profiles: [], slackEvents: { version: 1 } });
|
||||
if (operation === "slack-events") {
|
||||
expect(claims.sh).toBe(createHash("sha256").update(body.enrollment!).digest("base64url"));
|
||||
const enrollment = JSON.parse(body.enrollment!) as Record<string, unknown>;
|
||||
calls.push({ operation, enrollment: { ...enrollment, signingSecret: undefined } });
|
||||
await beforeEnrollmentReceipt?.();
|
||||
const { active, signingSecret: _secret, ...identity } = enrollment;
|
||||
const bindingId = `slack-event:${createHash("sha256").update(JSON.stringify(identity)).digest("hex")}`;
|
||||
return Response.json(active === false ? { retirement: { version: 1, instanceId, ...identity } }
|
||||
: { enrollment: { version: 1, instanceId, ...identity, bindingId } });
|
||||
}
|
||||
calls.push({ operation });
|
||||
if (operation === "event-lease") {
|
||||
if (!events.length) return Response.json({ events: [], eventWakeVersion: 1 });
|
||||
const leased = events.splice(0);
|
||||
return Response.json({ leaseId: "fixture-lease", eventWakeVersion: 1,
|
||||
sealed: seal({ v: 1, instanceId, environment: "test", leaseId: "fixture-lease", events: leased }, "github.code") });
|
||||
}
|
||||
if (operation === "event-ack") {
|
||||
expect(claims.sh).toBe(createHash("sha256").update(body.acknowledgement!).digest("base64url"));
|
||||
const value = JSON.parse(body.acknowledgement!) as { deliveryIds: string[] };
|
||||
acknowledged.push(...value.deliveryIds); return Response.json({ acknowledged: value.deliveryIds.length });
|
||||
}
|
||||
throw new Error(`Unexpected synthetic connector operation ${operation}`);
|
||||
} catch (error) {
|
||||
requestErrors.push(error);
|
||||
console.error("Synthetic Slack connector rejected its request", error);
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
function install() {
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_BASE_URL", config.baseUrl);
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID", instanceId);
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_ENVIRONMENT", "test");
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SIGN_PRIVATE_KEY", config.signPrivateKey);
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SEAL_PRIVATE_KEY", config.sealPrivateKey);
|
||||
}
|
||||
function enqueue(identity: Record<string, unknown>, signingSecret: string, eventId: string) {
|
||||
const acceptedAt = new Date().toISOString(), timestamp = String(Math.floor(Date.now() / 1000));
|
||||
const body = Buffer.from(JSON.stringify({ type: "event_callback", team_id: identity.teamId, api_app_id: identity.appId, event_id: eventId,
|
||||
event: { type: "app_mention", text: "<@UBOT> fixture callback", user: "UOPERATOR", channel: "CTEST", ts: `${Date.now() / 1000}`, event_ts: `${Date.now() / 1000}` } }));
|
||||
const id = `slack_${String(identity.bindingId).slice("slack-event:".length)}_${createHash("sha256").update(eventId).digest("hex")}`;
|
||||
const packet = { version: 1, instanceId, environment: "test", ...identity, eventId, acceptedAt, rawBody: body.toString("base64"), timestamp,
|
||||
signature: `v0=${createHmac("sha256", signingSecret).update(`v0:${timestamp}:`).update(body).digest("hex")}` };
|
||||
events.push({ id, provider: "slack", event: "event_callback", installationId: identity.bindingId, bindingIds: [identity.bindingId], createdAt: acceptedAt,
|
||||
payload: { slackEvent: { bindingId: identity.bindingId, sealed: seal(packet, "slack.events.v1") } } });
|
||||
return { id, packet, body };
|
||||
}
|
||||
return { config, request, requestErrors, calls, acknowledged, events, install, enqueue, connector: createPaperclipCloudConnector({ config, request }),
|
||||
setUnavailable(value: boolean) { unavailable = value; },
|
||||
beforeReceipt(work: (() => Promise<void>) | undefined) { beforeEnrollmentReceipt = work; } };
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
const key = Symbol.for('paperclip.slack.fixture.prewire');
|
||||
if (!process[key]) {
|
||||
const net = require('node:net');
|
||||
const tls = require('node:tls');
|
||||
const http = require('node:http');
|
||||
const https = require('node:https');
|
||||
const fs = require('node:fs');
|
||||
const loopback = new Set(['127.0.0.1', 'localhost', '::1', '[::1]']);
|
||||
const listeners = new Set();
|
||||
const pgTargets = new Set();
|
||||
const pgPorts = new Set([5432, 55448]);
|
||||
for (const name of ['PAPERCLIP_SLACK_TEST_DATABASE_URL', 'SLACK_EVENT_TEST_DATABASE_URL']) {
|
||||
if (!process.env[name]) continue;
|
||||
let url;
|
||||
try { url = new URL(process.env[name]); } catch { throw new Error('Invalid Slack fixture database target'); }
|
||||
if (!['postgres:', 'postgresql:'].includes(url.protocol) || !loopback.has(url.hostname)
|
||||
|| !/^[a-zA-Z0-9_]+_test$/.test(decodeURIComponent(url.pathname.slice(1)))
|
||||
|| ['options', 'database', 'dbname'].some(k => url.searchParams.has(k))) throw new Error('Invalid Slack fixture database target');
|
||||
const port = Number(url.port || 5432);
|
||||
pgPorts.add(port);
|
||||
pgTargets.add(`${url.hostname}:${port}`);
|
||||
}
|
||||
const sockets = new Set();
|
||||
const observed = new WeakSet();
|
||||
const attempts = [];
|
||||
let peakPgSockets = 0;
|
||||
const record = value => {
|
||||
if (process.env.SLACK_GATE_PROCESS_LOG) fs.appendFileSync(process.env.SLACK_GATE_PROCESS_LOG, JSON.stringify(value) + '\n');
|
||||
};
|
||||
const fail = (host, port) => {
|
||||
attempts.push({ host: String(host), port: Number(port) || null });
|
||||
process.exitCode = 98;
|
||||
throw new Error('Slack fixture blocked an unowned network target before transport');
|
||||
};
|
||||
const check = (host = 'localhost', port, protocol) => {
|
||||
host = String(host).toLowerCase();
|
||||
if (!loopback.has(host)) fail(host, port);
|
||||
if (protocol && !['http:', 'https:'].includes(protocol)) fail(host, port);
|
||||
if (port && pgPorts.has(Number(port)) && !pgTargets.has(`${host}:${Number(port)}`)) fail(host, port);
|
||||
if (port && !pgTargets.has(`${host}:${Number(port)}`) && !listeners.has(Number(port))) fail(host, port);
|
||||
};
|
||||
const checkArgs = args => {
|
||||
const first = args[0];
|
||||
if (first instanceof URL || typeof first === 'string' && /^https?:/.test(first)) {
|
||||
const url = first instanceof URL ? first : new URL(first);
|
||||
check(url.hostname, url.port || (url.protocol === 'https:' ? 443 : 80), url.protocol);
|
||||
} else if (first && typeof first === 'object') {
|
||||
if (first.path && !first.port && !first.host && !first.hostname) fail('unix-socket', 0);
|
||||
check(first.hostname || first.host, first.port, first.protocol);
|
||||
} else if (typeof first === 'number') check(typeof args[1] === 'string' ? args[1] : undefined, first);
|
||||
else fail('unknown-target', 0);
|
||||
};
|
||||
const listen = net.Server.prototype.listen;
|
||||
net.Server.prototype.listen = function (...args) {
|
||||
this.once('listening', () => {
|
||||
const address = this.address();
|
||||
if (address && typeof address === 'object' && loopback.has(address.address)) {
|
||||
listeners.add(address.port);
|
||||
this.once('close', () => listeners.delete(address.port));
|
||||
}
|
||||
});
|
||||
return Reflect.apply(listen, this, args);
|
||||
};
|
||||
for (const module of [http, https]) for (const name of ['request', 'get']) {
|
||||
const original = module[name];
|
||||
module[name] = function (...args) { checkArgs(args); return Reflect.apply(original, this, args); };
|
||||
}
|
||||
const connect = net.Socket.prototype.connect;
|
||||
net.Socket.prototype.connect = function (...args) {
|
||||
const normalized = Array.isArray(args[0]) ? args[0] : args;
|
||||
checkArgs(normalized);
|
||||
const first = normalized[0];
|
||||
const port = typeof first === 'object' ? first.port : first;
|
||||
if (pgPorts.has(Number(port)) && !observed.has(this)) {
|
||||
observed.add(this);
|
||||
this.once('connect', () => { sockets.add(this); peakPgSockets = Math.max(peakPgSockets, sockets.size); });
|
||||
this.once('close', () => sockets.delete(this));
|
||||
}
|
||||
return Reflect.apply(connect, this, args);
|
||||
};
|
||||
const tlsConnect = tls.connect;
|
||||
tls.connect = function (...args) { checkArgs(args); return Reflect.apply(tlsConnect, this, args); };
|
||||
const fetch = globalThis.fetch;
|
||||
if (fetch) globalThis.fetch = function (input, init) {
|
||||
checkArgs([input instanceof Request ? input.url : input]);
|
||||
return fetch(input, init);
|
||||
};
|
||||
const stats = () => ({ pid: process.pid, peakPgSockets, remainingPgSockets: sockets.size, unexpectedAttempts: attempts.slice() });
|
||||
process[key] = Object.freeze({ pid: process.pid, stats });
|
||||
globalThis[key] = process[key];
|
||||
process.env.SLACK_GATE_GUARD_PID = String(process.pid);
|
||||
record({ phase: 'repository-preload', pid: process.pid });
|
||||
process.on('exit', () => record({ phase: 'exit', ...stats(), exitCode: process.exitCode || 0 }));
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
import http from "node:http";
|
||||
import https from "node:https";
|
||||
import { expect, vi } from "vitest";
|
||||
|
||||
export function guardSlackFixtureNetwork(options: { requirePreload?: boolean } = {}) {
|
||||
const prewire = Reflect.get(process, Symbol.for("paperclip.slack.fixture.prewire")) as {
|
||||
pid: number;
|
||||
stats(): { pid: number; peakPgSockets: number; remainingPgSockets: number; unexpectedAttempts: unknown[] };
|
||||
} | undefined;
|
||||
if (options.requirePreload) {
|
||||
expect(prewire?.pid).toBe(process.pid);
|
||||
expect(process.env.SLACK_GATE_GUARD_PID).toBe(String(process.pid));
|
||||
}
|
||||
if (process.env.SLACK_GATE_EXPECT_PRELOAD === "1") {
|
||||
expect(process.env.SLACK_GATE_GUARD_PID).toBe(String(process.pid));
|
||||
console.info(JSON.stringify({ slackFixtureWorkerPid: process.pid, prewireGuardPid: process.env.SLACK_GATE_GUARD_PID }));
|
||||
}
|
||||
const blocked: string[] = [];
|
||||
const check = (value: unknown) => {
|
||||
const url = value instanceof URL ? value : typeof value === "string" ? new URL(value) : undefined;
|
||||
const options = value as { hostname?: string; host?: string } | undefined;
|
||||
const host = url?.hostname ?? options?.hostname ?? options?.host ?? "localhost";
|
||||
if (!["localhost", "127.0.0.1", "::1", "[::1]"].includes(host)) {
|
||||
blocked.push(host);
|
||||
throw new Error("Unexpected off-host Slack fixture request");
|
||||
}
|
||||
};
|
||||
const realHttp = http.request, realHttps = https.request, realFetch = globalThis.fetch;
|
||||
const httpSpy = vi.spyOn(http, "request").mockImplementation(((...args: unknown[]) => {
|
||||
check(args[0]); return Reflect.apply(realHttp, http, args);
|
||||
}) as typeof http.request);
|
||||
const httpsSpy = vi.spyOn(https, "request").mockImplementation(((...args: unknown[]) => {
|
||||
check(args[0]); return Reflect.apply(realHttps, https, args);
|
||||
}) as typeof https.request);
|
||||
const fetchSpy = vi.spyOn(globalThis, "fetch").mockImplementation((input, init) => {
|
||||
check(input instanceof Request ? input.url : input); return realFetch(input, init);
|
||||
});
|
||||
return { assertQuiet: () => {
|
||||
expect(blocked).toEqual([]);
|
||||
if (prewire) expect(prewire.stats().unexpectedAttempts).toEqual([]);
|
||||
const stats = prewire?.stats;
|
||||
if (process.env.SLACK_GATE_EXPECT_PRELOAD === "1") {
|
||||
expect(stats).toBeTypeOf("function");
|
||||
console.info(JSON.stringify({ slackFixtureWorkerPid: process.pid, networkStats: stats!() }));
|
||||
}
|
||||
}, restore: () => {
|
||||
fetchSpy.mockRestore(); httpsSpy.mockRestore(); httpSpy.mockRestore();
|
||||
} };
|
||||
}
|
||||
|
||||
export function stubSlackFixtureTransport(adapter: unknown) {
|
||||
const client = (adapter as { _client: {
|
||||
axios: { defaults: { adapter: unknown } };
|
||||
buildResult(response: unknown): Promise<unknown>;
|
||||
} })._client;
|
||||
const failures: unknown[] = [];
|
||||
const buildResult = client.buildResult.bind(client);
|
||||
client.buildResult = async response => {
|
||||
try { return await buildResult(response); }
|
||||
catch (error) { failures.push(error); throw error; }
|
||||
};
|
||||
const transport = vi.fn(async (config: { url: string; baseURL: string }) => {
|
||||
const url = new URL(config.url, config.baseURL);
|
||||
if (url.hostname !== "127.0.0.1") throw new Error("Slack fixture API origin changed");
|
||||
let data: unknown;
|
||||
if (url.pathname.endsWith("/users.info")) data = { ok: true, user: { id: "UOPERATOR", name: "operator", is_bot: false, profile: { display_name: "Operator" } } };
|
||||
else if (url.pathname.endsWith("/conversations.info")) data = { ok: true, channel: { id: "CTEST", name: "fixture" } };
|
||||
else throw new Error(`Unexpected Slack fixture API method ${url.pathname}`);
|
||||
return { data, status: 200, statusText: "OK", headers: {}, config, request: { path: url.pathname } };
|
||||
});
|
||||
client.axios.defaults.adapter = transport;
|
||||
return Object.assign(transport, { assertNoErrors: async () => {
|
||||
await Promise.all(transport.mock.results.map(result => result.value));
|
||||
expect(failures).toEqual([]);
|
||||
} });
|
||||
}
|
||||
@@ -301,6 +301,66 @@ describe("residual recovery production matrix", () => {
|
||||
}, { timeout: 45_000 });
|
||||
await competing.stop(); extraStops.pop(); await quiet();
|
||||
});
|
||||
it("recovers a fresh run from a prior boot at lease expiry instead of waiting for the stale-age threshold", async () => {
|
||||
// RC-27: a Retry admitted just before a kill records a controller lease but
|
||||
// no process. The prior boot's expired lease is the ownership proof.
|
||||
const f = await fixture(), expiry = new Date(Date.now() + 3_000);
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agent.id, status: "running",
|
||||
runtimeMode: "legacy", responsibleUserId: f.actorId, controllerBootId: randomUUID(), controllerLeaseExpiresAt: expiry,
|
||||
executionStage: "preparing", updatedAt: new Date(), startedAt: new Date() }).returning();
|
||||
await start(true);
|
||||
expect((await getRun(run.id)).status).toBe("running");
|
||||
await vi.waitFor(async () => expect((await getRun(run.id)).status).toBe("failed"), { timeout: 15_000 });
|
||||
expect(Date.now() - expiry.getTime()).toBeLessThan(15_000);
|
||||
expect(await getRun(run.id)).toMatchObject({ errorCode: "process_lost",
|
||||
resultJson: expect.objectContaining({ executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }) });
|
||||
await vi.waitFor(async () => {
|
||||
const retries = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, run.id));
|
||||
expect(retries).toHaveLength(1);
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
expect((await getRun(retries[0].id)).status).toBe("succeeded");
|
||||
}, { timeout: 45_000 });
|
||||
await quiet();
|
||||
});
|
||||
it("keeps the age guard when the prior boot dispatched between the scan and the revocation", async () => {
|
||||
// Stage the race: the scanned row says `preparing`, and the row committed
|
||||
// with the revocation says `dispatching` (the old controller's last renewal).
|
||||
const f = await fixture(), expiry = new Date(Date.now() + 2_000);
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agent.id, status: "running",
|
||||
runtimeMode: "legacy", responsibleUserId: f.actorId, controllerBootId: randomUUID(), controllerLeaseExpiresAt: expiry,
|
||||
executionStage: "preparing", updatedAt: new Date(), startedAt: new Date() }).returning();
|
||||
await db.execute(sql.raw(`create or replace function rc27_dispatch_on_revoke() returns trigger language plpgsql as $$
|
||||
begin if new.id = '${run.id}' and new.controller_boot_id is distinct from old.controller_boot_id
|
||||
then new.execution_stage := 'dispatching'; end if; return new; end $$`));
|
||||
await db.execute(sql`create trigger rc27_dispatch_on_revoke before update on heartbeat_runs
|
||||
for each row execute function rc27_dispatch_on_revoke()`);
|
||||
extraStops.push(async () => {
|
||||
await db.execute(sql`drop trigger if exists rc27_dispatch_on_revoke on heartbeat_runs`);
|
||||
await db.execute(sql`drop function if exists rc27_dispatch_on_revoke()`);
|
||||
});
|
||||
await start(true);
|
||||
await vi.waitFor(async () => expect((await getRun(run.id)).controllerBootId).not.toBe(run.controllerBootId), { timeout: 15_000 });
|
||||
await realDelay(5_000); await Promise.allSettled([...passes]);
|
||||
expect(await getRun(run.id)).toMatchObject({ status: "running", executionStage: "dispatching", errorCode: null });
|
||||
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, run.id))).toEqual([]);
|
||||
});
|
||||
it("keeps the stale-age guard for this boot, a recorded process, or a dispatched adapter", async () => {
|
||||
const f = await fixture(), now = Date.now(), expired = new Date(now - 1_000);
|
||||
const [sameBoot] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agent.id, status: "running",
|
||||
runtimeMode: "legacy", responsibleUserId: f.actorId, controllerBootId: legacyControllerBootId, controllerLeaseExpiresAt: expired,
|
||||
executionStage: "preparing", updatedAt: new Date(now), startedAt: new Date(now) }).returning();
|
||||
const [withProcess] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agent.id, status: "running",
|
||||
runtimeMode: "legacy", responsibleUserId: f.actorId, controllerBootId: randomUUID(), controllerLeaseExpiresAt: expired,
|
||||
executionStage: "dispatching", processPid: 2 ** 22 + 17, updatedAt: new Date(now), startedAt: new Date(now) }).returning();
|
||||
// A dispatched adapter may hold remote work without a local process.
|
||||
const [dispatched] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agent.id, status: "running",
|
||||
runtimeMode: "legacy", responsibleUserId: f.actorId, controllerBootId: randomUUID(), controllerLeaseExpiresAt: expired,
|
||||
executionStage: "dispatching", updatedAt: new Date(now), startedAt: new Date(now) }).returning();
|
||||
await start(); await advance(4 * 60_000);
|
||||
expect((await getRun(sameBoot.id)).status).toBe("running");
|
||||
expect((await getRun(withProcess.id)).status).toBe("running");
|
||||
expect((await getRun(dispatched.id)).status).toBe("running");
|
||||
});
|
||||
it.each(["confirmed", "already_retired", "missing_terminal", "unproven_terminal", "ack_failure", "late_terminal"] as const)(
|
||||
"settles an API-only Stop after deferred controller retirement without inventing proof: %s", async mode => {
|
||||
const f = await fixture(), runId = randomUUID(), sessionId = randomUUID(), runnerId = randomUUID();
|
||||
|
||||
@@ -0,0 +1,433 @@
|
||||
import { createHmac, randomUUID } from "node:crypto";
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { SLACK_CHAT_BOT_SCOPES } from "@paperclipai/shared";
|
||||
import { activityLog, agentRuntimeState, agentTaskSessions, agentWakeupRequests, agents, applyPendingMigrations, authUsers,
|
||||
chatDeliveries, chatEndpoints, chatExternalPrincipals, companySecretBindings, chatIdentityLinks, chatSlackRegistrations, companies, companyMemberships,
|
||||
companySecrets, companySkills, costEvents, financeEvents, heartbeatRunEvents, heartbeatRuns, issueComments, issues, toolApplications, toolConnections,
|
||||
closeCheckpointDb, closeFileStorageDb, closeLivenessDb, createDb } from "@paperclipai/db";
|
||||
import { databaseWorkPending, reconcileDatabaseWork, subscribeDatabaseWork } from "@paperclipai/db/work-signals";
|
||||
import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
import { createSlackCloudConnectorFixture } from "./helpers/slack-cloud-connector-fixture.js";
|
||||
import { chatChannelService } from "../services/chat-channels.js";
|
||||
import { toolAccessService } from "../services/tool-access.js";
|
||||
import { secretService } from "../services/secrets.js";
|
||||
import { ChatRuntime } from "../services/chat-runtime.js";
|
||||
import { heartbeatService } from "../services/heartbeat.js";
|
||||
import { githubConnectionEventService } from "../services/github-connection-events.js";
|
||||
import { chatCredentialMutationLease } from "../services/chat-credential-mutation-lease.js";
|
||||
import { createSlackCloudEnrollment, registerSlackCloudEnrollmentOwner, reconcilePendingSlackCloudEnrollments } from "../services/chat-channels/slack/cloud-enrollment.js";
|
||||
import { conflict } from "../errors.js";
|
||||
import { guardSlackFixtureNetwork, stubSlackFixtureTransport } from "./helpers/slack-cloud/transport.js";
|
||||
import type { InternalSetupState } from "../services/chat-channels/runtime-types.js";
|
||||
import type { ChatSdkMessageCallbackEvent } from "../services/chat-sdk-runtime.js";
|
||||
|
||||
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_SLACK_OWNER_TEST_DATABASE_URL");
|
||||
if (process.env.SLACK_GATE_EXPECT_PRELOAD === "1" && !external) throw new Error("Guarded local Slack owner tests require their exact external test database");
|
||||
const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
(support.supported ? describe : describe.skip)("Slack Cloud default owner and SQL fences", () => {
|
||||
let db: ReturnType<typeof createDb>;
|
||||
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
|
||||
let service: ReturnType<typeof chatChannelService>;
|
||||
let heartbeat: ReturnType<typeof heartbeatService>;
|
||||
let cloud: ReturnType<typeof createSlackCloudConnectorFixture>;
|
||||
let network: ReturnType<typeof guardSlackFixtureNetwork>;
|
||||
let directory: string;
|
||||
let companyId: string, actorId: string, agentId: string, applicationId: string;
|
||||
let runtime: ChatRuntime;
|
||||
let providerFetch: typeof fetch;
|
||||
const callbacks: ChatSdkMessageCallbackEvent[] = [];
|
||||
const transports: ReturnType<typeof stubSlackFixtureTransport>[] = [];
|
||||
const releases: (() => void)[] = [];
|
||||
const heldLeases: Promise<unknown>[] = [];
|
||||
const deferred: (() => void | Promise<void>)[] = [];
|
||||
const secret = "0123456789abcdef0123456789abcdef";
|
||||
const row = async (id: string) => (await db.select().from(chatEndpoints).where(eq(chatEndpoints.id, id)))[0]!;
|
||||
const setup = async (id: string) => (await row(id)).setup as InternalSetupState;
|
||||
beforeAll(async () => {
|
||||
if (!external) temporary = await startEmbeddedPostgresTestDatabase("slack-cloud-owner-");
|
||||
const url = external ?? temporary!.connectionString;
|
||||
await applyPendingMigrations(url);
|
||||
db = createDb(url, { maxConnections: 2, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
|
||||
}, 60_000);
|
||||
beforeEach(async () => {
|
||||
callbacks.length = 0; transports.length = 0; releases.length = 0; heldLeases.length = 0; deferred.length = 0;
|
||||
network = guardSlackFixtureNetwork({ requirePreload: process.env.SLACK_GATE_EXPECT_PRELOAD === "1" });
|
||||
directory = await mkdtemp(path.join(os.tmpdir(), "slack-owner-default-"));
|
||||
vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY_FILE", path.join(directory, "master.key"));
|
||||
companyId = randomUUID(); actorId = `slack-owner-${randomUUID()}`; agentId = randomUUID(); applicationId = randomUUID();
|
||||
cloud = createSlackCloudConnectorFixture(`instance-${randomUUID()}`, companyId, actorId); cloud.install();
|
||||
vi.spyOn(globalThis, "fetch").mockImplementation(cloud.request);
|
||||
const now = new Date();
|
||||
await db.insert(companies).values({ id: companyId, name: "Slack default owner", issuePrefix: `S${companyId.slice(0, 7)}` });
|
||||
await db.insert(authUsers).values({ id: actorId, name: "Slack owner", email: `${actorId}@example.test`, emailVerified: true, createdAt: now, updatedAt: now });
|
||||
await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: actorId, status: "active", membershipRole: "operator" });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Slack fixture", adapterType: "process", status: "idle",
|
||||
runtimeConfig: { heartbeat: { wakeOnDemand: false, intervalSec: 0 } } });
|
||||
await db.insert(companyMemberships).values({ companyId, principalType: "agent", principalId: agentId, status: "active", membershipRole: "member" });
|
||||
await db.insert(toolApplications).values({ id: applicationId, companyId, name: "Slack", type: "chat" });
|
||||
runtime = makeRuntime();
|
||||
const slackFetch: typeof fetch = async (input, init) => {
|
||||
const url = new URL(input instanceof Request ? input.url : String(input));
|
||||
if (url.origin !== "https://slack.com") throw new Error(`Unexpected Slack fixture origin ${url.origin}`);
|
||||
if (url.pathname === "/api/auth.test") {
|
||||
expect(init?.method).toBe("POST");
|
||||
const token = new Headers(init?.headers).get("authorization") ?? "";
|
||||
const suffix = token.slice(token.lastIndexOf("-") + 1);
|
||||
return Response.json({ ok: true, team_id: `T${suffix}`, team: "Fixture", user_id: `U${suffix}`, user: "fixture" },
|
||||
{ headers: { "x-oauth-scopes": SLACK_CHAT_BOT_SCOPES.join(",") } });
|
||||
}
|
||||
if (url.pathname === "/api/conversations.list") return Response.json({ ok: true,
|
||||
channels: [{ id: "CTEST", name: "fixture", is_member: true }], response_metadata: { next_cursor: "" } });
|
||||
if (["/api/reactions.add", "/api/reactions.remove"].includes(url.pathname)) return Response.json({ ok: true });
|
||||
throw new Error(`Unexpected Slack fixture method ${url.pathname}`);
|
||||
};
|
||||
providerFetch = slackFetch;
|
||||
heartbeat = heartbeatService(db);
|
||||
service = chatChannelService(db, { runtime, heartbeat, fetch: slackFetch, publicBaseUrl: "https://tenant.example",
|
||||
deferWebhookProcessing: true, scheduleDeferredWork: work => { deferred.push(work); } });
|
||||
});
|
||||
afterEach(async () => {
|
||||
const errors: unknown[] = [];
|
||||
for (const release of releases) release();
|
||||
for (const work of heldLeases) { try { await work; } catch (error) { errors.push(error); } }
|
||||
try { await service?.shutdown(); } catch (error) { errors.push(error); }
|
||||
try { await heartbeat?.drainActiveRunExecutions(); } catch (error) { errors.push(error); }
|
||||
try {
|
||||
await db.delete(chatEndpoints).where(eq(chatEndpoints.companyId, companyId));
|
||||
await db.delete(financeEvents).where(eq(financeEvents.companyId, companyId));
|
||||
await db.delete(costEvents).where(eq(costEvents.companyId, companyId));
|
||||
await db.delete(activityLog).where(eq(activityLog.companyId, companyId));
|
||||
await db.delete(agentTaskSessions).where(eq(agentTaskSessions.companyId, companyId));
|
||||
await db.delete(agentRuntimeState).where(eq(agentRuntimeState.companyId, companyId));
|
||||
await db.delete(heartbeatRunEvents).where(eq(heartbeatRunEvents.companyId, companyId));
|
||||
await db.delete(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId));
|
||||
await db.delete(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, companyId));
|
||||
await db.delete(issueComments).where(eq(issueComments.companyId, companyId));
|
||||
await db.delete(issues).where(eq(issues.companyId, companyId));
|
||||
await db.delete(agents).where(eq(agents.companyId, companyId));
|
||||
await db.delete(companyMemberships).where(eq(companyMemberships.companyId, companyId));
|
||||
await db.delete(companySecrets).where(eq(companySecrets.companyId, companyId));
|
||||
await db.delete(companySkills).where(eq(companySkills.companyId, companyId));
|
||||
await db.delete(companies).where(eq(companies.id, companyId));
|
||||
await db.delete(authUsers).where(eq(authUsers.id, actorId));
|
||||
} catch (error) { errors.push(error); }
|
||||
try { network.assertQuiet(); } catch (error) { errors.push(error); }
|
||||
network.restore(); vi.restoreAllMocks(); vi.unstubAllEnvs();
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
if (errors.length) throw new AggregateError(errors, "Slack owner fixture cleanup failed; original body failure is reported separately by Vitest");
|
||||
});
|
||||
afterAll(async () => {
|
||||
const closed = await Promise.allSettled([closeCheckpointDb(db, 1), closeFileStorageDb(db, 1), closeLivenessDb(db, 1)]);
|
||||
await db?.$client.end({ timeout: 1 }); await temporary?.cleanup();
|
||||
const errors = closed.flatMap(value => value.status === "rejected" ? [value.reason] : []);
|
||||
if (errors.length) throw new AggregateError(errors, "Slack owner auxiliary shutdown failed");
|
||||
});
|
||||
function makeRuntime() {
|
||||
const runtime = new ChatRuntime();
|
||||
const replace = runtime.replaceEndpoint.bind(runtime);
|
||||
vi.spyOn(runtime, "replaceEndpoint").mockImplementation(async input => {
|
||||
if (input.providerConfig.provider !== "slack") throw new Error("Unexpected fixture provider");
|
||||
const value = await replace({ ...input, providerConfig: { ...input.providerConfig,
|
||||
credentials: { ...input.providerConfig.credentials, apiUrl: "http://127.0.0.1:1/api/" } },
|
||||
callbacks: { ...input.callbacks, onMessage: async event => { callbacks.push(event); await input.callbacks.onMessage(event); } } });
|
||||
transports.push(stubSlackFixtureTransport(value.getProviderAdapter())); return value;
|
||||
});
|
||||
return runtime;
|
||||
}
|
||||
async function createEndpoint(suffix: string) {
|
||||
const endpointId = randomUUID(), connectionId = randomUUID(), publicId = randomUUID().replaceAll("-", "").padEnd(43, "a");
|
||||
await db.insert(toolConnections).values({ id: connectionId, companyId, applicationId, name: `Slack ${suffix}`, uid: endpointId,
|
||||
transport: "chat_sdk", connectionPurpose: "channel", enabled: true, status: "active" });
|
||||
await db.insert(chatEndpoints).values({ id: endpointId, companyId, connectionId, assignedAgentId: agentId, publicId,
|
||||
provider: "slack", status: "draft", sponsorUserId: actorId, allowGroupChats: true, setup: { step: "provider_setup" } });
|
||||
await db.insert(chatSlackRegistrations).values({ companyId, endpointId, requestId: randomUUID(), status: "configured", appId: `A${suffix}`,
|
||||
manifest: {}, manifestHash: "fixture", callbackUri: `https://tenant.example/api/chat-webhooks/${publicId}/slack`, createdByUserId: actorId });
|
||||
await service.configure(endpointId, { action: "configure", credentials: { botToken: `xoxb-fixture-${suffix}`, signingSecret: secret } }, actorId);
|
||||
expect((await setup(endpointId)).slackCloudIntent).toMatchObject({ companyId, endpointId, connectionId,
|
||||
instanceId: cloud.config.instanceId, subject: actorId, active: true });
|
||||
return endpointId;
|
||||
}
|
||||
async function confirm(endpointId: string) {
|
||||
const owner = githubConnectionEventService(db, { connector: cloud.connector });
|
||||
await owner.pollOnce({ forDelivery: true });
|
||||
expect((await setup(endpointId)).slackCloudIntent).toBeUndefined();
|
||||
expect((await setup(endpointId)).slackCloudEvents).toBeDefined();
|
||||
return owner;
|
||||
}
|
||||
async function activate(endpointId: string) {
|
||||
const endpoint = await row(endpointId);
|
||||
const accountId = endpoint.providerAccountId;
|
||||
if (!accountId) throw new Error("Normal Slack configure did not confirm its workspace");
|
||||
const payload = JSON.stringify({ type: "url_verification", team_id: accountId, api_app_id: `A${accountId.slice(1)}`, challenge: "fixture-challenge" });
|
||||
const timestamp = String(Math.floor(Date.now() / 1000));
|
||||
const response = await service.handleWebhook(endpoint.publicId, "slack", new Request(`https://tenant.example/api/chat-webhooks/${endpoint.publicId}/slack`, {
|
||||
method: "POST", headers: { "content-type": "application/json", "x-slack-request-timestamp": timestamp,
|
||||
"x-slack-signature": `v0=${createHmac("sha256", secret).update(`v0:${timestamp}:${payload}`).digest("hex")}` }, body: payload }));
|
||||
expect(response.status).toBe(200); await response.body?.cancel();
|
||||
await service.configure(endpointId, { action: "verify" }, actorId);
|
||||
const [principal] = await db.insert(chatExternalPrincipals).values({ companyId, provider: "slack", providerAccountId: accountId,
|
||||
externalId: "UOPERATOR", kind: "user", displayName: "Operator" }).returning();
|
||||
await db.insert(chatIdentityLinks).values({ companyId, endpointId, principalId: principal!.id, paperclipUserId: actorId, status: "linked", confirmedAt: new Date() });
|
||||
expect((await service.test(endpointId, { optionalSlackTestForUser: actorId })).status).toBe("active");
|
||||
}
|
||||
async function holdCredentials(endpointId: string) {
|
||||
let acquired!: () => void;
|
||||
const ready = new Promise<void>(resolve => { acquired = resolve; });
|
||||
const release = new Promise<void>(resolve => { releases.push(resolve); });
|
||||
heldLeases.push(chatCredentialMutationLease(db)(await row(endpointId), async lease => { await lease.assertOwned(); acquired(); await release; }));
|
||||
await ready;
|
||||
}
|
||||
|
||||
it("normal configure stages SQL intent, worker unseals into registered SDK and old generations cannot reenter", async () => {
|
||||
const endpointId = await createEndpoint("BOT");
|
||||
await confirm(endpointId); await activate(endpointId); await confirm(endpointId);
|
||||
const endpoint = await row(endpointId), saved = await setup(endpointId), enrolled = saved.slackCloudEvents!;
|
||||
const callbackEvidence = structuredClone(saved.slackCallbackSurfaces);
|
||||
const delivered = cloud.enqueue({ companyId, endpointId, publicId: endpoint.publicId, connectionId: endpoint.connectionId,
|
||||
runtimeGeneration: saved.runtimeGeneration, teamId: enrolled.teamId, appId: enrolled.appId, bindingId: enrolled.bindingId }, secret, "EvDefaultOwner");
|
||||
const owner = githubConnectionEventService(db, { connector: cloud.connector });
|
||||
expect((await owner.pollOnce({ forDelivery: true })).processed).toBe(1);
|
||||
expect(callbacks).toHaveLength(1);
|
||||
expect(callbacks[0]!.message.author.userId).toBe("UOPERATOR");
|
||||
expect(cloud.acknowledged).toEqual([delivered.id]);
|
||||
expect((await setup(endpointId)).slackCallbackSurfaces).toEqual(callbackEvidence);
|
||||
expect(await db.select().from(chatDeliveries).where(eq(chatDeliveries.endpointId, endpointId))).toHaveLength(1);
|
||||
for (const transport of transports) await transport.assertNoErrors();
|
||||
await service.configure(endpointId, { action: "pause" }, actorId);
|
||||
const oldRequest = new Request("https://tenant.example/old-receipt", { method: "POST", body: new Uint8Array(delivered.body),
|
||||
headers: { "content-type": "application/json", "x-slack-request-timestamp": delivered.packet.timestamp, "x-slack-signature": delivered.packet.signature } });
|
||||
const rejected = await service.handleWebhook(endpoint.publicId, "slack", oldRequest, { slackCloudGeneration: Number(saved.runtimeGeneration) });
|
||||
expect(await rejected.text()).toBe("superseded");
|
||||
expect(callbacks).toHaveLength(1);
|
||||
await owner.pollOnce({ forDelivery: true });
|
||||
expect((await setup(endpointId)).slackCloudIntent).toBeUndefined();
|
||||
expect((await setup(endpointId)).slackCloudRetiredGeneration).toBeGreaterThan(Number(saved.runtimeGeneration));
|
||||
}, 60_000);
|
||||
|
||||
it("two due registrations settle B while A still owns a busy credential lease", async () => {
|
||||
const a = await createEndpoint("BUSY"); await confirm(a); await activate(a); await confirm(a);
|
||||
const b = await createEndpoint("READY"); await confirm(b); await activate(b); await confirm(b);
|
||||
await service.configure(a, { action: "pause" }, actorId);
|
||||
const before = structuredClone((await setup(a)).slackCloudIntent!);
|
||||
await service.configure(b, { action: "pause" }, actorId);
|
||||
await holdCredentials(a);
|
||||
const start = cloud.calls.length;
|
||||
const owner = githubConnectionEventService(db, { connector: cloud.connector });
|
||||
await owner.pollOnce({ forDelivery: true });
|
||||
await owner.pollOnce({ forDelivery: true });
|
||||
expect((await setup(b)).slackCloudIntent).toBeUndefined();
|
||||
expect((await setup(b)).slackCloudRetiredGeneration).toBeDefined();
|
||||
const retained = (await setup(a)).slackCloudIntent!;
|
||||
expect(retained).toMatchObject({ ...before, attempts: 1, failureCode: "credential_lease_busy", nextAttemptAt: expect.any(String) });
|
||||
expect(Date.parse(retained.nextAttemptAt!)).toBeGreaterThan(Date.now());
|
||||
expect(cloud.calls.slice(start).filter(call => call.enrollment?.endpointId === a)).toEqual([]);
|
||||
expect(owner.nextPollAt()).not.toBeNull();
|
||||
releases.splice(0).forEach(release => release()); await Promise.all(heldLeases);
|
||||
const delay = Math.max(0, Date.parse(retained.nextAttemptAt!) - Date.now());
|
||||
await new Promise(resolve => setTimeout(resolve, delay));
|
||||
await owner.pollOnce();
|
||||
expect((await setup(a)).slackCloudIntent).toBeUndefined();
|
||||
expect((await setup(a)).slackCloudRetiredGeneration).toBeDefined();
|
||||
}, 60_000);
|
||||
|
||||
it("normal configuration rollback leaves no registration debt or retirement request", async () => {
|
||||
const endpointId = await createEndpoint("ROLLBACK"); await confirm(endpointId); await activate(endpointId); await confirm(endpointId);
|
||||
const before = await row(endpointId), requests = cloud.calls.length;
|
||||
await service.shutdown(); runtime = makeRuntime();
|
||||
let rejectAfterIntent = false;
|
||||
const stop = subscribeDatabaseWork(db, DELIVERY_QUEUES.connectionEvents, event => { if (event === "intent") rejectAfterIntent = true; });
|
||||
service = chatChannelService(db, { runtime, heartbeat, fetch: providerFetch, publicBaseUrl: "https://tenant.example",
|
||||
deferWebhookProcessing: true, scheduleDeferredWork: work => { deferred.push(work); }, renewCredentialMutationLease: async () => !rejectAfterIntent });
|
||||
try {
|
||||
await expect(service.configure(endpointId, { action: "pause" }, actorId)).rejects.toMatchObject({ code: "CHAT_CREDENTIAL_LEASE_LOST" });
|
||||
expect((await row(endpointId)).status).toBe(before.status);
|
||||
expect(await setup(endpointId)).toEqual(before.setup);
|
||||
expect(cloud.calls).toHaveLength(requests);
|
||||
} finally { stop(); }
|
||||
const owner = githubConnectionEventService(db, { connector: cloud.connector });
|
||||
await owner.pollOnce({ forDelivery: true });
|
||||
expect(cloud.calls.filter(call => call.enrollment?.active === false)).toEqual([]);
|
||||
}, 60_000);
|
||||
|
||||
it.each(["intent", "generation"] as const)("real SQL refuses an enrollment response after newer %s custody wins", async change => {
|
||||
const endpointId = await createEndpoint(`CAS${change}`);
|
||||
const original = (await setup(endpointId)).slackCloudIntent!;
|
||||
const successor = { ...original, id: randomUUID(), runtimeGeneration: original.runtimeGeneration + (change === "generation" ? 1 : 0) };
|
||||
cloud.beforeReceipt(async () => {
|
||||
await db.update(chatEndpoints).set({ setup: sql`jsonb_set(jsonb_set(${chatEndpoints.setup}, '{slackCloudIntent}', ${JSON.stringify(successor)}::jsonb), '{runtimeGeneration}', ${JSON.stringify(successor.runtimeGeneration)}::jsonb)` })
|
||||
.where(and(eq(chatEndpoints.id, endpointId), eq(chatEndpoints.companyId, companyId)));
|
||||
});
|
||||
const producer = createSlackCloudEnrollment(db, { resolveCredentials: async () => ({ signingSecret: secret }),
|
||||
runtimeGeneration: value => Number(value.runtimeGeneration), connector: cloud.connector });
|
||||
await expect(chatCredentialMutationLease(db)(await row(endpointId), lease => producer(endpointId, undefined, lease))).rejects.toThrow("changed");
|
||||
expect((await setup(endpointId)).slackCloudIntent).toEqual(successor);
|
||||
expect((await setup(endpointId)).slackCloudEvents).toBeUndefined();
|
||||
}, 60_000);
|
||||
|
||||
it("malformed sealed Slack receipt is acknowledged once instead of retained as an owner outage", async () => {
|
||||
const endpointId = await createEndpoint("INVALID"); await confirm(endpointId);
|
||||
const endpoint = await row(endpointId), saved = await setup(endpointId), enrolled = saved.slackCloudEvents!;
|
||||
const malformed = cloud.enqueue({ companyId, endpointId: "bad", publicId: endpoint.publicId, connectionId: endpoint.connectionId,
|
||||
runtimeGeneration: saved.runtimeGeneration, teamId: enrolled.teamId, appId: enrolled.appId, bindingId: enrolled.bindingId }, secret, "EvMalformed");
|
||||
const owner = githubConnectionEventService(db, { connector: cloud.connector });
|
||||
const result = await owner.pollOnce({ forDelivery: true });
|
||||
expect(result.ignored).toBe(1); expect(result.failed).toBe(0);
|
||||
expect(cloud.acknowledged).toEqual([malformed.id]);
|
||||
expect(callbacks).toEqual([]);
|
||||
}, 60_000);
|
||||
|
||||
it.each(["intent", "generation", "malformed-generation"] as const)(
|
||||
"busy retry CAS preserves a concurrent %s winner and rethrows the exact owner error", async change => {
|
||||
const endpointId = await createEndpoint(`BUSYCAS${change}`);
|
||||
const before = await setup(endpointId), original = before.slackCloudIntent!;
|
||||
const successor = change === "intent" ? { ...original, id: randomUUID() } : original;
|
||||
const generation: unknown = change === "malformed-generation" ? { invalid: true }
|
||||
: Number(before.runtimeGeneration) + (change === "generation" ? 1 : 0);
|
||||
const busy = conflict("Synthetic owner credential lane is busy", { code: "chat_endpoint_credentials_busy" });
|
||||
const requests = cloud.calls.length;
|
||||
const pendingBefore = databaseWorkPending(db, DELIVERY_QUEUES.connectionEvents), inheritedSignals: string[] = [];
|
||||
const stopInherited = subscribeDatabaseWork(db, DELIVERY_QUEUES.connectionEvents, event => { inheritedSignals.push(event); });
|
||||
try {
|
||||
console.info(JSON.stringify({ slackRetrySignalProbe: "subscription", endpointId, pendingBefore, immediateSignals: [...inheritedSignals] }));
|
||||
expect(inheritedSignals).toEqual(pendingBefore ? ["intent"] : []);
|
||||
await reconcileDatabaseWork(db, DELIVERY_QUEUES.connectionEvents);
|
||||
const pendingAfter = databaseWorkPending(db, DELIVERY_QUEUES.connectionEvents);
|
||||
console.info(JSON.stringify({ slackRetrySignalProbe: "reconciled", endpointId, pendingBefore, pendingAfter, inheritedSignals: [...inheritedSignals] }));
|
||||
expect(pendingAfter).toBe(false);
|
||||
} finally { stopInherited(); }
|
||||
const signals: string[] = [];
|
||||
const stopSignals = subscribeDatabaseWork(db, DELIVERY_QUEUES.connectionEvents, event => { signals.push(event); });
|
||||
const unregister = registerSlackCloudEnrollmentOwner(db, async id => {
|
||||
expect(id).toBe(endpointId);
|
||||
await db.update(chatEndpoints).set({ setup: sql`jsonb_set(jsonb_set(${chatEndpoints.setup}, '{slackCloudIntent}', ${JSON.stringify(successor)}::jsonb), '{runtimeGeneration}', ${JSON.stringify(generation)}::jsonb)` })
|
||||
.where(and(eq(chatEndpoints.id, endpointId), eq(chatEndpoints.companyId, companyId)));
|
||||
throw busy;
|
||||
});
|
||||
try {
|
||||
await expect(reconcilePendingSlackCloudEnrollments(db, new Date())).rejects.toBe(busy);
|
||||
const after = await setup(endpointId);
|
||||
expect(after.slackCloudIntent).toEqual(successor);
|
||||
expect(after.runtimeGeneration).toEqual(generation);
|
||||
expect(after.slackCloudEvents).toBeUndefined();
|
||||
expect(cloud.calls).toHaveLength(requests);
|
||||
expect(signals).toEqual([]);
|
||||
} finally { unregister(); stopSignals(); }
|
||||
}, 60_000);
|
||||
|
||||
it.each(["unknown-error", "unrelated-conflict"] as const)(
|
||||
"the %s owner failure propagates without a busy retry rewrite", async kind => {
|
||||
const endpointId = await createEndpoint(`UNKNOWN${kind}`), before = await setup(endpointId);
|
||||
const original = kind === "unknown-error" ? new Error("Synthetic unknown owner failure")
|
||||
: conflict("Synthetic different authority failure", { code: "chat_endpoint_runtime_superseded" });
|
||||
const requests = cloud.calls.length;
|
||||
const unregister = registerSlackCloudEnrollmentOwner(db, async id => { expect(id).toBe(endpointId); throw original; });
|
||||
try {
|
||||
await expect(reconcilePendingSlackCloudEnrollments(db, new Date())).rejects.toBe(original);
|
||||
expect(await setup(endpointId)).toEqual(before);
|
||||
expect(cloud.calls).toHaveLength(requests);
|
||||
} finally { unregister(); }
|
||||
}, 60_000);
|
||||
|
||||
it("an injected retry transaction failure preserves the original busy error and persistence cause", async () => {
|
||||
const endpointId = await createEndpoint("RETRYWRITE"), before = await setup(endpointId);
|
||||
const busy = conflict("Synthetic owner credential lane is busy", { code: "chat_endpoint_credentials_busy" });
|
||||
const writeError = new Error("Synthetic retry transaction unavailable"), requests = cloud.calls.length;
|
||||
const unregister = registerSlackCloudEnrollmentOwner(db, async id => { expect(id).toBe(endpointId); throw busy; });
|
||||
const transaction = vi.spyOn(db, "transaction").mockRejectedValueOnce(writeError);
|
||||
try {
|
||||
const error = await reconcilePendingSlackCloudEnrollments(db, new Date()).then(() => undefined, error => error);
|
||||
expect(error).toBeInstanceOf(AggregateError);
|
||||
if (!(error instanceof AggregateError)) throw new Error("Expected retry failure to retain both causes", { cause: error });
|
||||
expect(error.errors).toHaveLength(2);
|
||||
expect(error.errors[0]).toBe(busy);
|
||||
expect(error.errors[1]).toBe(writeError);
|
||||
expect(transaction).toHaveBeenCalledOnce();
|
||||
expect(await setup(endpointId)).toEqual(before);
|
||||
expect(cloud.calls).toHaveLength(requests);
|
||||
} finally { transaction.mockRestore(); unregister(); }
|
||||
}, 60_000);
|
||||
|
||||
it("public tool removal retains Slack retirement custody through broker outage and ordinary owner recovery", async () => {
|
||||
const endpointId = await createEndpoint("TOOLREMOVE"); await confirm(endpointId); await activate(endpointId); await confirm(endpointId);
|
||||
const originalEndpoint = await row(endpointId), originalSetup = await setup(endpointId);
|
||||
const [connection] = await db.select().from(toolConnections).where(eq(toolConnections.id, originalEndpoint.connectionId));
|
||||
const signingRef = connection!.credentialSecretRefs.find(ref => ref.configPath === "credentials.signingSecret");
|
||||
if (!signingRef) throw new Error("Normal Slack configure did not persist its signing credential binding");
|
||||
const credentialOwner = { consumerType: "tool_connection" as const, consumerId: connection!.id,
|
||||
configPath: "credentials.signingSecret", actorType: "system" as const };
|
||||
await expect(secretService(db).resolveSecretValue(companyId, signingRef.secretId, "latest", credentialOwner)).resolves.toBe(secret);
|
||||
cloud.setUnavailable(true);
|
||||
const requests = vi.mocked(cloud.request).mock.calls.length;
|
||||
const tools = toolAccessService(db);
|
||||
const removed = await tools.archiveConnection(connection!.id, companyId, { actorType: "user", actorId });
|
||||
expect(removed.connection).toMatchObject({ id: connection!.id, status: "archived", enabled: false });
|
||||
const archived = await row(endpointId);
|
||||
expect(archived.status).toBe("archived");
|
||||
const [revoked] = await db.select().from(toolConnections).where(eq(toolConnections.id, connection!.id));
|
||||
expect(revoked!.credentialSecretRefs).toEqual([]);
|
||||
expect(revoked!.credentialRefs).toEqual([]);
|
||||
expect(await db.select().from(companySecretBindings).where(and(eq(companySecretBindings.companyId, companyId),
|
||||
eq(companySecretBindings.targetType, "tool_connection"), eq(companySecretBindings.targetId, connection!.id)))).toEqual([]);
|
||||
await expect(secretService(db).resolveSecretValue(companyId, signingRef.secretId, "latest", credentialOwner)).rejects.toThrow();
|
||||
expect(vi.mocked(cloud.request).mock.calls).toHaveLength(requests);
|
||||
const intent = (await setup(endpointId)).slackCloudIntent;
|
||||
expect(intent).toMatchObject({ version: 1, active: false, companyId, endpointId, connectionId: connection!.id,
|
||||
publicId: originalEndpoint.publicId, runtimeGeneration: Number(originalSetup.runtimeGeneration),
|
||||
subject: originalSetup.slackCloudSubject, instanceId: originalSetup.slackCloudInstanceId, attempts: 0 });
|
||||
expect(JSON.stringify(intent)).not.toContain(secret);
|
||||
await tools.archiveConnection(connection!.id, companyId, { actorType: "user", actorId });
|
||||
expect((await setup(endpointId)).slackCloudIntent).toEqual(intent);
|
||||
const owner = githubConnectionEventService(db, { connector: cloud.connector });
|
||||
await expect(owner.pollOnce({ forDelivery: true })).rejects.toMatchObject({ status: 503 });
|
||||
const retry = (await setup(endpointId)).slackCloudIntent!;
|
||||
expect(retry).toMatchObject({ ...intent, attempts: 1, failureCode: "broker_unavailable", nextAttemptAt: expect.any(String) });
|
||||
const retryAt = Date.parse(retry.nextAttemptAt!);
|
||||
expect(retryAt).toBeGreaterThan(Date.now());
|
||||
cloud.setUnavailable(false); cloud.install();
|
||||
await new Promise(resolve => setTimeout(resolve, Math.max(0, retryAt - Date.now())));
|
||||
await owner.pollOnce();
|
||||
expect((await setup(endpointId)).slackCloudIntent).toBeUndefined();
|
||||
expect((await setup(endpointId)).slackCloudRetiredGeneration).toBe(Number(originalSetup.runtimeGeneration));
|
||||
expect(cloud.calls.filter(call => call.operation === "slack-events" && call.enrollment?.active === false
|
||||
&& call.enrollment.endpointId === endpointId)).toHaveLength(1);
|
||||
expect(callbacks).toEqual([]);
|
||||
}, 60_000);
|
||||
|
||||
it("public tool removal rolls back both archive writes without dropping current Slack custody or credentials", async () => {
|
||||
const endpointId = await createEndpoint("TOOLROLLBACK"); await confirm(endpointId); await activate(endpointId); await confirm(endpointId);
|
||||
const before = await row(endpointId);
|
||||
const [connection] = await db.select().from(toolConnections).where(eq(toolConnections.id, before.connectionId));
|
||||
const failure = new Error("Synthetic removal transaction rollback"), requests = vi.mocked(cloud.request).mock.calls.length;
|
||||
const realTransaction = db.transaction.bind(db);
|
||||
const transaction = vi.spyOn(db, "transaction").mockImplementationOnce(async work => realTransaction(async tx => {
|
||||
await work(tx);
|
||||
const [archivedEndpoint] = await tx.select().from(chatEndpoints).where(eq(chatEndpoints.id, endpointId));
|
||||
const [archivedConnection] = await tx.select().from(toolConnections).where(eq(toolConnections.id, connection!.id));
|
||||
expect(archivedEndpoint!.status).toBe("archived");
|
||||
expect(archivedConnection).toMatchObject({ status: "archived", enabled: false });
|
||||
throw failure;
|
||||
}));
|
||||
try {
|
||||
await expect(toolAccessService(db).archiveConnection(connection!.id, companyId, { actorType: "user", actorId })).rejects.toBe(failure);
|
||||
expect(await row(endpointId)).toMatchObject({ status: before.status, setup: before.setup });
|
||||
const [after] = await db.select().from(toolConnections).where(eq(toolConnections.id, connection!.id));
|
||||
expect(after).toMatchObject({ status: connection!.status, enabled: connection!.enabled,
|
||||
credentialSecretRefs: connection!.credentialSecretRefs, credentialRefs: connection!.credentialRefs });
|
||||
expect(vi.mocked(cloud.request).mock.calls).toHaveLength(requests);
|
||||
} finally { transaction.mockRestore(); }
|
||||
}, 60_000);
|
||||
|
||||
it("public tool removal refuses a foreign company before Slack archival or retirement staging", async () => {
|
||||
const endpointId = await createEndpoint("TOOLFOREIGN"); await confirm(endpointId);
|
||||
const before = await row(endpointId), requests = vi.mocked(cloud.request).mock.calls.length;
|
||||
await expect(toolAccessService(db).archiveConnection(before.connectionId, randomUUID(), { actorType: "user", actorId })).rejects.toMatchObject({ status: 404 });
|
||||
expect(await row(endpointId)).toEqual(before);
|
||||
expect(vi.mocked(cloud.request).mock.calls).toHaveLength(requests);
|
||||
}, 60_000);
|
||||
});
|
||||
@@ -381,15 +381,27 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
|
||||
// window. A company hint must not make that real deadline wait one second.
|
||||
await coordinator!.stop(); coordinator = undefined;
|
||||
const dueAt = Date.now() + 500;
|
||||
const actualWakeup = wakeup.getMockImplementation()!;
|
||||
let otherWakeEnteredAt: number | null = null;
|
||||
wakeup.mockImplementation((id, options) => {
|
||||
// Capture dispatch ENTRY before the original mock performs any DB I/O.
|
||||
if (id === other.agentId) otherWakeEnteredAt ??= Date.now();
|
||||
return actualWakeup(id, options);
|
||||
});
|
||||
await db.update(statusCards).set({ nextEvalAt: new Date(dueAt), nextEvalReason: "refresh" }).where(eq(statusCards.id, other.id));
|
||||
await start(service); await flush();
|
||||
await issueService(db).addComment(otherIssue.id, "Another company's coalesced human update", { userId: "board-user" });
|
||||
await issueService(db).addComment((await db.select({ id: issues.id }).from(issues).where(eq(issues.companyId, originalCompany)))[0]!.id,
|
||||
"More changes while another company is due", { userId: "board-user" });
|
||||
await flush();
|
||||
const armedBeforeAdvance = coordinator!.nextWakeAt();
|
||||
// A null deadline can mean the due work is already running. Any still
|
||||
// armed deadline must preserve the known earlier obligation.
|
||||
if (armedBeforeAdvance !== null) expect(armedBeforeAdvance).toBeLessThanOrEqual(dueAt);
|
||||
await vi.advanceTimersByTimeAsync(500); await flush();
|
||||
await vi.waitFor(() => expect(wakeup).toHaveBeenCalledOnce());
|
||||
expect(Date.now()).toBeLessThan(dueAt + 500);
|
||||
expect(otherWakeEnteredAt).not.toBeNull();
|
||||
expect(otherWakeEnteredAt!).toBeLessThan(dueAt + 500);
|
||||
expect(wakeup.mock.calls[0]![0]).not.toBe(originalAgent);
|
||||
const generated = (await statusCardService(db).getById(other.id))!;
|
||||
expect(generated.generatingIssueId).toBeTruthy();
|
||||
|
||||
@@ -8,7 +8,7 @@ import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import express from "express";
|
||||
import { DurablePrpControlPlane } from "../vendor/paperclip-runner/index.js";
|
||||
import { registerRunnerPrpAuthority, runnerPrpWebSocketInternals } from "../realtime/runner-prp-ws.js";
|
||||
import { resolveLocalRuntimeApiUrl } from "../runtime-api.js";
|
||||
import { LocalRuntimeApiListenerClosedError, resolveLocalRuntimeApiUrl } from "../runtime-api.js";
|
||||
import { createPaperclipHttpServer } from "./server.js";
|
||||
import { privateHostnameGuard } from "../middleware/private-hostname-guard.js";
|
||||
|
||||
@@ -206,6 +206,30 @@ describe("Paperclip HTTP server runner transport", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("distinguishes a listener closed for shutdown from one that never served", async () => {
|
||||
expect(() => resolveLocalRuntimeApiUrl()).toThrow("local_runtime_api_listener_unavailable");
|
||||
expect(() => resolveLocalRuntimeApiUrl()).not.toThrow(LocalRuntimeApiListenerClosedError);
|
||||
const server = createPaperclipHttpServer((_req, res) => res.end("closing"), { apiUrl: "http://ignored.example.test" });
|
||||
server.listen(0, "127.0.0.1");
|
||||
await once(server, "listening");
|
||||
const client = await fetch(resolveLocalRuntimeApiUrl(), { keepalive: true });
|
||||
// `close()` stops admission at once; its close event waits for open requests.
|
||||
const closed = new Promise<void>(resolve => server.close(() => resolve()));
|
||||
expect(() => resolveLocalRuntimeApiUrl()).toThrow(LocalRuntimeApiListenerClosedError);
|
||||
await client.text();
|
||||
server.closeAllConnections();
|
||||
await closed;
|
||||
expect(() => resolveLocalRuntimeApiUrl()).toThrow(LocalRuntimeApiListenerClosedError);
|
||||
expect(() => resolveLocalRuntimeApiUrl()).toThrow("local_runtime_api_listener_unavailable");
|
||||
server.listen(0, "127.0.0.1");
|
||||
await once(server, "listening");
|
||||
try {
|
||||
expect(resolveLocalRuntimeApiUrl()).toMatch(/^http:\/\/127\.0\.0\.1:\d+$/);
|
||||
} finally {
|
||||
await new Promise<void>(resolve => server.close(() => resolve()));
|
||||
}
|
||||
});
|
||||
|
||||
it.each(["127.0.0.1", "::1", "::ffff:127.0.0.1"])("routes a guarded native read through the real %s listener", async host => {
|
||||
await guardedNativeRead(host);
|
||||
});
|
||||
|
||||
@@ -4,15 +4,37 @@ import { isIP } from "node:net";
|
||||
|
||||
const runtimeApiListeners = new Set<Server>();
|
||||
const registeredRuntimeApiListeners = new WeakSet<Server>();
|
||||
let runtimeApiListenerClosed = false;
|
||||
|
||||
/**
|
||||
* The process had a runtime API listener and has since closed it, which only
|
||||
* happens while it shuts down. Work that needs a callback origin must hand off
|
||||
* to the next controller instead of failing as a provider or startup error.
|
||||
*/
|
||||
export class LocalRuntimeApiListenerClosedError extends Error {
|
||||
constructor() {
|
||||
// Keep the established failure code for classifiers and persisted rows.
|
||||
super("local_runtime_api_listener_unavailable");
|
||||
this.name = "LocalRuntimeApiListenerClosedError";
|
||||
}
|
||||
}
|
||||
|
||||
/** Track listener ownership without treating configured or advertised URLs as authority. */
|
||||
export function registerLocalRuntimeApiServer(server: Server): void {
|
||||
if (!registeredRuntimeApiListeners.has(server)) {
|
||||
registeredRuntimeApiListeners.add(server);
|
||||
server.on("listening", () => runtimeApiListeners.add(server));
|
||||
server.on("close", () => runtimeApiListeners.delete(server));
|
||||
server.on("listening", () => {
|
||||
runtimeApiListeners.add(server);
|
||||
runtimeApiListenerClosed = false;
|
||||
});
|
||||
server.on("close", () => {
|
||||
if (runtimeApiListeners.delete(server)) runtimeApiListenerClosed = true;
|
||||
});
|
||||
}
|
||||
if (server.listening) {
|
||||
runtimeApiListeners.add(server);
|
||||
runtimeApiListenerClosed = false;
|
||||
}
|
||||
if (server.listening) runtimeApiListeners.add(server);
|
||||
}
|
||||
|
||||
function normalizeHost(value: string | null | undefined): string {
|
||||
@@ -94,6 +116,11 @@ export function chooseLocalRuntimeApiUrl(input: {
|
||||
|
||||
export function resolveLocalRuntimeApiUrl(): string {
|
||||
const listeners = [...runtimeApiListeners].filter(server => server.listening);
|
||||
// `server.close()` clears `listening` at once; its close event waits for
|
||||
// admitted requests. Either state means this process stopped serving.
|
||||
if (listeners.length === 0 && (runtimeApiListenerClosed || runtimeApiListeners.size > 0)) {
|
||||
throw new LocalRuntimeApiListenerClosedError();
|
||||
}
|
||||
if (listeners.length !== 1) {
|
||||
throw new Error(listeners.length ? "local_runtime_api_listener_ambiguous" : "local_runtime_api_listener_unavailable");
|
||||
}
|
||||
@@ -103,7 +130,10 @@ export function resolveLocalRuntimeApiUrl(): string {
|
||||
}
|
||||
|
||||
export const runtimeApiInternals = {
|
||||
resetForTests(): void { runtimeApiListeners.clear(); },
|
||||
resetForTests(): void {
|
||||
runtimeApiListeners.clear();
|
||||
runtimeApiListenerClosed = false;
|
||||
},
|
||||
};
|
||||
|
||||
export function choosePrimaryRuntimeApiUrl(input: {
|
||||
|
||||
@@ -174,7 +174,7 @@ export function aiConnectionService(db: Db) {
|
||||
: ("connected" as const),
|
||||
},
|
||||
];
|
||||
});
|
||||
}).sort((a, b) => a.id.localeCompare(b.id) || a.grantId.localeCompare(b.grantId));
|
||||
}
|
||||
async function setDefault(
|
||||
companyId: string,
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import { githubReviewCheckService } from "./chat-github-checks.js";
|
||||
import { createSlackCloudEventIngress } from "./chat-channels/slack/cloud-ingress.js";
|
||||
import { createSlackCloudEnrollment, stageSlackCloudEnrollmentIntent, registerSlackCloudEnrollmentOwner } from "./chat-channels/slack/cloud-enrollment.js";
|
||||
import { leasedChatProvider } from "./chat-channels/runtime/receiver-policy.js";
|
||||
import { chatReceiverSchedule } from "./chat-channels/runtime/receiver-schedule.js";
|
||||
import { registerChatReceiverWork } from "./chat-channels/runtime/receiver-work.js";
|
||||
@@ -842,6 +844,10 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {
|
||||
}
|
||||
|
||||
const withCredentialMutationLease = chatCredentialMutationLease(db, options);
|
||||
const unregisterSlackCloudIngress = createSlackCloudEventIngress(db, {
|
||||
withCredentialMutationLease, resolveCredentials, runtimeGeneration: endpoint => runtimeGeneration(endpoint.setup),
|
||||
handleWebhook: (publicId, provider, request, generation) => handleWebhook(publicId, provider, request, { slackCloudGeneration: generation }),
|
||||
});
|
||||
const { list, get, create, update } = createChatEndpoints(db, {
|
||||
endpointRecord, CAPABILITIES, PROVIDER_LABELS, withCredentialMutationLease,
|
||||
getPublicBaseUrl, getWebhookPublicBaseUrl,
|
||||
@@ -2936,10 +2942,11 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {
|
||||
githubIngressActionId: string;
|
||||
githubIngressFence: LifecycleRuntimeFence;
|
||||
superseded?: boolean;
|
||||
},
|
||||
} | { slackCloudGeneration: number },
|
||||
) {
|
||||
const serviceReceivedAtMs = Date.now();
|
||||
const githubScope = createGitHubWebhookScope(provider, internalContext, options.githubWebhookResponseBudgetMs, GITHUB_WEBHOOK_RESPONSE_BUDGET_MS);
|
||||
const slackCloudGeneration = internalContext && "slackCloudGeneration" in internalContext ? internalContext.slackCloudGeneration : undefined;
|
||||
const githubScope = createGitHubWebhookScope(provider, internalContext && "githubIngressActionId" in internalContext ? internalContext : undefined, options.githubWebhookResponseBudgetMs, GITHUB_WEBHOOK_RESPONSE_BUDGET_MS);
|
||||
const { replayingDurableGitHubIngress, ignoreSupersededIngress, matchesGitHubIngressFence, githubResponseDeadlineAt } = githubScope;
|
||||
const endpoint = await db
|
||||
.select()
|
||||
@@ -2962,6 +2969,8 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {
|
||||
throw notFound("Chat endpoint not found");
|
||||
}
|
||||
recordChatWebhookStage("endpoint_resolved", endpoint.id);
|
||||
if (provider === "slack" && slackCloudGeneration !== undefined && runtimeGeneration(endpoint.setup) !== slackCloudGeneration)
|
||||
return new Response("superseded", { status: 200 });
|
||||
// A pause is a durable ingress fence. Providers generally retry non-2xx
|
||||
// webhooks, so acknowledge late callbacks without recreating a runtime or
|
||||
// admitting any Paperclip mutation.
|
||||
@@ -2998,6 +3007,8 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {
|
||||
code: "chat_endpoint_runtime_superseded",
|
||||
});
|
||||
}
|
||||
if (provider === "slack" && slackCloudGeneration !== undefined && runtimeContext.generation !== slackCloudGeneration)
|
||||
return new Response("superseded", { status: 200 });
|
||||
// runtimeFor can return a newer instance than the leased preflight saw.
|
||||
// Keep the exact staged epoch; callbacks on a matching instance retain
|
||||
// their normal transactional fences if a pause wins after this check.
|
||||
@@ -3403,7 +3414,14 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {
|
||||
: null;
|
||||
}
|
||||
|
||||
const slackCloudEnrollment = createSlackCloudEnrollment(db, { resolveCredentials, runtimeGeneration });
|
||||
const unregisterSlackCloudEnrollment = registerSlackCloudEnrollmentOwner(db, async endpointId => {
|
||||
const record = await endpointRecord(endpointId);
|
||||
if (!record) throw new Error("Slack enrollment endpoint is absent");
|
||||
await withCredentialMutationLease(record.endpoint, lease => slackCloudEnrollment(endpointId, undefined, lease));
|
||||
});
|
||||
const { configure, configureWithCredentialLease, runtimeConfig } = createChatConfiguration(db, {
|
||||
stageConfigurationEffects: stageSlackCloudEnrollmentIntent,
|
||||
endpointRecord, get, withCredentialMutationLease, getWebhookPublicBaseUrl, invalidateRuntime,
|
||||
nativeBotIdentityMatches, assertNativeBotIdentityAvailable, isNativeBotIdentityUniqueViolation, nativeBotIdentityConflict,
|
||||
legacyGitHubLabelsMatchVerifiedCredentials, normalizedCredentials, verifyCredentials,
|
||||
@@ -3611,6 +3629,8 @@ return {
|
||||
unregisterSlackTaskAuthority();
|
||||
voice.unregisterAgentTools();
|
||||
unregisterGitHubCloudIngress();
|
||||
unregisterSlackCloudIngress();
|
||||
unregisterSlackCloudEnrollment();
|
||||
unregisterCommittedResponseAuthority();
|
||||
await Promise.allSettled([...publicationEndpointTasks.values()]);
|
||||
await Promise.allSettled([...backgroundMessageTasks]);
|
||||
|
||||
@@ -100,6 +100,7 @@ type ChatConfigurationOptions = Pick<ReturnType<typeof createChatCredentials>, "
|
||||
githubWizard: Pick<ReturnType<typeof createGitHubAppSetup>["githubWizard"], "repairCloud">;
|
||||
nativeBotIdentityClaimBarrier?: () => Promise<void>;
|
||||
allowLocalVoiceBoard?: boolean;
|
||||
stageConfigurationEffects?: (transaction: Db, endpointId: string, actorUserId: string | null | undefined) => Promise<void>;
|
||||
};
|
||||
export function createChatConfiguration(db: Db, options: ChatConfigurationOptions) {
|
||||
const { endpointRecord, get, withCredentialMutationLease, getWebhookPublicBaseUrl, invalidateRuntime, nativeBotIdentityMatches, assertNativeBotIdentityAvailable, isNativeBotIdentityUniqueViolation, nativeBotIdentityConflict, legacyGitHubLabelsMatchVerifiedCredentials, normalizedCredentials, verifyCredentials, persistCredentials, clearCredentials, resolveCredentials, prepareProviderInventory, reconcileProviderResourceRows, reconcileDiscordCommands, runtimeFor, runtimeContextForRecord, stageTelegramMaintenance, processTelegramMaintenance, recordTelegramStopSubscription, fetchImpl, redactError, runtimeGeneration, leasedChatProvider, PROVIDER_LABELS, SUPPLIED_CREDENTIAL_KEYS, PROVIDER_CREDENTIAL_CHECK_TIMEOUT_MS, isSupportedTelegramWebhookBaseUrl } = options;
|
||||
@@ -176,6 +177,14 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption
|
||||
const record = await endpointRecord(endpointId);
|
||||
if (!record) throw notFound("Chat endpoint not found");
|
||||
const endpoint = record.endpoint;
|
||||
const mutate = <T>(work: (tx: Parameters<Parameters<Db["transaction"]>[0]>[0]) => Promise<T>) => db.transaction(async tx => {
|
||||
const result = await work(tx);
|
||||
if (endpoint.provider === "slack" && options.stageConfigurationEffects) {
|
||||
await options.stageConfigurationEffects(tx as unknown as Db, endpointId, actorUserId);
|
||||
await credentialLease.assertOwned(tx);
|
||||
}
|
||||
return result;
|
||||
});
|
||||
if (input.action === "pause") {
|
||||
if (endpoint.status !== "active") {
|
||||
throw conflict("Only an active chat connection can be paused", {
|
||||
@@ -183,7 +192,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption
|
||||
});
|
||||
}
|
||||
await managementContext?.reauthorize();
|
||||
await db.transaction(async (tx) => {
|
||||
await mutate(async (tx) => {
|
||||
await credentialLease.assertOwned(tx);
|
||||
const pausedAt = new Date();
|
||||
const current = await tx
|
||||
@@ -299,7 +308,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption
|
||||
let activatedEndpoint!: EndpointRow;
|
||||
try {
|
||||
await managementContext?.reauthorize();
|
||||
await db.transaction(async (tx) => {
|
||||
await mutate(async (tx) => {
|
||||
await credentialLease.assertOwned(tx);
|
||||
const resumedAt = new Date();
|
||||
const current = await tx
|
||||
@@ -384,7 +393,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption
|
||||
const failure = redactError(error);
|
||||
const failedAt = new Date();
|
||||
await managementContext?.reauthorize();
|
||||
await db.transaction(async (tx) => {
|
||||
await mutate(async (tx) => {
|
||||
await credentialLease.assertOwned(tx);
|
||||
await notifyChatEndpointConfiguration(tx, endpoint);
|
||||
await notifyChatEndpointWork(tx, endpoint.connectionId, endpoint.companyId);
|
||||
@@ -478,7 +487,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption
|
||||
});
|
||||
}
|
||||
await managementContext?.reauthorize();
|
||||
await db.transaction(async (tx) => {
|
||||
await mutate(async (tx) => {
|
||||
await credentialLease.assertOwned(tx);
|
||||
const current = await tx
|
||||
.select({ setup: chatEndpoints.setup, status: chatEndpoints.status })
|
||||
@@ -724,7 +733,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption
|
||||
if (!endpoint.botExternalId) {
|
||||
try {
|
||||
await managementContext?.reauthorize();
|
||||
const claimed = await db.transaction(async (tx) => {
|
||||
const claimed = await mutate(async (tx) => {
|
||||
await credentialLease.assertOwned(tx);
|
||||
await notifyChatEndpointConfiguration(tx, endpoint);
|
||||
await notifyChatEndpointWork(tx, endpoint.connectionId, endpoint.companyId);
|
||||
@@ -793,7 +802,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption
|
||||
(input.action === "configure" || input.action === "reconnect");
|
||||
try {
|
||||
await managementContext?.reauthorize();
|
||||
await db.transaction(async (tx) => {
|
||||
await mutate(async (tx) => {
|
||||
await credentialLease.assertOwned(tx);
|
||||
const current = await tx
|
||||
.select({ setup: chatEndpoints.setup })
|
||||
@@ -892,7 +901,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption
|
||||
const failure = redactError(error);
|
||||
await invalidateRuntime(endpoint.id).catch(() => undefined);
|
||||
await managementContext?.reauthorize();
|
||||
await db.transaction(async (tx) => {
|
||||
await mutate(async (tx) => {
|
||||
await credentialLease.assertOwned(tx);
|
||||
await notifyChatEndpointConfiguration(tx, endpoint);
|
||||
await notifyChatEndpointWork(tx, endpoint.connectionId, endpoint.companyId);
|
||||
@@ -922,7 +931,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption
|
||||
}
|
||||
|
||||
await managementContext?.reauthorize();
|
||||
await db.transaction(async (tx) => {
|
||||
await mutate(async (tx) => {
|
||||
await credentialLease.assertOwned(tx);
|
||||
await logActivity(tx as unknown as Db, {
|
||||
companyId: endpoint.companyId,
|
||||
|
||||
@@ -48,6 +48,34 @@ export type InternalSetupState = ChatEndpointSetupState & {
|
||||
runtimeGeneration?: number;
|
||||
/** Binds pre-install URL verification to the signing secret later configured. */
|
||||
slackVerificationSigningFingerprint?: string;
|
||||
|
||||
slackCloudSubject?: string;
|
||||
slackCloudInstanceId?: string;
|
||||
slackCloudRetiredGeneration?: number;
|
||||
slackCloudIntent?: {
|
||||
version: 1;
|
||||
id: string;
|
||||
subject: string;
|
||||
instanceId: string;
|
||||
companyId: string;
|
||||
endpointId: string;
|
||||
publicId: string;
|
||||
connectionId: string;
|
||||
runtimeGeneration: number;
|
||||
active: boolean;
|
||||
teamId?: string;
|
||||
appId?: string;
|
||||
attempts: number;
|
||||
nextAttemptAt: string | null;
|
||||
failureCode?: string;
|
||||
};
|
||||
slackCloudEvents?: {
|
||||
version: 1;
|
||||
bindingId: string;
|
||||
runtimeGeneration: number;
|
||||
teamId: string;
|
||||
appId: string;
|
||||
};
|
||||
slackCallbackSurfaces?: Partial<
|
||||
Record<SlackCallbackSurface, SlackCallbackObservation>
|
||||
>;
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
import { generateKeyPairSync } from "node:crypto";
|
||||
import { registerSlackCloudEnrollmentOwner } from "./cloud-enrollment.js";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { afterEach, expect, it, vi } from "vitest";
|
||||
import { githubConnectionEventService } from "../../github-connection-events.js";
|
||||
import type { PaperclipCloudConnector } from "../../paperclip-cloud-connector.js";
|
||||
import { registerSlackCloudEventIngress } from "./cloud-ingress.js";
|
||||
import { SlackCloudReceiptError, type SlackCloudEvent } from "./cloud-event.js";
|
||||
|
||||
afterEach(() => vi.unstubAllEnvs());
|
||||
function fixture(handler: (event: SlackCloudEvent) => Promise<void>, pendingEnrollment = false) {
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_BASE_URL", "");
|
||||
let reads = 0;
|
||||
const db = { select: () => {
|
||||
const rows = [[], [], [{ companyId: "company" }], [{ id: "endpoint", setup: { slackCloudIntent: { nextAttemptAt: "2000-01-01T00:00:00Z" } } }]][reads++ % (pendingEnrollment ? 4 : 3)]!;
|
||||
const query = { from: () => query, innerJoin: () => query, where: () => query, orderBy: () => query, limit: async () => rows,
|
||||
then: (resolve: (rows: unknown[]) => unknown) => Promise.resolve(rows).then(resolve) };
|
||||
return query;
|
||||
} } as unknown as Db;
|
||||
const event: SlackCloudEvent = { id: "slack-receipt", provider: "slack", event: "event_callback", installationId: "binding", bindingIds: ["binding"],
|
||||
createdAt: new Date().toISOString(), payload: {} };
|
||||
const connector = { leaseEvents: vi.fn(async () => ({ leaseId: "lease", events: [event] })),
|
||||
acknowledgeEvents: vi.fn(async () => 1), eventWakeVersion: () => 1 as const };
|
||||
const unregister = registerSlackCloudEventIngress(db, handler);
|
||||
let time = Date.now();
|
||||
const service = githubConnectionEventService(db, { connector: connector as unknown as PaperclipCloudConnector, now: () => new Date(time) });
|
||||
return { db, service, connector, unregister, event, time: () => time, advance: (ms: number) => { time += ms; } };
|
||||
}
|
||||
|
||||
it("actual connection-event owner routes Slack and retries a lost ACK through the registered owner", async () => {
|
||||
const handler = vi.fn(async () => {}), f = fixture(handler);
|
||||
try {
|
||||
f.connector.acknowledgeEvents.mockRejectedValueOnce(new Error("ack response lost"));
|
||||
await expect(f.service.pollOnce({ forDelivery: true })).rejects.toThrow("ack response lost");
|
||||
expect((await f.service.pollOnce({ forDelivery: true })).processed).toBe(1);
|
||||
expect(handler).toHaveBeenCalledTimes(2);
|
||||
expect(f.connector.leaseEvents).toHaveBeenCalledWith({ subject: "github-bot-events", companyId: "company", eventWakeVersion: 1 });
|
||||
expect(f.connector.acknowledgeEvents).toHaveBeenLastCalledWith(expect.objectContaining({ deliveryIds: [f.event.id] }));
|
||||
} finally { f.unregister(); }
|
||||
});
|
||||
|
||||
it.each(["retry", "attention", "invalid"] as const)("actual owner distinguishes %s receipt failure before ACK", async disposition => {
|
||||
const f = fixture(async () => { throw new SlackCloudReceiptError("synthetic_code", disposition); });
|
||||
try {
|
||||
const result = await f.service.pollOnce({ forDelivery: true });
|
||||
expect(result.failed).toBe(disposition === "invalid" ? 0 : 1);
|
||||
expect(f.connector.acknowledgeEvents).toHaveBeenCalledTimes(disposition === "invalid" ? 1 : 0);
|
||||
} finally { f.unregister(); }
|
||||
});
|
||||
|
||||
it.each(["chat_endpoint_credentials_busy", "temporary enrollment owner failure"])("drains and ACKs before retaining %s on a bounded existing-worker retry", async message => {
|
||||
const handler = vi.fn(async () => {}), f = fixture(handler, true);
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_BASE_URL", "https://cloud.example.test");
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID", "instance");
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_ENVIRONMENT", "test");
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SIGN_PRIVATE_KEY", generateKeyPairSync("ed25519").privateKey.export({ format: "jwk" }).d!);
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SEAL_PRIVATE_KEY", generateKeyPairSync("x25519").privateKey.export({ format: "jwk" }).d!);
|
||||
const owner = vi.fn(async () => {
|
||||
expect(handler).toHaveBeenCalled();
|
||||
expect(f.connector.acknowledgeEvents).toHaveBeenCalled();
|
||||
throw new Error(message);
|
||||
});
|
||||
const unregister = registerSlackCloudEnrollmentOwner(f.db, owner);
|
||||
try {
|
||||
expect((await f.service.pollOnce({ forDelivery: true })).processed).toBe(1);
|
||||
expect(owner).toHaveBeenCalledTimes(1);
|
||||
expect(f.service.nextPollAt()).toBe(f.time() + 5_000);
|
||||
await f.service.pollOnce();
|
||||
expect(owner).toHaveBeenCalledTimes(1);
|
||||
expect(f.connector.leaseEvents).toHaveBeenCalledTimes(1);
|
||||
f.advance(5_000);
|
||||
expect((await f.service.pollOnce()).processed).toBe(1);
|
||||
expect(owner).toHaveBeenCalledTimes(2);
|
||||
} finally { unregister(); f.unregister(); }
|
||||
});
|
||||
@@ -0,0 +1,451 @@
|
||||
import { guardSlackFixtureNetwork, stubSlackFixtureTransport } from "../../../__tests__/helpers/slack-cloud/transport.js";
|
||||
import { createHmac, generateKeyPairSync, randomUUID } from "node:crypto";
|
||||
import { once } from "node:events";
|
||||
import { createServer } from "node:http";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import type { ToolCredentialSecretRef } from "@paperclipai/shared";
|
||||
import { expect, it, vi } from "vitest";
|
||||
import { activityLog, agentRuntimeState, agentTaskSessions, agentWakeupRequests, agents, applyPendingMigrations, authUsers, chatConversations, chatDeliveries, chatExternalPrincipals, chatIdentityLinks, chatEndpointResources, chatEndpoints, chatMessageLinks, chatSlackRegistrations, heartbeatRunEvents, heartbeatRuns, companies, companyMemberships, companySecrets, companySkills, costEvents, financeEvents, closeCheckpointDb, closeFileStorageDb, closeLivenessDb, createDb, issueComments, issues, toolApplications, toolConnections } from "@paperclipai/db";
|
||||
import { resolveExternalTestDatabaseUrl } from "../../../lib/dedicated-test-database-url.js";
|
||||
import { subscribeDatabaseWork } from "@paperclipai/db/work-signals";
|
||||
import { DELIVERY_QUEUES } from "../../delivery-work-notifications.js";
|
||||
import { chatCredentialMutationLease } from "../../chat-credential-mutation-lease.js";
|
||||
import { createPaperclipCloudConnector, paperclipCloudConnectorConfigFromEnv } from "../../paperclip-cloud-connector.js";
|
||||
import { chatChannelService } from "../../chat-channels.js";
|
||||
import { ChatRuntime } from "../../chat-runtime.js";
|
||||
import { heartbeatService } from "../../heartbeat.js";
|
||||
import { githubConnectionEventService } from "../../github-connection-events.js";
|
||||
import { secretService } from "../../secrets.js";
|
||||
import type { ChatSdkMessageCallbackEvent } from "../../chat-sdk-runtime.js";
|
||||
import { createChatSdkEndpointRuntime } from "../../chat-sdk-runtime.js";
|
||||
import { createSlackCloudEnrollment, stageSlackCloudEnrollmentIntent } from "./cloud-enrollment.js";
|
||||
import { createSlackCloudEventIngress, dispatchSlackCloudEvent } from "./cloud-ingress.js";
|
||||
import type { InternalSetupState } from "../runtime-types.js";
|
||||
|
||||
const appUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_SLACK_TEST_DATABASE_URL");
|
||||
const cloudUrl = resolveExternalTestDatabaseUrl("SLACK_EVENT_TEST_DATABASE_URL");
|
||||
const cloudDist = process.env.SLACK_TEST_CLOUD_DIST;
|
||||
|
||||
it.skipIf(!appUrl || !cloudUrl || !cloudDist)("signed Slack enrollment and actual event owner preserve one domain reply through lost ACK and removal retries", async () => {
|
||||
const network = guardSlackFixtureNetwork({ requirePreload: true });
|
||||
const load = (file: string) => import(/* @vite-ignore */ pathToFileURL(path.resolve(cloudDist!, file)).href);
|
||||
const [{ PostgresCloudHarnessRegistry }, { PostgresConnectorStore }, { ConnectorBroker }, { InMemoryProviderSecretStore },
|
||||
{ providerRegistryFromEnvironment }, { CloudStackProvisioner }, { FakeCloudStackProvider }, { Pool }] = await Promise.all([
|
||||
load("registry/postgres.js"), load("connections/postgres.js"), load("connections/broker.js"), load("providers/neon.js"),
|
||||
load("connections/providers.js"), load("provisioner/provisioner.js"), load("providers/fake.js"),
|
||||
import(/* @vite-ignore */ pathToFileURL(path.resolve(cloudDist!, "../node_modules/pg/lib/index.js")).href),
|
||||
]);
|
||||
await applyPendingMigrations(appUrl!);
|
||||
const db = createDb(appUrl!, { maxConnections: 2, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
|
||||
const pool = new Pool({ connectionString: cloudUrl!, max: 2 });
|
||||
|
||||
const registry = Reflect.construct(PostgresCloudHarnessRegistry, [pool]) as {
|
||||
applyMigrations(folder: string, timeout: number): Promise<void>;
|
||||
hydrate(): Promise<void>;
|
||||
close(): Promise<void>;
|
||||
};
|
||||
try {
|
||||
await registry.applyMigrations(path.resolve(cloudDist!, "../migrations"), 30_000);
|
||||
await registry.hydrate();
|
||||
}
|
||||
catch (error) { await db.$client.end({ timeout: 1 }); await registry.close(); throw error; }
|
||||
const store = new PostgresConnectorStore(pool), secrets = new InMemoryProviderSecretStore();
|
||||
const id = randomUUID(), companyId = randomUUID(), endpointId = randomUUID(), agentId = randomUUID(), connectionId = randomUUID(), applicationId = randomUUID();
|
||||
const actorId = `slack-owner-${id}`;
|
||||
const instanceId = `instance-${id}`, stackId = `slack-${id}`, publicId = "a".repeat(43), signingSecret = "0123456789abcdef0123456789abcdef";
|
||||
const signing = generateKeyPairSync("ed25519"), sealing = generateKeyPairSync("x25519"), now = new Date();
|
||||
let broker: InstanceType<typeof ConnectorBroker>;
|
||||
let enrollmentRequests = 0;
|
||||
const server = createServer((req, res) => {
|
||||
if (req.url === "/v1/connector/slack-events") enrollmentRequests++;
|
||||
if (brokerUnavailable) { res.writeHead(503, { "content-type": "application/json" }); res.end(JSON.stringify({ error: "fixture_broker_unavailable" })); return; }
|
||||
if (loseEnrollmentResponse && req.url === "/v1/connector/slack-events") {
|
||||
loseEnrollmentResponse = false;
|
||||
res.end = (() => { res.destroy(); return res; }) as typeof res.end;
|
||||
}
|
||||
void broker.handle(req, res, new URL(req.url!, base)).catch((error: unknown) => {
|
||||
if (!res.headersSent) res.writeHead(500); res.end(String(error));
|
||||
}); });
|
||||
let base = "";
|
||||
let brokerUnavailable = false, loseEnrollmentResponse = false;
|
||||
let unregister: (() => void) | undefined;
|
||||
let runtime: ReturnType<typeof createChatSdkEndpointRuntime> | undefined;
|
||||
const deliveryIds: string[] = [];
|
||||
let normalService: ReturnType<typeof chatChannelService> | undefined;
|
||||
let secretDirectory: string | undefined;
|
||||
let normalHeartbeat: ReturnType<typeof heartbeatService> | undefined;
|
||||
let releaseEnrollmentLease: (() => void) | undefined;
|
||||
let heldEnrollmentLease: Promise<void> | undefined;
|
||||
try {
|
||||
await new CloudStackProvisioner({ provider: new FakeCloudStackProvider(), registry }).provisionDevelopmentStack({
|
||||
idempotencyKey: id, accountGroup: { id: `account-${id}`, slug: stackId, displayName: "Slack fixture" },
|
||||
owner: { id: actorId, email: `${id}@example.test` },
|
||||
stack: { stackId, accountGroupId: `account-${id}`, stackSlug: stackId, displayName: "Slack fixture", kind: "demo", primaryHost: `${stackId}.example.test`, region: "us-east-1" },
|
||||
versionPlan: { appVersion: "fixture", schemaVersion: "fixture", rolloutTrack: "stable" },
|
||||
sleepPolicy: { desiredState: "awake" }, primaryCompany: { paperclipCompanyId: companyId, displayName: "Fixture", defaultBudgetCents: 0 },
|
||||
});
|
||||
await store.putCloudStackInstance({ id: instanceId, kind: "cloud_stack", companyStackId: stackId, environment: "test",
|
||||
signPublicKey: signing.publicKey.export({ format: "jwk" }).x!, sealPublicKey: sealing.publicKey.export({ format: "jwk" }).x!,
|
||||
origins: ["https://tenant.example"], activeOrigin: "https://tenant.example", label: "Fixture", status: "active", createdAt: now, updatedAt: now });
|
||||
server.listen(0, "127.0.0.1"); await once(server, "listening");
|
||||
const address = server.address(); if (!address || typeof address === "string") throw new Error("Missing fixture address");
|
||||
base = `http://127.0.0.1:${address.port}`;
|
||||
const wake = vi.fn(async () => {});
|
||||
broker = new ConnectorBroker({ publicOrigin: base, environment: "test", store,
|
||||
providers: providerRegistryFromEnvironment({}, "test"), secrets: { resolve: async () => undefined },
|
||||
authenticate: async () => ({ globalUserId: actorId }), canApproveInstance: async (principal: { globalUserId: string }) => principal.globalUserId === actorId,
|
||||
slackEvents: { secrets, secretRefPrefix: "secret:///test/slack" },
|
||||
eventWakeEnrollmentEnabled: true, eventWakeDeliveryEnabled: true, wakeCloudStack: wake });
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID", instanceId);
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SIGN_PRIVATE_KEY", signing.privateKey.export({ format: "jwk" }).d!);
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SEAL_PRIVATE_KEY", sealing.privateKey.export({ format: "jwk" }).d!);
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_ENVIRONMENT", "test");
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_BASE_URL", base);
|
||||
await db.insert(companies).values({ id: companyId, name: "Slack", issuePrefix: `S${id.slice(0, 7)}` });
|
||||
await db.insert(authUsers).values({ id: actorId, name: "Slack owner", email: `${id}@owner.example.test`, emailVerified: true, createdAt: now, updatedAt: now });
|
||||
await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: actorId, status: "active", membershipRole: "operator" });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Slack", adapterType: "process", status: "idle", runtimeConfig: { heartbeat: { wakeOnDemand: true, intervalSec: 0 } } });
|
||||
await db.insert(companyMemberships).values({ companyId, principalType: "agent", principalId: agentId, status: "active", membershipRole: "member" });
|
||||
await db.insert(toolApplications).values({ id: applicationId, companyId, name: "Slack", type: "chat" });
|
||||
await db.insert(toolConnections).values({ id: connectionId, companyId, applicationId, name: "Slack", uid: id,
|
||||
transport: "chat_sdk", connectionPurpose: "channel", enabled: true, status: "active" });
|
||||
await db.insert(chatEndpoints).values({ id: endpointId, companyId, connectionId, assignedAgentId: agentId,
|
||||
publicId, provider: "slack", status: "active", providerAccountId: "TTEST", setup: { runtimeGeneration: 1 } as InternalSetupState });
|
||||
await db.insert(chatSlackRegistrations).values({ companyId, endpointId, requestId: randomUUID(), status: "configured",
|
||||
manifest: {}, manifestHash: "fixture", callbackUri: "https://tenant.example/callback", appId: "ATEST", createdByUserId: actorId });
|
||||
const withLease = chatCredentialMutationLease(db);
|
||||
const resolveCredentials = vi.fn(async () => ({ signingSecret }));
|
||||
const produce = createSlackCloudEnrollment(db, { resolveCredentials,
|
||||
runtimeGeneration: setup => Number(setup.runtimeGeneration ?? 0) });
|
||||
const endpoint = async () => (await db.select().from(chatEndpoints).where(eq(chatEndpoints.id, endpointId)))[0]!;
|
||||
await db.transaction(tx => stageSlackCloudEnrollmentIntent(tx as unknown as typeof db, endpointId, actorId));
|
||||
await withLease(await endpoint(), lease => produce(endpointId, actorId, lease));
|
||||
const saved = (await endpoint()).setup as InternalSetupState;
|
||||
expect(saved.slackCloudEvents?.runtimeGeneration).toBe(1);
|
||||
expect(JSON.stringify(saved)).not.toContain(signingSecret);
|
||||
const rawBody = Buffer.from(JSON.stringify({ type: "event_callback", event_id: "EvPaired", team_id: "TTEST", api_app_id: "ATEST",
|
||||
event: { type: "app_mention", text: "<@UBOT> hello", user: "UOPERATOR", channel: "CTEST", ts: "1788.500", event_ts: "1788.500" } }));
|
||||
const timestamp = String(Math.floor(Date.now() / 1000));
|
||||
const request = { rawBody, timestamp, signature: `v0=${createHmac("sha256", signingSecret).update(`v0:${timestamp}:`).update(rawBody).digest("hex")}` };
|
||||
const current = await broker.slackEvents.current(instanceId, publicId);
|
||||
await expect(broker.slackEvents.receive(current, request)).rejects.toThrow("SLACK_DURABLE_WAKE_UNAVAILABLE");
|
||||
const connector = createPaperclipCloudConnector({ config: paperclipCloudConnectorConfigFromEnv()! });
|
||||
expect(await connector.leaseEvents({ subject: actorId, companyId, eventWakeVersion: 1 })).toBeNull();
|
||||
expect(await broker.slackEvents.receive(current, request)).toEqual({ status: 200, duplicate: false });
|
||||
await vi.waitFor(() => expect(wake).toHaveBeenCalledOnce());
|
||||
const leased = await connector.leaseEvents({ subject: actorId, companyId, eventWakeVersion: 1 });
|
||||
expect(leased?.failedEventIds).toBeUndefined(); expect(leased?.events).toHaveLength(1);
|
||||
const event = leased!.events[0]!; deliveryIds.push(event.id);
|
||||
if (event.provider !== "slack") throw new Error("Unexpected event owner");
|
||||
await expect(dispatchSlackCloudEvent(db, event)).rejects.toThrow("not ready");
|
||||
const onMessage = vi.fn(async (_event: ChatSdkMessageCallbackEvent) => {});
|
||||
runtime = createChatSdkEndpointRuntime({ companyId, endpointId, logger: "silent", callbacks: { onMessage },
|
||||
persistence: { read: async () => null, compareAndSet: async () => true, deleteIfVersion: async () => true },
|
||||
providerConfig: { provider: "slack", userName: "paperclip-agent", credentials: { apiUrl: "http://127.0.0.1:1/api/", botToken: "xoxb-fixture", botUserId: "UBOT", signingSecret } } });
|
||||
await runtime.initialize();
|
||||
const transport = stubSlackFixtureTransport(runtime.getProviderAdapter());
|
||||
unregister = createSlackCloudEventIngress(db, { withCredentialMutationLease: withLease, resolveCredentials,
|
||||
runtimeGeneration: value => Number((value.setup as InternalSetupState).runtimeGeneration ?? 0),
|
||||
handleWebhook: async (_publicId, _provider, req, generation) => {
|
||||
expect(generation).toBe(1);
|
||||
|
||||
await withLease(await endpoint(), async lease => { await lease.assertOwned(); });
|
||||
return runtime!.handleWebhook(req);
|
||||
} });
|
||||
await dispatchSlackCloudEvent(db, event);
|
||||
expect(onMessage).toHaveBeenCalledOnce();
|
||||
expect(onMessage).toHaveBeenCalledWith(expect.objectContaining({ message: expect.objectContaining({ author: expect.objectContaining({ userId: "UOPERATOR", userName: "Operator" }) }) }));
|
||||
expect(transport).toHaveBeenCalled();
|
||||
await transport.assertNoErrors();
|
||||
expect(await connector.acknowledgeEvents({ subject: actorId, companyId, leaseId: leased!.leaseId, deliveryIds: [event.id] })).toBe(1);
|
||||
expect(await connector.leaseEvents({ subject: actorId, companyId, eventWakeVersion: 1 })).toBeNull();
|
||||
await db.update(chatEndpoints).set({ setup: { ...saved, runtimeGeneration: 2 } as InternalSetupState }).where(eq(chatEndpoints.id, endpointId));
|
||||
await dispatchSlackCloudEvent(db, event);
|
||||
expect(onMessage).toHaveBeenCalledOnce();
|
||||
await db.transaction(tx => stageSlackCloudEnrollmentIntent(tx as unknown as typeof db, endpointId, actorId));
|
||||
await withLease(await endpoint(), lease => produce(endpointId, actorId, lease));
|
||||
const replacement = ((await endpoint()).setup as InternalSetupState).slackCloudEvents!;
|
||||
expect(replacement.runtimeGeneration).toBe(2); expect(replacement.bindingId).not.toBe(saved.slackCloudEvents!.bindingId);
|
||||
const firstMessage = onMessage.mock.calls[0]![0];
|
||||
const [task] = await db.insert(issues).values({ companyId, title: "Existing Slack conversation", status: "in_progress",
|
||||
assigneeAgentId: agentId, createdByUserId: actorId, identifier: `S${id.slice(0, 7)}-1`, issueNumber: 1 }).returning();
|
||||
const [resource] = await db.insert(chatEndpointResources).values({ companyId, endpointId, type: "channel",
|
||||
providerResourceId: firstMessage.thread.channelId, label: "Slack fixture", availability: "available", enabled: true }).returning();
|
||||
await db.insert(chatConversations).values({ companyId, endpointId, issueId: task!.id, resourceId: resource!.id,
|
||||
externalConversationId: firstMessage.thread.channelId, externalThreadId: firstMessage.thread.id,
|
||||
externalLabel: "Slack fixture", isDirectMessage: firstMessage.thread.isDM });
|
||||
secretDirectory = await mkdtemp(path.join(os.tmpdir(), "slack-owner-secrets-"));
|
||||
vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY_FILE", path.join(secretDirectory, "master.key"));
|
||||
const markerPath = path.join(secretDirectory, "normal-process-marker.json");
|
||||
const markerText = `slack-domain-once-${id}`;
|
||||
await db.update(agents).set({ adapterConfig: { command: process.execPath, cwd: secretDirectory,
|
||||
args: ["--require", fileURLToPath(new URL("../../../__tests__/helpers/slack-cloud/prewire.cjs", import.meta.url)), "-e", `if(process.env.SLACK_GATE_GUARD_PID!==String(process.pid))throw Error("child prewire guard missing");require("node:fs").writeFileSync(${JSON.stringify(markerPath)},JSON.stringify({pid:process.pid,runId:process.env.PAPERCLIP_RUN_ID,marker:${JSON.stringify(markerText)}}),{flag:"ax"});console.log(${JSON.stringify(markerText)});`],
|
||||
timeoutSec: 10, graceSec: 1 } }).where(eq(agents.id, agentId));
|
||||
const vault = secretService(db), refs: ToolCredentialSecretRef[] = [];
|
||||
for (const [key, value] of Object.entries({ botToken: "xoxb-fixture", signingSecret })) {
|
||||
const secret = await vault.create(companyId, { name: `slack-${key}-${id}`, provider: "local_encrypted", value }, { userId: actorId });
|
||||
await vault.createBinding({ companyId, secretId: secret.id, targetType: "tool_connection", targetId: connectionId, configPath: `credentials.${key}` });
|
||||
refs.push({ secretId: secret.id, versionSelector: "latest", configPath: `credentials.${key}`, required: true, projectionClass: "unclassified" });
|
||||
}
|
||||
await db.update(toolConnections).set({ credentialSecretRefs: refs }).where(eq(toolConnections.id, connectionId));
|
||||
const [linkedPrincipal] = await db.insert(chatExternalPrincipals).values({ companyId, provider: "slack", providerAccountId: "TTEST",
|
||||
externalId: "UOPERATOR", kind: "user", displayName: "Operator" }).returning();
|
||||
await db.insert(chatIdentityLinks).values({ companyId, endpointId, principalId: linkedPrincipal!.id, paperclipUserId: actorId,
|
||||
status: "linked", confirmedAt: now });
|
||||
const canonicalCallback = "https://tenant.example/api/chat-webhooks/" + publicId + "/slack";
|
||||
await db.update(chatEndpoints).set({ botExternalId: "UBOT", sponsorUserId: actorId, allowGroupChats: true,
|
||||
setup: { ...((await endpoint()).setup as InternalSetupState), step: "complete", webhookVerifiedAt: now.toISOString(),
|
||||
slackCallbackSurfaces: { events: { url: canonicalCallback, observedAt: now.toISOString() } } } as InternalSetupState }).where(eq(chatEndpoints.id, endpointId));
|
||||
const callbackEvidence = structuredClone(((await endpoint()).setup as InternalSetupState).slackCallbackSurfaces);
|
||||
const actualRuntime = new ChatRuntime(), sdkCallbacks: ChatSdkMessageCallbackEvent[] = [];
|
||||
const transports: ReturnType<typeof stubSlackFixtureTransport>[] = [];
|
||||
const replaceEndpoint = actualRuntime.replaceEndpoint.bind(actualRuntime);
|
||||
vi.spyOn(actualRuntime, "replaceEndpoint").mockImplementation(async input => {
|
||||
if (input.providerConfig.provider !== "slack") throw new Error("Unexpected fixture provider");
|
||||
const value = await replaceEndpoint({ ...input, providerConfig: { ...input.providerConfig,
|
||||
credentials: { ...input.providerConfig.credentials, apiUrl: "http://127.0.0.1:1/api/" } },
|
||||
callbacks: { ...input.callbacks, onMessage: async message => { sdkCallbacks.push(message); await input.callbacks.onMessage(message); } } });
|
||||
transports.push(stubSlackFixtureTransport(value.getProviderAdapter()));
|
||||
return value;
|
||||
});
|
||||
normalHeartbeat = heartbeatService(db);
|
||||
const receiptFetch: typeof fetch = async (input, init) => {
|
||||
const url = input instanceof Request ? input.url : String(input);
|
||||
if (!["https://slack.com/api/reactions.add", "https://slack.com/api/reactions.remove"].includes(url)) throw new Error(`Unexpected normal Slack owner request ${url}`);
|
||||
expect(init?.method).toBe("POST");
|
||||
expect(JSON.parse(String(init?.body))).toMatchObject({ channel: "CTEST", name: "eyes" });
|
||||
return Response.json({ ok: true });
|
||||
};
|
||||
const deferred: (() => void | Promise<void>)[] = [];
|
||||
normalService = chatChannelService(db, { runtime: actualRuntime, publicBaseUrl: "https://tenant.example",
|
||||
deferWebhookProcessing: true, scheduleDeferredWork: work => { deferred.push(work); }, heartbeat: normalHeartbeat, fetch: receiptFetch });
|
||||
const ownerMessageTs = (Date.now() / 1000).toFixed(6);
|
||||
const ownerRawBody = Buffer.from(JSON.stringify({ type: "event_callback", event_id: "EvActualOwner", team_id: "TTEST", api_app_id: "ATEST",
|
||||
event: { type: "app_mention", text: "<@UBOT> one durable reply", user: "UOPERATOR", channel: "CTEST", ts: ownerMessageTs, event_ts: ownerMessageTs, thread_ts: "1788.500" } }));
|
||||
const ownerTimestamp = String(Math.floor(Date.now() / 1000));
|
||||
const ownerRequest = { rawBody: ownerRawBody, timestamp: ownerTimestamp,
|
||||
signature: `v0=${createHmac("sha256", signingSecret).update(`v0:${ownerTimestamp}:`).update(ownerRawBody).digest("hex")}` };
|
||||
await broker.slackEvents.receive(await broker.slackEvents.current(instanceId, publicId), ownerRequest);
|
||||
let lostAck = true;
|
||||
const captured: NonNullable<Awaited<ReturnType<typeof connector.leaseEvents>>>["events"] = [];
|
||||
const ownerConnector = { ...connector, leaseEvents: async (input: Parameters<typeof connector.leaseEvents>[0]) => {
|
||||
const batch = await connector.leaseEvents(input); if (batch) captured.push(...batch.events); return batch;
|
||||
}, acknowledgeEvents: async (input: Parameters<typeof connector.acknowledgeEvents>[0]) => {
|
||||
if (lostAck) { lostAck = false; throw new Error("fixture ACK request lost before broker settlement"); }
|
||||
return connector.acknowledgeEvents(input);
|
||||
} };
|
||||
const busyEndpointId = randomUUID(), busyConnectionId = randomUUID();
|
||||
await db.insert(toolConnections).values({ id: busyConnectionId, companyId, applicationId, name: "Retirement waiting for credential mutation", uid: busyEndpointId,
|
||||
transport: "chat_sdk", connectionPurpose: "channel", enabled: false, status: "disabled" });
|
||||
await db.insert(chatEndpoints).values({ id: busyEndpointId, companyId, connectionId: busyConnectionId, assignedAgentId: agentId,
|
||||
publicId: "f".repeat(43), provider: "slack", status: "archived", setup: { runtimeGeneration: 1 } as InternalSetupState });
|
||||
await db.transaction(tx => stageSlackCloudEnrollmentIntent(tx as unknown as typeof db, busyEndpointId, actorId));
|
||||
const [busyEndpoint] = await db.select().from(chatEndpoints).where(eq(chatEndpoints.id, busyEndpointId));
|
||||
const originalBusyIntent = structuredClone((busyEndpoint!.setup as InternalSetupState).slackCloudIntent!);
|
||||
let acquired!: () => void;
|
||||
const acquiredPromise = new Promise<void>(resolve => { acquired = resolve; });
|
||||
const releasePromise = new Promise<void>(resolve => { releaseEnrollmentLease = resolve; });
|
||||
heldEnrollmentLease = withLease(busyEndpoint!, async lease => { await lease.assertOwned(); acquired(); await releasePromise; });
|
||||
await acquiredPromise;
|
||||
const eventOwner = githubConnectionEventService(db, { connector: ownerConnector });
|
||||
await expect(eventOwner.pollOnce({ forDelivery: true })).rejects.toThrow("ACK request lost");
|
||||
expect(captured).toHaveLength(1); const actualReceiptId = captured[0]!.id; deliveryIds.push(actualReceiptId);
|
||||
const beforeRedelivery = await db.select().from(chatDeliveries).where(eq(chatDeliveries.endpointId, endpointId));
|
||||
expect(beforeRedelivery).toHaveLength(1);
|
||||
const [retainedBusy] = await db.select().from(chatEndpoints).where(eq(chatEndpoints.id, busyEndpointId));
|
||||
const busyIntent = (retainedBusy!.setup as InternalSetupState).slackCloudIntent!;
|
||||
expect(busyIntent).toEqual({ ...originalBusyIntent, attempts: 1, failureCode: "credential_lease_busy", nextAttemptAt: expect.any(String) });
|
||||
const originalRetryAt = Date.parse(busyIntent.nextAttemptAt!);
|
||||
expect(Number.isFinite(originalRetryAt)).toBe(true);
|
||||
expect(originalRetryAt).toBeGreaterThan(Date.now());
|
||||
expect(originalRetryAt).toBeLessThanOrEqual(Date.now() + 5_000);
|
||||
expect(eventOwner.nextPollAt()).not.toBeNull();
|
||||
expect(eventOwner.nextPollAt()!).toBeGreaterThan(Date.now());
|
||||
expect(eventOwner.nextPollAt()!).toBeLessThanOrEqual(Date.now() + 5_000);
|
||||
releaseEnrollmentLease!(); await heldEnrollmentLease;
|
||||
|
||||
const expired = await pool.query("UPDATE cloud_harness.connector_webhook_delivery_targets SET lease_expires_at=$3 WHERE connector_instance_id=$1 AND delivery_id=$2 AND status='leased' RETURNING delivery_id",
|
||||
[instanceId, actualReceiptId, new Date(Date.now() - 1)]);
|
||||
expect(expired.rowCount).toBe(1);
|
||||
expect((await eventOwner.pollOnce({ forDelivery: true })).processed).toBe(1);
|
||||
expect(captured.map(value => value.id)).toEqual([actualReceiptId, actualReceiptId]);
|
||||
const [beforeRetry] = await db.select().from(chatEndpoints).where(eq(chatEndpoints.id, busyEndpointId));
|
||||
if (Date.now() < originalRetryAt) expect((beforeRetry!.setup as InternalSetupState).slackCloudIntent).toEqual(busyIntent);
|
||||
await new Promise(resolve => setTimeout(resolve, Math.max(0, originalRetryAt - Date.now())));
|
||||
await eventOwner.pollOnce();
|
||||
const [settledBusy] = await db.select().from(chatEndpoints).where(eq(chatEndpoints.id, busyEndpointId));
|
||||
expect((settledBusy!.setup as InternalSetupState).slackCloudIntent).toBeUndefined();
|
||||
expect((settledBusy!.setup as InternalSetupState).slackCloudRetiredGeneration).toBe(1);
|
||||
expect((settledBusy!.setup as InternalSetupState).slackCloudRetiredGeneration).toBe(originalBusyIntent.runtimeGeneration);
|
||||
expect(captured.map(value => value.id)).toEqual([actualReceiptId, actualReceiptId]);
|
||||
|
||||
expect(captured[1]!.createdAt).toBe(captured[0]!.createdAt); expect(captured[1]!.payload).toEqual(captured[0]!.payload);
|
||||
expect(sdkCallbacks).toHaveLength(2); expect(sdkCallbacks[0]!.message.id).toBe(sdkCallbacks[1]!.message.id);
|
||||
expect(sdkCallbacks[0]!.thread.id).toBe(sdkCallbacks[1]!.thread.id);
|
||||
const afterRedelivery = await db.select().from(chatDeliveries).where(eq(chatDeliveries.endpointId, endpointId));
|
||||
expect(afterRedelivery).toHaveLength(1); expect(afterRedelivery[0]!.providerEventId).toBe(beforeRedelivery[0]!.providerEventId);
|
||||
expect(deferred.length).toBeGreaterThan(0);
|
||||
await deferred.shift()!();
|
||||
await normalService.processQueuedDeliveries();
|
||||
const links = await db.select().from(chatMessageLinks).where(and(eq(chatMessageLinks.endpointId, endpointId), eq(chatMessageLinks.direction, "inbound")));
|
||||
expect(links).toHaveLength(1); expect(links[0]!.deliveryId).toBe(afterRedelivery[0]!.id);
|
||||
const comments = await db.select().from(issueComments).where(eq(issueComments.issueId, task!.id));
|
||||
expect(comments.filter(value => value.id === links[0]!.commentId)).toHaveLength(1);
|
||||
expect(comments.find(value => value.id === links[0]!.commentId)?.body).toContain("one durable reply");
|
||||
const wakeReceipts = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.agentId, agentId));
|
||||
expect(wakeReceipts).toHaveLength(1);
|
||||
await normalHeartbeat.drainActiveRunExecutions();
|
||||
const completedRuns = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.agentId, agentId));
|
||||
expect(completedRuns).toHaveLength(1);
|
||||
expect(completedRuns[0]).toMatchObject({ status: "succeeded", exitCode: 0, wakeupRequestId: wakeReceipts[0]!.id });
|
||||
expect(completedRuns[0]!.startedAt).not.toBeNull();
|
||||
expect(completedRuns[0]!.finishedAt).not.toBeNull();
|
||||
expect(completedRuns[0]!.stdoutExcerpt).toContain(markerText);
|
||||
const marker = JSON.parse(await readFile(markerPath, "utf8")) as { pid: number; runId: string; marker: string };
|
||||
expect(marker).toMatchObject({ runId: completedRuns[0]!.id, marker: markerText });
|
||||
expect(marker.pid).not.toBe(process.pid);
|
||||
expect(((await endpoint()).setup as InternalSetupState).slackCallbackSurfaces).toEqual(callbackEvidence);
|
||||
const signingRef = refs.find(ref => ref.configPath === "credentials.signingSecret")!;
|
||||
await vault.update(signingRef.secretId, { status: "disabled" });
|
||||
try {
|
||||
await expect(dispatchSlackCloudEvent(db, captured[1]!)).rejects.toMatchObject({
|
||||
code: "signing_credential_unavailable", disposition: "attention",
|
||||
});
|
||||
expect(sdkCallbacks).toHaveLength(2);
|
||||
} finally { await vault.update(signingRef.secretId, { status: "active" }); }
|
||||
|
||||
expect(await connector.leaseEvents({ subject: actorId, companyId, eventWakeVersion: 1 })).toBeNull();
|
||||
for (const transport of transports) await transport.assertNoErrors();
|
||||
await normalService.shutdown(); normalService = undefined;
|
||||
let rejectAfterIntent = false;
|
||||
const beforeRollbackRequests = enrollmentRequests;
|
||||
const stopSignals = subscribeDatabaseWork(db, DELIVERY_QUEUES.connectionEvents, event => { if (event === "intent") rejectAfterIntent = true; });
|
||||
normalService = chatChannelService(db, { heartbeat: normalHeartbeat, fetch: receiptFetch, publicBaseUrl: "https://tenant.example",
|
||||
scheduleDeferredWork: work => { deferred.push(work); }, renewCredentialMutationLease: async () => !rejectAfterIntent });
|
||||
try {
|
||||
await expect(normalService.configure(endpointId, { action: "pause" }, actorId)).rejects.toMatchObject({ code: "CHAT_CREDENTIAL_LEASE_LOST" });
|
||||
expect((await endpoint()).status).toBe("active");
|
||||
expect(((await endpoint()).setup as InternalSetupState).runtimeGeneration).toBe(2);
|
||||
expect(((await endpoint()).setup as InternalSetupState).slackCloudIntent).toBeUndefined();
|
||||
expect(enrollmentRequests).toBe(beforeRollbackRequests);
|
||||
} finally { stopSignals(); await normalService.shutdown(); normalService = undefined; }
|
||||
normalService = chatChannelService(db, { heartbeat: normalHeartbeat, fetch: receiptFetch, publicBaseUrl: "https://tenant.example",
|
||||
scheduleDeferredWork: work => { deferred.push(work); } });
|
||||
await db.transaction(async tx => {
|
||||
await tx.update(chatEndpoints).set({ setup: { ...((await endpoint()).setup as InternalSetupState), runtimeGeneration: 3 } as InternalSetupState }).where(eq(chatEndpoints.id, endpointId));
|
||||
await stageSlackCloudEnrollmentIntent(tx as unknown as typeof db, endpointId, actorId);
|
||||
});
|
||||
loseEnrollmentResponse = true;
|
||||
await withLease(await endpoint(), lease => produce(endpointId, actorId, lease));
|
||||
expect((await broker.slackEvents.current(instanceId, publicId)).runtimeGeneration).toBe(3);
|
||||
expect(((await endpoint()).setup as InternalSetupState).slackCloudEvents!.runtimeGeneration).toBe(2);
|
||||
expect(((await endpoint()).setup as InternalSetupState).slackCloudIntent).toMatchObject({ active: true, runtimeGeneration: 3, failureCode: "broker_unavailable" });
|
||||
await withLease(await endpoint(), async lease => {
|
||||
let transactionChecks = 0;
|
||||
await produce(endpointId, actorId, { assertOwned: async transaction => {
|
||||
await lease.assertOwned(transaction);
|
||||
if (transaction && ++transactionChecks === 2) throw new Error("fixture final enrollment guard lost");
|
||||
} });
|
||||
});
|
||||
expect(((await endpoint()).setup as InternalSetupState).slackCloudEvents!.runtimeGeneration).toBe(2);
|
||||
const casProducer = createSlackCloudEnrollment(db, { runtimeGeneration: setup => Number(setup.runtimeGeneration), resolveCredentials: async () => {
|
||||
await db.update(chatEndpoints).set({ setup: sql`jsonb_set(${chatEndpoints.setup}, '{runtimeGeneration}', '4'::jsonb)` }).where(eq(chatEndpoints.id, endpointId));
|
||||
return { signingSecret };
|
||||
} });
|
||||
await expect(withLease(await endpoint(), lease => casProducer(endpointId, actorId, lease))).rejects.toThrow("changed");
|
||||
expect(((await endpoint()).setup as InternalSetupState).slackCloudEvents!.runtimeGeneration).toBe(2);
|
||||
brokerUnavailable = true;
|
||||
expect((await normalService.configure(endpointId, { action: "pause" }, actorId)).status).toBe("paused");
|
||||
expect(((await endpoint()).setup as InternalSetupState).slackCloudIntent).toMatchObject({ active: false, runtimeGeneration: 5, attempts: 0 });
|
||||
await expect(eventOwner.pollOnce({ forDelivery: true })).rejects.toThrow("status=503");
|
||||
expect(((await endpoint()).setup as InternalSetupState).slackCloudIntent).toMatchObject({ failureCode: "broker_unavailable", attempts: 1 });
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_BASE_URL", "");
|
||||
expect((await normalService.configure(endpointId, { action: "remove" }, actorId)).status).toBe("archived");
|
||||
expect(((await endpoint()).setup as InternalSetupState).slackCloudIntent).toMatchObject({ active: false, runtimeGeneration: 6, subject: actorId, attempts: 0 });
|
||||
expect(((await endpoint()).setup as InternalSetupState).slackCloudIntent).toMatchObject({ active: false, runtimeGeneration: 6, failureCode: "connector_configuration_unavailable", nextAttemptAt: null });
|
||||
expect((await db.select().from(toolConnections).where(eq(toolConnections.id, connectionId)))[0]!.credentialSecretRefs).toEqual([]);
|
||||
expect((await db.select().from(chatSlackRegistrations).where(eq(chatSlackRegistrations.endpointId, endpointId)))[0]!.status).toBe("removed");
|
||||
await normalService.shutdown(); normalService = undefined;
|
||||
brokerUnavailable = false;
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_BASE_URL", base);
|
||||
normalService = chatChannelService(db, { heartbeat: normalHeartbeat, fetch: receiptFetch, publicBaseUrl: "https://tenant.example",
|
||||
scheduleDeferredWork: work => { deferred.push(work); } });
|
||||
const reconstructedOwner = githubConnectionEventService(db, { connector });
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID", `replacement-${instanceId}`);
|
||||
await reconstructedOwner.pollOnce({ forDelivery: true });
|
||||
expect(((await endpoint()).setup as InternalSetupState).slackCloudIntent).toMatchObject({ instanceId, failureCode: "connector_configuration_unavailable", nextAttemptAt: null });
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID", instanceId);
|
||||
await reconstructedOwner.pollOnce({ forDelivery: true });
|
||||
expect(reconstructedOwner.nextPollAt()).toBeNull();
|
||||
expect(((await endpoint()).setup as InternalSetupState).slackCloudIntent).toBeUndefined();
|
||||
expect(((await endpoint()).setup as InternalSetupState).slackCloudRetiredGeneration).toBe(6);
|
||||
expect(await broker.slackEvents.current(instanceId, publicId)).toBeUndefined();
|
||||
expect((await store.listWebhookBindings(instanceId)).find((value: { receivesEvents?: boolean }) => value.receivesEvents === false).providerMetadata.retirement.runtimeGeneration).toBe(6);
|
||||
await expect(broker.slackEvents.enroll(await store.getInstance(instanceId), actorId, { companyId, endpointId, publicId, connectionId,
|
||||
runtimeGeneration: 3, teamId: "TTEST", appId: "ATEST", signingSecret })).rejects.toThrow("GENERATION_CHANGED");
|
||||
|
||||
const pendingIds = [randomUUID(), randomUUID()];
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_BASE_URL", "");
|
||||
for (const [index, pendingId] of pendingIds.entries()) {
|
||||
const pendingConnectionId = randomUUID();
|
||||
await db.insert(toolConnections).values({ id: pendingConnectionId, companyId, applicationId, name: "Retired Slack", uid: pendingId,
|
||||
transport: "chat_sdk", connectionPurpose: "channel", enabled: false, status: "disabled" });
|
||||
await db.insert(chatEndpoints).values({ id: pendingId, companyId, connectionId: pendingConnectionId, assignedAgentId: agentId,
|
||||
publicId: (index === 0 ? "d" : "e").repeat(43), provider: "slack", status: "archived", providerAccountId: "TTEST",
|
||||
setup: { runtimeGeneration: 1, slackCloudInstanceId: instanceId, slackCloudSubject: actorId } as InternalSetupState });
|
||||
await db.transaction(tx => stageSlackCloudEnrollmentIntent(tx as unknown as typeof db, pendingId, actorId));
|
||||
}
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_BASE_URL", base);
|
||||
await reconstructedOwner.pollOnce({ forDelivery: true });
|
||||
const remaining = await db.select().from(chatEndpoints).where(and(eq(chatEndpoints.companyId, companyId),
|
||||
sql`${chatEndpoints.setup}->'slackCloudIntent' is not null`));
|
||||
expect(remaining).toHaveLength(1);
|
||||
expect(pendingIds).toContain(remaining[0]!.id);
|
||||
expect(reconstructedOwner.nextPollAt()).not.toBeNull();
|
||||
expect(reconstructedOwner.nextPollAt()!).toBeLessThanOrEqual(Date.now());
|
||||
await reconstructedOwner.pollOnce({ forDelivery: true });
|
||||
expect(await db.select().from(chatEndpoints).where(and(eq(chatEndpoints.companyId, companyId),
|
||||
sql`${chatEndpoints.setup}->'slackCloudIntent' is not null`))).toHaveLength(0);
|
||||
expect(reconstructedOwner.nextPollAt()).toBeNull();
|
||||
|
||||
} catch (error) {
|
||||
console.error("Slack paired fixture body failure", error);
|
||||
throw error;
|
||||
} finally {
|
||||
releaseEnrollmentLease?.(); await heldEnrollmentLease;
|
||||
await normalHeartbeat?.drainActiveRunExecutions();
|
||||
await normalService?.shutdown();
|
||||
unregister?.(); await runtime?.shutdown();
|
||||
if (server.listening) { server.close(); await once(server, "close"); }
|
||||
vi.unstubAllEnvs();
|
||||
try {
|
||||
await db.delete(chatEndpoints).where(eq(chatEndpoints.companyId, companyId));
|
||||
await db.delete(financeEvents).where(eq(financeEvents.companyId, companyId));
|
||||
await db.delete(costEvents).where(eq(costEvents.companyId, companyId));
|
||||
await db.delete(activityLog).where(eq(activityLog.companyId, companyId));
|
||||
await db.delete(agentTaskSessions).where(eq(agentTaskSessions.companyId, companyId));
|
||||
await db.delete(agentRuntimeState).where(eq(agentRuntimeState.companyId, companyId));
|
||||
await db.delete(heartbeatRunEvents).where(eq(heartbeatRunEvents.companyId, companyId));
|
||||
await db.delete(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId));
|
||||
await db.delete(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, companyId));
|
||||
await db.delete(issueComments).where(eq(issueComments.companyId, companyId));
|
||||
await db.delete(issues).where(eq(issues.companyId, companyId));
|
||||
await db.delete(agents).where(eq(agents.companyId, companyId));
|
||||
await db.delete(companyMemberships).where(eq(companyMemberships.companyId, companyId));
|
||||
await db.delete(companySecrets).where(eq(companySecrets.companyId, companyId));
|
||||
await db.delete(companySkills).where(eq(companySkills.companyId, companyId));
|
||||
await db.delete(companies).where(eq(companies.id, companyId));
|
||||
await db.delete(authUsers).where(eq(authUsers.id, actorId));
|
||||
await pool.query("DELETE FROM cloud_harness.connector_instances WHERE id=$1", [instanceId]);
|
||||
await pool.query("DELETE FROM cloud_harness.connector_webhook_deliveries WHERE id=ANY($1::text[])", [deliveryIds]);
|
||||
} finally { await closeCheckpointDb(db, 1); await closeFileStorageDb(db, 1); await closeLivenessDb(db, 1);
|
||||
await db.$client.end({ timeout: 1 }); await registry.close();
|
||||
if (secretDirectory) await rm(secretDirectory, { recursive: true, force: true });
|
||||
try { network.assertQuiet(); } finally { network.restore(); }
|
||||
}
|
||||
}
|
||||
}, 60_000);
|
||||
@@ -0,0 +1,214 @@
|
||||
import { HttpError } from "../../../errors.js";
|
||||
import { generateKeyPairSync } from "node:crypto";
|
||||
import { chatEndpoints, chatSlackRegistrations, type Db } from "@paperclipai/db";
|
||||
import { afterEach, expect, it, vi } from "vitest";
|
||||
import { PaperclipCloudConnectorError } from "../../paperclip-cloud-connector.js";
|
||||
import { createSlackCloudEnrollment, stageSlackCloudEnrollmentIntent } from "./cloud-enrollment.js";
|
||||
import type { InternalSetupState } from "../runtime-types.js";
|
||||
import type { CredentialMutationLeaseGuard } from "../../chat-credential-mutation-lease.js";
|
||||
|
||||
afterEach(() => vi.unstubAllEnvs());
|
||||
function configureCloud() {
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_BASE_URL", "https://cloud.example.test");
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID", "instance");
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_ENVIRONMENT", "test");
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SIGN_PRIVATE_KEY", generateKeyPairSync("ed25519").privateKey.export({ format: "jwk" }).d!);
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SEAL_PRIVATE_KEY", generateKeyPairSync("x25519").privateKey.export({ format: "jwk" }).d!);
|
||||
}
|
||||
function jsonValues(value: unknown): unknown[] {
|
||||
if (typeof value === "string") { try { return [JSON.parse(value)]; } catch { return []; } }
|
||||
if (!value || typeof value !== "object") return [];
|
||||
const chunks = Reflect.get(value, "queryChunks");
|
||||
return Array.isArray(chunks) ? chunks.flatMap(jsonValues) : [];
|
||||
}
|
||||
function fixture() {
|
||||
configureCloud();
|
||||
const endpoint = { id: "00000000-0000-4000-8000-000000000001", companyId: "company", connectionId: "connection", publicId: "a".repeat(43),
|
||||
provider: "slack", status: "active", providerAccountId: "TTEST", setup: { runtimeGeneration: 1 } as InternalSetupState };
|
||||
const connection = { status: "active", enabled: true };
|
||||
let rejectWrite = false;
|
||||
const update = vi.fn((_table: object) => ({ set: (values: { setup: unknown }) => ({ where: () => {
|
||||
const write = () => {
|
||||
if (rejectWrite) return [];
|
||||
const priorIntent = endpoint.setup.slackCloudIntent;
|
||||
for (const value of jsonValues(values.setup)) {
|
||||
if (value && typeof value === "object" && "active" in value && "id" in value) {
|
||||
endpoint.setup.slackCloudIntent = value as NonNullable<InternalSetupState["slackCloudIntent"]>;
|
||||
|
||||
}
|
||||
else if (value && typeof value === "object" && "bindingId" in value) { delete endpoint.setup.slackCloudIntent; delete endpoint.setup.slackCloudRetiredGeneration; endpoint.setup.slackCloudEvents = value as NonNullable<InternalSetupState["slackCloudEvents"]>; endpoint.setup.slackCloudInstanceId = priorIntent!.instanceId; endpoint.setup.slackCloudSubject = priorIntent!.subject; }
|
||||
else if (typeof value === "number") { delete endpoint.setup.slackCloudIntent; endpoint.setup.slackCloudRetiredGeneration = value; endpoint.setup.slackCloudInstanceId = priorIntent!.instanceId; endpoint.setup.slackCloudSubject = priorIntent!.subject; }
|
||||
}
|
||||
return [{ id: endpoint.id }];
|
||||
};
|
||||
return { then: (resolve: (rows: unknown[]) => unknown) => Promise.resolve(write()).then(resolve), returning: async () => write() };
|
||||
} }) }));
|
||||
const db = { update, select: (fields?: Record<string, unknown>) => {
|
||||
let table: object, joined = false;
|
||||
const rows = () => table === chatSlackRegistrations ? [{ appId: "ATEST" }] : joined
|
||||
? [{ endpoint: structuredClone(endpoint), connection: structuredClone(connection) }]
|
||||
: fields?.setup ? [{ setup: structuredClone(endpoint.setup) }] : table === chatEndpoints ? [structuredClone(endpoint)] : [];
|
||||
const query = { from: (value: object) => { table = value; return query; }, innerJoin: () => { joined = true; return query; },
|
||||
where: () => query, limit: async () => rows(), then: (resolve: (rows: unknown[]) => unknown) => Promise.resolve(rows()).then(resolve) };
|
||||
return query;
|
||||
}, transaction: async (work: (tx: unknown) => Promise<unknown>) => {
|
||||
const previous = structuredClone(endpoint.setup);
|
||||
try { return await work(db); } catch (error) { endpoint.setup = previous; throw error; }
|
||||
} } as unknown as Db;
|
||||
const resolveCredentials = vi.fn(async () => ({ signingSecret: "synthetic" }));
|
||||
const connector = { slackEventsAvailable: vi.fn(async () => true), slackEventEnrollment: vi.fn(async (input: { enrollment: Record<string, unknown> }): Promise<unknown> => {
|
||||
const { active: _active, signingSecret: _secret, ...identity } = input.enrollment;
|
||||
return { version: 1, instanceId: "instance", ...identity, ...(input.enrollment.active ? { bindingId: `slack-event:${"b".repeat(64)}` } : {}) };
|
||||
}) };
|
||||
const lease = { assertOwned: vi.fn(async () => {}) } as unknown as CredentialMutationLeaseGuard;
|
||||
const run = createSlackCloudEnrollment(db, { resolveCredentials, runtimeGeneration: setup => Number(setup.runtimeGeneration), connector });
|
||||
const stage = () => db.transaction(tx => stageSlackCloudEnrollmentIntent(tx as unknown as Db, endpoint.id, "original-authorized-user"));
|
||||
return { db, endpoint, connection, update, connector, resolveCredentials, lease, stage, run,
|
||||
rejectWrites: () => { rejectWrite = true; } };
|
||||
}
|
||||
|
||||
it("stages nonsecret responsibility before network and clears only an exactly confirmed current intent", async () => {
|
||||
const f = fixture(); await f.stage();
|
||||
expect(f.connector.slackEventEnrollment).not.toHaveBeenCalled();
|
||||
expect(JSON.stringify(f.endpoint.setup.slackCloudIntent)).not.toContain("synthetic");
|
||||
await f.run(f.endpoint.id, undefined, f.lease);
|
||||
expect(f.endpoint.setup.slackCloudIntent).toBeUndefined();
|
||||
expect(f.endpoint.setup.slackCloudEvents?.runtimeGeneration).toBe(1);
|
||||
expect(f.connector.slackEventEnrollment).toHaveBeenCalledWith(expect.objectContaining({ subject: "original-authorized-user" }));
|
||||
});
|
||||
|
||||
it.each(["paused", "archived"])("retains credential-free %s removal responsibility after local cleanup", async status => {
|
||||
const f = fixture(); f.endpoint.status = status; f.connection.enabled = false; f.endpoint.setup.runtimeGeneration = 2;
|
||||
await f.stage(); expect(f.endpoint.setup.slackCloudIntent?.active).toBe(false);
|
||||
await f.run(f.endpoint.id, undefined, f.lease);
|
||||
expect(f.resolveCredentials).not.toHaveBeenCalled();
|
||||
expect(f.endpoint.setup.slackCloudRetiredGeneration).toBe(2);
|
||||
expect(f.endpoint.setup.slackCloudIntent).toBeUndefined();
|
||||
});
|
||||
|
||||
it("outage preserves a bounded pending retry; current broker denial preserves visible attention debt", async () => {
|
||||
const f = fixture(); await f.stage();
|
||||
f.connector.slackEventsAvailable.mockRejectedValueOnce(new PaperclipCloudConnectorError("offline", "CONNECTOR_UNAVAILABLE"));
|
||||
await f.run(f.endpoint.id, undefined, f.lease);
|
||||
expect(f.endpoint.setup.slackCloudIntent).toMatchObject({ attempts: 1, failureCode: "broker_unavailable" });
|
||||
expect(Date.parse(f.endpoint.setup.slackCloudIntent!.nextAttemptAt!)).toBeGreaterThan(Date.now());
|
||||
f.connector.slackEventEnrollment.mockRejectedValueOnce(new PaperclipCloudConnectorError("denied", "CONNECTOR_REQUEST_FAILED", 403));
|
||||
await f.run(f.endpoint.id, undefined, f.lease);
|
||||
expect(f.endpoint.setup.slackCloudIntent).toMatchObject({ failureCode: "authority_required", nextAttemptAt: null });
|
||||
});
|
||||
|
||||
it("receipt identity mismatch and failed transaction guard cannot publish a live descriptor", async () => {
|
||||
const f = fixture(); await f.stage();
|
||||
f.connector.slackEventEnrollment.mockResolvedValueOnce({ version: 1, runtimeGeneration: 999 });
|
||||
await f.run(f.endpoint.id, undefined, f.lease);
|
||||
expect(f.endpoint.setup.slackCloudEvents).toBeUndefined(); expect(f.endpoint.setup.slackCloudIntent).toBeDefined();
|
||||
f.rejectWrites(); await expect(f.run(f.endpoint.id, undefined, f.lease)).rejects.toThrow("changed");
|
||||
expect(f.endpoint.setup.slackCloudEvents).toBeUndefined(); expect(f.endpoint.setup.slackCloudIntent).toBeDefined();
|
||||
});
|
||||
|
||||
it("an absent Cloud configuration creates no new intent or background retry", async () => {
|
||||
const f = fixture(); for (const key of ["BASE_URL", "INSTANCE_ID", "SIGN_PRIVATE_KEY", "SEAL_PRIVATE_KEY", "ENVIRONMENT"]) vi.stubEnv(`PAPERCLIP_CLOUD_CONNECTOR_${key}`, "");
|
||||
await f.stage(); expect(f.update).not.toHaveBeenCalled(); expect(f.endpoint.setup.slackCloudIntent).toBeUndefined();
|
||||
});
|
||||
|
||||
it.each(["SLACK_ENROLLMENT_GENERATION_CHANGED", "SLACK_ENROLLMENT_DISABLED"])("closed broker reason %s distinguishes permanent authority from rollout retry", async reason => {
|
||||
const f = fixture(); await f.stage();
|
||||
f.connector.slackEventEnrollment.mockRejectedValueOnce(new PaperclipCloudConnectorError("rejected", "REAUTHORIZATION_REQUIRED", 409, undefined, reason));
|
||||
await f.run(f.endpoint.id, undefined, f.lease);
|
||||
expect(f.endpoint.setup.slackCloudEvents).toBeUndefined();
|
||||
expect(f.endpoint.setup.slackCloudIntent!.nextAttemptAt === null).toBe(reason === "SLACK_ENROLLMENT_GENERATION_CHANGED");
|
||||
expect(f.endpoint.setup.slackCloudIntent!.failureCode).toBe(reason === "SLACK_ENROLLMENT_DISABLED" ? "enrollment_disabled" : "authority_required");
|
||||
});
|
||||
|
||||
it("losing Cloud identity retains configured retirement responsibility as attention without an empty retry", async () => {
|
||||
const f = fixture(); await f.stage(); await f.run(f.endpoint.id, undefined, f.lease);
|
||||
f.endpoint.status = "archived"; f.connection.enabled = false; f.endpoint.setup.runtimeGeneration = 2;
|
||||
for (const key of ["BASE_URL", "INSTANCE_ID", "SIGN_PRIVATE_KEY", "SEAL_PRIVATE_KEY", "ENVIRONMENT"]) vi.stubEnv(`PAPERCLIP_CLOUD_CONNECTOR_${key}`, "");
|
||||
await f.stage();
|
||||
expect(f.endpoint.setup.slackCloudIntent).toMatchObject({ active: false, instanceId: "instance", runtimeGeneration: 2,
|
||||
failureCode: "connector_configuration_unavailable", nextAttemptAt: null });
|
||||
expect(f.connector.slackEventEnrollment).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it.each([404, 422])("unavailable signing credential %s becomes retained attention instead of broker retry", async status => {
|
||||
const f = fixture(); await f.stage();
|
||||
f.resolveCredentials.mockRejectedValueOnce(new HttpError(status, "synthetic credential failure"));
|
||||
await f.run(f.endpoint.id, undefined, f.lease);
|
||||
expect(f.endpoint.setup.slackCloudIntent).toMatchObject({ failureCode: "signing_credential_unavailable", nextAttemptAt: null });
|
||||
expect(f.connector.slackEventEnrollment).not.toHaveBeenCalled();
|
||||
expect(f.endpoint.setup.slackCloudEvents).toBeUndefined();
|
||||
});
|
||||
|
||||
it("temporary signing-credential store failure keeps the bounded retry", async () => {
|
||||
const f = fixture(); await f.stage();
|
||||
f.resolveCredentials.mockRejectedValueOnce(new Error("temporary database failure"));
|
||||
await f.run(f.endpoint.id, undefined, f.lease);
|
||||
expect(f.endpoint.setup.slackCloudIntent?.failureCode).toBe("broker_unavailable");
|
||||
expect(Date.parse(f.endpoint.setup.slackCloudIntent!.nextAttemptAt!)).toBeGreaterThan(Date.now());
|
||||
expect(f.connector.slackEventEnrollment).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("only positive old retirement and an explicit new generation can stage a replacement instance", async () => {
|
||||
const f = fixture(); await f.stage(); await f.run(f.endpoint.id, undefined, f.lease);
|
||||
const oldDescriptor = structuredClone(f.endpoint.setup.slackCloudEvents);
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID", "replacement");
|
||||
f.endpoint.setup.runtimeGeneration = 2;
|
||||
await f.stage();
|
||||
expect(f.endpoint.setup.slackCloudIntent).toMatchObject({ instanceId: "instance", subject: "original-authorized-user",
|
||||
failureCode: "instance_replacement_requires_retirement", nextAttemptAt: null });
|
||||
expect(f.endpoint.setup.slackCloudEvents).toEqual(oldDescriptor);
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID", "instance");
|
||||
f.endpoint.status = "paused"; f.connection.enabled = false; f.endpoint.setup.runtimeGeneration = 3;
|
||||
await f.stage(); await f.run(f.endpoint.id, undefined, f.lease);
|
||||
expect(f.endpoint.setup.slackCloudRetiredGeneration).toBe(3);
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID", "replacement");
|
||||
f.endpoint.status = "active"; f.connection.enabled = true; f.endpoint.setup.runtimeGeneration = 4;
|
||||
await f.stage();
|
||||
expect(f.endpoint.setup.slackCloudIntent).toMatchObject({ instanceId: "replacement", runtimeGeneration: 4 });
|
||||
expect(f.endpoint.setup.slackCloudInstanceId).toBe("instance");
|
||||
expect(f.endpoint.setup.slackCloudSubject).toBe("original-authorized-user");
|
||||
expect(f.endpoint.setup.slackCloudEvents).toEqual(oldDescriptor);
|
||||
f.connector.slackEventEnrollment.mockImplementationOnce(async input => {
|
||||
const { active: _active, signingSecret: _secret, ...identity } = input.enrollment;
|
||||
return { version: 1, instanceId: "replacement", ...identity, bindingId: `slack-event:${"c".repeat(64)}` };
|
||||
});
|
||||
await f.run(f.endpoint.id, undefined, f.lease);
|
||||
expect(f.endpoint.setup.slackCloudInstanceId).toBe("replacement");
|
||||
expect(f.endpoint.setup.slackCloudIntent).toBeUndefined();
|
||||
expect(f.endpoint.setup.slackCloudRetiredGeneration).toBeUndefined();
|
||||
expect(f.endpoint.setup.slackCloudEvents?.runtimeGeneration).toBe(4);
|
||||
});
|
||||
|
||||
it("a new config without an explicit actor cannot adopt even a retired old instance", async () => {
|
||||
const f = fixture(); await f.stage(); await f.run(f.endpoint.id, undefined, f.lease);
|
||||
f.endpoint.status = "paused"; f.connection.enabled = false; f.endpoint.setup.runtimeGeneration = 2;
|
||||
await f.stage(); await f.run(f.endpoint.id, undefined, f.lease);
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID", "replacement");
|
||||
f.endpoint.status = "active"; f.connection.enabled = true; f.endpoint.setup.runtimeGeneration = 3;
|
||||
await stageSlackCloudEnrollmentIntent(f.db, f.endpoint.id, undefined);
|
||||
expect(f.endpoint.setup.slackCloudIntent).toMatchObject({ instanceId: "instance", subject: "original-authorized-user",
|
||||
nextAttemptAt: null, failureCode: "instance_replacement_requires_retirement" });
|
||||
});
|
||||
|
||||
it("removal after an unconfirmed replacement targets its pending instance without new credentials", async () => {
|
||||
const f = fixture(); await f.stage(); await f.run(f.endpoint.id, undefined, f.lease);
|
||||
f.endpoint.status = "paused"; f.connection.enabled = false; f.endpoint.setup.runtimeGeneration = 2;
|
||||
await f.stage(); await f.run(f.endpoint.id, undefined, f.lease);
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID", "replacement");
|
||||
f.endpoint.status = "active"; f.connection.enabled = true; f.endpoint.setup.runtimeGeneration = 3;
|
||||
await f.stage();
|
||||
expect(f.endpoint.setup.slackCloudInstanceId).toBe("instance");
|
||||
f.endpoint.status = "archived"; f.connection.enabled = false; f.endpoint.setup.runtimeGeneration = 4;
|
||||
await f.stage();
|
||||
expect(f.endpoint.setup.slackCloudIntent).toMatchObject({ active: false, instanceId: "replacement", runtimeGeneration: 4 });
|
||||
f.resolveCredentials.mockClear();
|
||||
f.connector.slackEventEnrollment.mockImplementationOnce(async input => {
|
||||
const { active: _active, ...identity } = input.enrollment;
|
||||
return { version: 1, instanceId: "replacement", ...identity };
|
||||
});
|
||||
await f.run(f.endpoint.id, undefined, f.lease);
|
||||
expect(f.resolveCredentials).not.toHaveBeenCalled();
|
||||
expect(f.endpoint.setup.slackCloudIntent).toBeUndefined();
|
||||
expect(f.endpoint.setup.slackCloudInstanceId).toBe("replacement");
|
||||
expect(f.endpoint.setup.slackCloudRetiredGeneration).toBe(4);
|
||||
});
|
||||
@@ -0,0 +1,218 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { and, eq, or, sql } from "drizzle-orm";
|
||||
import { chatEndpoints, chatSlackRegistrations, toolConnections, type Db } from "@paperclipai/db";
|
||||
import { createPaperclipCloudConnector, paperclipCloudConnectorConfigFromEnv, PaperclipCloudConnectorError, type PaperclipCloudConnector } from "../../paperclip-cloud-connector.js";
|
||||
import type { CredentialMutationLeaseGuard } from "../../chat-credential-mutation-lease.js";
|
||||
import type { createChatCredentials } from "../credentials.js";
|
||||
import type { InternalSetupState } from "../runtime-types.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "../../delivery-work-notifications.js";
|
||||
import { logger } from "../../../middleware/logger.js";
|
||||
import { HttpError } from "../../../errors.js";
|
||||
|
||||
import { resolveSlackCloudSigningSecret } from "./cloud-signing-secret.js";
|
||||
import { SlackCloudReceiptError } from "./cloud-event.js";
|
||||
|
||||
type Intent = NonNullable<InternalSetupState["slackCloudIntent"]>;
|
||||
class SlackEnrollmentFailure extends Error {
|
||||
constructor(readonly code: string) { super(code); }
|
||||
}
|
||||
|
||||
type Owner = (endpointId: string) => Promise<void>;
|
||||
const owners = new WeakMap<Db, Owner>();
|
||||
|
||||
export function registerSlackCloudEnrollmentOwner(db: Db, owner: Owner) {
|
||||
owners.set(db, owner);
|
||||
return () => { if (owners.get(db) === owner) owners.delete(db); };
|
||||
}
|
||||
|
||||
function slackCloudEnrollmentIsActive(endpoint: { status: string }, connection: { status: string; enabled: boolean }) {
|
||||
return ["active", "verifying", "attention"].includes(endpoint.status) && connection.status === "active" && connection.enabled;
|
||||
}
|
||||
|
||||
export async function stageSlackCloudEnrollmentIntent(db: Db, endpointId: string, actorUserId: string | null | undefined) {
|
||||
let config: ReturnType<typeof paperclipCloudConnectorConfigFromEnv>;
|
||||
try { config = paperclipCloudConnectorConfigFromEnv(); } catch { config = null; }
|
||||
const [current] = await db.select({ endpoint: chatEndpoints, connection: toolConnections }).from(chatEndpoints)
|
||||
.innerJoin(toolConnections, and(eq(toolConnections.id, chatEndpoints.connectionId), eq(toolConnections.companyId, chatEndpoints.companyId)))
|
||||
.where(eq(chatEndpoints.id, endpointId)).limit(1);
|
||||
if (!current || current.endpoint.provider !== "slack") return;
|
||||
const endpoint = current.endpoint, setup = endpoint.setup as InternalSetupState;
|
||||
const generation = Number(setup.runtimeGeneration ?? 0);
|
||||
const active = slackCloudEnrollmentIsActive(endpoint, current.connection);
|
||||
const custodyInstance = setup.slackCloudIntent?.instanceId ?? setup.slackCloudInstanceId;
|
||||
const custodySubject = setup.slackCloudIntent?.subject ?? setup.slackCloudSubject;
|
||||
const oldRetired = !setup.slackCloudIntent && setup.slackCloudInstanceId
|
||||
&& Number.isSafeInteger(setup.slackCloudRetiredGeneration) && Number(setup.slackCloudRetiredGeneration) >= 1
|
||||
&& Number(setup.slackCloudRetiredGeneration) >= (setup.slackCloudEvents?.runtimeGeneration ?? 1);
|
||||
const changingInstance = !!custodyInstance && !!config && custodyInstance !== config.instanceId;
|
||||
const mayReplace = active && changingInstance && oldRetired && !!actorUserId
|
||||
&& generation > Number(setup.slackCloudRetiredGeneration);
|
||||
const blocked = changingInstance && !mayReplace;
|
||||
const instanceId = mayReplace ? config!.instanceId : custodyInstance ?? config?.instanceId;
|
||||
const subject = blocked || !active ? custodySubject ?? actorUserId : actorUserId ?? custodySubject;
|
||||
if (!instanceId || !subject || !Number.isSafeInteger(generation) || generation < 1) return;
|
||||
if (setup.slackCloudIntent?.runtimeGeneration === generation && setup.slackCloudIntent.active === active
|
||||
&& setup.slackCloudIntent.instanceId === instanceId && setup.slackCloudIntent.subject === subject) return;
|
||||
if (!blocked && setup.slackCloudInstanceId === instanceId && setup.slackCloudSubject === subject
|
||||
&& (active ? setup.slackCloudEvents?.runtimeGeneration === generation : setup.slackCloudRetiredGeneration === generation)) return;
|
||||
const [registration] = active ? await db.select({ appId: chatSlackRegistrations.appId }).from(chatSlackRegistrations)
|
||||
.where(and(eq(chatSlackRegistrations.endpointId, endpointId), eq(chatSlackRegistrations.companyId, endpoint.companyId))).limit(1) : [];
|
||||
const appId = registration?.appId ?? setup.slackCloudEvents?.appId;
|
||||
if (active && (!appId || !endpoint.providerAccountId)) return;
|
||||
const intent: Intent = { version: 1, id: randomUUID(), subject, instanceId, companyId: endpoint.companyId, endpointId,
|
||||
publicId: endpoint.publicId, connectionId: endpoint.connectionId, runtimeGeneration: generation, active,
|
||||
...(active ? { appId: appId!, teamId: endpoint.providerAccountId! } : {}), attempts: 0, nextAttemptAt: config && !blocked ? new Date().toISOString() : null,
|
||||
...(blocked ? { failureCode: "instance_replacement_requires_retirement" } : !config ? { failureCode: "connector_configuration_unavailable" } : {}) };
|
||||
await notifyDeliveryWork(db, DELIVERY_QUEUES.connectionEvents);
|
||||
await db.update(chatEndpoints).set({
|
||||
lastError: config && !blocked ? sql`${chatEndpoints.lastError}` : "Slack Cloud event registration requires attention",
|
||||
setup: sql`jsonb_set(${chatEndpoints.setup}, '{slackCloudIntent}', ${JSON.stringify(intent)}::jsonb)` })
|
||||
.where(and(eq(chatEndpoints.id, endpointId), eq(chatEndpoints.companyId, endpoint.companyId)));
|
||||
}
|
||||
|
||||
export async function reconcilePendingSlackCloudEnrollments(db: Db, now: Date): Promise<number | null> {
|
||||
const config = paperclipCloudConnectorConfigFromEnv();
|
||||
if (!config) return null;
|
||||
const recoverable = and(
|
||||
sql`${chatEndpoints.setup}->'slackCloudIntent'->>'nextAttemptAt' is null`,
|
||||
sql`${chatEndpoints.setup}->'slackCloudIntent'->>'failureCode' = 'connector_configuration_unavailable'`,
|
||||
sql`${chatEndpoints.setup}->'slackCloudIntent'->>'instanceId' = ${config.instanceId}`,
|
||||
);
|
||||
const scheduled = and(eq(chatEndpoints.provider, "slack"), or(
|
||||
sql`${chatEndpoints.setup}->'slackCloudIntent'->>'nextAttemptAt' is not null`, recoverable));
|
||||
const order = sql`coalesce(${chatEndpoints.setup}->'slackCloudIntent'->>'nextAttemptAt', '')`;
|
||||
const pending = await db.select().from(chatEndpoints).where(scheduled).orderBy(order).limit(1);
|
||||
const owner = owners.get(db);
|
||||
const deadline = (intent: Intent | undefined) => {
|
||||
if (!intent) return Number.POSITIVE_INFINITY;
|
||||
if (intent.nextAttemptAt === null) return intent.failureCode === "connector_configuration_unavailable" && intent.instanceId === config.instanceId
|
||||
? now.getTime() : Number.POSITIVE_INFINITY;
|
||||
const parsed = Date.parse(intent.nextAttemptAt);
|
||||
return Number.isFinite(parsed) ? parsed : now.getTime();
|
||||
};
|
||||
let next = Number.POSITIVE_INFINITY;
|
||||
for (const endpoint of pending) {
|
||||
const intent = (endpoint.setup as InternalSetupState).slackCloudIntent!;
|
||||
let due = deadline(intent);
|
||||
if (!Number.isFinite(due)) continue;
|
||||
if (due <= now.getTime()) {
|
||||
if (owner) {
|
||||
try { await owner(endpoint.id); }
|
||||
catch (error) {
|
||||
if (error instanceof HttpError && error.status === 409 && error.details && typeof error.details === "object"
|
||||
&& "code" in error.details && error.details.code === "chat_endpoint_credentials_busy") {
|
||||
try {
|
||||
if (intent.companyId === endpoint.companyId && intent.endpointId === endpoint.id
|
||||
&& intent.publicId === endpoint.publicId && intent.connectionId === endpoint.connectionId
|
||||
&& intent.runtimeGeneration === Number((endpoint.setup as InternalSetupState).runtimeGeneration ?? 0)) {
|
||||
await db.transaction(async tx => {
|
||||
const sameIntent = and(eq(chatEndpoints.id, endpoint.id), eq(chatEndpoints.companyId, intent.companyId),
|
||||
eq(chatEndpoints.provider, "slack"), eq(chatEndpoints.publicId, intent.publicId), eq(chatEndpoints.connectionId, intent.connectionId),
|
||||
sql`${chatEndpoints.setup}->'runtimeGeneration' = ${JSON.stringify(intent.runtimeGeneration)}::jsonb`,
|
||||
sql`${chatEndpoints.setup}->'slackCloudIntent' = ${JSON.stringify(intent)}::jsonb`);
|
||||
const [owned] = await tx.select({ id: chatEndpoints.id }).from(chatEndpoints).where(sameIntent).for("update").limit(1);
|
||||
if (!owned) return;
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.connectionEvents);
|
||||
const retry = { ...intent, attempts: Math.min(intent.attempts + 1, 1_000_000), failureCode: "credential_lease_busy",
|
||||
nextAttemptAt: new Date(Date.now() + Math.min(300_000, 5_000 * 2 ** Math.min(intent.attempts, 6))).toISOString() };
|
||||
const updated = await tx.update(chatEndpoints).set({
|
||||
setup: sql`jsonb_set(${chatEndpoints.setup}, '{slackCloudIntent}', ${JSON.stringify(retry)}::jsonb)`,
|
||||
}).where(sameIntent).returning({ id: chatEndpoints.id });
|
||||
if (updated.length !== 1) throw new Error("Slack enrollment busy retry custody changed");
|
||||
});
|
||||
}
|
||||
} catch (retryError) { throw new AggregateError([error, retryError], "Slack enrollment busy retry persistence failed"); }
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
else logger.warn({ endpointId: endpoint.id, reason: "slack_enrollment_owner_not_ready" }, "Slack enrollment responsibility retained");
|
||||
const [updated] = await db.select({ setup: chatEndpoints.setup }).from(chatEndpoints).where(eq(chatEndpoints.id, endpoint.id)).limit(1);
|
||||
const retained = (updated?.setup as InternalSetupState | undefined)?.slackCloudIntent;
|
||||
due = deadline(retained);
|
||||
if (!owner) due = now.getTime() + 5_000;
|
||||
}
|
||||
next = Math.min(next, due);
|
||||
}
|
||||
if (pending.length > 0) {
|
||||
const [earliest] = await db.select({ setup: chatEndpoints.setup }).from(chatEndpoints).where(scheduled).orderBy(order).limit(1);
|
||||
next = Math.min(next, deadline((earliest?.setup as InternalSetupState | undefined)?.slackCloudIntent));
|
||||
}
|
||||
return Number.isFinite(next) ? next : null;
|
||||
}
|
||||
|
||||
export function createSlackCloudEnrollment(db: Db, options: {
|
||||
resolveCredentials: ReturnType<typeof createChatCredentials>["resolveCredentials"];
|
||||
runtimeGeneration(setup: InternalSetupState): number;
|
||||
connector?: Pick<PaperclipCloudConnector, "slackEventsAvailable" | "slackEventEnrollment">;
|
||||
}) {
|
||||
return async (endpointId: string, _actorUserId: string | null | undefined, lease: CredentialMutationLeaseGuard) => {
|
||||
const [current] = await db.select({ endpoint: chatEndpoints, connection: toolConnections }).from(chatEndpoints)
|
||||
.innerJoin(toolConnections, and(eq(toolConnections.id, chatEndpoints.connectionId), eq(toolConnections.companyId, chatEndpoints.companyId)))
|
||||
.where(eq(chatEndpoints.id, endpointId)).limit(1);
|
||||
if (!current || current.endpoint.provider !== "slack") return;
|
||||
const endpoint = current.endpoint, setup = endpoint.setup as InternalSetupState, intent = setup.slackCloudIntent;
|
||||
const config = paperclipCloudConnectorConfigFromEnv();
|
||||
if (!config || !intent) return;
|
||||
const identity = { companyId: intent.companyId, endpointId: intent.endpointId, publicId: intent.publicId,
|
||||
connectionId: intent.connectionId, runtimeGeneration: intent.runtimeGeneration };
|
||||
const active = slackCloudEnrollmentIsActive(endpoint, current.connection);
|
||||
if (identity.companyId !== endpoint.companyId || identity.endpointId !== endpoint.id || identity.publicId !== endpoint.publicId
|
||||
|| identity.connectionId !== endpoint.connectionId || identity.runtimeGeneration !== options.runtimeGeneration(setup) || intent.active !== active) {
|
||||
await db.transaction(async tx => {
|
||||
await lease.assertOwned(tx);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.connectionEvents);
|
||||
await tx.update(chatEndpoints).set({
|
||||
setup: sql`jsonb_set(${chatEndpoints.setup}, '{slackCloudIntent}', ${JSON.stringify({ ...intent, nextAttemptAt: null, failureCode: "intent_authority_changed" })}::jsonb)`,
|
||||
lastError: "Slack Cloud event registration requires attention",
|
||||
}).where(and(eq(chatEndpoints.id, endpoint.id), eq(chatEndpoints.companyId, endpoint.companyId), sql`${chatEndpoints.setup}->'slackCloudIntent'->>'id'=${intent.id}`));
|
||||
await lease.assertOwned(tx);
|
||||
});
|
||||
logger.warn({ endpointId, reason: "intent_authority_changed" }, "Slack enrollment responsibility retained");
|
||||
return;
|
||||
}
|
||||
const connector = options.connector ?? createPaperclipCloudConnector({ config });
|
||||
const commit = async (change: { receipt?: Record<string, unknown>; failure?: string; attention?: boolean; retryAt?: number }) => db.transaction(async tx => {
|
||||
await lease.assertOwned(tx);
|
||||
const next = change.failure ? { ...intent, attempts: Math.min(intent.attempts + 1, 1_000_000), failureCode: change.failure,
|
||||
nextAttemptAt: change.attention ? null : new Date(Math.max(Date.now() + Math.min(300_000, 5_000 * 2 ** Math.min(intent.attempts, 6)), change.retryAt ?? 0)).toISOString() } : null;
|
||||
const field = intent.active ? "slackCloudEvents" : "slackCloudRetiredGeneration";
|
||||
const enrolled = intent.active ? { version: 1, bindingId: change.receipt?.bindingId, runtimeGeneration: intent.runtimeGeneration, teamId: intent.teamId, appId: intent.appId } : intent.runtimeGeneration;
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.connectionEvents);
|
||||
const updated = await tx.update(chatEndpoints).set({
|
||||
lastError: change.attention ? "Slack Cloud event registration requires attention" : change.receipt
|
||||
? sql`case when ${chatEndpoints.lastError} = 'Slack Cloud event registration requires attention' then null else ${chatEndpoints.lastError} end` : sql`${chatEndpoints.lastError}`,
|
||||
setup: next
|
||||
? sql`jsonb_set(${chatEndpoints.setup}, '{slackCloudIntent}', ${JSON.stringify(next)}::jsonb)`
|
||||
: sql`jsonb_set(jsonb_set(jsonb_set(${chatEndpoints.setup} - 'slackCloudIntent' - 'slackCloudRetiredGeneration', ARRAY[${field}], ${JSON.stringify(enrolled)}::jsonb), '{slackCloudInstanceId}', ${JSON.stringify(intent.instanceId)}::jsonb), '{slackCloudSubject}', ${JSON.stringify(intent.subject)}::jsonb)` })
|
||||
.where(and(eq(chatEndpoints.id, endpointId), eq(chatEndpoints.companyId, endpoint.companyId),
|
||||
sql`${chatEndpoints.setup}->'slackCloudIntent'->>'id'=${intent.id}`,
|
||||
sql`coalesce((${chatEndpoints.setup}->>'runtimeGeneration')::integer,0)=${intent.runtimeGeneration}`)).returning({ id: chatEndpoints.id });
|
||||
if (updated.length !== 1) throw new Error("Slack configuration changed before event enrollment committed");
|
||||
await lease.assertOwned(tx);
|
||||
});
|
||||
try {
|
||||
await lease.assertOwned();
|
||||
if (config.instanceId !== intent.instanceId) { await commit({ failure: "instance_authority_changed", attention: true }); return; }
|
||||
if (!await connector.slackEventsAvailable()) { await commit({ failure: "unsupported_capability" }); return; }
|
||||
const signingSecret = intent.active ? await resolveSlackCloudSigningSecret(options.resolveCredentials, endpoint, db, ["signingSecret"]) : undefined;
|
||||
const receipt = await connector.slackEventEnrollment({ subject: intent.subject, companyId: intent.companyId,
|
||||
enrollment: { active: intent.active, ...identity, ...(intent.active ? { teamId: intent.teamId, appId: intent.appId, signingSecret: signingSecret! } : {}) } });
|
||||
if (!receipt || typeof receipt !== "object" || Array.isArray(receipt)) throw new SlackEnrollmentFailure("receipt_unconfirmed");
|
||||
const value = receipt as Record<string, unknown>;
|
||||
if (value.version !== 1 || value.instanceId !== intent.instanceId || Object.entries(identity).some(([key, expected]) => value[key] !== expected)
|
||||
|| (intent.active && (value.teamId !== intent.teamId || value.appId !== intent.appId || typeof value.bindingId !== "string" || !/^slack-event:[a-f0-9]{64}$/.test(value.bindingId))))
|
||||
throw new SlackEnrollmentFailure("receipt_identity_changed");
|
||||
await commit({ receipt: value });
|
||||
} catch (error) {
|
||||
const conflict = error instanceof PaperclipCloudConnectorError && error.status === 409
|
||||
&& ["SLACK_ENROLLMENT_GENERATION_CHANGED", "SLACK_ENROLLMENT_IDENTITY_CHANGED", "SLACK_ENROLLMENT_CHANGED", "SLACK_ENROLLMENT_UNSETTLED"].includes(error.brokerReason ?? "");
|
||||
const credentialAttention = error instanceof SlackCloudReceiptError && error.code === "signing_credential_unavailable" && error.disposition === "attention";
|
||||
const attention = credentialAttention || error instanceof SlackEnrollmentFailure || conflict || (error instanceof PaperclipCloudConnectorError && [400, 401, 403].includes(error.status ?? 0));
|
||||
const code = credentialAttention ? "signing_credential_unavailable" : error instanceof SlackEnrollmentFailure ? error.code : attention ? "authority_required"
|
||||
: error instanceof PaperclipCloudConnectorError && error.brokerReason === "SLACK_ENROLLMENT_DISABLED" ? "enrollment_disabled" : "broker_unavailable";
|
||||
await commit({ failure: code, attention, retryAt: error instanceof PaperclipCloudConnectorError ? error.retryAt : undefined });
|
||||
logger.warn({ endpointId, reason: code }, "Slack enrollment responsibility retained");
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
import { guardSlackFixtureNetwork, stubSlackFixtureTransport } from "../../../__tests__/helpers/slack-cloud/transport.js";
|
||||
import { createHmac, createHash } from "node:crypto";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { verifySlackSignature } from "@chat-adapter/slack/webhook";
|
||||
import { createChatSdkEndpointRuntime } from "../../chat-sdk-runtime.js";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import type { EndpointRow } from "../runtime-types.js";
|
||||
import { createSlackWebhookCallbacks } from "./webhook-callbacks.js";
|
||||
import { receiveSlackCloudEvent, slackCloudAcceptedAt, type CurrentSlackCloudBinding, type SlackCloudEvent } from "./cloud-event.js";
|
||||
|
||||
const acceptedAt = Date.parse("2099-01-01T00:00:00Z");
|
||||
function fixture() {
|
||||
const binding: CurrentSlackCloudBinding = { companyId: "company", endpointId: "endpoint", publicId: "a".repeat(43),
|
||||
connectionId: "connection", runtimeGeneration: 1, teamId: "TTEST", appId: "ATEST", bindingId: `slack-event:${"b".repeat(64)}`,
|
||||
signingSecret: "0123456789abcdef0123456789abcdef" };
|
||||
const rawBody = Buffer.from(JSON.stringify({ type: "event_callback", event_id: "EvFIRST", team_id: "TTEST", api_app_id: "ATEST", event: { type: "app_mention", text: "<@UBOT> hello", user: "UOPERATOR", channel: "CTEST", ts: "1788.500", event_ts: "1788.500" } }));
|
||||
const timestamp = String(acceptedAt / 1000);
|
||||
const signature = `v0=${createHmac("sha256", binding.signingSecret).update(`v0:${timestamp}:`).update(rawBody).digest("hex")}`;
|
||||
const { signingSecret: _secret, ...identity } = binding;
|
||||
const event: SlackCloudEvent = { id: `slack_${"b".repeat(64)}_${createHash("sha256").update("EvFIRST").digest("hex")}`,
|
||||
provider: "slack", event: "event_callback", installationId: binding.bindingId, bindingIds: [binding.bindingId],
|
||||
createdAt: new Date(acceptedAt).toISOString(), payload: { slackEvent: { version: 1, ...identity,
|
||||
rawBody: rawBody.toString("base64"), timestamp, signature, eventId: "EvFIRST", acceptedAt: new Date(acceptedAt).toISOString() } } };
|
||||
return { binding, event, rawBody, timestamp, signature };
|
||||
}
|
||||
|
||||
describe("authenticated Slack Cloud receipt consumer", () => {
|
||||
it("crosses the real pinned Chat runtime after cold-start delay and preserves ordinary external freshness rejection", async () => {
|
||||
const network = guardSlackFixtureNetwork();
|
||||
const f = fixture(), onMessage = vi.fn(async () => {});
|
||||
const runtime = createChatSdkEndpointRuntime({ companyId: f.binding.companyId, endpointId: f.binding.endpointId,
|
||||
logger: "silent", callbacks: { onMessage }, persistence: { read: async () => null,
|
||||
compareAndSet: async () => true, deleteIfVersion: async () => true },
|
||||
providerConfig: { provider: "slack", userName: "paperclip-agent",
|
||||
credentials: { apiUrl: "http://127.0.0.1:1/api/", botToken: "xoxb-synthetic", botUserId: "UBOT", signingSecret: f.binding.signingSecret } } });
|
||||
try {
|
||||
await runtime.initialize();
|
||||
const transport = stubSlackFixtureTransport(runtime.getProviderAdapter());
|
||||
await receiveSlackCloudEvent(f.event, f.binding, request => runtime.handleWebhook(request), acceptedAt + 600_000);
|
||||
expect(onMessage).toHaveBeenCalledOnce();
|
||||
expect(onMessage).toHaveBeenCalledWith(expect.objectContaining({ message: expect.objectContaining({ author: expect.objectContaining({ userId: "UOPERATOR", userName: "Operator" }) }) }));
|
||||
expect(transport).toHaveBeenCalled();
|
||||
await transport.assertNoErrors();
|
||||
expect(onMessage).toHaveBeenCalledWith(expect.objectContaining({ provider: "slack", endpointId: f.binding.endpointId }));
|
||||
const response = await runtime.handleWebhook(new Request("https://external.invalid", { method: "POST",
|
||||
headers: { "content-type": "application/json", "x-slack-request-timestamp": f.timestamp, "x-slack-signature": f.signature },
|
||||
body: new Uint8Array(f.rawBody) }));
|
||||
expect(response.status).toBe(401);
|
||||
expect(onMessage).toHaveBeenCalledOnce();
|
||||
} finally {
|
||||
await runtime.shutdown();
|
||||
try { network.assertQuiet(); } finally { network.restore(); }
|
||||
}
|
||||
});
|
||||
it("replays original evidence after a cold-start delay without refreshing its timestamp", async () => {
|
||||
const f = fixture();
|
||||
let delivered: Request | undefined;
|
||||
await receiveSlackCloudEvent(f.event, f.binding, async request => {
|
||||
delivered = request;
|
||||
expect(slackCloudAcceptedAt(request)).toBe(acceptedAt);
|
||||
expect(request.headers.get("x-slack-request-timestamp")).toBe(f.timestamp);
|
||||
expect(request.headers.get("x-slack-signature")).toBe(f.signature);
|
||||
const body = await request.text();
|
||||
expect(body).toBe(f.rawBody.toString());
|
||||
|
||||
await verifySlackSignature(body, request.headers, { signingSecret: f.binding.signingSecret,
|
||||
now: () => slackCloudAcceptedAt(request) ?? acceptedAt + 600_000 });
|
||||
await expect(verifySlackSignature(body, request.headers, { signingSecret: f.binding.signingSecret,
|
||||
now: () => acceptedAt + 600_000 })).rejects.toThrow("too old");
|
||||
return new Response("durably accepted");
|
||||
}, acceptedAt + 600_000);
|
||||
expect(slackCloudAcceptedAt(delivered!)).toBeUndefined();
|
||||
expect(slackCloudAcceptedAt(new Request("https://external.invalid"))).toBeUndefined();
|
||||
});
|
||||
|
||||
it("ignores removed or superseded generations without invoking product admission", async () => {
|
||||
const f = fixture(), handler = vi.fn();
|
||||
await receiveSlackCloudEvent(f.event, null, handler, acceptedAt);
|
||||
await receiveSlackCloudEvent(f.event, { ...f.binding, runtimeGeneration: 2 }, handler, acceptedAt);
|
||||
expect(handler).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("retains current work on signature/body/receipt-time failure and on failed durable admission", async () => {
|
||||
const f = fixture(), handler = vi.fn(async () => new Response(null, { status: 503 }));
|
||||
await expect(receiveSlackCloudEvent(f.event, f.binding, handler, acceptedAt)).rejects.toThrow("callback_unaccepted");
|
||||
await expect(receiveSlackCloudEvent(f.event, { ...f.binding, signingSecret: "rotated" }, handler, acceptedAt)).rejects.toThrow("original_authentication_failed");
|
||||
await expect(receiveSlackCloudEvent({ ...f.event, createdAt: new Date(acceptedAt + 600_000).toISOString() }, f.binding, handler, acceptedAt)).rejects.toThrow("invalid_acceptance_time");
|
||||
await expect(receiveSlackCloudEvent(f.event, f.binding, handler, acceptedAt + 31 * 24 * 60 * 60_000)).rejects.toThrow("invalid_acceptance_time");
|
||||
expect(handler).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
it("private replay preserves canonical Slack callback evidence while ordinary external observation remains active", async () => {
|
||||
const f = fixture();
|
||||
const transaction = vi.fn(async () => { throw new Error("external callback observation"); });
|
||||
const db = { transaction } as unknown as Db;
|
||||
const callbacks = createSlackWebhookCallbacks(db, { resolveCredentials: async () => ({}), runtimeCallbackEndpoint: async () => null,
|
||||
getWebhookPublicBaseUrl: () => "https://canonical.example", scheduleMessageProcessing: () => {}, slackRegistration: { notifyVerified: async () => {} } });
|
||||
const endpoint = { setup: { slackCallbackSurfaces: { events: { url: "https://canonical.example/slack", observedAt: "2099-01-01T00:00:00Z" } } } } as unknown as EndpointRow;
|
||||
const before = structuredClone(endpoint.setup);
|
||||
const context = { credentialFingerprint: "fixture", generation: 1, localEpoch: 1, version: "fixture" };
|
||||
await receiveSlackCloudEvent(f.event, f.binding, async request => {
|
||||
await callbacks.recordSlackWebhookCallback(endpoint, request, request.clone(), context);
|
||||
return new Response(null, { status: 200 });
|
||||
}, acceptedAt);
|
||||
expect(endpoint.setup).toEqual(before); expect(transaction).not.toHaveBeenCalled();
|
||||
const external = new Request("https://canonical.example/slack", { method: "POST", headers: { "content-type": "application/json" }, body: new Uint8Array(f.rawBody) });
|
||||
await expect(callbacks.recordSlackWebhookCallback(endpoint, external, external.clone(), context)).rejects.toThrow("external callback observation");
|
||||
expect(transaction).toHaveBeenCalledOnce();
|
||||
});
|
||||
@@ -0,0 +1,77 @@
|
||||
import { createHash, createHmac, timingSafeEqual } from "node:crypto";
|
||||
|
||||
const acceptedTimes = new WeakMap<Request, number>();
|
||||
export const slackCloudAcceptedAt = (request: Request): number | undefined => acceptedTimes.get(request);
|
||||
|
||||
export class SlackCloudReceiptError extends Error {
|
||||
constructor(readonly code: string, readonly disposition: "invalid" | "attention" | "retry") { super(code); }
|
||||
}
|
||||
|
||||
export interface SlackCloudEvent {
|
||||
id: string;
|
||||
provider: string;
|
||||
event: string;
|
||||
installationId: string | null;
|
||||
bindingIds: string[];
|
||||
createdAt: string;
|
||||
payload: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface CurrentSlackCloudBinding {
|
||||
companyId: string;
|
||||
endpointId: string;
|
||||
publicId: string;
|
||||
connectionId: string;
|
||||
runtimeGeneration: number;
|
||||
teamId: string;
|
||||
appId: string;
|
||||
bindingId: string;
|
||||
signingSecret: string;
|
||||
}
|
||||
|
||||
export async function receiveSlackCloudEvent(event: SlackCloudEvent, current: CurrentSlackCloudBinding | null,
|
||||
handleWebhook: (request: Request) => Promise<Response>, now = Date.now()): Promise<void> {
|
||||
if (!current) return;
|
||||
const value = event.payload.slackEvent;
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) throw new SlackCloudReceiptError("invalid_receipt", "invalid");
|
||||
const packet = value as Record<string, unknown>;
|
||||
if (event.provider !== "slack" || event.event !== "event_callback" || packet.version !== 1)
|
||||
throw new SlackCloudReceiptError("unsupported_receipt", "attention");
|
||||
if (packet.bindingId !== current.bindingId || packet.companyId !== current.companyId
|
||||
|| packet.endpointId !== current.endpointId || packet.publicId !== current.publicId
|
||||
|| packet.connectionId !== current.connectionId || packet.runtimeGeneration !== current.runtimeGeneration
|
||||
|| packet.teamId !== current.teamId || packet.appId !== current.appId) return;
|
||||
if (event.installationId !== current.bindingId || !event.bindingIds.includes(current.bindingId)
|
||||
|| typeof packet.rawBody !== "string" || packet.rawBody.length > 1_398_104
|
||||
|| !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(packet.rawBody)
|
||||
|| typeof packet.timestamp !== "string" || !/^[0-9]{1,16}$/.test(packet.timestamp)
|
||||
|| typeof packet.signature !== "string" || !/^v0=[a-f0-9]{64}$/.test(packet.signature))
|
||||
throw new SlackCloudReceiptError("invalid_receipt", "invalid");
|
||||
const acceptedAt = Date.parse(event.createdAt);
|
||||
|
||||
if (packet.acceptedAt !== event.createdAt || !Number.isFinite(acceptedAt) || acceptedAt > now + 300_000 || now - acceptedAt >= 30 * 24 * 60 * 60_000
|
||||
|| Math.abs(Math.floor(acceptedAt / 1000) - Number(packet.timestamp)) > 300)
|
||||
throw new SlackCloudReceiptError("invalid_acceptance_time", "attention");
|
||||
const body = Buffer.from(packet.rawBody, "base64");
|
||||
if (body.byteLength > 1_048_576) throw new SlackCloudReceiptError("receipt_too_large", "invalid");
|
||||
const signature = createHmac("sha256", current.signingSecret).update(`v0:${packet.timestamp}:`).update(body).digest();
|
||||
if (!timingSafeEqual(signature, Buffer.from(packet.signature.slice(3), "hex")))
|
||||
throw new SlackCloudReceiptError("original_authentication_failed", "attention");
|
||||
let payload: unknown;
|
||||
try { payload = JSON.parse(body.toString("utf8")); } catch { throw new SlackCloudReceiptError("invalid_event", "invalid"); }
|
||||
if (!payload || typeof payload !== "object" || Array.isArray(payload)) throw new SlackCloudReceiptError("invalid_event", "invalid");
|
||||
const envelope = payload as Record<string, unknown>;
|
||||
if (envelope.type !== "event_callback" || envelope.team_id !== current.teamId || envelope.api_app_id !== current.appId
|
||||
|| typeof envelope.event_id !== "string" || !/^[A-Za-z0-9_-]{1,200}$/.test(envelope.event_id) || envelope.event_id !== packet.eventId
|
||||
|| event.id !== `slack_${current.bindingId.slice("slack-event:".length)}_${createHash("sha256").update(envelope.event_id).digest("hex")}`)
|
||||
throw new SlackCloudReceiptError("event_identity_changed", "invalid");
|
||||
const request = new Request("https://paperclip.invalid/slack-cloud-ingress", { method: "POST",
|
||||
headers: { "content-type": "application/json", "x-slack-request-timestamp": packet.timestamp,
|
||||
"x-slack-signature": packet.signature }, body: new Uint8Array(body) });
|
||||
acceptedTimes.set(request, acceptedAt);
|
||||
try {
|
||||
const response = await handleWebhook(request);
|
||||
await response.body?.cancel();
|
||||
if (!response.ok) throw new SlackCloudReceiptError("callback_unaccepted", "retry");
|
||||
} finally { acceptedTimes.delete(request); }
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
import { HttpError } from "../../../errors.js";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { expect, it, vi } from "vitest";
|
||||
import { createSlackCloudEventIngress, dispatchSlackCloudEvent, registerSlackCloudEventIngress } from "./cloud-ingress.js";
|
||||
import type { SlackCloudEvent } from "./cloud-event.js";
|
||||
|
||||
it("missing or failed runtime owner retains the leased event; retired owner cleanup cannot remove its replacement", async () => {
|
||||
const db = {} as Db;
|
||||
const event: SlackCloudEvent = { id: "receipt", provider: "slack", event: "event_callback", installationId: "binding",
|
||||
bindingIds: ["binding"], createdAt: "2099-01-01T00:00:00Z", payload: {} };
|
||||
await expect(dispatchSlackCloudEvent(db, event)).rejects.toThrow("not ready");
|
||||
const stopped = vi.fn(async () => { throw new Error("admission stopped"); });
|
||||
const unregisterOld = registerSlackCloudEventIngress(db, stopped);
|
||||
await expect(dispatchSlackCloudEvent(db, event)).rejects.toThrow("admission stopped");
|
||||
const replacement = vi.fn(async () => {});
|
||||
const unregisterNew = registerSlackCloudEventIngress(db, replacement);
|
||||
unregisterOld();
|
||||
await dispatchSlackCloudEvent(db, event);
|
||||
expect(replacement).toHaveBeenCalledWith(event);
|
||||
unregisterNew();
|
||||
await expect(dispatchSlackCloudEvent(db, event)).rejects.toThrow("not ready");
|
||||
});
|
||||
|
||||
it.each([404, 422])("current ingress keeps credential failure %s as unacknowledged attention before callback admission", async status => {
|
||||
const endpoint = { id: "00000000-0000-4000-8000-000000000001", companyId: "company", publicId: "public", connectionId: "connection",
|
||||
provider: "slack", status: "active", providerAccountId: "TTEST", setup: { slackCloudEvents: {
|
||||
version: 1, bindingId: "binding", runtimeGeneration: 1, teamId: "TTEST", appId: "ATEST" } } };
|
||||
const db = { select: () => {
|
||||
let joined = false;
|
||||
const query = { from: () => query, where: () => query, innerJoin: () => { joined = true; return query; },
|
||||
limit: async () => joined ? [{ endpoint, connection: { enabled: true, status: "active" } }] : [endpoint] };
|
||||
return query;
|
||||
} } as unknown as Db;
|
||||
const handleWebhook = vi.fn(async () => new Response());
|
||||
const withCredentialMutationLease: Parameters<typeof createSlackCloudEventIngress>[1]["withCredentialMutationLease"] =
|
||||
async (_endpoint, work) => work({ assertOwned: async () => {} } as Parameters<typeof work>[0]);
|
||||
const unregister = createSlackCloudEventIngress(db, { withCredentialMutationLease, runtimeGeneration: () => 1, handleWebhook,
|
||||
resolveCredentials: async () => { throw new HttpError(status, "synthetic credential failure"); } });
|
||||
try {
|
||||
await expect(dispatchSlackCloudEvent(db, { id: "receipt", provider: "slack", event: "event_callback", installationId: "binding",
|
||||
bindingIds: ["binding"], createdAt: new Date().toISOString(), payload: { slackEvent: {
|
||||
endpointId: endpoint.id, companyId: endpoint.companyId, publicId: endpoint.publicId,
|
||||
connectionId: endpoint.connectionId, runtimeGeneration: 1 } } }))
|
||||
.rejects.toMatchObject({ code: "signing_credential_unavailable", disposition: "attention" });
|
||||
expect(handleWebhook).not.toHaveBeenCalled();
|
||||
} finally { unregister(); }
|
||||
});
|
||||
@@ -0,0 +1,63 @@
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { chatEndpoints, toolConnections } from "@paperclipai/db";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import type { chatCredentialMutationLease } from "../../chat-credential-mutation-lease.js";
|
||||
import type { createChatCredentials } from "../credentials.js";
|
||||
import type { EndpointRow, InternalSetupState } from "../runtime-types.js";
|
||||
import { SlackCloudReceiptError, receiveSlackCloudEvent, type CurrentSlackCloudBinding, type SlackCloudEvent } from "./cloud-event.js";
|
||||
|
||||
import { resolveSlackCloudSigningSecret } from "./cloud-signing-secret.js";
|
||||
|
||||
type Handler = (event: SlackCloudEvent) => Promise<void>;
|
||||
const handlers = new WeakMap<Db, Handler>();
|
||||
|
||||
export function registerSlackCloudEventIngress(db: Db, handler: Handler) {
|
||||
handlers.set(db, handler);
|
||||
return () => { if (handlers.get(db) === handler) handlers.delete(db); };
|
||||
}
|
||||
|
||||
export async function dispatchSlackCloudEvent(db: Db, event: SlackCloudEvent): Promise<void> {
|
||||
const handler = handlers.get(db);
|
||||
if (!handler) throw new Error("Slack Cloud ingress is not ready");
|
||||
await handler(event);
|
||||
}
|
||||
|
||||
export function createSlackCloudEventIngress(db: Db, options: {
|
||||
withCredentialMutationLease: ReturnType<typeof chatCredentialMutationLease>;
|
||||
resolveCredentials: ReturnType<typeof createChatCredentials>["resolveCredentials"];
|
||||
runtimeGeneration(endpoint: EndpointRow): number;
|
||||
handleWebhook(publicId: string, provider: "slack", request: Request, expectedGeneration: number): Promise<Response>;
|
||||
}) {
|
||||
return registerSlackCloudEventIngress(db, async event => {
|
||||
const value = event.payload.slackEvent;
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) throw new SlackCloudReceiptError("invalid_receipt", "invalid");
|
||||
const packet = value as Record<string, unknown>;
|
||||
if (typeof packet.endpointId !== "string" || !/^[a-f0-9-]{36}$/i.test(packet.endpointId))
|
||||
throw new SlackCloudReceiptError("invalid_receipt", "invalid");
|
||||
const [initial] = await db.select().from(chatEndpoints).where(eq(chatEndpoints.id, packet.endpointId)).limit(1);
|
||||
if (!initial || initial.provider !== "slack" || ["archived", "paused", "revoked"].includes(initial.status)) return;
|
||||
const binding = await options.withCredentialMutationLease(initial, async (lease): Promise<CurrentSlackCloudBinding | null> => {
|
||||
const [current] = await db.select({ endpoint: chatEndpoints, connection: toolConnections }).from(chatEndpoints)
|
||||
.innerJoin(toolConnections, and(eq(toolConnections.id, chatEndpoints.connectionId), eq(toolConnections.companyId, chatEndpoints.companyId)))
|
||||
.where(eq(chatEndpoints.id, initial.id)).limit(1);
|
||||
if (!current || current.endpoint.provider !== "slack" || ["archived", "paused", "revoked"].includes(current.endpoint.status)
|
||||
|| current.connection.status !== "active" || !current.connection.enabled) return null;
|
||||
const endpoint = current.endpoint;
|
||||
if (packet.companyId !== endpoint.companyId || packet.publicId !== endpoint.publicId
|
||||
|| packet.connectionId !== endpoint.connectionId || packet.runtimeGeneration !== options.runtimeGeneration(endpoint)) return null;
|
||||
const enrolled = (endpoint.setup as InternalSetupState).slackCloudEvents;
|
||||
|
||||
if (!enrolled) throw new Error("Slack Cloud enrollment is not locally committed");
|
||||
if (enrolled.version !== 1 || enrolled.runtimeGeneration !== options.runtimeGeneration(endpoint)
|
||||
|| enrolled.teamId !== endpoint.providerAccountId) throw new Error("Slack Cloud enrollment is unsettled");
|
||||
const signingSecret = await resolveSlackCloudSigningSecret(options.resolveCredentials, endpoint, db, ["signingSecret"]);
|
||||
await lease.assertOwned();
|
||||
return { companyId: endpoint.companyId, endpointId: endpoint.id,
|
||||
publicId: endpoint.publicId, connectionId: endpoint.connectionId, runtimeGeneration: options.runtimeGeneration(endpoint),
|
||||
bindingId: enrolled.bindingId, teamId: enrolled.teamId, appId: enrolled.appId, signingSecret };
|
||||
});
|
||||
if (!binding) return;
|
||||
await receiveSlackCloudEvent(event, binding,
|
||||
request => options.handleWebhook(binding.publicId, "slack", request, binding.runtimeGeneration));
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,202 @@
|
||||
import { createHmac, generateKeyPairSync, randomUUID } from "node:crypto";
|
||||
import { once } from "node:events";
|
||||
import { createServer } from "node:http";
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { expect, it, vi } from "vitest";
|
||||
import { SLACK_CHAT_BOT_SCOPES, type ToolCredentialSecretRef } from "@paperclipai/shared";
|
||||
import { activityLog, agents, applyPendingMigrations, authUsers, chatEndpoints, chatSlackRegistrations, closeCheckpointDb, closeFileStorageDb, closeLivenessDb, companies, companyMemberships, companySecrets, createDb, toolApplications, toolConnections } from "@paperclipai/db";
|
||||
import { resolveExternalTestDatabaseUrl } from "../../../lib/dedicated-test-database-url.js";
|
||||
import { heartbeatService } from "../../heartbeat.js";
|
||||
import { chatChannelService } from "../../chat-channels.js";
|
||||
import { createDeliveryWorkCoordinator } from "../../delivery-work-coordinator.js";
|
||||
import { githubConnectionEventService } from "../../github-connection-events.js";
|
||||
import { registerGitHubConnectionEventWork } from "../../github-connection-work.js";
|
||||
import { createPaperclipCloudConnector, paperclipCloudConnectorConfigFromEnv } from "../../paperclip-cloud-connector.js";
|
||||
import { secretService } from "../../secrets.js";
|
||||
import type { InternalSetupState } from "../runtime-types.js";
|
||||
import { guardSlackFixtureNetwork } from "../../../__tests__/helpers/slack-cloud/transport.js";
|
||||
|
||||
const appUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_SLACK_TEST_DATABASE_URL");
|
||||
const cloudUrl = resolveExternalTestDatabaseUrl("SLACK_EVENT_TEST_DATABASE_URL");
|
||||
const cloudDist = process.env.SLACK_TEST_CLOUD_DIST;
|
||||
|
||||
it.skipIf(!appUrl || !cloudUrl || !cloudDist)("normal notifications enroll the first consumer and authorized reconfigure replaces only positively retired instance custody", async () => {
|
||||
const network = guardSlackFixtureNetwork({ requirePreload: true });
|
||||
const load = (file: string) => import(/* @vite-ignore */ pathToFileURL(path.resolve(cloudDist!, file)).href);
|
||||
const [{ PostgresCloudHarnessRegistry }, { PostgresConnectorStore }, { ConnectorBroker }, { InMemoryProviderSecretStore },
|
||||
{ providerRegistryFromEnvironment }, { CloudStackProvisioner }, { FakeCloudStackProvider }, { Pool }] = await Promise.all([
|
||||
load("registry/postgres.js"), load("connections/postgres.js"), load("connections/broker.js"), load("providers/neon.js"),
|
||||
load("connections/providers.js"), load("provisioner/provisioner.js"), load("providers/fake.js"),
|
||||
import(/* @vite-ignore */ pathToFileURL(path.resolve(cloudDist!, "../node_modules/pg/lib/index.js")).href),
|
||||
]);
|
||||
await applyPendingMigrations(appUrl!);
|
||||
const db = createDb(appUrl!, { maxConnections: 2, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
|
||||
const pool = new Pool({ connectionString: cloudUrl!, max: 2 });
|
||||
const registry = Reflect.construct(PostgresCloudHarnessRegistry, [pool]) as {
|
||||
applyMigrations(folder: string, timeout: number): Promise<void>; hydrate(): Promise<void>; close(): Promise<void>;
|
||||
};
|
||||
try { await registry.applyMigrations(path.resolve(cloudDist!, "../migrations"), 30_000); await registry.hydrate(); }
|
||||
catch (error) { await db.$client.end({ timeout: 1 }); await registry.close(); throw error; }
|
||||
const store = new PostgresConnectorStore(pool), secrets = new InMemoryProviderSecretStore();
|
||||
const id = randomUUID(), companyId = randomUUID(), endpointId = randomUUID(), connectionId = randomUUID(), applicationId = randomUUID(), agentId = randomUUID();
|
||||
const actor = `owner-${id}`, instanceA = `a-${id}`, instanceB = `b-${id}`, publicId = "h".repeat(43);
|
||||
const stackIds = [instanceA, instanceB], signingSecret = "0123456789abcdef0123456789abcdef";
|
||||
const keys = stackIds.map(() => ({ sign: generateKeyPairSync("ed25519"), seal: generateKeyPairSync("x25519") }));
|
||||
let broker: InstanceType<typeof ConnectorBroker>, base = "", loseEnrollmentResponse = false;
|
||||
const requests: { path: string; issuer?: string }[] = [];
|
||||
const server = createServer((req, res) => {
|
||||
requests.push({ path: req.url! });
|
||||
if (loseEnrollmentResponse && req.url === "/v1/connector/slack-events") {
|
||||
loseEnrollmentResponse = false;
|
||||
res.end = (() => { res.destroy(); return res; }) as typeof res.end;
|
||||
}
|
||||
void broker.handle(req, res, new URL(req.url!, base)).catch((error: unknown) => {
|
||||
if (!res.headersSent) res.writeHead(500); res.end(String(error));
|
||||
});
|
||||
});
|
||||
const directory = await mkdtemp(path.join(os.tmpdir(), "slack-transition-"));
|
||||
let service: ReturnType<typeof chatChannelService> | undefined;
|
||||
let heartbeat: ReturnType<typeof heartbeatService> | undefined;
|
||||
let coordinator: ReturnType<typeof createDeliveryWorkCoordinator> | undefined;
|
||||
let worker: ReturnType<typeof registerGitHubConnectionEventWork> | undefined;
|
||||
const errors: unknown[] = [];
|
||||
const deliveryIds: string[] = [];
|
||||
const endpoint = async () => (await db.select().from(chatEndpoints).where(eq(chatEndpoints.id, endpointId)))[0]!;
|
||||
const setup = async () => (await endpoint()).setup as InternalSetupState;
|
||||
const configureIdentity = (index: number) => {
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID", stackIds[index]!);
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SIGN_PRIVATE_KEY", keys[index]!.sign.privateKey.export({ format: "jwk" }).d!);
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SEAL_PRIVATE_KEY", keys[index]!.seal.privateKey.export({ format: "jwk" }).d!);
|
||||
};
|
||||
try {
|
||||
for (const [index, stackId] of stackIds.entries()) {
|
||||
await new CloudStackProvisioner({ provider: new FakeCloudStackProvider(), registry }).provisionDevelopmentStack({
|
||||
idempotencyKey: stackId, accountGroup: { id: `account-${stackId}`, slug: stackId, displayName: "Transition" },
|
||||
owner: { id: actor, email: `${id}@example.test` },
|
||||
stack: { stackId, accountGroupId: `account-${stackId}`, stackSlug: stackId, displayName: "Transition", kind: "demo", primaryHost: `${stackId}.example.test`, region: "us-east-1" },
|
||||
versionPlan: { appVersion: "fixture", schemaVersion: "fixture", rolloutTrack: "stable" },
|
||||
sleepPolicy: { desiredState: "awake" }, primaryCompany: { paperclipCompanyId: index === 0 ? companyId : randomUUID(), displayName: "Transition", defaultBudgetCents: 0 },
|
||||
});
|
||||
await store.putCloudStackInstance({ id: stackId, kind: "cloud_stack", companyStackId: stackId, ownerUserId: actor, environment: "test",
|
||||
signPublicKey: keys[index]!.sign.publicKey.export({ format: "jwk" }).x!, sealPublicKey: keys[index]!.seal.publicKey.export({ format: "jwk" }).x!,
|
||||
origins: ["https://tenant.example"], activeOrigin: "https://tenant.example", label: "Transition", status: "active", createdAt: new Date(), updatedAt: new Date() });
|
||||
}
|
||||
server.listen(0, "127.0.0.1"); await once(server, "listening");
|
||||
const address = server.address(); if (!address || typeof address === "string") throw new Error("Missing fixture address");
|
||||
base = `http://127.0.0.1:${address.port}`;
|
||||
broker = new ConnectorBroker({ publicOrigin: base, environment: "test", store,
|
||||
providers: providerRegistryFromEnvironment({}, "test"), secrets: { resolve: async () => undefined },
|
||||
authenticate: async () => ({ globalUserId: actor }), slackEvents: { secrets, secretRefPrefix: "secret:///test/transition" },
|
||||
eventWakeEnrollmentEnabled: true, eventWakeDeliveryEnabled: true, wakeCloudStack: async () => {} });
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_BASE_URL", base); vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_ENVIRONMENT", "test");
|
||||
vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY_FILE", path.join(directory, "master.key")); configureIdentity(0);
|
||||
await db.insert(companies).values({ id: companyId, name: "Transition", issuePrefix: `T${id.slice(0, 7)}` });
|
||||
await db.insert(authUsers).values({ id: actor, name: "Owner", email: `${id}@owner.example.test`, emailVerified: true, createdAt: new Date(), updatedAt: new Date() });
|
||||
await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: actor, status: "active", membershipRole: "operator" });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Transition", adapterType: "process", status: "idle" });
|
||||
await db.insert(toolApplications).values({ id: applicationId, companyId, name: "Transition", type: "chat" });
|
||||
await db.insert(toolConnections).values({ id: connectionId, companyId, applicationId, name: "Transition", uid: id,
|
||||
transport: "chat_sdk", connectionPurpose: "channel", enabled: false, status: "disabled" });
|
||||
await db.insert(chatEndpoints).values({ id: endpointId, companyId, connectionId, assignedAgentId: agentId, publicId,
|
||||
provider: "slack", status: "paused", providerAccountId: "TTEST", botExternalId: "UBOT", setup: { runtimeGeneration: 0, step: "complete" } as InternalSetupState });
|
||||
await db.insert(chatSlackRegistrations).values({ companyId, endpointId, requestId: randomUUID(), status: "configured", appId: "ATEST",
|
||||
manifest: {}, manifestHash: "fixture", callbackUri: "https://tenant.example/callback", createdByUserId: actor });
|
||||
const vault = secretService(db), refs: ToolCredentialSecretRef[] = [];
|
||||
for (const [key, value] of Object.entries({ botToken: "xoxb-fixture", signingSecret })) {
|
||||
const secret = await vault.create(companyId, { name: `${key}-${id}`, provider: "local_encrypted", value }, { userId: actor });
|
||||
await vault.createBinding({ companyId, secretId: secret.id, targetType: "tool_connection", targetId: connectionId, configPath: `credentials.${key}` });
|
||||
refs.push({ secretId: secret.id, versionSelector: "latest", configPath: `credentials.${key}`, required: true, projectionClass: "unclassified" });
|
||||
}
|
||||
await db.update(toolConnections).set({ credentialSecretRefs: refs }).where(eq(toolConnections.id, connectionId));
|
||||
const fetchImpl: typeof fetch = async input => {
|
||||
const url = input instanceof Request ? new URL(input.url) : new URL(String(input));
|
||||
if (url.href === "https://slack.com/api/auth.test") return Response.json({ ok: true, team_id: "TTEST", user_id: "UBOT", user: "fixture" }, { headers: { "x-oauth-scopes": SLACK_CHAT_BOT_SCOPES.join(",") } });
|
||||
if (url.origin === "https://slack.com" && url.pathname === "/api/conversations.list") return Response.json({ ok: true, channels: [], response_metadata: {} });
|
||||
throw new Error(`Unexpected transition provider request ${url}`);
|
||||
};
|
||||
heartbeat = heartbeatService(db);
|
||||
service = chatChannelService(db, { heartbeat, fetch: fetchImpl, publicBaseUrl: "https://tenant.example" });
|
||||
const events = githubConnectionEventService(db);
|
||||
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) });
|
||||
worker = registerGitHubConnectionEventWork(db, coordinator, events, () => true); await worker.ready;
|
||||
await vi.waitFor(() => expect(coordinator!.nextWakeAt()).toBeNull());
|
||||
const leaseCount = () => requests.filter(value => value.path === "/v1/connector/events/lease").length;
|
||||
expect(leaseCount()).toBe(0);
|
||||
await service.configure(endpointId, { action: "resume" }, actor);
|
||||
await vi.waitFor(async () => expect((await setup()).slackCloudEvents?.runtimeGeneration).toBe(1), { timeout: 10_000 });
|
||||
await vi.waitFor(() => expect(leaseCount()).toBeGreaterThan(0), { timeout: 10_000 });
|
||||
expect((await store.getInstance(instanceA)).eventWakeVersion).toBe(1);
|
||||
expect((await setup()).slackCloudInstanceId).toBe(instanceA);
|
||||
await service.configure(endpointId, { action: "pause" }, actor);
|
||||
await vi.waitFor(async () => expect((await setup()).slackCloudRetiredGeneration).toBe(2), { timeout: 10_000 });
|
||||
expect(await broker.slackEvents.current(instanceA, publicId)).toBeUndefined();
|
||||
expect((await store.listWebhookBindings(instanceA)).every((binding: { receivesEvents: boolean }) => !binding.receivesEvents)).toBe(true);
|
||||
const retired = await setup(); configureIdentity(1);
|
||||
const foreign = createPaperclipCloudConnector({ config: paperclipCloudConnectorConfigFromEnv()! });
|
||||
await expect(foreign.slackEventEnrollment({ subject: `foreign-${actor}`, companyId, enrollment: {
|
||||
active: true, companyId, endpointId, publicId, connectionId, runtimeGeneration: 3, teamId: "TTEST", appId: "ATEST", signingSecret,
|
||||
} })).rejects.toMatchObject({ status: 403 });
|
||||
expect(await broker.slackEvents.current(instanceB, publicId)).toBeUndefined();
|
||||
expect((await setup()).slackCloudIntent).toBeUndefined();
|
||||
expect((await setup()).slackCloudInstanceId).toBe(instanceA);
|
||||
expect((await setup()).slackCloudSubject).toBe(actor);
|
||||
await service.configure(endpointId, { action: "resume" }, actor);
|
||||
const pending = await setup();
|
||||
expect(pending.slackCloudIntent).toMatchObject({ instanceId: instanceB, runtimeGeneration: 3, subject: actor });
|
||||
expect(pending.slackCloudInstanceId).toBe(instanceA);
|
||||
expect(pending.slackCloudEvents).toEqual(retired.slackCloudEvents);
|
||||
await vi.waitFor(async () => expect((await setup()).slackCloudInstanceId).toBe(instanceB), { timeout: 10_000 });
|
||||
expect((await setup()).slackCloudIntent).toBeUndefined();
|
||||
expect((await setup()).slackCloudRetiredGeneration).toBeUndefined();
|
||||
expect((await broker.slackEvents.current(instanceB, publicId)).runtimeGeneration).toBe(3);
|
||||
expect(await broker.slackEvents.current(instanceA, publicId)).toBeUndefined();
|
||||
await service.configure(endpointId, { action: "pause" }, actor);
|
||||
await vi.waitFor(async () => expect((await setup()).slackCloudRetiredGeneration).toBe(4), { timeout: 10_000 });
|
||||
loseEnrollmentResponse = true;
|
||||
await service.configure(endpointId, { action: "resume" }, actor);
|
||||
await vi.waitFor(async () => expect((await setup()).slackCloudIntent).toMatchObject({ instanceId: instanceB,
|
||||
runtimeGeneration: 5, failureCode: "broker_unavailable" }), { timeout: 10_000 });
|
||||
expect((await broker.slackEvents.current(instanceB, publicId)).runtimeGeneration).toBe(5);
|
||||
expect((await setup()).slackCloudEvents?.runtimeGeneration).toBe(3);
|
||||
const rawBody = Buffer.from(JSON.stringify({ type: "event_callback", event_id: `EvLostReplacement-${id}`, team_id: "TTEST", api_app_id: "ATEST",
|
||||
event: { type: "app_mention", text: "removed before admission", user: "UOPERATOR", channel: "CTEST", ts: "1788.600" } }));
|
||||
const timestamp = String(Math.floor(Date.now() / 1000));
|
||||
await broker.slackEvents.receive(await broker.slackEvents.current(instanceB, publicId), { rawBody, timestamp,
|
||||
signature: `v0=${createHmac("sha256", signingSecret).update(`v0:${timestamp}:`).update(rawBody).digest("hex")}` });
|
||||
deliveryIds.push(...(await pool.query("SELECT delivery_id FROM cloud_harness.connector_webhook_delivery_targets WHERE connector_instance_id=$1", [instanceB])).rows.map((row: { delivery_id: string }) => row.delivery_id));
|
||||
await service.configure(endpointId, { action: "remove" }, actor);
|
||||
expect((await db.select().from(toolConnections).where(eq(toolConnections.id, connectionId)))[0]!.credentialSecretRefs).toEqual([]);
|
||||
await vi.waitFor(async () => expect((await setup()).slackCloudRetiredGeneration).toBe(6), { timeout: 10_000 });
|
||||
expect((await setup()).slackCloudIntent).toBeUndefined();
|
||||
expect((await setup()).slackCloudInstanceId).toBe(instanceB);
|
||||
expect(await broker.slackEvents.current(instanceB, publicId)).toBeUndefined();
|
||||
expect((await pool.query("SELECT delivery_id FROM cloud_harness.connector_webhook_delivery_targets WHERE connector_instance_id=$1", [instanceB])).rows).toEqual([]);
|
||||
await expect(broker.slackEvents.enroll(await store.getInstance(instanceB), actor, { companyId, endpointId, publicId, connectionId,
|
||||
runtimeGeneration: 5, teamId: "TTEST", appId: "ATEST", signingSecret })).rejects.toThrow("GENERATION_CHANGED");
|
||||
expect(errors).toEqual([]);
|
||||
} catch (error) { console.error("Slack instance transition body failure", error); throw error; }
|
||||
finally {
|
||||
await worker?.stop(); await coordinator?.stop(); await service?.shutdown(); await heartbeat?.drainActiveRunExecutions();
|
||||
if (server.listening) { server.close(); await once(server, "close"); }
|
||||
vi.unstubAllEnvs();
|
||||
try {
|
||||
await db.delete(chatEndpoints).where(eq(chatEndpoints.companyId, companyId));
|
||||
await db.delete(activityLog).where(eq(activityLog.companyId, companyId));
|
||||
await db.delete(agents).where(eq(agents.companyId, companyId));
|
||||
await db.delete(companyMemberships).where(eq(companyMemberships.companyId, companyId));
|
||||
await db.delete(companySecrets).where(eq(companySecrets.companyId, companyId));
|
||||
await db.delete(companies).where(eq(companies.id, companyId));
|
||||
await db.delete(authUsers).where(eq(authUsers.id, actor));
|
||||
await pool.query("DELETE FROM cloud_harness.connector_instances WHERE id=ANY($1::text[])", [stackIds]);
|
||||
await pool.query("DELETE FROM cloud_harness.connector_webhook_deliveries WHERE id=ANY($1::text[])", [deliveryIds]);
|
||||
} finally {
|
||||
await closeCheckpointDb(db, 1); await closeFileStorageDb(db, 1); await closeLivenessDb(db, 1);
|
||||
await db.$client.end({ timeout: 1 }); await registry.close(); await rm(directory, { recursive: true, force: true });
|
||||
try { network.assertQuiet(); } finally { network.restore(); }
|
||||
}
|
||||
}
|
||||
}, 60_000);
|
||||
@@ -0,0 +1,19 @@
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { expect, it, vi } from "vitest";
|
||||
import { HttpError } from "../../../errors.js";
|
||||
import type { EndpointRow } from "../runtime-types.js";
|
||||
import { resolveSlackCloudSigningSecret } from "./cloud-signing-secret.js";
|
||||
|
||||
it.each([400, 401, 403, 404, 422])("credential resolution %s cannot authenticate or acknowledge a receipt", async status => {
|
||||
const resolve = vi.fn(async (): Promise<Record<string, string>> => { throw new HttpError(status, "synthetic"); });
|
||||
await expect(resolveSlackCloudSigningSecret(resolve, {} as EndpointRow, {} as Db, ["signingSecret"]))
|
||||
.rejects.toMatchObject({ code: "signing_credential_unavailable", disposition: "attention" });
|
||||
});
|
||||
|
||||
it("empty credential is attention, while transient failures retain their retry classification", async () => {
|
||||
await expect(resolveSlackCloudSigningSecret(async () => ({}), {} as EndpointRow))
|
||||
.rejects.toMatchObject({ code: "signing_credential_unavailable", disposition: "attention" });
|
||||
const transient = new Error("database temporarily unavailable");
|
||||
await expect(resolveSlackCloudSigningSecret(async () => { throw transient; }, {} as EndpointRow)).rejects.toBe(transient);
|
||||
await expect(resolveSlackCloudSigningSecret(async () => ({ signingSecret: "synthetic" }), {} as EndpointRow)).resolves.toBe("synthetic");
|
||||
});
|
||||
@@ -0,0 +1,18 @@
|
||||
import { HttpError } from "../../../errors.js";
|
||||
import type { createChatCredentials } from "../credentials.js";
|
||||
import { SlackCloudReceiptError } from "./cloud-event.js";
|
||||
|
||||
type Resolver = ReturnType<typeof createChatCredentials>["resolveCredentials"];
|
||||
|
||||
export async function resolveSlackCloudSigningSecret(resolve: Resolver, ...args: Parameters<Resolver>): Promise<string> {
|
||||
let credentials: Record<string, string>;
|
||||
try {
|
||||
credentials = await resolve(...args);
|
||||
} catch (error) {
|
||||
if (error instanceof HttpError && [400, 401, 403, 404, 422].includes(error.status))
|
||||
throw new SlackCloudReceiptError("signing_credential_unavailable", "attention");
|
||||
throw error;
|
||||
}
|
||||
if (!credentials.signingSecret) throw new SlackCloudReceiptError("signing_credential_unavailable", "attention");
|
||||
return credentials.signingSecret;
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { expect, it } from "vitest";
|
||||
|
||||
const guard = fileURLToPath(new URL("../../../__tests__/helpers/slack-cloud/prewire.cjs", import.meta.url));
|
||||
const launcher = fileURLToPath(new URL("../../../../scripts/test-slack-cloud-postgres.mjs", import.meta.url));
|
||||
const cleanEnv = () => ({ PATH: process.env.PATH, HOME: process.env.HOME, NODE_OPTIONS: "" });
|
||||
|
||||
it.each([
|
||||
"require('node:http').get('http://example.invalid')",
|
||||
"require('node:https').get('https://example.invalid')",
|
||||
"require('node:net').connect(443,'example.invalid')",
|
||||
"require('node:tls').connect(443,'example.invalid')",
|
||||
"fetch('https://example.invalid')",
|
||||
"require('node:net').connect(55448,'127.0.0.1')",
|
||||
"require('node:http').get('http://127.0.0.1:1/')",
|
||||
])("the checked-in guard fails caught unowned transport attempts: %s", operation => {
|
||||
const child = spawnSync(process.execPath, ["--require", guard, "-e",
|
||||
`let dnsCalls=0; require('node:dns').lookup=()=>{dnsCalls++;throw Error('DNS must not run');}; try { ${operation}; } catch {} console.log(JSON.stringify({...globalThis[Symbol.for('paperclip.slack.fixture.prewire')].stats(),dnsCalls}));`],
|
||||
{ env: cleanEnv(), encoding: "utf8", timeout: 5_000 });
|
||||
expect(child.error).toBeUndefined();
|
||||
expect(child.status).toBe(98);
|
||||
const stats = JSON.parse(child.stdout.trim());
|
||||
expect(stats.pid).toBe(child.pid);
|
||||
expect(stats.unexpectedAttempts).toHaveLength(1);
|
||||
expect(stats.peakPgSockets).toBe(0);
|
||||
expect(stats.dnsCalls).toBe(0);
|
||||
});
|
||||
|
||||
it("the same guard allows an actual owned loopback HTTP server and initializes before the child script", () => {
|
||||
const child = spawnSync(process.execPath, ["--require", guard, "-e", `
|
||||
const http=require('node:http');
|
||||
if(process.env.SLACK_GATE_GUARD_PID!==String(process.pid))throw Error('preload missing');
|
||||
const server=http.createServer((req,res)=>res.end('owned response'));
|
||||
server.listen(0,'127.0.0.1',()=>http.get('http://127.0.0.1:'+server.address().port,response=>{
|
||||
let body='';response.on('data',chunk=>body+=chunk);response.on('end',()=>server.close(()=>{
|
||||
if(body!=='owned response')throw Error('response missing');
|
||||
console.log(JSON.stringify(globalThis[Symbol.for('paperclip.slack.fixture.prewire')].stats()));
|
||||
}));
|
||||
}));`], { env: cleanEnv(), encoding: "utf8", timeout: 5_000 });
|
||||
expect(child.error).toBeUndefined();
|
||||
expect(child.status).toBe(0);
|
||||
expect(JSON.parse(child.stdout.trim())).toMatchObject({ pid: child.pid, peakPgSockets: 0, unexpectedAttempts: [] });
|
||||
});
|
||||
|
||||
it.each([
|
||||
{},
|
||||
{ PAPERCLIP_SLACK_TEST_DATABASE_URL: "postgres://user@127.0.0.1/live", SLACK_EVENT_TEST_DATABASE_URL: "postgres://user@127.0.0.1/cloud_test" },
|
||||
{ PAPERCLIP_SLACK_TEST_DATABASE_URL: "postgres://user@127.0.0.1/same_test", SLACK_EVENT_TEST_DATABASE_URL: "postgres://user@127.0.0.1/same_test" },
|
||||
{ PAPERCLIP_SLACK_TEST_DATABASE_URL: "postgres://user@example.invalid/app_test", SLACK_EVENT_TEST_DATABASE_URL: "postgres://user@127.0.0.1/cloud_test" },
|
||||
])("the repository launcher rejects unsafe or absent fixture settings before starting Vitest", settings => {
|
||||
const child = spawnSync(process.execPath, [launcher], { env: { ...cleanEnv(), ...settings }, encoding: "utf8", timeout: 5_000 });
|
||||
expect(child.error).toBeUndefined();
|
||||
expect(child.status).toBe(1);
|
||||
expect(child.stderr).toMatch(/test database|loopback|distinct/);
|
||||
expect(child.stdout).not.toContain("RUN");
|
||||
});
|
||||
@@ -1,3 +1,4 @@
|
||||
import { slackCloudAcceptedAt } from "./cloud-event.js";
|
||||
import { isProvisionedSlackSetup } from "@paperclipai/shared";
|
||||
import { slackRequestSignatureIsValid, slackRequestWorkspaceId, slackCallbackMatchesPublicUrl, inspectSlackCallback } from "./callback-verification.js";
|
||||
import type { InternalSetupState } from "../runtime-types.js";
|
||||
@@ -53,6 +54,7 @@ export function createSlackWebhookCallbacks(db: Db, options: SlackWebhookOptions
|
||||
return null;
|
||||
}
|
||||
async function recordSlackWebhookCallback(endpoint: EndpointRow, request: Request, slackCallbackInspection: Request, runtimeContext: RuntimeContext): Promise<void> {
|
||||
if (slackCloudAcceptedAt(request) !== undefined) return;
|
||||
const eventBody = await slackCallbackInspection.clone().text();
|
||||
const callback = await inspectSlackCallback(slackCallbackInspection);
|
||||
if (callback) {
|
||||
|
||||
@@ -17,6 +17,8 @@ import {
|
||||
} from "@chat-adapter/discord";
|
||||
import type { SlackAdapterConfig } from "@chat-adapter/slack";
|
||||
import { PaperclipSlackAdapter } from "./chat-channels/slack/adapter.js";
|
||||
import { verifySlackSignature } from "@chat-adapter/slack/webhook";
|
||||
import { slackCloudAcceptedAt } from "./chat-channels/slack/cloud-event.js";
|
||||
import {
|
||||
createTeamsAdapter,
|
||||
type TeamsAdapter,
|
||||
@@ -1361,6 +1363,13 @@ function createProviderAdapter(
|
||||
case "slack": {
|
||||
const adapterConfig: SlackAdapterConfig = {
|
||||
...config.credentials,
|
||||
webhookVerifier: async (request, body) => {
|
||||
await verifySlackSignature(body, request.headers, {
|
||||
signingSecret: config.credentials.signingSecret,
|
||||
now: () => slackCloudAcceptedAt(request) ?? Date.now(),
|
||||
});
|
||||
return true;
|
||||
},
|
||||
logger: resolvedLogger,
|
||||
mode: "webhook",
|
||||
nativeStreaming: true,
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
import { SlackCloudReceiptError } from "./chat-channels/slack/cloud-event.js";
|
||||
import { reconcilePendingSlackCloudEnrollments } from "./chat-channels/slack/cloud-enrollment.js";
|
||||
import { notifyExternalObjectRefresh } from "./external-object-refresh-notifications.js";
|
||||
import { notifyGitHubCredentialChange } from "./github-connection-notifications.js";
|
||||
import { notifyToolConnectionHealth } from "./tool-connection-health-notifications.js";
|
||||
import { dispatchGitHubBotCloudEvent } from "./chat-github-cloud-ingress.js";
|
||||
import { dispatchSlackCloudEvent } from "./chat-channels/slack/cloud-ingress.js";
|
||||
import { chatEndpoints } from "@paperclipai/db";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { isDeepStrictEqual } from "node:util";
|
||||
@@ -224,6 +227,9 @@ export function githubConnectionEventService(
|
||||
) {
|
||||
const now = options.now ?? (() => new Date());
|
||||
let nextPollAt = 0;
|
||||
let nextEnrollmentAt: number | null = null;
|
||||
let nextEnrollmentDiagnosticAt = 0;
|
||||
const slackDiagnostics = new Map<string, number>();
|
||||
let configuredConnector: {
|
||||
config: NonNullable<ReturnType<typeof paperclipCloudConnectorConfigFromEnv>>;
|
||||
connector: PaperclipCloudConnector;
|
||||
@@ -503,8 +509,27 @@ export function githubConnectionEventService(
|
||||
nextPollAt = Number.POSITIVE_INFINITY;
|
||||
return { leased: 0, processed: 0, duplicate: 0, ignored: 0, failed: 0 };
|
||||
}
|
||||
try {
|
||||
return await drainConnectorEvents(connector);
|
||||
} finally {
|
||||
try {
|
||||
nextEnrollmentAt = await reconcilePendingSlackCloudEnrollments(db, now());
|
||||
} catch {
|
||||
const time = now().getTime();
|
||||
nextEnrollmentAt = time + 5_000;
|
||||
if (time >= nextEnrollmentDiagnosticAt) {
|
||||
nextEnrollmentDiagnosticAt = time + 300_000;
|
||||
logger.warn({ reason: "enrollment_reconcile_unavailable" }, "Slack enrollment responsibility retained");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function drainConnectorEvents(connector: PaperclipCloudConnector): Promise<GitHubConnectionEventPollResult> {
|
||||
const bindings = await activeBindings();
|
||||
const bots = await db.select().from(chatEndpoints).where(and(eq(chatEndpoints.provider, "github"), sql`${chatEndpoints.setup}->'github'->>'cloudRegistrationId' is not null`, sql`${chatEndpoints.status} not in ('archived', 'disabled')`, sql`exists (select 1 from companies where id = ${chatEndpoints.companyId} and status <> 'archived')`));
|
||||
const githubBots = await db.select().from(chatEndpoints).where(and(eq(chatEndpoints.provider, "github"), sql`${chatEndpoints.setup}->'github'->>'cloudRegistrationId' is not null`, sql`${chatEndpoints.status} not in ('archived', 'disabled')`, sql`exists (select 1 from companies where id = ${chatEndpoints.companyId} and status <> 'archived')`));
|
||||
const slackBots = await db.select().from(chatEndpoints).where(and(eq(chatEndpoints.provider, "slack"), sql`${chatEndpoints.setup}->'slackCloudEvents' is not null`, sql`${chatEndpoints.status} in ('active','verifying','attention')`, sql`exists (select 1 from companies where id = ${chatEndpoints.companyId} and status <> 'archived')`));
|
||||
const bots = [...githubBots, ...slackBots];
|
||||
let identity: { subject: string; companyId: string } | null = bindings[0] ?? (bots[0] ? { subject: "github-bot-events", companyId: bots[0].companyId } : null);
|
||||
const hasConsumer = bindings.length > 0 || bots.length > 0;
|
||||
if (!identity) {
|
||||
@@ -517,7 +542,7 @@ export function githubConnectionEventService(
|
||||
if (grant && subject) identity = { subject, companyId: grant.companyId };
|
||||
}
|
||||
if (!identity) {
|
||||
const [history] = await db.select().from(chatEndpoints).where(and(eq(chatEndpoints.provider, "github"), sql`${chatEndpoints.setup}->'github'->>'cloudRegistrationId' is not null`)).limit(1);
|
||||
const [history] = await db.select().from(chatEndpoints).where(sql`(${chatEndpoints.provider} = 'github' AND ${chatEndpoints.setup}->'github'->>'cloudRegistrationId' is not null) OR (${chatEndpoints.provider} = 'slack' AND ${chatEndpoints.setup}->'slackCloudEvents' is not null)`).limit(1);
|
||||
if (history) identity = { subject: "github-bot-events", companyId: history.companyId };
|
||||
}
|
||||
if (!identity) {
|
||||
@@ -543,6 +568,23 @@ export function githubConnectionEventService(
|
||||
};
|
||||
const acknowledge: string[] = [];
|
||||
for (const event of lease.events) {
|
||||
if (event.provider === "slack") {
|
||||
try { await dispatchSlackCloudEvent(db, event); acknowledge.push(event.id); result.processed += 1; }
|
||||
catch (error) {
|
||||
const code = error instanceof SlackCloudReceiptError ? error.code : "owner_or_admission_unavailable";
|
||||
const disposition = error instanceof SlackCloudReceiptError ? error.disposition : "retry";
|
||||
const time = now().getTime();
|
||||
if (time - (slackDiagnostics.get(event.id) ?? Number.NEGATIVE_INFINITY) >= 300_000) {
|
||||
if (slackDiagnostics.size >= 1000) slackDiagnostics.delete(slackDiagnostics.keys().next().value!);
|
||||
slackDiagnostics.set(event.id, time);
|
||||
logger.warn({ deliveryId: event.id.slice(0, 200), reason: code, disposition,
|
||||
recoveryExpiresAt: Number.isFinite(Date.parse(event.createdAt)) ? new Date(Date.parse(event.createdAt) + 30 * 24 * 60 * 60_000).toISOString() : null }, "Slack connector receipt requires attention");
|
||||
}
|
||||
if (disposition === "invalid") { acknowledge.push(event.id); result.ignored += 1; }
|
||||
else result.failed += 1;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (event.payload.githubApp !== undefined) {
|
||||
try { await dispatchGitHubBotCloudEvent(db, event); acknowledge.push(event.id); result.processed += 1; }
|
||||
catch { result.failed += 1; }
|
||||
@@ -583,14 +625,14 @@ export function githubConnectionEventService(
|
||||
}
|
||||
|
||||
return {
|
||||
nextPollAt: () => Number.isFinite(nextPollAt) ? nextPollAt : null,
|
||||
nextPollAt: () => { const next = Math.min(nextPollAt, nextEnrollmentAt ?? Number.POSITIVE_INFINITY); return Number.isFinite(next) ? next : null; },
|
||||
pollOnce(input: { forSetup?: boolean; forDelivery?: boolean } = {}): Promise<GitHubConnectionEventPollResult> {
|
||||
// A manager actively awaiting a dedicated App's signed ping must not
|
||||
// inherit a personal connection's old empty-queue backoff. This is a
|
||||
// bounded readiness probe, not an additional background polling loop.
|
||||
if (inFlight) return inFlight;
|
||||
const time = now().getTime();
|
||||
if (!input.forDelivery && time < (input.forSetup ? nextSetupPollAt : nextPollAt)) {
|
||||
if (!input.forDelivery && time < (input.forSetup ? nextSetupPollAt : Math.min(nextPollAt, nextEnrollmentAt ?? Number.POSITIVE_INFINITY))) {
|
||||
return Promise.resolve({ leased: 0, processed: 0, duplicate: 0, ignored: 0, failed: 0 });
|
||||
}
|
||||
nextSetupPollAt = time + 5_000;
|
||||
|
||||
@@ -2471,8 +2471,23 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
|
||||
const legacyLease = await liveLegacyControllerExpiresAt(db, run);
|
||||
if (legacyLease) { scope.onDeferredDeadline?.(legacyLease.getTime()); continue; }
|
||||
|
||||
// A legacy controller proves it is alive only by renewing its lease, and
|
||||
// the check above found no live lease (a missing expiry also lands here;
|
||||
// revocation below then refuses the row). When the lease belongs to a
|
||||
// previous boot, setup never reached adapter dispatch, and no process
|
||||
// was recorded, the owner is gone and no provider work can be running:
|
||||
// recover now instead of waiting for the age threshold. A dispatched
|
||||
// adapter (for example a remote HTTP request) keeps the age guard. The
|
||||
// stage is rechecked from the state committed with the revocation below.
|
||||
const priorBootSetupExpired =
|
||||
run.runtimeMode === "legacy" &&
|
||||
run.executionStage === "preparing" &&
|
||||
!!run.controllerBootId &&
|
||||
run.controllerBootId !== legacyControllerBootId &&
|
||||
!run.processPid &&
|
||||
!run.processGroupId;
|
||||
// Apply staleness threshold to avoid false positives
|
||||
if (staleThresholdMs > 0) {
|
||||
if (staleThresholdMs > 0 && !priorBootSetupExpired) {
|
||||
const refTime = run.updatedAt ? new Date(run.updatedAt).getTime() : 0;
|
||||
if (now.getTime() - refTime < staleThresholdMs) {
|
||||
scope.onDeferredDeadline?.(refTime + staleThresholdMs);
|
||||
@@ -2543,6 +2558,15 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
|
||||
(!monitorNextCheckAt || monitorNextCheckAt.getTime() <= now.getTime());
|
||||
const revokedState = await revokeExpiredLegacyControllerWithState(db, run);
|
||||
if (!revokedState) continue;
|
||||
// The earlier snapshot may predate a final renewal that admitted the
|
||||
// adapter. Revocation leaves a fresh foreign lease, so a later pass
|
||||
// applies the age guard to that dispatched run.
|
||||
if (priorBootSetupExpired && staleThresholdMs > 0 && (revokedState.executionStage !== "preparing"
|
||||
|| revokedState.processPid || revokedState.processGroupId)) {
|
||||
const refTime = run.updatedAt ? new Date(run.updatedAt).getTime() : 0;
|
||||
scope.onDeferredDeadline?.(Math.max(refTime + staleThresholdMs, Date.now() + 60_000));
|
||||
continue;
|
||||
}
|
||||
// A legacy controller persists "dispatching" in the same lease renewal
|
||||
// that admits the adapter. An expired lease from another boot that is
|
||||
// still "preparing" proves a restart stopped setup before provider work.
|
||||
|
||||
@@ -25,7 +25,7 @@ import {
|
||||
import { nativeRuntimeContextFixture } from "./runtime-context.test-fixture.js";
|
||||
|
||||
const toolDefinitions = vi.hoisted(() => vi.fn());
|
||||
vi.mock("../../runtime-api.js", () => ({ resolveLocalRuntimeApiUrl: () => "http://127.0.0.1:43219" }));
|
||||
vi.mock("../../runtime-api.js", async importOriginal => ({ ...await importOriginal<typeof import("../../runtime-api.js")>(), resolveLocalRuntimeApiUrl: () => "http://127.0.0.1:43219" }));
|
||||
vi.mock("./paperclip-runner-tool-authority.js", () => ({
|
||||
PaperclipRunnerToolAuthority: class {
|
||||
readonly binding = {};
|
||||
@@ -185,7 +185,11 @@ describe.skipIf(!recoveryPostgresSupport.supported)("follow-up after a recovered
|
||||
runner: { execute: vi.fn(async () => { throw new Error("admission must not reach the sandbox"); }) },
|
||||
sandboxLeaseAcquisition: { outcome, providerLeaseId: `sandbox-${runId}` } } as never,
|
||||
}).then(() => "constructed", (error: Error) => error.message);
|
||||
return { runId, logs, start, selectedRunId: selected?.id ?? null };
|
||||
// A local Runner keeps its controller root on this host between turns.
|
||||
const startLocal = () => createRunnerdBackend({
|
||||
db, execution, runnerInstanceId: next.runnerInstanceId, onLog: async (_stream, chunk) => { logs.push(chunk); },
|
||||
}).then(() => "constructed", (error: Error) => error.message);
|
||||
return { runId, logs, start, startLocal, selectedRunId: selected?.id ?? null };
|
||||
}
|
||||
|
||||
async function quarantined() {
|
||||
@@ -465,6 +469,104 @@ describe.skipIf(!recoveryPostgresSupport.supported)("follow-up after a recovered
|
||||
expect(followUp.logs.join("")).not.toContain("finished without a saved runner checkpoint");
|
||||
}, 60_000);
|
||||
|
||||
/** A spawned process that stands in for the prior local Runner. */
|
||||
async function runnerProcess() {
|
||||
const child = spawn(process.execPath, ["-e", "setTimeout(() => {}, 60_000)"], { stdio: "ignore" });
|
||||
await new Promise(resolve => child.once("spawn", resolve));
|
||||
const startedAt = await readProcessStartedAt(child.pid!);
|
||||
expect(startedAt).toBeTruthy();
|
||||
return { child, pid: child.pid!, processStartedAt: new Date(startedAt!) };
|
||||
}
|
||||
|
||||
async function stoppedRunnerProcess() {
|
||||
const runner = await runnerProcess();
|
||||
const exited = new Promise(resolve => runner.child.once("exit", resolve));
|
||||
runner.child.kill("SIGKILL");
|
||||
await exited;
|
||||
return runner;
|
||||
}
|
||||
|
||||
/** The state a local per-turn Runner leaves when its result was accepted but
|
||||
* its suspend command never finished: the controller journal holds a pending
|
||||
* runner.suspend and the Runner's own receipt is still `ready`. */
|
||||
async function seedUnsuspendedLocalOwner(runner: { pid: number; processStartedAt: Date }) {
|
||||
const prior = await seedRecoveredPriorRun("pending-command");
|
||||
await db.update(heartbeatRuns).set({ processPid: runner.pid, processGroupId: runner.pid, processStartedAt: runner.processStartedAt })
|
||||
.where(eq(heartbeatRuns.id, prior.runId));
|
||||
const followUp = await admitFollowUp(prior);
|
||||
// Learn the scope key from a held attempt, then keep its root in place.
|
||||
expect(await followUp.start()).toMatch(/^runner_state_identity_mismatch: prior_owner_terminal_state_indeterminate/);
|
||||
const [copy] = await quarantined();
|
||||
const root = join(stateDirectory, copy!.split(".")[0]!);
|
||||
await rename(join(stateDirectory, "quarantine", copy!), root);
|
||||
await rm(join(stateDirectory, "quarantine"), { recursive: true, force: true });
|
||||
await mkdir(join(root, "runner"), { recursive: true });
|
||||
await writeFile(join(root, "runner", "runner-state.json"), JSON.stringify({
|
||||
schema: "paperclip.runner.durable.state.v1", lifecycle: "ready", ...prior.identity }));
|
||||
return { prior, followUp, root };
|
||||
}
|
||||
|
||||
it("retires a finished local owner whose Runner stopped before it proved suspension", async () => {
|
||||
const { prior, followUp, root } = await seedUnsuspendedLocalOwner(await stoppedRunnerProcess());
|
||||
expect(await followUp.startLocal()).toBe(ADMITTED);
|
||||
expect(await quarantined()).toEqual([]);
|
||||
expect(existsSync(join(root, "runner"))).toBe(false);
|
||||
expect(await archivedOwners(root)).toEqual([prior.runId]);
|
||||
expect(followUp.logs.join("")).toContain(`Run ${prior.runId} finished without a saved runner checkpoint`);
|
||||
}, 60_000);
|
||||
|
||||
it("keeps a finished local owner held while its recorded Runner process is alive", async () => {
|
||||
const runner = await runnerProcess();
|
||||
try {
|
||||
const { followUp } = await seedUnsuspendedLocalOwner(runner);
|
||||
expect(await followUp.startLocal()).toBe(stranded("runner_process_alive"));
|
||||
expect(followUp.logs.join("")).not.toContain("finished without a saved runner checkpoint");
|
||||
} finally {
|
||||
runner.child.kill("SIGKILL");
|
||||
}
|
||||
}, 60_000);
|
||||
|
||||
it.each(["no identity", "no process group"] as const)("keeps a finished local owner held with %s", async scenario => {
|
||||
const { prior, followUp } = await seedUnsuspendedLocalOwner(await stoppedRunnerProcess());
|
||||
await db.update(heartbeatRuns).set(scenario === "no identity"
|
||||
? { processPid: null, processGroupId: null, processStartedAt: null } : { processGroupId: null })
|
||||
.where(eq(heartbeatRuns.id, prior.runId));
|
||||
expect(await followUp.startLocal()).toBe(stranded("runner_process_identity_missing"));
|
||||
}, 60_000);
|
||||
|
||||
it("retires a finished local owner from its saved stop receipt after settlement cleared the process fields", async () => {
|
||||
const runner = await stoppedRunnerProcess();
|
||||
const { prior, followUp, root } = await seedUnsuspendedLocalOwner(runner);
|
||||
await db.insert(heartbeatRunEvents).values({ companyId, agentId, runId: prior.runId, seq: 900, eventType: "native.local_process_stopped",
|
||||
stream: "system", payload: { processPid: runner.pid, processGroupId: runner.pid, processStartedAt: runner.processStartedAt.toISOString() } });
|
||||
await db.update(heartbeatRuns).set({ processPid: null, processGroupId: null, processStartedAt: null })
|
||||
.where(eq(heartbeatRuns.id, prior.runId));
|
||||
expect(await followUp.startLocal()).toBe(ADMITTED);
|
||||
expect(await archivedOwners(root)).toEqual([prior.runId]);
|
||||
}, 60_000);
|
||||
|
||||
it("keeps a stopped local owner held while a provider process from its run events is alive", async () => {
|
||||
const provider = await runnerProcess();
|
||||
try {
|
||||
const { prior, followUp } = await seedUnsuspendedLocalOwner(await stoppedRunnerProcess());
|
||||
await db.insert(heartbeatRunEvents).values({ companyId, agentId, runId: prior.runId, seq: 901, eventType: "provider.event",
|
||||
stream: "system", payload: { prpEvent: { payload: { processId: provider.pid,
|
||||
processStartedAt: provider.processStartedAt.toISOString() } } } });
|
||||
expect(await followUp.startLocal()).toBe(stranded("provider_process_alive"));
|
||||
} finally {
|
||||
provider.child.kill("SIGKILL");
|
||||
}
|
||||
}, 60_000);
|
||||
|
||||
it("keeps a stopped local owner held by an unfinished non-lifecycle command", async () => {
|
||||
const { followUp, root } = await seedUnsuspendedLocalOwner(await stoppedRunnerProcess());
|
||||
const path = join(root, "control-plane", "control-plane-state.json");
|
||||
const state = JSON.parse(await readFile(path, "utf8"));
|
||||
state.commands.find((command: { type: string }) => command.type === "turn.start").status = "pending";
|
||||
await writeFile(path, JSON.stringify(state));
|
||||
expect(await followUp.startLocal()).toBe(stranded("journal_commands_unsettled(turn.start.pending)"));
|
||||
}, 60_000);
|
||||
|
||||
it.each(["uncommitted result", "committed without result", "live controller lease", "unexpired controller lease", "recovery pending", "pending command",
|
||||
"unanswered tool input", "foreign quarantine"] as const)("keeps the prior owner held with %s", async scenario => {
|
||||
const prior = await seedRecoveredPriorRun(scenario === "pending command" ? "pending-command"
|
||||
|
||||
@@ -5,8 +5,48 @@ import {
|
||||
evaluateNativeControllerTakeover,
|
||||
evaluateNativeProviderProcesses,
|
||||
nextNativeProviderAttempt,
|
||||
proveNativeLocalRunnerStopped,
|
||||
} from "./native-restart-recovery.js";
|
||||
|
||||
describe("local runner stop proof", () => {
|
||||
const startedAt = new Date("2026-11-06T17:58:54.000Z");
|
||||
const base = { processPid: 72454, processGroupId: 72454, processStartedAt: startedAt };
|
||||
const probe = (alive: number[], starts: Record<number, Date | null>, groups: number[] = []) => ({
|
||||
isProcessAlive: (pid: number) => alive.includes(pid),
|
||||
isProcessGroupAlive: (group: number) => groups.includes(group),
|
||||
readProcessStartedAt: async (pid: number) => starts[pid] ?? null,
|
||||
});
|
||||
|
||||
it.each([
|
||||
["the runner and its group exited", base, probe([], {}), { stopped: true }],
|
||||
["the runner PID now belongs to another process", base,
|
||||
probe([72454], { 72454: new Date(startedAt.getTime() + 60_000) }), { stopped: true }],
|
||||
["the recorded runner is alive", base, probe([72454], { 72454: startedAt }),
|
||||
{ stopped: false, reason: "runner_process_alive" }],
|
||||
["the live PID start time is unknown", base, probe([72454], {}),
|
||||
{ stopped: false, reason: "runner_process_identity_unverifiable" }],
|
||||
["the runner group is alive", base, probe([], {}, [72454]),
|
||||
{ stopped: false, reason: "runner_process_group_alive" }],
|
||||
["no runner PID was recorded", { ...base, processPid: null }, probe([], {}),
|
||||
{ stopped: false, reason: "runner_process_identity_missing" }],
|
||||
["no runner start time was recorded", { ...base, processStartedAt: null }, probe([], {}),
|
||||
{ stopped: false, reason: "runner_process_identity_missing" }],
|
||||
["no runner process group was recorded", { ...base, processGroupId: null }, probe([], {}),
|
||||
{ stopped: false, reason: "runner_process_identity_missing" }],
|
||||
["a stop receipt proved the runner absent", { ...base, processStartedAt: null, runnerStopRecorded: true },
|
||||
probe([], {}), { stopped: true }],
|
||||
] as const)("decides when %s", async (_case, identity, checks, expected) => {
|
||||
await expect(proveNativeLocalRunnerStopped({ ...identity, ...checks })).resolves.toEqual(expected);
|
||||
});
|
||||
|
||||
it("refuses while a recorded provider process is alive", async () => {
|
||||
const providerStartedAt = new Date("2026-11-06T17:58:56.000Z");
|
||||
await expect(proveNativeLocalRunnerStopped({ ...base,
|
||||
providerProcesses: [{ pid: 72460, processStartedAt: providerStartedAt }],
|
||||
...probe([72460], { 72460: providerStartedAt }) })).resolves.toEqual({ stopped: false, reason: "provider_process_alive" });
|
||||
});
|
||||
});
|
||||
|
||||
describe("native restart recovery classification", () => {
|
||||
it.each([false, true])("recognizes verified staged remote evidence with target snapshot=%s", hasCheckpoint => {
|
||||
const evidence = { remoteSandbox: true, runnerPidAlive: false, runnerGroupAlive: false,
|
||||
|
||||
@@ -336,6 +336,118 @@ export async function evaluateNativeProviderProcesses(input: {
|
||||
};
|
||||
}
|
||||
|
||||
/** Run event types that can carry a provider process identity. */
|
||||
export const NATIVE_PROVIDER_PROCESS_EVENT_TYPES = [
|
||||
"harness.ready",
|
||||
"session.started",
|
||||
"session.resumed",
|
||||
"session.updated",
|
||||
"turn.started",
|
||||
"provider.event",
|
||||
"provider.rpc_result",
|
||||
] as const;
|
||||
|
||||
function parsedProcessStart(value: unknown): Date | null {
|
||||
const parsed = typeof value === "string" ? new Date(value) : null;
|
||||
return parsed && !Number.isNaN(parsed.getTime()) ? parsed : null;
|
||||
}
|
||||
|
||||
/** Provider process identities from a saved checkpoint and recent run events. */
|
||||
export function nativeProviderProcessIdentities(
|
||||
checkpoint: Record<string, unknown>,
|
||||
providerEvents: ReadonlyArray<{ payload: Record<string, unknown> | null }>,
|
||||
): NativeProviderProcessIdentity[] {
|
||||
const identities: NativeProviderProcessIdentity[] = [];
|
||||
const checkpointProcess =
|
||||
checkpoint.process && typeof checkpoint.process === "object" && !Array.isArray(checkpoint.process)
|
||||
? (checkpoint.process as Record<string, unknown>)
|
||||
: {};
|
||||
for (const [field, startedAtField] of [
|
||||
["providerPid", "providerProcessStartedAt"],
|
||||
["codexPid", "codexProcessStartedAt"],
|
||||
["sidecarPid", "sidecarProcessStartedAt"],
|
||||
["agentPid", "agentProcessStartedAt"],
|
||||
] as const) {
|
||||
const value = checkpointProcess[field];
|
||||
if (typeof value === "number" && Number.isInteger(value) && value > 0) {
|
||||
identities.push({ pid: value, processStartedAt: parsedProcessStart(checkpointProcess[startedAtField]) });
|
||||
}
|
||||
}
|
||||
for (const providerEvent of providerEvents) {
|
||||
const prpEvent = providerEvent.payload?.prpEvent;
|
||||
const event = prpEvent && typeof prpEvent === "object" && !Array.isArray(prpEvent)
|
||||
? (prpEvent as Record<string, unknown>) : {};
|
||||
const eventPayload = event.payload && typeof event.payload === "object" && !Array.isArray(event.payload)
|
||||
? (event.payload as Record<string, unknown>) : {};
|
||||
const processId = eventPayload.processId;
|
||||
if (typeof processId === "number" && Number.isInteger(processId) && processId > 0) {
|
||||
identities.push({ pid: processId,
|
||||
processStartedAt: parsedProcessStart(eventPayload.processStartedAt ?? eventPayload.providerProcessStartedAt) });
|
||||
}
|
||||
}
|
||||
return identities;
|
||||
}
|
||||
|
||||
export type NativeLocalRunnerStopProof =
|
||||
| { stopped: true }
|
||||
| {
|
||||
stopped: false;
|
||||
reason:
|
||||
| "runner_process_identity_missing"
|
||||
| "runner_process_alive"
|
||||
| "runner_process_identity_unverifiable"
|
||||
| "runner_process_group_alive"
|
||||
| "provider_process_alive"
|
||||
| "provider_process_identity_unverifiable";
|
||||
};
|
||||
|
||||
/**
|
||||
* Proves that a finished local runner and its recorded provider processes
|
||||
* stopped, with the same exact PID and start-time checks as restart recovery.
|
||||
* A PID that now belongs to a different process counts as stopped. A live
|
||||
* process with the recorded identity, a live process group, or a live PID
|
||||
* whose start time cannot be compared is not proof.
|
||||
*/
|
||||
export async function proveNativeLocalRunnerStopped(input: {
|
||||
processPid: number | null;
|
||||
processGroupId: number | null;
|
||||
processStartedAt: Date | null;
|
||||
providerProcesses?: readonly NativeProviderProcessIdentity[];
|
||||
/** A current, server-authored stop receipt already proved this exact Runner and group absent. */
|
||||
runnerStopRecorded?: boolean;
|
||||
isProcessAlive?: (pid: number) => boolean;
|
||||
isProcessGroupAlive?: (processGroupId: number) => boolean;
|
||||
readProcessStartedAt?: (pid: number) => Promise<Date | null>;
|
||||
}): Promise<NativeLocalRunnerStopProof> {
|
||||
const refuse = (reason: Extract<NativeLocalRunnerStopProof, { stopped: false }>["reason"]) =>
|
||||
({ stopped: false as const, reason });
|
||||
// A local Runner leads its own process group. Without both identities, a
|
||||
// live child in that group could outlive the Runner unseen.
|
||||
if (!input.processPid || !Number.isInteger(input.processPid) || input.processPid <= 1 || !input.processGroupId
|
||||
|| (!input.processStartedAt && !input.runnerStopRecorded)) {
|
||||
return refuse("runner_process_identity_missing");
|
||||
}
|
||||
if (!input.runnerStopRecorded) {
|
||||
const runner = await evaluateNativeProviderProcesses({
|
||||
identities: [{ pid: input.processPid, processStartedAt: input.processStartedAt }],
|
||||
isProcessAlive: input.isProcessAlive,
|
||||
readProcessStartedAt: input.readProcessStartedAt,
|
||||
});
|
||||
if (runner.livePids.length > 0) return refuse("runner_process_alive");
|
||||
if (runner.ambiguousLivePids.length > 0) return refuse("runner_process_identity_unverifiable");
|
||||
}
|
||||
const groupAlive = input.isProcessGroupAlive ?? processGroupIsAlive;
|
||||
if (groupAlive(input.processGroupId)) return refuse("runner_process_group_alive");
|
||||
const providers = await evaluateNativeProviderProcesses({
|
||||
identities: (input.providerProcesses ?? []).filter(identity => identity.pid !== input.processPid),
|
||||
isProcessAlive: input.isProcessAlive,
|
||||
readProcessStartedAt: input.readProcessStartedAt,
|
||||
});
|
||||
if (providers.livePids.length > 0) return refuse("provider_process_alive");
|
||||
if (providers.ambiguousLivePids.length > 0) return refuse("provider_process_identity_unverifiable");
|
||||
return { stopped: true };
|
||||
}
|
||||
|
||||
export function classifyNativeRunnerRecoveryEvidence(input: {
|
||||
runnerPidAlive: boolean;
|
||||
runnerGroupAlive: boolean;
|
||||
@@ -898,79 +1010,12 @@ export async function claimNativeRestartRecoveries(input: {
|
||||
.where(
|
||||
and(
|
||||
eq(heartbeatRunEvents.runId, row.run.id),
|
||||
inArray(heartbeatRunEvents.eventType, [
|
||||
"harness.ready",
|
||||
"session.started",
|
||||
"session.resumed",
|
||||
"session.updated",
|
||||
"turn.started",
|
||||
"provider.event",
|
||||
"provider.rpc_result",
|
||||
]),
|
||||
inArray(heartbeatRunEvents.eventType, [...NATIVE_PROVIDER_PROCESS_EVENT_TYPES]),
|
||||
),
|
||||
)
|
||||
.orderBy(desc(heartbeatRunEvents.id))
|
||||
.limit(100);
|
||||
const checkpointProcess =
|
||||
checkpointRecord.process &&
|
||||
typeof checkpointRecord.process === "object" &&
|
||||
!Array.isArray(checkpointRecord.process)
|
||||
? (checkpointRecord.process as Record<string, unknown>)
|
||||
: {};
|
||||
const providerProcessIdentities: NativeProviderProcessIdentity[] = [];
|
||||
const checkpointProcessFields = [
|
||||
["providerPid", "providerProcessStartedAt"],
|
||||
["codexPid", "codexProcessStartedAt"],
|
||||
["sidecarPid", "sidecarProcessStartedAt"],
|
||||
["agentPid", "agentProcessStartedAt"],
|
||||
] as const;
|
||||
for (const [field, startedAtField] of checkpointProcessFields) {
|
||||
const value = checkpointProcess[field];
|
||||
if (typeof value === "number" && Number.isInteger(value) && value > 0) {
|
||||
const rawStartedAt = checkpointProcess[startedAtField];
|
||||
const parsedStartedAt =
|
||||
typeof rawStartedAt === "string" ? new Date(rawStartedAt) : null;
|
||||
providerProcessIdentities.push({
|
||||
pid: value,
|
||||
processStartedAt:
|
||||
parsedStartedAt && !Number.isNaN(parsedStartedAt.getTime())
|
||||
? parsedStartedAt
|
||||
: null,
|
||||
});
|
||||
}
|
||||
}
|
||||
for (const providerEvent of providerEvents) {
|
||||
const prpEvent = providerEvent.payload?.prpEvent;
|
||||
const event =
|
||||
prpEvent && typeof prpEvent === "object" && !Array.isArray(prpEvent)
|
||||
? (prpEvent as Record<string, unknown>)
|
||||
: {};
|
||||
const eventPayload =
|
||||
event.payload &&
|
||||
typeof event.payload === "object" &&
|
||||
!Array.isArray(event.payload)
|
||||
? (event.payload as Record<string, unknown>)
|
||||
: {};
|
||||
const processId = eventPayload.processId;
|
||||
if (
|
||||
typeof processId === "number" &&
|
||||
Number.isInteger(processId) &&
|
||||
processId > 0
|
||||
) {
|
||||
const rawStartedAt =
|
||||
eventPayload.processStartedAt ??
|
||||
eventPayload.providerProcessStartedAt;
|
||||
const parsedStartedAt =
|
||||
typeof rawStartedAt === "string" ? new Date(rawStartedAt) : null;
|
||||
providerProcessIdentities.push({
|
||||
pid: processId,
|
||||
processStartedAt:
|
||||
parsedStartedAt && !Number.isNaN(parsedStartedAt.getTime())
|
||||
? parsedStartedAt
|
||||
: null,
|
||||
});
|
||||
}
|
||||
}
|
||||
const providerProcessIdentities = nativeProviderProcessIdentities(checkpointRecord, providerEvents);
|
||||
const providerProcesses = await evaluateNativeProviderProcesses({
|
||||
identities: (remoteProcess ? [] : providerProcessIdentities).filter(
|
||||
(identity) => identity.pid !== row.run.processPid,
|
||||
|
||||
@@ -275,7 +275,7 @@ const state = vi.hoisted(() => ({
|
||||
release: null as null | (() => void),
|
||||
}));
|
||||
|
||||
vi.mock("../../runtime-api.js", () => ({ resolveLocalRuntimeApiUrl: state.localRuntimeApiUrl }));
|
||||
vi.mock("../../runtime-api.js", async importOriginal => ({ ...await importOriginal<typeof import("../../runtime-api.js")>(), resolveLocalRuntimeApiUrl: state.localRuntimeApiUrl }));
|
||||
|
||||
const accountingEvents = vi.hoisted(() => ({ committed: undefined as undefined | ((event: PrpEvent) => Promise<void>), duplicate: undefined as undefined | ((event: PrpEvent) => Promise<void>) }));
|
||||
vi.mock("./paperclip-control-plane-port.js", async importOriginal => {
|
||||
@@ -418,6 +418,7 @@ import {
|
||||
resolveNativeBilling,
|
||||
parseRemoteRunnerProcessIdentity,
|
||||
REMOTE_RUNNER_CHILD_LAUNCH_SCRIPT,
|
||||
REMOTE_RUNNER_AUTHORITY_ARCHIVE_SCRIPT,
|
||||
verifyRemoteRunnerReattachment,
|
||||
readRemoteProviderPackManifest,
|
||||
prepareRemoteProviderPackOnTarget,
|
||||
@@ -15313,6 +15314,132 @@ describe("runnerd provider runtime wiring", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it.each([
|
||||
["moves live state into a new archive", "live", null, 0, "live"],
|
||||
["returns state that is already archived", null, "archived", 0, "archived"],
|
||||
["drops a byte-identical live copy beside the archive", "same", "same", 0, "same"],
|
||||
["keeps a different live copy beside the archive", "live", "archived", 1, null],
|
||||
["fails when no state exists", null, null, 1, null],
|
||||
] as const)("remote authority archive script %s", async (_name, live, archived, exitCode, printed) => {
|
||||
const root = await mkdtemp(join(tmpdir(), "paperclip-authority-archive-script-"));
|
||||
try {
|
||||
const source = join(root, "runner", "runner-state.json");
|
||||
const archiveDirectory = join(root, "authority-epochs", "epoch-aaaaaaaaaaaaaaaaaaaaaaaa");
|
||||
const target = join(archiveDirectory, "runner-state.json");
|
||||
await mkdir(dirname(source), { recursive: true });
|
||||
if (live !== null) await writeFile(source, live);
|
||||
if (archived !== null) {
|
||||
await mkdir(archiveDirectory, { recursive: true, mode: 0o700 });
|
||||
await writeFile(target, archived);
|
||||
}
|
||||
const result = spawnSync("sh", ["-c", REMOTE_RUNNER_AUTHORITY_ARCHIVE_SCRIPT,
|
||||
"paperclip-runner-authority-archive", source, archiveDirectory, target], { encoding: "utf8" });
|
||||
expect(result.status).toBe(exitCode);
|
||||
if (printed === null) {
|
||||
// A refused archive leaves both sides untouched.
|
||||
if (live !== null) expect(await readFile(source, "utf8")).toBe(live);
|
||||
if (archived !== null) expect(await readFile(target, "utf8")).toBe(archived);
|
||||
return;
|
||||
}
|
||||
expect(Buffer.from(result.stdout.replace(/\s+/g, ""), "base64").toString("utf8")).toBe(printed);
|
||||
expect(await readFile(target, "utf8")).toBe(printed);
|
||||
await expect(lstat(source)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
} finally {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it.each([true, false])("prepares a resumed sandbox after a resumed rotation archived its Runner state (archived: %s)", async archived => {
|
||||
const remoteCwd = join(isolatedStateDirectory, "remote-resumed-rotation");
|
||||
const current: NativeExecutionInputV1 = { ...execution,
|
||||
binding: { ...execution.binding, companyId: randomUUID(), agentId: randomUUID(), issueId: randomUUID(), runId: randomUUID() },
|
||||
session: { ...execution.session, normalizedSessionId: randomUUID(), lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } },
|
||||
};
|
||||
const runnerInstanceId = "resumed-rotation-runner";
|
||||
const priorIdentity = { runnerInstanceId, environmentLeaseId: "lease-resumed", runId: randomUUID(),
|
||||
normalizedSessionId: current.session.normalizedSessionId!, turnId: "turn-prior", itemId: "item-prior" };
|
||||
const remoteExecute = vi.fn(async (command: { command: string; args?: string[] }) => {
|
||||
if (command.args?.[0] === "--build-metadata") return { exitCode: 0, timedOut: false, stderr: "", stdout: JSON.stringify({
|
||||
schema: "paperclip-runner/runnerd-build-metadata/v1", binaryName: "paperclip-runnerd",
|
||||
packageName: "@paperclipai/paperclip-runner", binaryContractVersion: 2,
|
||||
durableSessionCapabilities: ["unlimited_runtime", "connection_lease_renewal"], prpTransportModes: ["listen_ws"],
|
||||
}) };
|
||||
if (command.args?.[0] === "--version") return { exitCode: 0, timedOut: false, stdout: "codex-cli 0.160.0", stderr: "" };
|
||||
// The archive script and the harness inspection run for real against
|
||||
// the local directory that stands in for the sandbox filesystem.
|
||||
if (command.args?.[1]?.includes("base64")) {
|
||||
try { return { exitCode: 0, timedOut: false, stdout: execFileSync("sh", command.args, { encoding: "utf8" }), stderr: "" }; }
|
||||
catch { return { exitCode: 1, timedOut: false, stdout: "", stderr: "" }; }
|
||||
}
|
||||
return { exitCode: 0, timedOut: false, stdout: "", stderr: "" };
|
||||
});
|
||||
const syncIn = vi.fn(async () => undefined);
|
||||
const controllerArtifact = join(isolatedStateDirectory, "rotation-fixture-runnerd");
|
||||
await writeFile(controllerArtifact, "fixture bootstrap artifact");
|
||||
await createRunnerdBackend({ db: leaseDb(current), execution: current, runnerInstanceId, runnerIngressAuthorized: true,
|
||||
runnerRemoteBinaryPath: controllerArtifact, runnerRemoteCodexPath: controllerArtifact,
|
||||
runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd, environmentId: "environment",
|
||||
leaseId: priorIdentity.environmentLeaseId, providerKey: "daytona", reusableLeaseConfigured: true,
|
||||
effectiveCapabilities: { runnerWebSocketIngress: true },
|
||||
sandboxLeaseAcquisition: { outcome: "resumed", providerLeaseId: "retained-sandbox" },
|
||||
runner: { execute: remoteExecute, syncIn },
|
||||
getRunnerIngressEndpoint: async () => ({ kind: "authenticated_websocket", websocketUrl: "wss://sandbox.invalid/runner",
|
||||
secretHeaders: [{ name: "Authorization", value: "fixture-ingress" }], generation: "1", refresh: vi.fn(), close: vi.fn(async () => undefined) }),
|
||||
} as never,
|
||||
});
|
||||
state.createBackend.mock.calls.at(-1)![1].codexTransportFactory!();
|
||||
const options = state.createTransport.mock.calls.at(-1)![0] as RunnerTransportOptions & {
|
||||
prepareExternalRunnerState: () => Promise<void>;
|
||||
archiveExternalRunnerState: (input: { archiveKey: string; priorIdentity: typeof priorIdentity }) => Promise<Record<string, unknown>>;
|
||||
controlPlaneRegistration: NonNullable<import("../../vendor/paperclip-runner/index.js").RunnerExecutionDriver["controlPlaneRegistration"]>;
|
||||
};
|
||||
// The resumed sandbox keeps the prior run's suspended Runner and provider state.
|
||||
const remoteSession = join(remoteCwd, ".paperclip-runtime/paperclip-runner/sessions",
|
||||
createHash("sha256").update(current.session.normalizedSessionId!).digest("hex"));
|
||||
const runnerState = durableRunnerState(priorIdentity, "suspended");
|
||||
await mkdir(join(remoteSession, "runner"), { recursive: true });
|
||||
await mkdir(join(remoteSession, "filesystem/codex-home"), { recursive: true });
|
||||
await writeFile(join(remoteSession, "runner/runner-state.json"), JSON.stringify(runnerState));
|
||||
await writeFile(join(remoteSession, "runner/codex-provider-state.json"), JSON.stringify({
|
||||
schema: "paperclip.runner.codex-provider-state.v1", lifecycle: "prepared", threadId: "retained-provider-thread",
|
||||
providerSessionId: null, activeProviderTurnId: null, config: { provider: "codex", driver: "codex_app_server" },
|
||||
}));
|
||||
// A retained failover backup makes a missing live state a hard failure.
|
||||
const backupRoot = join(options.stateDirectory!, "failover-backups/current");
|
||||
for (const directory of resolveNativeHarnessPersistenceProfile(current).directories)
|
||||
await mkdir(join(backupRoot, directory.name), { recursive: true });
|
||||
await writeFile(join(backupRoot, "runner/runner-state.json"), JSON.stringify(runnerState));
|
||||
await writeFile(join(backupRoot, "manifest.json"), JSON.stringify(buildNativeHarnessBackupManifest({
|
||||
backupRoot, execution: current, runnerInstanceId, sourceProviderLeaseId: "retained-sandbox",
|
||||
providerSessionIdentity: { providerSessionId: "retained-provider-thread", providerBackendSessionId: null, providerSessionIdentity: null },
|
||||
})));
|
||||
const archiveKey = "b".repeat(24);
|
||||
const archivedPath = join(remoteSession, "authority-epochs", `epoch-${archiveKey}`, "runner-state.json");
|
||||
if (archived) {
|
||||
// A resumed rotation archives the state without preparing the host first.
|
||||
await expect(options.archiveExternalRunnerState({ archiveKey, priorIdentity })).resolves.toEqual(runnerState);
|
||||
await expect(options.prepareExternalRunnerState()).resolves.toBeUndefined();
|
||||
// Host preparation used the archived snapshot and staged nothing back.
|
||||
await expect(lstat(join(remoteSession, "runner/runner-state.json"))).rejects.toMatchObject({ code: "ENOENT" });
|
||||
expect(JSON.parse(await readFile(archivedPath, "utf8"))).toEqual(runnerState);
|
||||
expect(syncIn).not.toHaveBeenCalled();
|
||||
// The replacement Runner stops before it writes runner-state.json. Close
|
||||
// checkpointing must inspect live state, never the retired archive.
|
||||
const controller = new DurablePrpControlPlane({ stateDirectory: join(options.stateDirectory!, "control-plane"),
|
||||
identity: options.prpIdentity! as ConstructorParameters<typeof DurablePrpControlPlane>[0]["identity"],
|
||||
expectedRunnerVersion: "rotation-fixture", expectedRunnerDigest: `sha256:${"a".repeat(64)}` });
|
||||
let registration: Awaited<ReturnType<typeof options.controlPlaneRegistration>> | undefined;
|
||||
onTestFinished(async () => { await registration?.release(); await controller.stop(); });
|
||||
registration = await options.controlPlaneRegistration(controller);
|
||||
await expect(registration.checkpoint!("settled")).rejects.toThrow(
|
||||
"runner_remote_checkpoint_incomplete: exact suspended harness state unavailable (unavailable)");
|
||||
} else {
|
||||
// Without a completed rotation, missing live state still fails closed.
|
||||
await rm(join(remoteSession, "runner/runner-state.json"));
|
||||
await expect(options.prepareExternalRunnerState()).rejects.toThrow("runner_harness_state_mismatch: resumed_unavailable");
|
||||
}
|
||||
});
|
||||
|
||||
it("makes remote authority archival idempotent and returns the archived state", async () => {
|
||||
const remoteExecute = vi.fn();
|
||||
const remoteTarget = {
|
||||
|
||||
@@ -5,7 +5,7 @@ import { logger } from "../../middleware/logger.js";
|
||||
import { canRestoreMigratedNativeSession } from "./native-workspace-migration.js";
|
||||
import { NATIVE_PROVIDER_DURABLE_EVENT_TYPES, nativeHarnessBackupManifestSha256, retiredSourceAuthorizesNativeBackupRestore } from "./native-retired-backup-source.js";
|
||||
import { prepareCursorCloudRuntime } from "./cursor-cloud-runtime.js";
|
||||
import { resolveLocalRuntimeApiUrl } from "../../runtime-api.js";
|
||||
import { LocalRuntimeApiListenerClosedError, resolveLocalRuntimeApiUrl } from "../../runtime-api.js";
|
||||
import { providerFailureFromError } from "../../vendor/paperclip-runner/index.js";
|
||||
import { readProviderFailure } from "@paperclipai/shared";
|
||||
import { nativeCancellationHasUnsettledOwner, notifyNativeCancellation, waitForNativeCancellation } from "./native-cancellation-delivery.js";
|
||||
@@ -292,6 +292,10 @@ import {
|
||||
import {
|
||||
currentNativeControllerIdentity,
|
||||
evaluateNativeControllerTakeover,
|
||||
proveNativeLocalRunnerStopped,
|
||||
nativeProviderProcessIdentities,
|
||||
NATIVE_PROVIDER_PROCESS_EVENT_TYPES,
|
||||
type NativeLocalRunnerStopProof,
|
||||
isCurrentNativeControllerProcess,
|
||||
nextNativeProviderAttempt,
|
||||
type NativeControllerIdentity,
|
||||
@@ -5286,15 +5290,25 @@ function moveNativeContinuityState(root: string, archiveRoot: string): void {
|
||||
* Runner outbox can hold only post-terminal telemetry. Local lifecycle
|
||||
* evidence, even an unsuspended receipt, and unknown entries are owner
|
||||
* claims that this proof never overrides.
|
||||
*
|
||||
* `localRunner` names a local Runner that wrote its own lifecycle receipt in
|
||||
* this root. Its unsuspended receipt and its own unfinished lifecycle commands
|
||||
* (suspend, drain, shutdown) are claims of that one process. The caller must
|
||||
* separately prove that process stopped before it relies on this projection.
|
||||
*/
|
||||
type SettledRunnerdProjection = { bytes: string; refused?: never } | { bytes?: never; refused: string };
|
||||
|
||||
function readSettledRunnerdProjection(root: string, identity: Record<string, string>): SettledRunnerdProjection {
|
||||
const LOCAL_RUNNER_LIFECYCLE_COMMANDS = new Set(["runner.suspend", "runner.drain", "runner.shutdown"]);
|
||||
|
||||
function readSettledRunnerdProjection(root: string, identity: Record<string, string>,
|
||||
options: { localRunner?: boolean } = {}): SettledRunnerdProjection {
|
||||
const refuse = (refused: string): SettledRunnerdProjection => ({ refused });
|
||||
try {
|
||||
if (!isSafeNativeStateDirectory(root)) return refuse("root_unsafe");
|
||||
const lifecycle = runnerdAuthorityLifecycle(root, identity as RunnerdDurableIdentity);
|
||||
if (lifecycle !== "absent") return refuse(`runner_lifecycle_${lifecycle}`);
|
||||
if (lifecycle !== "absent" && !(options.localRunner && lifecycle === "not_suspended")) {
|
||||
return refuse(`runner_lifecycle_${lifecycle}`);
|
||||
}
|
||||
// Every legitimate writer's entry is classified in native-session-root-entries.ts.
|
||||
// Held entries are known evidence that needs its own settlement; anything
|
||||
// else is writer drift. Name each kind separately.
|
||||
@@ -5306,6 +5320,11 @@ function readSettledRunnerdProjection(root: string, identity: Record<string, str
|
||||
NATIVE_CONTROL_PLANE_STATE_MAX_BYTES, "runner_durable_identity_too_large").toString("utf8");
|
||||
const control = record(JSON.parse(bytes));
|
||||
const commands = (Array.isArray(control.commands) ? control.commands : []).map(record);
|
||||
// A stopped local Runner can no longer act on its own lifecycle commands.
|
||||
const settledCommands = options.localRunner && lifecycle === "not_suspended" && Array.isArray(control.commands)
|
||||
? commands.filter(command => !LOCAL_RUNNER_LIFECYCLE_COMMANDS.has(String(command.type)) || command.status === "completed")
|
||||
: commands;
|
||||
const settledControl = settledCommands === commands ? control : { ...control, commands: settledCommands };
|
||||
const answered = new Set(commands
|
||||
.filter(command => command.type === "semantic_tool.result" && command.status === "completed")
|
||||
.map(command => record(command.payload).sourceEventId));
|
||||
@@ -5315,11 +5334,11 @@ function readSettledRunnerdProjection(root: string, identity: Record<string, str
|
||||
if (Object.entries(identity).some(([key, value]) => record(control.identity)[key] !== value)) {
|
||||
return refuse("journal_identity_mismatch");
|
||||
}
|
||||
if (!isSettledRemoteWarmControlState(control, identity)) {
|
||||
if (!isSettledRemoteWarmControlState(settledControl, identity)) {
|
||||
// Closed vocabulary only: known command types and statuses, never payloads.
|
||||
const pending = commands.filter(command => command.status !== "completed")
|
||||
const unsettled = settledCommands.filter(command => command.status !== "completed")
|
||||
.map(command => `${retirementReasonName("commandType", command.type)}.${retirementReasonName("commandStatus", command.status)}`);
|
||||
return refuse(pending.length > 0 ? `journal_commands_unsettled(${[...new Set(pending)].slice(0, 4).join(",")})`
|
||||
return refuse(unsettled.length > 0 ? `journal_commands_unsettled(${[...new Set(unsettled)].slice(0, 4).join(",")})`
|
||||
: "journal_terminal_unsettled");
|
||||
}
|
||||
if ((control.committedEvents as unknown[]).map(record).some(event =>
|
||||
@@ -5363,6 +5382,12 @@ function retirementReasonName(kind: keyof typeof RETIREMENT_REASON_NAMES, value:
|
||||
* that authority as a continuity break instead. This run then bootstraps its
|
||||
* own authority, and the provider session follows the ordinary
|
||||
* resume-or-replace policy. Any missing proof keeps the original rejection.
|
||||
*
|
||||
* A local per-turn Runner can end the same way: the result is accepted but its
|
||||
* suspend command does not finish, so its own receipt stays unsuspended. That
|
||||
* receipt is a claim of one recorded Runner process. Once that process and its
|
||||
* recorded provider processes are proven stopped by exact PID and start time,
|
||||
* the claim has no owner left and the same retirement applies.
|
||||
*/
|
||||
async function retireFinishedPerTurnRunnerdOwner(input: {
|
||||
db: Db;
|
||||
@@ -5377,11 +5402,14 @@ async function retireFinishedPerTurnRunnerdOwner(input: {
|
||||
// existing recovery path reads. Plugin tasks own a separate saved checkpoint.
|
||||
const refuse = (refused: string) => ({ retired: false as const, refused });
|
||||
if (input.execution.session.lifecyclePolicy.mode !== "per_turn") return refuse("lifecycle_not_per_turn");
|
||||
if (target?.kind !== "remote" || target.transport !== "sandbox" || !target.runner || target.runnerTask) {
|
||||
return refuse("target_not_sandbox_runner");
|
||||
}
|
||||
if (!["created", "replacement"].includes(target.sandboxLeaseAcquisition?.outcome ?? "")) {
|
||||
return refuse(`sandbox_${retirementReasonName("sandboxOutcome", target.sandboxLeaseAcquisition?.outcome)}`);
|
||||
const localRunner = !target || target.kind === "local";
|
||||
if (!localRunner) {
|
||||
if (target.kind !== "remote" || target.transport !== "sandbox" || !target.runner || target.runnerTask) {
|
||||
return refuse("target_not_sandbox_runner");
|
||||
}
|
||||
if (!["created", "replacement"].includes(target.sandboxLeaseAcquisition?.outcome ?? "")) {
|
||||
return refuse(`sandbox_${retirementReasonName("sandboxOutcome", target.sandboxLeaseAcquisition?.outcome)}`);
|
||||
}
|
||||
}
|
||||
const identity: Record<string, string> = {};
|
||||
for (const key of ["runId", "normalizedSessionId", "runnerInstanceId", "environmentLeaseId", "turnId", "itemId"]) {
|
||||
@@ -5389,7 +5417,7 @@ async function retireFinishedPerTurnRunnerdOwner(input: {
|
||||
if (typeof value !== "string" || !value) return refuse("durable_identity_incomplete");
|
||||
identity[key] = value;
|
||||
}
|
||||
const projection = readSettledRunnerdProjection(input.root, identity);
|
||||
const projection = readSettledRunnerdProjection(input.root, identity, { localRunner });
|
||||
if (projection.refused !== undefined) return refuse(projection.refused);
|
||||
const controlBytes = projection.bytes;
|
||||
const { binding } = input.execution;
|
||||
@@ -5421,6 +5449,29 @@ async function retireFinishedPerTurnRunnerdOwner(input: {
|
||||
remoteLeaseFenced: readNativeControllerRecovery(profile.nativeControllerRecovery, profile.sessionCheckpoint) !== null });
|
||||
if (!stopped.allowed) return refuse(`controller_${retirementReasonName("controllerReason", stopped.reason)}`);
|
||||
const provenFinishedHere = finishedHere();
|
||||
const proveLocalRunnerStopped = async (): Promise<NativeLocalRunnerStopProof> => {
|
||||
const run = prior!.run;
|
||||
// Settlement may already have cleared the process fields after it saved a
|
||||
// server-authored stop receipt; that receipt is the identity to check.
|
||||
const released = run.processPid === null && run.processGroupId === null && run.processStartedAt === null;
|
||||
const receipt = released ? await readNativeLocalProcessStop(input.db, run.companyId, run.id) : null;
|
||||
if (released && !receipt) return { stopped: false, reason: "runner_process_identity_missing" };
|
||||
const providerEvents = await input.db.select({ payload: heartbeatRunEvents.payload }).from(heartbeatRunEvents)
|
||||
.where(and(eq(heartbeatRunEvents.companyId, run.companyId), eq(heartbeatRunEvents.runId, run.id),
|
||||
inArray(heartbeatRunEvents.eventType, [...NATIVE_PROVIDER_PROCESS_EVENT_TYPES])))
|
||||
.orderBy(desc(heartbeatRunEvents.id)).limit(100);
|
||||
return proveNativeLocalRunnerStopped({
|
||||
processPid: receipt?.processPid ?? run.processPid,
|
||||
processGroupId: receipt?.processGroupId ?? run.processGroupId,
|
||||
processStartedAt: receipt ? null : run.processStartedAt,
|
||||
runnerStopRecorded: receipt !== null,
|
||||
providerProcesses: nativeProviderProcessIdentities(record(profile.sessionCheckpoint), providerEvents),
|
||||
});
|
||||
};
|
||||
if (localRunner) {
|
||||
const runnerStopped = await proveLocalRunnerStopped();
|
||||
if (!runnerStopped.stopped) return refuse(runnerStopped.reason);
|
||||
}
|
||||
// Earlier rejected admissions can quarantine the same projection. Each copy
|
||||
// must pass the same proof: a rebuilt root never carries the local Runner
|
||||
// evidence that held an earlier attempt, so that evidence stays decisive.
|
||||
@@ -5429,13 +5480,14 @@ async function retireFinishedPerTurnRunnerdOwner(input: {
|
||||
if (!isSafeNativeStateDirectory(quarantine)) return refuse("quarantine_unsafe");
|
||||
for (const name of readdirSync(quarantine)) {
|
||||
if (!name.startsWith(`${basename(input.root)}.`)) continue;
|
||||
const copy = readSettledRunnerdProjection(resolve(quarantine, name), identity);
|
||||
const copy = readSettledRunnerdProjection(resolve(quarantine, name), identity, { localRunner });
|
||||
if (copy.refused !== undefined) return refuse(`quarantine_${copy.refused}`);
|
||||
}
|
||||
}
|
||||
// Database reads can await. Retire only the exact evidence proven above.
|
||||
if (readSettledRunnerdProjection(input.root, identity).bytes !== controlBytes) return refuse("journal_changed");
|
||||
if (readSettledRunnerdProjection(input.root, identity, { localRunner }).bytes !== controlBytes) return refuse("journal_changed");
|
||||
if (provenFinishedHere && !finishedHere()) return refuse("controller_restarted_here");
|
||||
if (localRunner && !(await proveLocalRunnerStopped()).stopped) return refuse("runner_process_restarted");
|
||||
const archivedAt = Date.now();
|
||||
const archiveRoot = resolve(input.root, "continuity-breaks", `${archivedAt}-${randomUUID()}`);
|
||||
mkdirSync(archiveRoot, { recursive: true, mode: 0o700 });
|
||||
@@ -6191,7 +6243,7 @@ function loadRunnerdDurableBinding(
|
||||
const identity = readRunnerdDurableIdentity(runnerdStateRoot(execution));
|
||||
// The run id is intentionally different during a continuation. Reuse only
|
||||
// the verified runner/lease binding from the same company-scoped durable
|
||||
// session root; rotateLocalAuthorityEpoch then requires that exact binding
|
||||
// session root; rotateAuthorityEpoch then requires that exact binding
|
||||
// before it can archive the prior per-run authority.
|
||||
if (!durableIdentityMatchesSession(identity, execution)) return null;
|
||||
return {
|
||||
@@ -10676,7 +10728,14 @@ async function executePaperclipNativeSessionWithinScope(
|
||||
} else {
|
||||
error = identityRedactor.redact(error);
|
||||
}
|
||||
if (nativeRunsDetachingForRestart.has(input.execution.binding.runId)) {
|
||||
// A closed runtime API listener means this process is shutting down; it
|
||||
// closes HTTP admission before it drains runs. For a local Runner, keep the
|
||||
// claim for the drain to suspend: the next controller proves the Runner
|
||||
// process identity and adopts the run without spending a failure or a
|
||||
// same-run retry. A remote Runner has no process evidence on this host, so
|
||||
// it keeps the bootstrap retry below.
|
||||
if (nativeRunsDetachingForRestart.has(input.execution.binding.runId)
|
||||
|| (error instanceof LocalRuntimeApiListenerClosedError && input.runnerExecutionTarget?.kind !== "remote")) {
|
||||
await leaseRenewal.stop().catch(() => undefined);
|
||||
liveQuestions.close();
|
||||
activeNativeSessions.delete(input.execution.binding.runId);
|
||||
@@ -12563,6 +12622,20 @@ const REMOTE_RUNNER_PROCESS_FINGERPRINT_SCRIPT =
|
||||
const REMOTE_RUNNER_IDENTITY_CHECK_SCRIPT =
|
||||
'set -eu; identity_path=$1; expected_nonce=$2; expected_runner_id=$3; expected_pid=$4; test -f "$identity_path" && test ! -L "$identity_path" || exit 3; { IFS= read -r nonce; IFS= read -r pid; IFS= read -r started_at; IFS= read -r runner_id; } < "$identity_path"; test "$nonce" = "$expected_nonce" && test "$runner_id" = "$expected_runner_id" && test "$pid" = "$expected_pid" && test -n "$started_at" || exit 4; kill -0 "$pid" 2>/dev/null || exit 3; if test -r "/proc/$pid/cmdline"; then command_line=$(tr "\\000" "\\n" < "/proc/$pid/cmdline"); printf "%s\\n" "$command_line" | grep -Fqx -- "--runner-id" || exit 4; printf "%s\\n" "$command_line" | grep -Fqx -- "$expected_runner_id" || exit 4; fi';
|
||||
|
||||
/**
|
||||
* Move the remote Runner state ($1) into its epoch archive directory ($2) as
|
||||
* $3, then print the archived snapshot. A retry succeeds when the state is
|
||||
* already archived, and when a byte-identical live copy sits beside the
|
||||
* archive (the same rules as the controller and the computer Launcher). Any
|
||||
* other live copy, or a host without `cmp`, fails closed.
|
||||
*/
|
||||
export const REMOTE_RUNNER_AUTHORITY_ARCHIVE_SCRIPT =
|
||||
'set -eu; if test -L "$2" || { test -e "$2" && test ! -d "$2"; }; then exit 1; fi; ' +
|
||||
'if test -f "$1" && test ! -L "$1" && test ! -e "$3" && test ! -L "$3"; then umask 077; install -d -m 0700 "$2"; mv -- "$1" "$3"; ' +
|
||||
'elif test ! -e "$1" && test ! -L "$1" && test -f "$3" && test ! -L "$3"; then :; ' +
|
||||
'elif test -f "$1" && test ! -L "$1" && test -f "$3" && test ! -L "$3" && command -v cmp >/dev/null 2>&1 && cmp -s "$1" "$3"; then rm -f -- "$1"; ' +
|
||||
'else exit 1; fi; base64 < "$3"';
|
||||
|
||||
export const REMOTE_RUNNER_CHILD_LAUNCH_SCRIPT =
|
||||
'set -eu; identity_path=$1; identity_nonce=$2; runner_instance_id=$3; diagnostics_directory=$4; shift 4; umask 077; test ! -L "$diagnostics_directory"; if test -e "$diagnostics_directory"; then test -d "$diagnostics_directory"; else mkdir -p -- "$diagnostics_directory"; fi; chmod 0700 "$diagnostics_directory"; started_at=$(date -u +"%Y-%m-%dT%H:%M:%S.%3NZ"); pid=$$; process_fingerprint=""; if ' +
|
||||
REMOTE_RUNNER_PROCESS_FINGERPRINT_SCRIPT +
|
||||
@@ -13990,6 +14063,10 @@ async function createRunnerdBackendWithinSessionClaim(
|
||||
: sourceRuntimeContext;
|
||||
let remotePrepared = false;
|
||||
let remoteHarnessStatePrepared = false;
|
||||
// Set when this invocation's authority rotation moved the live Runner state
|
||||
// into its epoch archive. Later host preparation validates that snapshot
|
||||
// instead of requiring, or re-staging, the retired live state.
|
||||
let archivedRotationRunnerStatePath: string | null = null;
|
||||
let selectedRemoteMode: "dial_wss" | "listen_ws" | null = null;
|
||||
let remoteCaBundleMapping: { sourcePath: string; targetPath: string } | null =
|
||||
null;
|
||||
@@ -14103,7 +14180,14 @@ async function createRunnerdBackendWithinSessionClaim(
|
||||
remotePrepared = true;
|
||||
};
|
||||
|
||||
const inspectRemoteHarnessState = async (): Promise<{
|
||||
/**
|
||||
* Inspect the remote harness state. Only in-place host preparation may use
|
||||
* this invocation's archived rotation snapshot; every other caller,
|
||||
* including close checkpointing, must inspect the live Runner state.
|
||||
*/
|
||||
const inspectRemoteHarnessState = async (
|
||||
source: "live" | "host_preparation" = "live",
|
||||
): Promise<{
|
||||
complete: boolean;
|
||||
runnerState: Record<string, unknown> | null;
|
||||
providerSessionIdentity: Record<string, unknown> | null;
|
||||
@@ -14117,17 +14201,20 @@ async function createRunnerdBackendWithinSessionClaim(
|
||||
incompleteReason: "unavailable",
|
||||
};
|
||||
}
|
||||
const archivedRunnerStatePath = source === "host_preparation"
|
||||
? archivedRotationRunnerStatePath : null;
|
||||
const runnerStatePath = archivedRunnerStatePath ??
|
||||
posix.join(remoteStateDirectory, "runner-state.json");
|
||||
const escapedRunnerState = runnerStatePath.replaceAll("'", "'\\''");
|
||||
const requirements = persistenceProfile.directories.flatMap((directory) => {
|
||||
const path = remotePersistencePath(directory);
|
||||
if (!path) return [];
|
||||
const escaped = path.replaceAll("'", "'\\''");
|
||||
return directory.name === "runner"
|
||||
? [`test -f '${escaped}/runner-state.json'`]
|
||||
: [`test -d '${escaped}'`];
|
||||
if (directory.name !== "runner") return [`test -d '${escaped}'`];
|
||||
return [archivedRunnerStatePath
|
||||
? `test -d '${escaped}' && test -f '${escapedRunnerState}'`
|
||||
: `test -f '${escaped}/runner-state.json'`];
|
||||
});
|
||||
const escapedRunnerState = posix
|
||||
.join(remoteStateDirectory, "runner-state.json")
|
||||
.replaceAll("'", "'\\''");
|
||||
const escapedProviderState = posix
|
||||
.join(remoteStateDirectory, runnerProviderStateFilename(input.execution))
|
||||
.replaceAll("'", "'\\''");
|
||||
@@ -14494,7 +14581,7 @@ async function createRunnerdBackendWithinSessionClaim(
|
||||
const state = await measureNativeRunnerSpan(
|
||||
input.trace,
|
||||
"sandbox.lease.resume",
|
||||
inspectRemoteHarnessState,
|
||||
() => inspectRemoteHarnessState("host_preparation"),
|
||||
{
|
||||
attributes: {
|
||||
provider: remoteTarget.providerKey ?? "sandbox",
|
||||
@@ -14558,7 +14645,7 @@ async function createRunnerdBackendWithinSessionClaim(
|
||||
);
|
||||
} else {
|
||||
const reuseStartedAtMs = Date.now();
|
||||
const state = await measureNativeRunnerSpan(input.trace, "harness_state.inspect", () => inspectRemoteHarnessState());
|
||||
const state = await measureNativeRunnerSpan(input.trace, "harness_state.inspect", () => inspectRemoteHarnessState("host_preparation"));
|
||||
if (
|
||||
state.complete &&
|
||||
state.runnerState &&
|
||||
@@ -14589,9 +14676,11 @@ async function createRunnerdBackendWithinSessionClaim(
|
||||
existsSync(localDirectory) &&
|
||||
!(await measureNativeRunnerSpan(input.trace, "harness_state.path_check", () => remoteRunnerPathExists({
|
||||
runner: remoteCommandRunner,
|
||||
// An archived rotation snapshot counts as present, so
|
||||
// retired Runner state is not staged beside it again.
|
||||
path:
|
||||
directory.name === "runner"
|
||||
? posix.join(remoteDirectory, "runner-state.json")
|
||||
? archivedRotationRunnerStatePath ?? posix.join(remoteDirectory, "runner-state.json")
|
||||
: remoteDirectory,
|
||||
kind:
|
||||
directory.name === "runner" ? "file" : "directory",
|
||||
@@ -14850,7 +14939,7 @@ async function createRunnerdBackendWithinSessionClaim(
|
||||
command: "sh",
|
||||
args: [
|
||||
"-c",
|
||||
'set -eu; if test -L "$2" || { test -e "$2" && test ! -d "$2"; }; then exit 1; fi; if test -f "$1" && test ! -L "$1" && test ! -e "$3" && test ! -L "$3"; then umask 077; install -d -m 0700 "$2"; mv -- "$1" "$3"; elif test ! -e "$1" && test ! -L "$1" && test -f "$3" && test ! -L "$3"; then :; else exit 1; fi; base64 < "$3"',
|
||||
REMOTE_RUNNER_AUTHORITY_ARCHIVE_SCRIPT,
|
||||
"paperclip-runner-authority-archive",
|
||||
sourcePath,
|
||||
archiveDirectory,
|
||||
@@ -14862,8 +14951,9 @@ async function createRunnerdBackendWithinSessionClaim(
|
||||
if (result.exitCode !== 0 || result.timedOut) {
|
||||
throw new Error("runner_remote_authority_archive_failed");
|
||||
}
|
||||
let archivedState: Record<string, unknown>;
|
||||
try {
|
||||
return record(
|
||||
archivedState = record(
|
||||
JSON.parse(
|
||||
Buffer.from(
|
||||
result.stdout.replace(/\s+/g, ""),
|
||||
@@ -14874,6 +14964,10 @@ async function createRunnerdBackendWithinSessionClaim(
|
||||
} catch {
|
||||
throw new Error("runner_remote_authority_archive_failed");
|
||||
}
|
||||
// A resumed rotation never prepared the host before this move, so
|
||||
// the launch's host preparation must find the state here.
|
||||
archivedRotationRunnerStatePath = archivedStatePath;
|
||||
return archivedState;
|
||||
}
|
||||
: undefined;
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@ export const NATIVE_CONTINUITY_STATE_ENTRIES: readonly string[] = [
|
||||
"failover-backups",
|
||||
"provider-pack-authority.json",
|
||||
// Runner transport: each cold authority rotation archives the settled prior
|
||||
// epoch here (rotateLocalAuthorityEpoch/rotateExternalAuthorityEpoch).
|
||||
// epoch here (rotateAuthorityEpoch).
|
||||
"authority-epochs",
|
||||
// Runner transport: execution-owned launch preparation keyed by identity.
|
||||
"launch-preparation.json",
|
||||
|
||||
@@ -584,11 +584,13 @@ describe("Paperclip Cloud connector", () => {
|
||||
return Response.json({
|
||||
active: true,
|
||||
status: "active",
|
||||
profiles: ["gmail.read", "drive.write", "unknown.profile", "gmail.read"],
|
||||
profiles: ["gmail.read", "drive.write", "unknown.profile", "slack.events.v1", "gmail.read"],
|
||||
slackEvents: { version: 1 },
|
||||
});
|
||||
});
|
||||
const connector = createPaperclipCloudConnector({ config: keys.config, request: request as typeof fetch });
|
||||
await expect(connector.getCapabilities()).resolves.toEqual(["gmail.read", "drive.write"]);
|
||||
await expect(connector.slackEventsAvailable()).resolves.toBe(true);
|
||||
});
|
||||
|
||||
it("fails capability discovery closed for inactive, legacy, malformed, or rejected status responses", async () => {
|
||||
@@ -699,10 +701,10 @@ function seal(
|
||||
recipientPublicKey: KeyObject,
|
||||
purpose: "initial" | "access" | "events",
|
||||
configValue: PaperclipCloudConnectorConfig,
|
||||
profile: keyof typeof GOOGLE_WORKSPACE_CONNECTOR_PROFILES | "slack.manager" | "github.bot" | "github.code",
|
||||
profile: keyof typeof GOOGLE_WORKSPACE_CONNECTOR_PROFILES | "slack.manager" | "slack.events.v1" | "github.bot" | "github.code",
|
||||
) {
|
||||
const provider = profile.startsWith("github.") ? "github" : profile === "slack.manager" ? "slack" : "google";
|
||||
const scopes = provider === "github" ? [] : profile === "slack.manager" ? SLACK_MANAGER_SCOPES : GOOGLE_WORKSPACE_CONNECTOR_PROFILES[profile as keyof typeof GOOGLE_WORKSPACE_CONNECTOR_PROFILES].scopes;
|
||||
const provider = profile.startsWith("github.") ? "github" : profile.startsWith("slack.") ? "slack" : "google";
|
||||
const scopes = provider === "github" || profile === "slack.events.v1" ? [] : profile === "slack.manager" ? SLACK_MANAGER_SCOPES : GOOGLE_WORKSPACE_CONNECTOR_PROFILES[profile as keyof typeof GOOGLE_WORKSPACE_CONNECTOR_PROFILES].scopes;
|
||||
const ephemeral = generateKeyPairSync("x25519");
|
||||
const ephemeralJwk = ephemeral.publicKey.export({ format: "jwk" }) as { x: string };
|
||||
const recipientJwk = recipientPublicKey.export({ format: "jwk" }) as { x: string };
|
||||
@@ -795,3 +797,66 @@ describe("dedicated App gateway delivery", () => {
|
||||
expect(JSON.stringify(result)).not.toContain('"ct"');
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
describe("sealed Slack event connector transport", () => {
|
||||
it("opens a Slack event through the normal outer lease without exposing ciphertext or changing callback bytes", async () => {
|
||||
const keys = config();
|
||||
const createdAt = "2026-11-05T10:00:00.000Z";
|
||||
const bindingId = `slack-event:${"a".repeat(64)}`;
|
||||
const packet = { version: 1, instanceId, environment: "staging", bindingId, acceptedAt: createdAt,
|
||||
rawBody: Buffer.from('{ "event_id": "EvFixture", "text": "original bytes" }').toString("base64"),
|
||||
timestamp: "1793872800", signature: `v0=${"b".repeat(64)}` };
|
||||
const inner = seal(packet, keys.sealPublicKey, "events", keys.config, "slack.events.v1");
|
||||
const outer = seal({ v: 1, instanceId, environment: "staging", leaseId: "lease-slack", events: [
|
||||
{ id: "receipt-slack", provider: "slack", event: "event_callback", createdAt, installationId: bindingId,
|
||||
bindingIds: [bindingId], payload: { slackEvent: { bindingId, sealed: inner } } },
|
||||
] }, keys.sealPublicKey, "events", keys.config, "github.code");
|
||||
const connector = createPaperclipCloudConnector({ config: keys.config,
|
||||
request: vi.fn(async () => Response.json({ leaseId: "lease-slack", sealed: outer })) as typeof fetch });
|
||||
const result = await connector.leaseEvents({ subject, companyId });
|
||||
expect(result?.failedEventIds).toBeUndefined();
|
||||
expect(result?.events).toHaveLength(1);
|
||||
expect(result?.events[0]?.payload.slackEvent).toEqual(packet);
|
||||
expect(JSON.stringify(inner)).not.toContain("original bytes");
|
||||
expect(JSON.stringify(result)).not.toContain('"ct"');
|
||||
});
|
||||
|
||||
it.each(["binding", "acceptedAt", "instance", "environment", "key", "profile"] as const)(
|
||||
"isolates an invalid inner Slack %s while retaining a healthy sibling", async failure => {
|
||||
const keys = config();
|
||||
const createdAt = "2026-11-05T10:00:00.000Z";
|
||||
const bindingId = `slack-event:${"a".repeat(64)}`;
|
||||
const packet = { version: 1, instanceId, environment: "staging", bindingId, acceptedAt: createdAt, rawBody: "e30=" };
|
||||
const changed = { ...packet, ...(failure === "binding" ? { bindingId: "other" } : {}),
|
||||
...(failure === "acceptedAt" ? { acceptedAt: "2026-11-05T10:00:00.001Z" } : {}),
|
||||
...(failure === "instance" ? { instanceId: "foreign" } : {}),
|
||||
...(failure === "environment" ? { environment: "production" } : {}) };
|
||||
const event = (id: string, inner: unknown) => ({ id, provider: "slack", event: "event_callback", createdAt,
|
||||
installationId: bindingId, bindingIds: [bindingId], payload: { slackEvent: { bindingId, sealed: inner } } });
|
||||
const bad = seal(changed, failure === "key" ? config().sealPublicKey : keys.sealPublicKey, "events", keys.config,
|
||||
failure === "profile" ? "slack.manager" : "slack.events.v1");
|
||||
const outer = seal({ v: 1, instanceId, environment: "staging", leaseId: "lease-slack", events: [
|
||||
event("bad", bad), event("healthy", seal(packet, keys.sealPublicKey, "events", keys.config, "slack.events.v1")),
|
||||
] }, keys.sealPublicKey, "events", keys.config, "github.code");
|
||||
const connector = createPaperclipCloudConnector({ config: keys.config,
|
||||
request: vi.fn(async () => Response.json({ leaseId: "lease-slack", sealed: outer })) as typeof fetch });
|
||||
await expect(connector.leaseEvents({ subject, companyId })).resolves.toMatchObject({
|
||||
leaseId: "lease-slack", failedEventIds: ["bad"], events: [{ id: "healthy", payload: { slackEvent: packet } }],
|
||||
});
|
||||
});
|
||||
|
||||
it.each([true, false])("selects the matching signed Slack operation receipt (active=%s)", async active => {
|
||||
const keys = config();
|
||||
const enrollment = { active, endpointId: "endpoint", runtimeGeneration: 7 };
|
||||
const enrolled = { version: 1, bindingId: "enrolled" }, retired = { version: 1, runtimeGeneration: 7 };
|
||||
const request = vi.fn(async () => Response.json({ enrollment: enrolled, retirement: retired }));
|
||||
const connector = createPaperclipCloudConnector({ config: keys.config, request: request as typeof fetch });
|
||||
await expect(connector.slackEventEnrollment({ subject, companyId, enrollment })).resolves.toEqual(active ? enrolled : retired);
|
||||
expect(request).toHaveBeenCalledOnce();
|
||||
const [url, init] = request.mock.calls[0] as unknown as [string | URL, RequestInit];
|
||||
expect(new URL(String(url)).pathname).toBe("/v1/connector/slack-events");
|
||||
expect(typeof init.body).toBe("string");
|
||||
expect(JSON.parse(String(init.body))).toMatchObject({ enrollment: JSON.stringify(enrollment) });
|
||||
});
|
||||
});
|
||||
@@ -35,7 +35,7 @@ export const GMAIL_CONNECTOR_SCOPES = [
|
||||
export { GOOGLE_WORKSPACE_CONNECTOR_PROFILES };
|
||||
|
||||
export type PaperclipCloudConnectorEnvironment = "development" | "test" | "staging" | "production";
|
||||
export type PaperclipCloudConnectorOperation = "status" | "session" | "claim" | "refresh" | "revoke" | "webhook-bind" | "event-lease" | "event-ack" | "github-app";
|
||||
export type PaperclipCloudConnectorOperation = "status" | "session" | "claim" | "refresh" | "revoke" | "webhook-bind" | "event-lease" | "event-ack" | "github-app" | "slack-events";
|
||||
export type PaperclipCloudConnectorProfileId = GoogleWorkspaceConnectorProfileId | GitHubConnectorProfileId | AsanaConnectorProfileId | "slack.manager" | "github.bot";
|
||||
export type PaperclipCloudConnectorProvider = "google" | "github" | "asana" | "slack";
|
||||
|
||||
@@ -75,7 +75,7 @@ export type SealedConnectorEvents = {
|
||||
leaseId: string;
|
||||
events: Array<{
|
||||
id: string;
|
||||
provider: "github";
|
||||
provider: "github" | "slack";
|
||||
event: string;
|
||||
action: string | null;
|
||||
installationId: string | null;
|
||||
@@ -86,12 +86,14 @@ export type SealedConnectorEvents = {
|
||||
}>;
|
||||
};
|
||||
|
||||
type SealedEnvelopeProfileId = PaperclipCloudConnectorProfileId | "slack.events.v1";
|
||||
|
||||
type SealedEnvelope = {
|
||||
v: 1;
|
||||
alg: "X25519-HKDF-SHA256-A256GCM";
|
||||
purpose: "initial" | "access" | "events";
|
||||
provider: PaperclipCloudConnectorProvider;
|
||||
profile: PaperclipCloudConnectorProfileId;
|
||||
profile: SealedEnvelopeProfileId;
|
||||
epk: string;
|
||||
iv: string;
|
||||
ct: string;
|
||||
@@ -115,10 +117,14 @@ type ConnectorResponse = {
|
||||
events?: unknown;
|
||||
acknowledged?: unknown;
|
||||
githubApps?: { version: number };
|
||||
slackEvents?: { version: number };
|
||||
enrollment?: unknown;
|
||||
retirement?: unknown;
|
||||
eventWakeVersion?: unknown;
|
||||
};
|
||||
|
||||
const ENDPOINTS: Record<PaperclipCloudConnectorOperation, string> = {
|
||||
"slack-events": "/v1/connector/slack-events",
|
||||
"github-app": "/v1/connector/github-apps",
|
||||
status: "/v1/connector/instance-status",
|
||||
session: "/v1/connector/sessions",
|
||||
@@ -147,6 +153,9 @@ const BROKER_REJECTION_REASONS = new Set([
|
||||
"AUDIENCE_MISMATCH", "OPERATION_MISMATCH", "PAYLOAD_MISMATCH",
|
||||
"MALFORMED_REQUEST", "UNSUPPORTED_ALGORITHM", "RATE_LIMITED",
|
||||
"PROVIDER_OPERATION_FAILED", "PROVIDER_REAUTHORIZATION_REQUIRED",
|
||||
"SLACK_ENROLLMENT_GENERATION_CHANGED", "SLACK_ENROLLMENT_IDENTITY_CHANGED",
|
||||
"SLACK_ENROLLMENT_CHANGED", "SLACK_ENROLLMENT_UNSETTLED", "SLACK_ENROLLMENT_DISABLED",
|
||||
"SLACK_COMPANY_MISMATCH", "SLACK_EVENTS_UNAVAILABLE",
|
||||
]);
|
||||
const UNKNOWN_BROKER_REASON = "UNKNOWN_BROKER_ERROR";
|
||||
|
||||
@@ -192,6 +201,7 @@ export class PaperclipCloudConnectorError extends Error {
|
||||
readonly code: string,
|
||||
readonly status?: number,
|
||||
readonly retryAt?: number,
|
||||
readonly brokerReason?: string,
|
||||
) {
|
||||
super(message);
|
||||
this.name = "PaperclipCloudConnectorError";
|
||||
@@ -280,7 +290,7 @@ export function createPaperclipCloudConnector(input: {
|
||||
async function call(
|
||||
operation: PaperclipCloudConnectorOperation,
|
||||
claims: { subject: string; companyId: string; profile?: PaperclipCloudConnectorProfileId; scopes?: string[]; returnUri?: string; returnState?: string; claimId?: string; redemptionId?: string; eventWakeVersion?: 0 | 1 },
|
||||
secret?: { field: "refreshToken" | "token" | "binding" | "acknowledgement"; value: string },
|
||||
secret?: { field: "refreshToken" | "token" | "binding" | "acknowledgement" | "enrollment"; value: string },
|
||||
): Promise<ConnectorResponse> {
|
||||
const endpoint = new URL(ENDPOINTS[operation], `${config.baseUrl}/`).toString();
|
||||
const issuedAt = Math.floor(now() / 1000);
|
||||
@@ -333,6 +343,7 @@ export function createPaperclipCloudConnector(input: {
|
||||
response.status === 409 ? "REAUTHORIZATION_REQUIRED" : "CONNECTOR_REQUEST_FAILED",
|
||||
response.status,
|
||||
providerRetryAt(response.headers.get("retry-after"), now()),
|
||||
reason,
|
||||
);
|
||||
}
|
||||
try {
|
||||
@@ -381,6 +392,14 @@ export function createPaperclipCloudConnector(input: {
|
||||
|
||||
return {
|
||||
eventWakeVersion: () => eventWakeVersion,
|
||||
async slackEventsAvailable() {
|
||||
const response = await call("status", { subject: "instance-capabilities", companyId: "instance-capabilities" });
|
||||
return response.active === true && response.slackEvents?.version === 1;
|
||||
},
|
||||
async slackEventEnrollment(values: { subject: string; companyId: string; enrollment: Record<string, unknown> }) {
|
||||
const response = await call("slack-events", values, { field: "enrollment", value: JSON.stringify(values.enrollment) });
|
||||
return values.enrollment.active === false ? response.retirement : response.enrollment;
|
||||
},
|
||||
async githubAppsAvailable() {
|
||||
const response = await call("status", { subject: "instance-capabilities", companyId: "instance-capabilities" });
|
||||
return response.active === true && response.githubApps?.version === 2;
|
||||
@@ -551,6 +570,16 @@ export function createPaperclipCloudConnector(input: {
|
||||
|| delivery.registrationId !== packet.registrationId) throw badEnvelope();
|
||||
event.payload = { githubApp: delivery };
|
||||
}
|
||||
if (event.provider === "slack") {
|
||||
const packet = event.payload.slackEvent;
|
||||
if (!isRecord(packet) || typeof packet.bindingId !== "string" || packet.sealed === undefined) throw badEnvelope();
|
||||
const sealed = parseEnvelope(packet.sealed, "events", "slack", "slack.events.v1");
|
||||
const delivery = decryptEnvelope(sealed, sealKey, config.instanceId, config.environment, "slack", "slack.events.v1", []);
|
||||
if (!isRecord(delivery) || delivery.version !== 1 || delivery.instanceId !== config.instanceId
|
||||
|| delivery.environment !== config.environment || delivery.bindingId !== packet.bindingId
|
||||
|| delivery.acceptedAt !== event.createdAt) throw badEnvelope();
|
||||
event.payload = { slackEvent: delivery };
|
||||
}
|
||||
events.push(event);
|
||||
} catch {
|
||||
failedEventIds.push(event.id);
|
||||
@@ -684,7 +713,7 @@ function parseEnvelope(
|
||||
value: unknown,
|
||||
purpose: SealedEnvelope["purpose"],
|
||||
provider: PaperclipCloudConnectorProvider,
|
||||
profile: PaperclipCloudConnectorProfileId,
|
||||
profile: SealedEnvelopeProfileId,
|
||||
): SealedEnvelope {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) throw badEnvelope();
|
||||
const candidate = value as Partial<SealedEnvelope>;
|
||||
@@ -737,7 +766,7 @@ function unsealEvents(
|
||||
if (parsed.v !== 1 || parsed.instanceId !== instanceId || parsed.environment !== environment
|
||||
|| typeof parsed.leaseId !== "string" || !Array.isArray(parsed.events)) throw badEnvelope();
|
||||
for (const event of parsed.events) {
|
||||
if (!isRecord(event) || typeof event.id !== "string" || event.provider !== "github"
|
||||
if (!isRecord(event) || typeof event.id !== "string" || !["github", "slack"].includes(event.provider)
|
||||
|| typeof event.event !== "string" || !isRecord(event.payload) || !Array.isArray(event.bindingIds)
|
||||
|| !event.bindingIds.every((id) => typeof id === "string")) throw badEnvelope();
|
||||
}
|
||||
@@ -754,7 +783,7 @@ function decryptEnvelope(
|
||||
instanceId: string,
|
||||
environment: string,
|
||||
provider: PaperclipCloudConnectorProvider,
|
||||
profile: PaperclipCloudConnectorProfileId,
|
||||
profile: SealedEnvelopeProfileId,
|
||||
scopes: readonly string[],
|
||||
): unknown {
|
||||
const ephemeralRaw = Buffer.from(envelope.epk, "base64url");
|
||||
|
||||
@@ -3,7 +3,8 @@ import { notifyVoiceChange } from "../../voice/voice-recovery-notifications.js";
|
||||
import { notifyChatEndpointWork } from "../../chat-work-notifications.js";
|
||||
import { notifyToolConnectionHealth } from "../../tool-connection-health-notifications.js";
|
||||
import { toConnection } from "./projection.js";
|
||||
import { actorBinding } from "../actor.js";
|
||||
import { actorBinding, connectionActorUserId } from "../actor.js";
|
||||
import { stageSlackCloudEnrollmentIntent } from "../../chat-channels/slack/cloud-enrollment.js";
|
||||
import { vercelConnectResourcesFor } from "../vercel-credentials.js";
|
||||
import type { ActorInfo } from "../types.js";
|
||||
import { chatCredentialMutationLease } from "../../chat-credential-mutation-lease.js";
|
||||
@@ -241,6 +242,8 @@ async function removeConnection(
|
||||
await lease.assertOwned(tx);
|
||||
await tx.update(chatEndpoints).set({ status: "archived", updatedAt: new Date() }).where(eq(chatEndpoints.id, endpoint.id));
|
||||
await tx.update(toolConnections).set({ status: "archived", enabled: false, updatedAt: new Date() }).where(eq(toolConnections.id, connection.id));
|
||||
await stageSlackCloudEnrollmentIntent(tx as unknown as Db, endpoint.id, connectionActorUserId(actor));
|
||||
await lease.assertOwned(tx);
|
||||
});
|
||||
await removeSlackRegistration(db, endpoint.id, lease);
|
||||
return removeConnectionUnlocked(connectionId, companyId, actor);
|
||||
|
||||
@@ -24,6 +24,16 @@ node scripts/typecheck-test-projects.mjs tests/restart-chaos # typecheck
|
||||
Options: `--triggers time,comment,retry,queued,tool` (fault timing, below),
|
||||
`--workloads A1,B1,B5,C1`, `--budget-cents` (company hard cap,
|
||||
default 3000), `--poll-ms` (2000–5000), `--keep` (keep the temporary instance).
|
||||
|
||||
`--rc06-masked` keeps the company budget amount but disables its hard stop on
|
||||
the isolated Chaos Lab company, so one stuck accounting row (RC-06) cannot
|
||||
wedge every later task. Masked tasks are marked `rc06-masked` in the ledger and
|
||||
summary, and each wave still records its spend (`wave_spend` in the timeline).
|
||||
A replay inherits the masks saved in the source wave's `plan.json`;
|
||||
`--replay <wave>/<task> --rc06-masked` adds the mask to a replay of an unmasked wave.
|
||||
Each fault receipt records the host 1-minute load average (`loadAvg1m`); treat
|
||||
timeout failures above about 16 as load-suspect.
|
||||
|
||||
## Daytona (`--env daytona`)
|
||||
|
||||
The control plane stays local; agent runs execute in Daytona sandboxes from an
|
||||
|
||||
@@ -30,7 +30,7 @@ export const DAYTONA_VCPU_BUDGET = 40;
|
||||
|
||||
export type ParsedCommand =
|
||||
| { mode: "wave"; options: WaveOptions; keep: boolean }
|
||||
| { mode: "replay"; ref: ReplayRef; keep: boolean; overrides: Partial<Pick<WaveOptions, "pollMs" | "budgetCents">> };
|
||||
| { mode: "replay"; ref: ReplayRef; keep: boolean; overrides: Partial<Pick<WaveOptions, "pollMs" | "budgetCents" | "masks">> };
|
||||
|
||||
export interface ReplayRef { waveId: string; taskId: string }
|
||||
|
||||
@@ -84,6 +84,7 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC
|
||||
"daytona-image": { type: "string" },
|
||||
"daytona-cpu": { type: "string" },
|
||||
keep: { type: "boolean" },
|
||||
"rc06-masked": { type: "boolean" },
|
||||
},
|
||||
});
|
||||
const keep = values.keep === true;
|
||||
@@ -94,7 +95,9 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC
|
||||
if (values[key] !== undefined) throw new Error(`--replay reproduces the saved plan; do not combine it with --${key}`);
|
||||
}
|
||||
return { mode: "replay", ref: parseReplayRef(values.replay), keep,
|
||||
overrides: { ...(values["poll-ms"] ? { pollMs } : {}), ...(values["budget-cents"] ? { budgetCents } : {}) } };
|
||||
overrides: { ...(values["poll-ms"] ? { pollMs } : {}), ...(values["budget-cents"] ? { budgetCents } : {}),
|
||||
// The saved plan already carries its masks; this adds the RC-06 mask to a replay of an unmasked wave.
|
||||
...(values["rc06-masked"] ? { masks: ["rc06"] } : {}) } };
|
||||
}
|
||||
const env = (values.env ?? "local") as EnvironmentKind;
|
||||
if (!ENVIRONMENT_KINDS.includes(env)) throw new Error(`Unknown env "${env}"; allowed: ${ENVIRONMENT_KINDS.join(", ")}`);
|
||||
@@ -118,6 +121,7 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC
|
||||
budgetCents,
|
||||
pollMs,
|
||||
...(env === "daytona" ? { daytonaImage, daytonaCpu: integer(values["daytona-cpu"], "daytona-cpu", DAYTONA_DEFAULT_CPU, 1, 4) } : {}),
|
||||
...(values["rc06-masked"] ? { masks: ["rc06"] } : {}),
|
||||
};
|
||||
if (env !== "daytona" && values["daytona-cpu"] !== undefined) throw new Error("--daytona-cpu applies only to --env daytona");
|
||||
if (options.daytonaCpu !== undefined) {
|
||||
|
||||
@@ -30,6 +30,8 @@ export interface LedgerLine {
|
||||
evidence: string | null;
|
||||
replay: string;
|
||||
recordedAt: string;
|
||||
/** Known signatures masked for this wave, e.g. ["rc06"]. */
|
||||
masks?: string[];
|
||||
}
|
||||
|
||||
export function ledgerLine(plan: WavePlan, record: TaskRecord, grade: TaskGrade, gitSha: string, evidence: string | null,
|
||||
@@ -46,6 +48,7 @@ export function ledgerLine(plan: WavePlan, record: TaskRecord, grade: TaskGrade,
|
||||
...(loads.length ? { maxLoadAvg1m: Math.max(...loads) } : {}),
|
||||
signature: grade.signature, retried: Boolean(record.retry), evidence, replay: replayCommand(plan.waveId, record.plan.taskId),
|
||||
recordedAt: new Date().toISOString(),
|
||||
...(plan.options.masks?.length ? { masks: [...plan.options.masks] } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -64,7 +67,7 @@ export function renderSummary(lines: readonly LedgerLine[], latestWave?: string)
|
||||
const cells = new Map<string, { pass: number; total: number; crossed: number }>();
|
||||
const signatures = new Map<string, { count: number; latest: LedgerLine }>();
|
||||
for (const line of lines) {
|
||||
const key = `${line.env} · ${line.adapter} · ${line.workload} · ${line.fault}${line.trigger ? ` ^${line.trigger}` : ""}`;
|
||||
const key = `${line.env} · ${line.adapter} · ${line.workload} · ${line.fault}${line.trigger ? ` ^${line.trigger}` : ""}${line.masks?.length ? ` · ${line.masks.map(mask => `${mask}-masked`).join(",")}` : ""}`;
|
||||
const cell = cells.get(key) ?? { pass: 0, total: 0, crossed: 0 };
|
||||
cell.total += 1; if (line.pass) cell.pass += 1; if (line.crossed) cell.crossed += 1;
|
||||
cells.set(key, cell);
|
||||
@@ -78,7 +81,7 @@ export function renderSummary(lines: readonly LedgerLine[], latestWave?: string)
|
||||
if (latestWave) {
|
||||
out.push(`## Latest wave ${latestWave}`, "", "| Task | Cell | Crossed | R1 | R2 | R3 | R4 | R5 | R6 | T | Result |", "| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |");
|
||||
for (const line of lines.filter(candidate => candidate.waveId === latestWave)) {
|
||||
out.push(`| ${line.taskId} ${line.issue ?? ""} | ${line.adapter} ${line.workload} ${line.fault}${line.trigger ? ` ^${line.trigger}${line.triggerMissed?.length ? " (missed)" : ""}` : ""} | ${line.crossed ? "yes" : "no"} | ${line.rules.R1} | ${line.rules.R2} | ${line.rules.R3} | ${line.rules.R4} | ${line.rules.R5} | ${line.rules.R6} | ${line.T} | ${line.pass ? "PASS" : `FAIL ${line.signature}`} |`);
|
||||
out.push(`| ${line.taskId} ${line.issue ?? ""} | ${line.adapter} ${line.workload} ${line.fault}${line.trigger ? ` ^${line.trigger}${line.triggerMissed?.length ? " (missed)" : ""}` : ""}${line.masks?.length ? ` (${line.masks.map(mask => `${mask}-masked`).join(", ")})` : ""} | ${line.crossed ? "yes" : "no"} | ${line.rules.R1} | ${line.rules.R2} | ${line.rules.R3} | ${line.rules.R4} | ${line.rules.R5} | ${line.rules.R6} | ${line.T} | ${line.pass ? "PASS" : `FAIL ${line.signature}`} |`);
|
||||
}
|
||||
out.push("");
|
||||
}
|
||||
|
||||
@@ -125,6 +125,18 @@ async function main(): Promise<number> {
|
||||
harnessError ??= error;
|
||||
interrupted = true;
|
||||
});
|
||||
// Spend is read once per wave: after the tasks, or during cleanup when the wave stopped early.
|
||||
let spendTarget: { api: ChaosApi; companyId: string } | null = null;
|
||||
let spendRecorded = false;
|
||||
const recordWaveSpend = async () => {
|
||||
if (!spendTarget || spendRecorded) return;
|
||||
spendRecorded = true;
|
||||
// Each wave owns its company, so all-time spend is exactly this wave, even across a month boundary.
|
||||
const spend = await spendTarget.api.get<Record<string, unknown>>(`/api/companies/${spendTarget.companyId}/costs/summary?period=all`)
|
||||
.catch(error => ({ error: String(error) }));
|
||||
timeline.add("wave_spend", { spend });
|
||||
console.info(`Wave spend: ${JSON.stringify(spend)}`);
|
||||
};
|
||||
let faultFailures = 0;
|
||||
let faultsQueued = 0;
|
||||
try {
|
||||
@@ -132,11 +144,12 @@ async function main(): Promise<number> {
|
||||
const api = new ChaosApi(supervisor.baseUrl);
|
||||
const lab = await seedChaosLab({ api, env: plan.options.env, daytonaImage: plan.options.daytonaImage, daytonaCpu: plan.options.daytonaCpu,
|
||||
adapters: [...new Set(plan.tasks.map(task => task.adapter))],
|
||||
concurrency: plan.options.concurrency, budgetCents: plan.options.budgetCents,
|
||||
concurrency: plan.options.concurrency, budgetCents: plan.options.budgetCents, masks: plan.options.masks,
|
||||
fixtureRoot: path.join(supervisor.paths.temporaryRoot, "fixtures"), waveId: plan.waveId, timeline,
|
||||
onEnvironment: owned => {
|
||||
sweeper = new DaytonaWaveSweeper({ apiKey: process.env.DAYTONA_API_KEY!, companyId: owned.companyId, environmentId: owned.environmentId, api });
|
||||
} });
|
||||
spendTarget = { api, companyId: lab.companyId };
|
||||
// The instance root holds the secrets master key and the database: oracle code may not read it.
|
||||
const protectedRoots = [os.homedir(), supervisor.paths.temporaryRoot, resultsRoot];
|
||||
const observers = plan.tasks.map(task => new TaskObserver(task, api, lab, timeline, supervisor, protectedRoots));
|
||||
@@ -193,6 +206,7 @@ async function main(): Promise<number> {
|
||||
// R6: let finalizers settle, then sweep for anything still pending.
|
||||
const quietFor = Date.now() - lastDoneAt;
|
||||
if (quietFor < QUIET_BEFORE_SWEEP_MS) await delay(QUIET_BEFORE_SWEEP_MS - quietFor);
|
||||
await recordWaveSpend();
|
||||
timeline.add("wave_sweep", {});
|
||||
if (sweeper) {
|
||||
sandboxSweep = await (sweeper as DaytonaWaveSweeper).sweep();
|
||||
@@ -238,6 +252,8 @@ async function main(): Promise<number> {
|
||||
harnessError = error;
|
||||
timeline.add("harness_error", { error: error instanceof Error ? error.stack ?? error.message : String(error) });
|
||||
} finally {
|
||||
// Best effort for an interrupted or failed wave; the server may already be down.
|
||||
if (supervisor.serverReady) await recordWaveSpend().catch(() => undefined);
|
||||
// A failure here (for example writing the report) must not skip stopping the controller and Postgres.
|
||||
try { if (sweeper) try {
|
||||
const owned = sweeper as DaytonaWaveSweeper;
|
||||
|
||||
@@ -67,6 +67,20 @@ describe("replay", () => {
|
||||
expect(again.waveId).toBe(`${plan.waveId}-replay-t03-1700000000001`);
|
||||
});
|
||||
|
||||
it("keeps the source wave's masks on replay and lets --rc06-masked add the RC-06 mask", () => {
|
||||
const masked = planWave("w-20261106T101112-s42", { ...plan.options, masks: ["rc06"] });
|
||||
expect(replayWave(masked, { waveId: masked.waveId, taskId: "t01" }, 1).options.masks).toEqual(["rc06"]);
|
||||
expect(replayWave(plan, { waveId: plan.waveId, taskId: "t01" }, 1).options.masks).toBeUndefined();
|
||||
const command = parseCommand(["--replay", `${plan.waveId}/t01`, "--rc06-masked"]);
|
||||
if (command.mode !== "replay") throw new Error("expected replay");
|
||||
expect(command.overrides.masks).toEqual(["rc06"]);
|
||||
expect(replayWave(plan, { waveId: plan.waveId, taskId: "t01" }, 1, command.overrides).options.masks).toEqual(["rc06"]);
|
||||
expect(replayWave(masked, { waveId: masked.waveId, taskId: "t01" }, 1, command.overrides).options.masks).toEqual(["rc06"]);
|
||||
const wave = parseCommand(["--rc06-masked", "--seed", "3"]);
|
||||
if (wave.mode !== "wave") throw new Error("expected wave");
|
||||
expect(wave.options.masks).toEqual(["rc06"]);
|
||||
});
|
||||
|
||||
it("replays the recorded fault sequence that crossed the task, other tasks' faults included", () => {
|
||||
const createdAt = 1_000_000;
|
||||
const faults = [
|
||||
|
||||
@@ -80,6 +80,12 @@ export function planTasks(options: Pick<WaveOptions, "seed" | "tasks" | "workloa
|
||||
return tasks;
|
||||
}
|
||||
|
||||
/** A replay keeps the source wave's masks and may add more; it never drops one. */
|
||||
function replayMasks(source: readonly string[] | undefined, added: readonly string[] | undefined) {
|
||||
const masks = [...new Set([...(source ?? []), ...(added ?? [])])];
|
||||
return masks.length ? { masks } : {};
|
||||
}
|
||||
|
||||
export function planWave(waveId: string, options: WaveOptions, createdAt = new Date()): WavePlan {
|
||||
return { schema: "paperclip.restart-chaos.wave-plan/v1", waveId, seed: options.seed,
|
||||
createdAt: createdAt.toISOString(), options, tasks: planTasks(options) };
|
||||
@@ -114,7 +120,7 @@ export function taskFaultSchedule(task: TaskPlan): ScheduledFault[] {
|
||||
* the replay fires that same sequence at the same offsets from launch.
|
||||
*/
|
||||
export function replayWave(source: WavePlan, ref: ReplayRef, now = Date.now(),
|
||||
overrides: Partial<Pick<WaveOptions, "pollMs" | "budgetCents">> = {}, observed: ScheduledFault[] = []): WavePlan {
|
||||
overrides: Partial<Pick<WaveOptions, "pollMs" | "budgetCents" | "masks">> = {}, observed: ScheduledFault[] = []): WavePlan {
|
||||
if (source.schema !== "paperclip.restart-chaos.wave-plan/v1") throw new Error("Unsupported wave plan schema");
|
||||
if (source.waveId !== ref.waveId) throw new Error(`Plan is for ${source.waveId}, not ${ref.waveId}`);
|
||||
const task = source.tasks.find(candidate => candidate.taskId === ref.taskId);
|
||||
@@ -125,7 +131,7 @@ export function replayWave(source: WavePlan, ref: ReplayRef, now = Date.now(),
|
||||
waveId: `${base}-replay-${task.taskId}-${now}`,
|
||||
seed: source.seed,
|
||||
createdAt: new Date(now).toISOString(),
|
||||
options: { ...source.options, ...overrides, tasks: 1, concurrency: 1 },
|
||||
options: { ...source.options, ...overrides, tasks: 1, concurrency: 1, ...replayMasks(source.options.masks, overrides.masks) },
|
||||
tasks: [{ ...task, ...(observed.length ? { replayFaults: observed.map(fault => ({ ...fault })) } : {}) }],
|
||||
replayOf: { ...ref },
|
||||
};
|
||||
|
||||
@@ -66,3 +66,41 @@ describe("seedChaosLab workspace isolation", () => {
|
||||
expect(calls.some(call => call.method === "POST" && call.route.endsWith("/projects"))).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("seedChaosLab RC-06 mask", () => {
|
||||
let fixtureRoot: string;
|
||||
beforeEach(async () => {
|
||||
fixtureRoot = await mkdtemp(path.join(os.tmpdir(), "rc06-seed-"));
|
||||
vi.stubEnv("OPENAI_API_KEY", "test-only-not-a-key");
|
||||
});
|
||||
afterEach(async () => {
|
||||
vi.unstubAllEnvs();
|
||||
await rm(fixtureRoot, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
const seed = (api: ChaosApi, timeline: Timeline, masks?: string[]) => seedChaosLab({ api, env: "local", adapters: ["codex"],
|
||||
concurrency: 2, budgetCents: 3_000, fixtureRoot, waveId: "w-test", timeline, ...(masks ? { masks } : {}) });
|
||||
const isPolicyCall = (call: Call) => call.method === "POST" && /^\/api\/companies\/[^/]+\/budgets\/policies$/.test(call.route);
|
||||
|
||||
it("keeps the budget amount and disables only the company hard stop", async () => {
|
||||
const { api, calls } = fakeApi(body => ({ ...body }));
|
||||
const timeline = new Timeline(null);
|
||||
await seed(api, timeline, ["rc06"]);
|
||||
const company = calls.find(call => call.method === "POST" && call.route === "/api/companies")!;
|
||||
expect((company.body as { budgetMonthlyCents: number }).budgetMonthlyCents).toBe(3_000);
|
||||
const policies = calls.filter(isPolicyCall);
|
||||
expect(policies).toHaveLength(1);
|
||||
expect(policies[0]!.route).toBe("/api/companies/id-1/budgets/policies");
|
||||
expect(policies[0]!.body).toEqual({ scopeType: "company", scopeId: "id-1", amount: 3_000, hardStopEnabled: false });
|
||||
expect(timeline.entries.filter(entry => entry.kind === "seed_mask_rc06"))
|
||||
.toEqual([expect.objectContaining({ companyId: "id-1", budgetCents: 3_000, hardStopEnabled: false })]);
|
||||
});
|
||||
|
||||
it("leaves the company hard stop on without the mask", async () => {
|
||||
const { api, calls } = fakeApi(body => ({ ...body }));
|
||||
const timeline = new Timeline(null);
|
||||
await seed(api, timeline);
|
||||
expect(calls.some(isPolicyCall)).toBe(false);
|
||||
expect(timeline.entries.some(entry => entry.kind === "seed_mask_rc06")).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -100,6 +100,7 @@ export async function seedChaosLab(input: {
|
||||
adapters: readonly AdapterId[];
|
||||
concurrency: number;
|
||||
budgetCents: number;
|
||||
masks?: readonly string[];
|
||||
fixtureRoot: string;
|
||||
waveId: string;
|
||||
timeline: Timeline;
|
||||
@@ -120,6 +121,13 @@ export async function seedChaosLab(input: {
|
||||
budgetMonthlyCents: input.budgetCents,
|
||||
});
|
||||
timeline.add("seed_company", { companyId: company.id });
|
||||
if (input.masks?.includes("rc06")) {
|
||||
// Keep the amount, disable only the hard stop on this isolated company (RC-06 mask).
|
||||
await api.post(`/api/companies/${company.id}/budgets/policies`, {
|
||||
scopeType: "company", scopeId: company.id, amount: input.budgetCents, hardStopEnabled: false,
|
||||
});
|
||||
timeline.add("seed_mask_rc06", { companyId: company.id, budgetCents: input.budgetCents, hardStopEnabled: false });
|
||||
}
|
||||
|
||||
const secretRefs: SecretReferenceMap = {};
|
||||
const credentialKeys = [...new Set([...input.adapters.map(adapter => CREDENTIAL_FOR_ADAPTER[adapter]),
|
||||
|
||||
@@ -32,6 +32,13 @@ export interface WaveOptions {
|
||||
daytonaImage?: string;
|
||||
/** `--env daytona`: vCPU per sandbox (see seed.ts chaosDaytonaShape). */
|
||||
daytonaCpu?: number;
|
||||
/**
|
||||
* Known product signatures this wave deliberately masks. "rc06" keeps the
|
||||
* company budget amount but disables its hard stop, so a stuck
|
||||
* accounting row (RC-06) cannot wedge every later task. Spend is still
|
||||
* recorded per wave. Absent on plans written before masks existed.
|
||||
*/
|
||||
masks?: string[];
|
||||
}
|
||||
|
||||
export interface TaskPlan {
|
||||
|
||||
@@ -37,8 +37,16 @@ export const reliabilityDefinitions = Object.freeze([
|
||||
// Until every case and the plan have an identity, selecting this plan fails
|
||||
// closed with `product_regression_testops_reliability_unregistered`.
|
||||
export const reliabilityRegistration = Object.freeze({
|
||||
planId: null,
|
||||
caseIds: Object.freeze({}),
|
||||
planId: 135,
|
||||
caseIds: Object.freeze({
|
||||
'interrupt-redirect': 885,
|
||||
'reassign-midrun': 886,
|
||||
'three-turn-revise': 887,
|
||||
'long-silent-command': 888,
|
||||
'tree-pause-release': 889,
|
||||
'queue-steer': 890,
|
||||
'queued-across-restart': 891,
|
||||
}),
|
||||
});
|
||||
|
||||
const fail = code => { throw new Error(`product_regression_testops_${code}`); };
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"inputSha256": "14049cb20d48f448b6f24e5445a75bf8d22ed3052fe9ff885f4c127775d958d3",
|
||||
"outputSha256": "53f6c7c7f09a5cf0442ee2992a656eb2336ab6b13e665219a0cd1247475f13d9",
|
||||
"patchSha256": "eb4adb2f3c8a2ebefe65d3100b581c5765318023d96aa8a80394c848d7299b9d",
|
||||
"inputSha256": "53f6c7c7f09a5cf0442ee2992a656eb2336ab6b13e665219a0cd1247475f13d9",
|
||||
"outputSha256": "5e2b012b50bb8b198384da4f5427c27acd948af405f89cde15bfa48bb10c0ad4",
|
||||
"patchSha256": "2abdd6ec8e374bfb20b7e168ef010858493a50a044360175e1af5a2ca393f909",
|
||||
"basis": "Generated with the trusted master lockfile refresh. Review this offline correction and its hashes with the dependency changes."
|
||||
}
|
||||
@@ -1,14 +1,15 @@
|
||||
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
|
||||
index 45b3aaf..a87f8e7 100644
|
||||
index a87f8e7..97cbf7b 100644
|
||||
--- a/pnpm-lock.yaml
|
||||
+++ b/pnpm-lock.yaml
|
||||
@@ -1306,6 +1306,9 @@ importers:
|
||||
lucide-react:
|
||||
specifier: ^1.48.0
|
||||
version: 1.48.0(react@19.2.8)
|
||||
+ mdast-util-from-markdown:
|
||||
+ specifier: ^2.0.3
|
||||
+ version: 2.0.3
|
||||
mermaid:
|
||||
specifier: ^12.1.0
|
||||
version: 12.1.0
|
||||
@@ -5048,7 +5048,9 @@ packages:
|
||||
'@smithy/core@3.35.1':
|
||||
resolution: {integrity: sha512-i4YPS4B6ts7bjn7UwLnGjiZdprOvHvgGobFZsYK3GIY3E5hIqtj0rReU69BcTpGp+fvtraSNXeG1l+jtJvF55w==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
- deprecated: Deprecated due to a bug in custom logger data redaction. See https://github.com/aws/aws-sdk-js-v3/issues/8393.
|
||||
+ deprecated: |-
|
||||
+ Deprecated due to a bug in custom logger data redaction. Upgrade to the latest version. If you used a custom logger with this
|
||||
+ version, review your logs and rotate any credentials or other secrets. See https://github.com/aws/aws-sdk-js-v3/issues/8393.
|
||||
|
||||
'@smithy/credential-provider-imds@4.5.2':
|
||||
resolution: {integrity: sha512-A9uSdn72ozbRUSit0eib0TW7nXuNPlaeM0zcGkJ+nE6tFcSDbnmtwoxbTCFBukVQcszDAyvsd7+rTduPTXpygg==}
|
||||
@@ -0,0 +1,44 @@
|
||||
import { expect, test } from "@playwright/test";
|
||||
|
||||
for (const theme of ["dark", "light"]) {
|
||||
for (const mobile of [false, true]) {
|
||||
for (const streaming of [false, true]) {
|
||||
test(`GitHub inline links [${theme}, ${mobile ? "mobile" : "desktop"}, ${streaming ? "streaming" : "ordinary"}]`, async ({ page }, testInfo) => {
|
||||
if (mobile) await page.setViewportSize({ width: 390, height: 844 });
|
||||
const errors: string[] = [];
|
||||
const providerRequests: string[] = [];
|
||||
page.on("pageerror", (error) => errors.push(error.message));
|
||||
page.on("request", (request) => {
|
||||
if (new URL(request.url()).hostname === "github.com") providerRequests.push(request.url());
|
||||
});
|
||||
const id = `tasks-inline-links--${streaming ? "streaming" : mobile ? "narrow" : "playground"}`;
|
||||
await page.goto(`/iframe.html?id=${id}&viewMode=story&globals=theme:${theme}`);
|
||||
await expect.poll(() => page.evaluate(() => {
|
||||
const channel = (window as unknown as {
|
||||
__STORYBOOK_ADDONS_CHANNEL__?: { data: Record<string, Array<{ status?: string }>> };
|
||||
}).__STORYBOOK_ADDONS_CHANNEL__;
|
||||
return channel?.data.storyFinished?.[0]?.status;
|
||||
})).toBe("success");
|
||||
const main = page.getByRole("main");
|
||||
const pr = main.getByRole("link", { name: "GitHub pull request: acme/web#241", exact: true });
|
||||
await expect(pr).toHaveAttribute("rel", "noopener noreferrer");
|
||||
if (streaming) {
|
||||
await expect(pr.locator(".conversation-stream-word")).toHaveText("acme/web#241");
|
||||
}
|
||||
const comment = main.getByRole("link", { name: "GitHub pull request: Jump to the review comment" });
|
||||
await expect(comment).toHaveAttribute("href", "https://github.com/acme/web/pull/241/files?diff=split#discussion_r123");
|
||||
await expect(comment).toHaveAttribute("title", "Review comment on the exact diff");
|
||||
await pr.focus();
|
||||
await expect(pr).toBeFocused();
|
||||
await pr.blur();
|
||||
const settings = main.getByRole("link", { name: "company settings" });
|
||||
await expect(settings).not.toHaveAttribute("target");
|
||||
await expect(main.locator("a[data-github-link] .lucide-external-link")).toHaveCount(0);
|
||||
expect(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)).toBe(true);
|
||||
expect(errors).toEqual([]);
|
||||
expect(providerRequests).toEqual([]);
|
||||
await page.screenshot({ path: testInfo.outputPath(`inline-links-${theme}-${mobile ? "mobile" : "desktop"}-${streaming ? "streaming" : "ordinary"}.png`), fullPage: true });
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
} from "@paperclipai/shared";
|
||||
import { ThemeProvider } from "../context/ThemeContext";
|
||||
import { MarkdownBody } from "./MarkdownBody";
|
||||
import { MarkdownWordStreaming } from "./MarkdownWordStreaming";
|
||||
import { queryKeys } from "../lib/queryKeys";
|
||||
import type { WorkspaceFileAvailabilityTarget } from "../lib/workspace-file-availability";
|
||||
|
||||
@@ -66,6 +67,7 @@ function renderMarkdown(
|
||||
children: string,
|
||||
seededIssues: Array<{ identifier: string; status: string; title?: string; cacheKey?: string }> = [],
|
||||
props: Partial<ComponentProps<typeof MarkdownBody>> = {},
|
||||
streaming = false,
|
||||
) {
|
||||
const queryClient = new QueryClient({
|
||||
defaultOptions: {
|
||||
@@ -87,7 +89,9 @@ function renderMarkdown(
|
||||
return renderToStaticMarkup(
|
||||
<QueryClientProvider client={queryClient}>
|
||||
<ThemeProvider>
|
||||
<MarkdownBody {...props}>{children}</MarkdownBody>
|
||||
<MarkdownWordStreaming.Provider value={streaming}>
|
||||
<MarkdownBody {...props}>{children}</MarkdownBody>
|
||||
</MarkdownWordStreaming.Provider>
|
||||
</ThemeProvider>
|
||||
</QueryClientProvider>,
|
||||
);
|
||||
@@ -607,23 +611,85 @@ describe("MarkdownBody", () => {
|
||||
expect(html).toContain('rel="noreferrer"');
|
||||
});
|
||||
|
||||
it("prefixes GitHub markdown links with the GitHub icon glued to the first character", () => {
|
||||
const html = renderMarkdown("[https://github.com/paperclipai/paperclip/pull/4099](https://github.com/paperclipai/paperclip/pull/4099)");
|
||||
it("gives pull request URLs a compact label and a distinct icon", () => {
|
||||
const href = "https://github.com/paperclipai/paperclip/pull/4099";
|
||||
const html = renderMarkdown(`[${href}](${href})`);
|
||||
|
||||
expect(html).toContain('<a href="https://github.com/paperclipai/paperclip/pull/4099"');
|
||||
expect(html).toContain('class="lucide lucide-github mr-1 inline h-3.5 w-3.5 align-(--va-0_125em)"');
|
||||
// The icon and first character "h" must sit in a no-wrap span so the
|
||||
// icon can never be orphaned on the previous line from the URL text.
|
||||
expect(html).toMatch(/<span style="white-space:nowrap">.*lucide-github.*?<\/svg>h<\/span>/);
|
||||
expect(html).toContain("ttps://github.com/paperclipai/paperclip/pull/4099");
|
||||
expect(html).toContain(`<a href="${href}"`);
|
||||
expect(html).toContain('data-github-link="pull_request"');
|
||||
expect(html).toContain("lucide-git-pull-request");
|
||||
expect(html).toContain("GitHub pull request:");
|
||||
expect(html).toMatch(/<span style="white-space:nowrap">.*lucide-git-pull-request.*?<\/svg>p<\/span>/);
|
||||
expect(html.replace(/<[^>]*>/g, "")).toContain("paperclipai/paperclip#4099");
|
||||
expect(html).not.toContain("lucide-external-link");
|
||||
});
|
||||
|
||||
it("prefixes GitHub autolinks with the GitHub icon", () => {
|
||||
const html = renderMarkdown("See https://github.com/paperclipai/paperclip/issues/1778");
|
||||
it("gives issue autolinks their own icon and keeps the exact destination", () => {
|
||||
const html = renderMarkdown("See https://github.com/paperclipai/paperclip/issues/1778?source=chat#issuecomment-123");
|
||||
|
||||
expect(html).toContain('<a href="https://github.com/paperclipai/paperclip/issues/1778"');
|
||||
expect(html).toContain('class="lucide lucide-github mr-1 inline h-3.5 w-3.5 align-(--va-0_125em)"');
|
||||
expect(html).toContain('href="https://github.com/paperclipai/paperclip/issues/1778?source=chat#issuecomment-123"');
|
||||
expect(html).toContain('data-github-link="issue"');
|
||||
expect(html).toContain("lucide-circle-dot");
|
||||
expect(html).toContain("GitHub issue:");
|
||||
expect(html.replace(/<[^>]*>/g, "")).toContain("paperclipai/paperclip#1778");
|
||||
});
|
||||
|
||||
it.each([
|
||||
["https://github.com/acme/web/pull/241", "acme/web#241"],
|
||||
["https://github.com/acme/web/issues/42", "acme/web#42"],
|
||||
["https://github.com/acme/web", "acme/web"],
|
||||
])("compacts streaming URL text while preserving its word wrapper: %s", (href, label) => {
|
||||
const html = renderMarkdown(href, [], {}, true);
|
||||
expect(html).toContain(`class="conversation-stream-word">${label}</span>`);
|
||||
expect(html).toContain(`href="${href}"`);
|
||||
expect(html.replace(/<[^>]*>/g, "")).not.toContain(href);
|
||||
});
|
||||
|
||||
it("preserves custom streaming labels and formatted URL labels", () => {
|
||||
const href = "https://github.com/acme/web/pull/241";
|
||||
const html = renderMarkdown(`[Review **the PR**](${href}) and [**${href}**](${href})`, [], {}, true);
|
||||
expect(html).toContain("<strong>");
|
||||
expect(html).toContain('class="conversation-stream-word"');
|
||||
const text = html.replace(/<[^>]*>/g, "");
|
||||
expect(text).toContain("Review the PR");
|
||||
expect(text).toContain(href);
|
||||
expect(text).not.toContain("acme/web#241");
|
||||
});
|
||||
|
||||
it("keeps custom labels, formatted text, and authored titles", () => {
|
||||
const html = renderMarkdown('[**Review** the PR](https://github.com/acme/web/pull/241 "Ready for review")');
|
||||
expect(html).toContain("<strong>Review</strong> the PR");
|
||||
expect(html).toContain('title="Ready for review"');
|
||||
expect(html).toContain('data-github-link="pull_request"');
|
||||
expect(html).toContain("lucide-git-pull-request");
|
||||
expect(html).not.toContain("acme/web#241");
|
||||
});
|
||||
|
||||
it("keeps GitHub branding for repositories and other GitHub pages", () => {
|
||||
const html = renderMarkdown("https://github.com/acme/web and [GitHub Actions](https://github.com/features/actions)");
|
||||
expect(html).toContain('data-github-link="repository"');
|
||||
expect(html.replace(/<[^>]*>/g, "")).toContain("acme/web");
|
||||
expect(html).toContain('data-github-link="github"');
|
||||
expect(html).toContain("lucide-github");
|
||||
});
|
||||
|
||||
it.each([
|
||||
"https://github.com/stars/torvalds",
|
||||
"https://github.com/education/teachers",
|
||||
"https://github.com/enterprises/acme",
|
||||
])("keeps GitHub site pages as branded full URL links: %s", (href) => {
|
||||
const html = renderMarkdown(href);
|
||||
expect(html).toContain('data-github-link="github"');
|
||||
expect(html).not.toContain('data-github-link="repository"');
|
||||
expect(html.replace(/<[^>]*>/g, "")).toContain(href);
|
||||
expect(html).toContain(`href="${href}"`);
|
||||
});
|
||||
|
||||
it("keeps full URL labels on lookalike hosts", () => {
|
||||
const html = renderMarkdown("https://github.com.example.com/acme/web/pull/241");
|
||||
expect(html).not.toContain("data-github-link");
|
||||
expect(html).toContain("lucide-external-link");
|
||||
expect(html.replace(/<[^>]*>/g, "")).toContain("https://github.com.example.com/acme/web/pull/241");
|
||||
});
|
||||
|
||||
;
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { MarkdownWordStreaming, rehypeStreamingWords } from "./MarkdownWordStreaming";
|
||||
import { isValidElement, memo, useContext, useCallback, useEffect, useId, useMemo, useRef, useState, type ReactNode } from "react";
|
||||
import { useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
import { Check, Copy, ExternalLink, MessageCircle, WrapText } from "lucide-react";
|
||||
import { Check, Copy, MessageCircle, WrapText } from "lucide-react";
|
||||
import { useCoordinatorPresentation } from "./coordinator/CoordinatorPresentation";
|
||||
import { useCoordinatorChatReference } from "./coordinator/CoordinatorChatReferences";
|
||||
import { SidebarDocumentLink } from "./SidebarDocumentLink";
|
||||
@@ -9,7 +9,7 @@ import { parseDocumentAnnotationHash } from "@/lib/document-annotation-hash";
|
||||
import Markdown, { defaultUrlTransform, type Components, type Options } from "react-markdown";
|
||||
import remarkGfm from "remark-gfm";
|
||||
import { cn } from "../lib/utils";
|
||||
import { GithubIcon } from "@/components/icons/github-icon";
|
||||
import { MarkdownUrlLink } from "./MarkdownUrlLink";
|
||||
import { Link, useCaseHref } from "@/lib/router";
|
||||
import { useTheme } from "../context/ThemeContext";
|
||||
import { useOptionalCompany } from "../context/CompanyContext";
|
||||
@@ -572,78 +572,6 @@ function createRemarkWikiLinks(options: {
|
||||
};
|
||||
}
|
||||
|
||||
function isGitHubUrl(href: string | null | undefined): boolean {
|
||||
if (!href) return false;
|
||||
try {
|
||||
const url = new URL(href);
|
||||
return url.protocol === "https:" && (url.hostname === "github.com" || url.hostname === "www.github.com");
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function isExternalHttpUrl(href: string | null | undefined): boolean {
|
||||
if (!href) return false;
|
||||
try {
|
||||
const url = new URL(href);
|
||||
if (url.protocol !== "http:" && url.protocol !== "https:") return false;
|
||||
if (typeof window === "undefined") return true;
|
||||
return url.origin !== window.location.origin;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function renderLinkBody(
|
||||
children: ReactNode,
|
||||
leadingIcon: ReactNode,
|
||||
trailingIcon: ReactNode,
|
||||
): ReactNode {
|
||||
if (!leadingIcon && !trailingIcon) return children;
|
||||
|
||||
// React-markdown can pass arrays/elements for styled link text; the nowrap
|
||||
// splitting below is intentionally limited to plain text links.
|
||||
if (typeof children === "string" && children.length > 0) {
|
||||
if (children.length === 1) {
|
||||
return (
|
||||
<span style={{ whiteSpace: "nowrap" }}>
|
||||
{leadingIcon}
|
||||
{children}
|
||||
{trailingIcon}
|
||||
</span>
|
||||
);
|
||||
}
|
||||
const first = children[0];
|
||||
const last = children[children.length - 1];
|
||||
const middle = children.slice(1, -1);
|
||||
return (
|
||||
<>
|
||||
{leadingIcon ? (
|
||||
<span style={{ whiteSpace: "nowrap" }}>
|
||||
{leadingIcon}
|
||||
{first}
|
||||
</span>
|
||||
) : first}
|
||||
{middle}
|
||||
{trailingIcon ? (
|
||||
<span style={{ whiteSpace: "nowrap" }}>
|
||||
{last}
|
||||
{trailingIcon}
|
||||
</span>
|
||||
) : last}
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<>
|
||||
{leadingIcon}
|
||||
{children}
|
||||
{trailingIcon}
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
function CodeBlock({
|
||||
children,
|
||||
preProps,
|
||||
@@ -1009,24 +937,10 @@ function MarkdownBodyImpl({
|
||||
</Link>
|
||||
) : <>{linkChildren}</>;
|
||||
}
|
||||
const isGitHubLink = isGitHubUrl(href);
|
||||
const isExternal = isExternalHttpUrl(href);
|
||||
const leadingIcon = isGitHubLink ? (
|
||||
<GithubIcon aria-hidden="true" className="mr-1 inline h-3.5 w-3.5 align-(--va-0_125em)" />
|
||||
) : null;
|
||||
const trailingIcon = isExternal && !isGitHubLink ? (
|
||||
<ExternalLink aria-hidden="true" className="ml-1 inline h-3 w-3 align-(--va-0_125em)" />
|
||||
) : null;
|
||||
return (
|
||||
<a
|
||||
href={href}
|
||||
{...(isExternal
|
||||
? { target: "_blank", rel: "noopener noreferrer" }
|
||||
: { rel: "noreferrer" })}
|
||||
style={mergeWrapStyle(linkStyle as React.CSSProperties | undefined)}
|
||||
>
|
||||
{renderLinkBody(linkChildren, leadingIcon, trailingIcon)}
|
||||
</a>
|
||||
<MarkdownUrlLink href={href} title={anchorProps.title} style={mergeWrapStyle(linkStyle as React.CSSProperties | undefined)}>
|
||||
{linkChildren}
|
||||
</MarkdownUrlLink>
|
||||
);
|
||||
},
|
||||
};
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
import { cloneElement, isValidElement, type CSSProperties, type ReactNode } from "react";
|
||||
import { CircleDot, ExternalLink, GitPullRequest } from "lucide-react";
|
||||
import { GithubIcon } from "./icons/github-icon";
|
||||
import { parseGitHubLink } from "@/lib/github-link";
|
||||
|
||||
function isExternalHttpUrl(href: string | undefined): boolean {
|
||||
if (!href) return false;
|
||||
try {
|
||||
const url = new URL(href);
|
||||
if (url.protocol !== "http:" && url.protocol !== "https:") return false;
|
||||
return typeof window === "undefined" || url.origin !== window.location.origin;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function renderLinkBody(children: ReactNode, leadingIcon: ReactNode, trailingIcon: ReactNode): ReactNode {
|
||||
if (!leadingIcon && !trailingIcon) return children;
|
||||
if (typeof children !== "string" || !children.length) return <>{leadingIcon}{children}{trailingIcon}</>;
|
||||
if (children.length === 1) {
|
||||
return <span style={{ whiteSpace: "nowrap" }}>{leadingIcon}{children}{trailingIcon}</span>;
|
||||
}
|
||||
return <>
|
||||
{leadingIcon ? <span style={{ whiteSpace: "nowrap" }}>{leadingIcon}{children[0]}</span> : children[0]}
|
||||
{children.slice(1, -1)}
|
||||
{trailingIcon ? <span style={{ whiteSpace: "nowrap" }}>{children.at(-1)}{trailingIcon}</span> : children.at(-1)}
|
||||
</>;
|
||||
}
|
||||
|
||||
function compactUrlLabel(children: ReactNode, href: string | undefined, label: string): ReactNode {
|
||||
if (children === href) return label;
|
||||
const child = Array.isArray(children) && children.length === 1 ? children[0] : children;
|
||||
if (
|
||||
isValidElement<{ className?: string; children?: ReactNode }>(child)
|
||||
&& child.type === "span"
|
||||
&& child.props.className?.split(/\s+/).includes("conversation-stream-word")
|
||||
&& child.props.children === href
|
||||
) {
|
||||
return cloneElement(child, undefined, label);
|
||||
}
|
||||
return children;
|
||||
}
|
||||
|
||||
export function MarkdownUrlLink({ href, children, title, style }: {
|
||||
href?: string;
|
||||
children: ReactNode;
|
||||
title?: string;
|
||||
style?: CSSProperties;
|
||||
}) {
|
||||
const github = parseGitHubLink(href);
|
||||
const external = isExternalHttpUrl(href);
|
||||
const Icon = github?.kind === "pull_request" ? GitPullRequest : github?.kind === "issue" ? CircleDot : GithubIcon;
|
||||
const label = github?.label ? compactUrlLabel(children, href, github.label) : children;
|
||||
const leadingIcon = github ? <Icon aria-hidden="true" className="mr-1 inline h-3.5 w-3.5 align-(--va-0_125em)" /> : null;
|
||||
const trailingIcon = external && !github ? <ExternalLink aria-hidden="true" className="ml-1 inline h-3 w-3 align-(--va-0_125em)" /> : null;
|
||||
return <a
|
||||
href={href}
|
||||
{...(external ? { target: "_blank", rel: "noopener noreferrer" } : { rel: "noreferrer" })}
|
||||
data-github-link={github?.kind}
|
||||
title={title ?? (github ? `${github.description}\n${href}` : undefined)}
|
||||
style={style}
|
||||
>
|
||||
{github ? <span className="sr-only">{github.kind === "pull_request" ? "GitHub pull request: " : github.kind === "issue" ? "GitHub issue: " : "GitHub: "}</span> : null}
|
||||
{renderLinkBody(label, leadingIcon, trailingIcon)}
|
||||
</a>;
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { parseGitHubLink } from "./github-link";
|
||||
|
||||
describe("GitHub inline link presentation", () => {
|
||||
it.each([
|
||||
["https://github.com/github/docs", "repository", "github/docs"],
|
||||
["https://github.com/paperclipai/paperclip", "repository", "paperclipai/paperclip"],
|
||||
["https://github.com/paperclipai/paperclip/", "repository", "paperclipai/paperclip"],
|
||||
["https://github.com/acme/web/pull/241", "pull_request", "acme/web#241"],
|
||||
["HTTPS://WWW.GITHUB.COM/Acme/Web/pull/241/files?diff=split#diff-a", "pull_request", "Acme/Web#241"],
|
||||
["https://github.com/acme/web/issues/42#issuecomment-123", "issue", "acme/web#42"],
|
||||
["https://github.com/acme/web/issues/42/?q=test", "issue", "acme/web#42"],
|
||||
])("classifies %s", (href, kind, label) => {
|
||||
expect(parseGitHubLink(href)).toMatchObject({ kind, label });
|
||||
});
|
||||
|
||||
it.each([
|
||||
"https://github.com", "https://github.com/paperclipai",
|
||||
"https://github.com/acme/web/tree/main", "https://github.com/acme/web/pull/nope",
|
||||
"https://github.com/acme/web/issues/0", "https://github.com/settings/profile",
|
||||
"https://github.com/features/actions", "https://github.com/orgs/acme",
|
||||
"https://github.com/stars/torvalds", "https://github.com/education/teachers",
|
||||
"https://github.com/enterprises/acme",
|
||||
"https://github.com/acme/web/pulls", "https://github.com/acme/web/issues",
|
||||
])("keeps GitHub pages without a resource match as branded links: %s", (href) => {
|
||||
expect(parseGitHubLink(href)).toEqual({ kind: "github", label: null, description: "GitHub" });
|
||||
});
|
||||
|
||||
it.each([
|
||||
undefined, "", "/acme/web/pull/241", "//github.com/acme/web/pull/241",
|
||||
"https://example.com/acme/web/pull/241", "https://github.com.example.com/acme/web/pull/241",
|
||||
"https://github.example.com/acme/web/pull/241", "https://github.com@evil.example/acme/web",
|
||||
"https://someone@github.com/acme/web", "https://github.com:8443/acme/web",
|
||||
"http://github.com/acme/web", "javascript:alert(1)", "not a URL",
|
||||
])("does not apply GitHub presentation to %s", (href) => {
|
||||
expect(parseGitHubLink(href)).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,52 @@
|
||||
export interface GitHubLink {
|
||||
kind: "github" | "repository" | "pull_request" | "issue";
|
||||
label: string | null;
|
||||
description: string;
|
||||
}
|
||||
|
||||
const RESERVED_PATHS = new Set([
|
||||
"about", "account", "apps", "business", "codespaces", "collections", "contact",
|
||||
"customer-stories", "education", "enterprise", "enterprises", "events", "explore", "features",
|
||||
"issues", "join", "login", "logout", "marketplace", "new", "notifications",
|
||||
"organizations", "orgs", "pricing", "pulls", "readme", "search", "security",
|
||||
"sessions", "settings", "site", "sponsors", "stars", "team", "topics", "trending", "users",
|
||||
]);
|
||||
|
||||
export function parseGitHubLink(href: string | null | undefined): GitHubLink | null {
|
||||
if (!href) return null;
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(href);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (
|
||||
url.protocol !== "https:"
|
||||
|| (url.hostname !== "github.com" && url.hostname !== "www.github.com")
|
||||
|| url.username || url.password || url.port
|
||||
) return null;
|
||||
|
||||
const fallback: GitHubLink = { kind: "github", label: null, description: "GitHub" };
|
||||
const segments = url.pathname.replace(/\/$/, "").slice(1).split("/");
|
||||
const [owner, repository, type, number] = segments;
|
||||
if (
|
||||
!owner || !repository || RESERVED_PATHS.has(owner.toLowerCase())
|
||||
|| !/^[a-z\d](?:[a-z\d-]*[a-z\d])?$/i.test(owner)
|
||||
|| !/^[a-z\d_.-]+$/i.test(repository)
|
||||
|| repository === "." || repository === ".."
|
||||
) return fallback;
|
||||
|
||||
const repo = `${owner}/${repository}`;
|
||||
if ((type === "pull" || type === "issues") && number && /^[1-9]\d*$/.test(number)) {
|
||||
const label = `${repo}#${number}`;
|
||||
return {
|
||||
kind: type === "pull" ? "pull_request" : "issue",
|
||||
label,
|
||||
description: `GitHub ${type === "pull" ? "pull request" : "issue"}: ${label}`,
|
||||
};
|
||||
}
|
||||
if (segments.length === 2) {
|
||||
return { kind: "repository", label: repo, description: `GitHub repository: ${repo}` };
|
||||
}
|
||||
return fallback;
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
import type { Meta, StoryObj } from "@storybook/react-vite";
|
||||
import { expect, within } from "storybook/test";
|
||||
import { MarkdownWordStreaming } from "@/components/MarkdownWordStreaming";
|
||||
import { MarkdownBody, type MarkdownExternalReferenceMap } from "@/components/MarkdownBody";
|
||||
|
||||
const references: MarkdownExternalReferenceMap = {
|
||||
"https://github.com/acme/web/pull/241": {
|
||||
providerKey: "github", objectType: "pull_request", displayKey: "GitHub pull request",
|
||||
statusCategory: "succeeded", statusIconKey: "git-merge", liveness: "fresh",
|
||||
statusLabel: "Merged", displayTitle: "Keep inline links readable",
|
||||
},
|
||||
};
|
||||
|
||||
function Sample({ label, markdown, resolved = false }: { label: string; markdown: string; resolved?: boolean }) {
|
||||
return <section className="flex min-w-0 flex-col gap-3">
|
||||
<h2 className="text-xs font-semibold text-muted-foreground">{label}</h2>
|
||||
<MarkdownBody linkIssueReferences={false} externalReferences={resolved ? references : undefined}>{markdown}</MarkdownBody>
|
||||
</section>;
|
||||
}
|
||||
|
||||
function InlineLinksGallery() {
|
||||
return <main className="mx-auto flex max-w-3xl flex-col gap-8 p-6">
|
||||
<header className="flex flex-col gap-2">
|
||||
<p className="text-xs text-muted-foreground">INLINE LINKS</p>
|
||||
<h1 className="text-2xl font-semibold">A little more context, right in the line.</h1>
|
||||
<p className="text-sm text-muted-foreground">GitHub references stay compact. Each resource has a familiar icon.</p>
|
||||
</header>
|
||||
<Sample label="In a conversation" markdown={
|
||||
"The fix is ready at https://github.com/acme/web/pull/241. It closes https://github.com/acme/web/issues/42 and updates https://github.com/acme/web. Please take a look before the next release."
|
||||
} />
|
||||
<Sample label="Repository · Pull request · Issue · GitHub" markdown={
|
||||
"https://github.com/paperclipai/paperclip\n\nhttps://github.com/paperclipai/paperclip/pull/4099\n\nhttps://github.com/paperclipai/paperclip/issues/1778\n\n[GitHub Actions](https://github.com/features/actions)"
|
||||
} />
|
||||
<Sample label="Your words stay yours" markdown={
|
||||
'[Review the changes](https://github.com/acme/web/pull/241) and [**read** the issue](https://github.com/acme/web/issues/42).\n\n[Jump to the review comment](https://github.com/acme/web/pull/241/files?diff=split#discussion_r123 "Review comment on the exact diff") or open the [repository](https://github.com/acme/web).'
|
||||
} />
|
||||
<Sample label="Connected metadata keeps its status and title" resolved markdown={
|
||||
"Already shipped: [Keep inline links readable](https://github.com/acme/web/pull/241)."
|
||||
} />
|
||||
<Sample label="Other destinations keep their usual treatment" markdown={
|
||||
"Read the [documentation](https://example.com/docs), open [company settings](/company/settings), or share https://github.com.example.com/acme/web/pull/241.\n\nCode stays literal: `https://github.com/acme/web/pull/241`."
|
||||
} />
|
||||
<Sample label="Long references still wrap" markdown={
|
||||
"Follow https://github.com/an-organization-with-a-long-name/a-repository-with-an-equally-long-name/pull/123456/files?diff=split#discussion_r123456."
|
||||
} />
|
||||
</main>;
|
||||
}
|
||||
|
||||
const meta = {
|
||||
id: "tasks-inline-links",
|
||||
title: "Tasks and conversations/Task creation and lifecycle/Inline links",
|
||||
component: InlineLinksGallery,
|
||||
parameters: {
|
||||
paperclip: {
|
||||
capabilities: ["tasks/discussion"],
|
||||
status: "current",
|
||||
implementation: "production",
|
||||
boundary: "fixtures",
|
||||
notes: "Mounts MarkdownBody with synthetic text and one host-resolved metadata fixture. Covers URL-derived GitHub repository, pull request and issue icons, authored labels, exact anchors, generic links, code, wrapping, streaming playback labels, and existing connected status. No remote metadata requests, provider actions, or production persistence.",
|
||||
},
|
||||
},
|
||||
} satisfies Meta<typeof InlineLinksGallery>;
|
||||
|
||||
export default meta;
|
||||
type Story = StoryObj<typeof meta>;
|
||||
|
||||
export const Playground: Story = {
|
||||
play: async ({ canvasElement }) => {
|
||||
const canvas = within(canvasElement);
|
||||
const pr = canvas.getByRole("link", { name: "GitHub pull request: acme/web#241" });
|
||||
await expect(pr).toHaveAttribute("href", "https://github.com/acme/web/pull/241");
|
||||
await expect(pr).toHaveAttribute("target", "_blank");
|
||||
await expect(pr.querySelector(".lucide-git-pull-request")).not.toBeNull();
|
||||
const issue = canvas.getByRole("link", { name: "GitHub issue: acme/web#42" });
|
||||
await expect(issue.querySelector(".lucide-circle-dot")).not.toBeNull();
|
||||
await expect(canvas.getByRole("link", { name: "GitHub pull request: Review the changes" })).toBeVisible();
|
||||
await expect(canvas.getByRole("link", { name: "GitHub pull request Merged: Keep inline links readable" })).toHaveAttribute("data-external-status", "succeeded");
|
||||
await expect(canvas.getByText("https://github.com/acme/web/pull/241", { selector: "code" })).toBeVisible();
|
||||
},
|
||||
};
|
||||
|
||||
export const Narrow: Story = {
|
||||
...Playground,
|
||||
parameters: { waitForViewport: true },
|
||||
globals: { viewport: { value: "mobile1", isRotated: false } },
|
||||
};
|
||||
|
||||
export const Streaming: Story = {
|
||||
...Playground,
|
||||
decorators: [(Story) => <MarkdownWordStreaming.Provider value><Story /></MarkdownWordStreaming.Provider>],
|
||||
};
|
||||
Reference in new issue
Block a user