PaperClipAI/server/src/services/native-runtime/evidence-classifier.ts

import { and, eq, or } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import {
  approvals,
  caseHandoffs,
  agents,
  heartbeatRunEvents,
  issueApprovals,
  issueAttachments,
  issueThreadInteractions,
  issueWorkProducts,
} from "@paperclipai/db";
import {
  normalizePrpResultSignals,
  type PrpIgnoredAttentionRequest,
  type PrpNormalizedAttentionRequest,
  type PrpVerificationReasonCode,
} from "../../vendor/paperclip-runner/index.js";

export type NativeEvidenceOutcome = "accepted" | "missing" | "rejected" | "unverifiable";

export interface NativeEvidenceRefAssessment {
  ref: string;
  kind: "event" | "work_product" | "approval" | "interaction" | "attachment" | "case_handoff" | "unknown";
  outcome: NativeEvidenceOutcome;
  reasonCode: string;
  durableRecordId: string | null;
}

export type NativeBlockerOwnerKind = "agent" | "user" | "system" | "external" | "board" | "unspecified";

export function nativeBlockerOwnerKind(owner: Record<string, unknown>): NativeBlockerOwnerKind {
  if (owner.owner === "board") return "board";
  return ["agent", "user", "system", "external", "board"].includes(String(owner.kind))
    ? owner.kind as NativeBlockerOwnerKind : "unspecified";
}

export interface NativeEvidenceAssessment {
  objectiveClaimSatisfied: boolean;
  objectiveSatisfied: boolean;
  allCriteriaSatisfied: boolean;
  verificationPassed: boolean;
  hasFailedVerification: boolean;
  hasBlockingRemainingWork: boolean;
  reportedDisposition: "done" | "blocked" | "needs_review" | "yielded";
  summary: string;
  contractRevisionMatches: boolean;
  criterionAssessments: Array<{
    criterionId: string;
    claimStatus: "satisfied" | "not_satisfied" | "unknown" | null;
    outcome: NativeEvidenceOutcome;
    evidenceRefs: NativeEvidenceRefAssessment[];
    reasonCode: string;
  }>;
  verificationAssessments: Array<{
    commandOrCheck: string;
    claimStatus: "passed" | "failed" | "not_run";
    outcome: NativeEvidenceOutcome;
    evidenceRef: NativeEvidenceRefAssessment | null;
    reasonCode: string;
    reportedReasonCode: PrpVerificationReasonCode | null;
    detail: string | null;
  }>;
  verificationCaveats: Array<{
    commandOrCheck: string;
    reasonCode: PrpVerificationReasonCode | null;
    detail: string | null;
  }>;
  acceptedEvidenceRefs: string[];
  missingRequirements: string[];
  rejectedEvidence: Array<{ ref: string; reasonCode: string }>;
  unverifiableEvidence: Array<{ ref: string; reasonCode: string }>;
  blocker: {
    unblockAction: string;
    boardOwned: boolean;
    ownerKind?: NativeBlockerOwnerKind;
    scope: "current_track" | "task_wide";
  } | null;
  continuation: {
    kind: "same_agent" | "retry" | "delegated_issue" | "response_wake" | "monitor" | "case_handoff";
    summary: string;
    idempotencyKey: string;
    handoffId?: string;
  } | null;
  /** Actionable, blocking requests only. */
  attentionRequests: PrpNormalizedAttentionRequest[];
  ignoredAttentionRequests: PrpIgnoredAttentionRequest[];
}

function record(value: unknown): Record<string, unknown> {
  return typeof value === "object" && value !== null && !Array.isArray(value)
    ? value as Record<string, unknown>
    : {};
}

function text(value: unknown): string | null {
  return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
}

function uuidRef(ref: string, prefixes: string[]): string | null {
  for (const prefix of prefixes) {
    if (!ref.startsWith(`${prefix}:`)) continue;
    const value = ref.slice(prefix.length + 1).trim();
    // Model-authored evidence often annotates a durable reference, for example
    // `interaction:<uuid> (request_confirmation, status accepted)`. Extract the
    // canonical UUID only; never pass the descriptive suffix (or arbitrary
    // malformed text) to a Postgres UUID comparison.
    const match = /^([0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})(?=$|[^0-9a-f-])/i.exec(value);
    return match?.[1] ?? null;
  }
  return null;
}

export function readNativeEventEvidenceOutcome(value: unknown): "accepted" | "rejected" | null {
  const event = record(value);
  if (event.sourceKind !== "control_plane") return null;
  const outcome = text(record(event.payload).evidenceOutcome);
  return outcome === "accepted" || outcome === "rejected" ? outcome : null;
}

async function classifyEvidenceRef(input: {
  db: Db;
  companyId: string;
  issueId: string;
  runId: string;
  ref: string;
  lockSources?: boolean;
}): Promise<NativeEvidenceRefAssessment> {
  const handoffId = uuidRef(input.ref, ["case-handoff"]);
  if (handoffId) {
    const query = input.db.select({ id: caseHandoffs.id, state: caseHandoffs.state }).from(caseHandoffs).where(and(
      eq(caseHandoffs.id, handoffId), eq(caseHandoffs.companyId, input.companyId), eq(caseHandoffs.issueId, input.issueId),
      eq(caseHandoffs.sourceRunId, input.runId),
    )).limit(1);
    const [row] = await (input.lockSources ? query.for("share", { noWait: true }) : query);
    return { ref: input.ref, kind: "case_handoff", outcome: !row ? "missing" : row.state === "cancelled" ? "rejected" : "accepted",
      reasonCode: !row ? "case_handoff_missing" : row.state === "cancelled" ? "case_handoff_cancelled" : "case_handoff_persisted", durableRecordId: row?.id ?? null };
  }
  const eventMatch = /^event:(\d+)$/.exec(input.ref);
  if (eventMatch) {
    const cursor = Number(eventMatch[1]);
    const query = input.db.select({
      id: heartbeatRunEvents.id,
      eventType: heartbeatRunEvents.eventType,
      payload: heartbeatRunEvents.payload,
    })
      .from(heartbeatRunEvents)
      .where(and(
        eq(heartbeatRunEvents.companyId, input.companyId),
        eq(heartbeatRunEvents.runId, input.runId),
        or(eq(heartbeatRunEvents.seq, cursor), eq(heartbeatRunEvents.sourceSeq, cursor)),
      )).limit(1);
    const [event] = await (input.lockSources ? query.for("share", { noWait: true }) : query);
    if (!event) {
      return { ref: input.ref, kind: "event", outcome: "missing", reasonCode: "durable_run_event_missing", durableRecordId: null };
    }
    const evidenceOutcome = readNativeEventEvidenceOutcome(record(event.payload).prpEvent);
    if (evidenceOutcome === "accepted") {
      return { ref: input.ref, kind: "event", outcome: "accepted", reasonCode: "control_plane_evidence_accepted", durableRecordId: String(event.id) };
    }
    if (evidenceOutcome === "rejected") {
      return { ref: input.ref, kind: "event", outcome: "rejected", reasonCode: "control_plane_evidence_rejected", durableRecordId: String(event.id) };
    }
    return {
      ref: input.ref,
      kind: "event",
      outcome: "unverifiable",
      reasonCode: event.eventType === "run.result.proposed"
        ? "model_result_event_is_claim_only"
        : "run_event_has_no_authoritative_evidence_verdict",
      durableRecordId: String(event.id),
    };
  }

  const workProductId = uuidRef(input.ref, ["work_product", "work-product", "artifact"]);
  if (workProductId) {
    const query = input.db.select({
      id: issueWorkProducts.id,
      status: issueWorkProducts.status,
      reviewState: issueWorkProducts.reviewState,
    }).from(issueWorkProducts).where(and(
      eq(issueWorkProducts.id, workProductId),
      eq(issueWorkProducts.companyId, input.companyId),
      eq(issueWorkProducts.issueId, input.issueId),
    )).limit(1);
    const [row] = await (input.lockSources ? query.for("share", { noWait: true }) : query);
    if (!row) return { ref: input.ref, kind: "work_product", outcome: "missing", reasonCode: "work_product_missing", durableRecordId: null };
    if (
      ["failed", "error", "cancelled", "rejected", "changes_requested"].includes(row.status)
      || ["rejected", "changes_requested"].includes(row.reviewState)
    ) {
      return { ref: input.ref, kind: "work_product", outcome: "rejected", reasonCode: "work_product_rejected", durableRecordId: row.id };
    }
    if (
      row.reviewState === "approved"
      || ["approved", "passed", "succeeded", "completed", "merged", "published"].includes(row.status)
    ) {
      return { ref: input.ref, kind: "work_product", outcome: "accepted", reasonCode: "work_product_authoritatively_accepted", durableRecordId: row.id };
    }
    return { ref: input.ref, kind: "work_product", outcome: "unverifiable", reasonCode: "work_product_not_yet_accepted", durableRecordId: row.id };
  }

  const approvalId = uuidRef(input.ref, ["approval"]);
  if (approvalId) {
    const query = input.db.select({ id: approvals.id, status: approvals.status })
      .from(issueApprovals)
      .innerJoin(approvals, eq(issueApprovals.approvalId, approvals.id))
      .where(and(
        eq(issueApprovals.companyId, input.companyId),
        eq(issueApprovals.issueId, input.issueId),
        eq(approvals.companyId, input.companyId),
        eq(approvals.id, approvalId),
      )).limit(1);
    const [row] = await (input.lockSources ? query.for("share", { noWait: true }) : query);
    if (!row) return { ref: input.ref, kind: "approval", outcome: "missing", reasonCode: "approval_missing", durableRecordId: null };
    if (row.status === "approved") return { ref: input.ref, kind: "approval", outcome: "accepted", reasonCode: "approval_approved", durableRecordId: row.id };
    if (["rejected", "cancelled"].includes(row.status)) return { ref: input.ref, kind: "approval", outcome: "rejected", reasonCode: "approval_rejected", durableRecordId: row.id };
    return { ref: input.ref, kind: "approval", outcome: "missing", reasonCode: "approval_pending", durableRecordId: row.id };
  }

  const interactionId = uuidRef(input.ref, ["interaction"]);
  if (interactionId) {
    const query = input.db.select({
      id: issueThreadInteractions.id,
      status: issueThreadInteractions.status,
      result: issueThreadInteractions.result,
    }).from(issueThreadInteractions).where(and(
      eq(issueThreadInteractions.id, interactionId),
      eq(issueThreadInteractions.companyId, input.companyId),
      eq(issueThreadInteractions.issueId, input.issueId),
    )).limit(1);
    const [row] = await (input.lockSources ? query.for("share", { noWait: true }) : query);
    if (!row) return { ref: input.ref, kind: "interaction", outcome: "missing", reasonCode: "interaction_missing", durableRecordId: null };
    const outcome = text(record(row.result).outcome);
    if (["accepted", "answered", "resolved"].includes(outcome ?? "") || ["accepted", "answered"].includes(row.status)) {
      return { ref: input.ref, kind: "interaction", outcome: "accepted", reasonCode: "interaction_resolved", durableRecordId: row.id };
    }
    if (["rejected", "expired", "cancelled"].includes(row.status) || ["rejected", "withdrawn", "issue_closed"].includes(outcome ?? "")) {
      return { ref: input.ref, kind: "interaction", outcome: "rejected", reasonCode: "interaction_rejected", durableRecordId: row.id };
    }
    return { ref: input.ref, kind: "interaction", outcome: "missing", reasonCode: "interaction_pending", durableRecordId: row.id };
  }

  const attachmentId = uuidRef(input.ref, ["attachment"]);
  if (attachmentId) {
    const query = input.db.select({ id: issueAttachments.id }).from(issueAttachments).where(and(
      eq(issueAttachments.id, attachmentId),
      eq(issueAttachments.companyId, input.companyId),
      eq(issueAttachments.issueId, input.issueId),
    )).limit(1);
    const [row] = await (input.lockSources ? query.for("share", { noWait: true }) : query);
    return row
      ? { ref: input.ref, kind: "attachment", outcome: "accepted", reasonCode: "attachment_persisted", durableRecordId: row.id }
      : { ref: input.ref, kind: "attachment", outcome: "missing", reasonCode: "attachment_missing", durableRecordId: null };
  }

  return {
    ref: input.ref,
    kind: "unknown",
    outcome: "unverifiable",
    reasonCode: "unsupported_evidence_reference",
    durableRecordId: null,
  };
}





export async function classifyNativeEvidence(input: {
  db: Db;
  companyId: string;
  issueId: string;
  runId: string;
  contract: Record<string, unknown>;
  result: Record<string, unknown>;

  lockSources?: boolean;
}): Promise<NativeEvidenceAssessment> {
  const claim = record(input.result.completionClaim);
  const claimedCriteria = Array.isArray(claim.criteria) ? claim.criteria.map(record) : [];
  const contractCriteria = Array.isArray(input.contract.criteria) ? input.contract.criteria.map(record) : [];
  const remaining = Array.isArray(claim.remainingWork) ? claim.remainingWork.map(record) : [];
  const normalizedSignals = normalizePrpResultSignals(input.result);
  const verification = normalizedSignals.verification;
  const reported = input.result.reportedWorkDisposition;
  const blocker = record(input.result.blocker);
  const blockerOwner = record(blocker.owner);
  const continuation = record(input.result.continuation);
  if (!["done", "blocked", "needs_review", "yielded"].includes(String(reported))) {
    throw new Error("native_result_invalid_disposition");
  }

  const contractRevision = text(input.contract.revision);
  const contractRevisionMatches = contractRevision !== null && text(claim.contractRevision) === contractRevision;
  const cache = new Map<string, Promise<NativeEvidenceRefAssessment>>();
  const resolveRef = (ref: string) => {
    let pending = cache.get(ref);
    if (!pending) {
      pending = classifyEvidenceRef({ ...input, ref });
      cache.set(ref, pending);
    }
    return pending;
  };

  const criterionAssessments = await Promise.all(contractCriteria.map(async (criterion) => {
    const criterionId = text(criterion.id) ?? "invalid-contract-criterion";
    const claimed = claimedCriteria.find((entry) => text(entry.criterionId) === criterionId) ?? null;
    const claimStatus = claimed && ["satisfied", "not_satisfied", "unknown"].includes(String(claimed.status))
      ? claimed.status as "satisfied" | "not_satisfied" | "unknown"
      : null;
    const refs = claimed && Array.isArray(claimed.evidenceRefs)
      ? claimed.evidenceRefs.flatMap((value) => text(value) ? [text(value)!] : [])
      : [];
    const evidenceRefs = await Promise.all(refs.map(resolveRef));
    let outcome: NativeEvidenceOutcome;
    let reasonCode: string;
    if (!contractRevisionMatches) {
      outcome = "rejected";
      reasonCode = "contract_revision_stale";
    } else if (!claimed) {
      outcome = "missing";
      reasonCode = "criterion_claim_missing";
    } else if (claimStatus === "not_satisfied") {
      outcome = "rejected";
      reasonCode = "criterion_reported_not_satisfied";
    } else if (claimStatus !== "satisfied") {
      outcome = "unverifiable";
      reasonCode = "criterion_claim_not_satisfied";
    } else if (refs.length === 0) {
      outcome = "missing";
      reasonCode = "criterion_evidence_missing";
    } else if (evidenceRefs.some((entry) => entry.outcome === "rejected")) {
      outcome = "rejected";
      reasonCode = "criterion_evidence_rejected";
    } else if (evidenceRefs.some((entry) => entry.outcome === "unverifiable")) {
      outcome = "unverifiable";
      reasonCode = "criterion_evidence_unverifiable";
    } else if (evidenceRefs.some((entry) => entry.outcome === "missing")) {
      outcome = "missing";
      reasonCode = "criterion_evidence_missing";
    } else {
      outcome = "accepted";
      reasonCode = "criterion_evidence_accepted";
    }
    return { criterionId, claimStatus, outcome, evidenceRefs, reasonCode };
  }));

  const verificationAssessments = await Promise.all(verification.map(async (entry) => {
    const commandOrCheck = entry.commandOrCheck;
    const claimStatus = entry.status;
    const ref = entry.artifactRef;
    const evidenceRef = ref ? await resolveRef(ref) : null;
    const common = { commandOrCheck, claimStatus, evidenceRef, reportedReasonCode: entry.reasonCode, detail: entry.detail };
    if (claimStatus === "failed") return { ...common, outcome: "rejected" as const, reasonCode: "verification_reported_failed" };
    if (claimStatus === "not_run") return { ...common, outcome: "missing" as const, reasonCode: "verification_not_run" };
    if (!evidenceRef) return { ...common, outcome: "unverifiable" as const, reasonCode: "verification_has_no_durable_reference" };
    return {
      ...common,
      outcome: evidenceRef.outcome,
      reasonCode: evidenceRef.outcome === "accepted" ? "verification_evidence_accepted" : evidenceRef.reasonCode,
    };
  }));

  const allRefs = [...cache.keys()].map((ref) => cache.get(ref)!);
  const resolvedRefs = await Promise.all(allRefs);
  const acceptedEvidenceRefs = resolvedRefs.filter((entry) => entry.outcome === "accepted").map((entry) => entry.ref);
  const missingRequirements = [
    ...criterionAssessments.filter((entry) => entry.outcome === "missing").map((entry) => entry.criterionId),
    ...verificationAssessments.filter((entry) => entry.outcome === "missing").map((entry) => entry.commandOrCheck),
  ];
  const rejectedEvidence = resolvedRefs.filter((entry) => entry.outcome === "rejected").map((entry) => ({ ref: entry.ref, reasonCode: entry.reasonCode }));
  const unverifiableEvidence = resolvedRefs.filter((entry) => entry.outcome === "unverifiable").map((entry) => ({ ref: entry.ref, reasonCode: entry.reasonCode }));
  const allCriteriaSatisfied = contractCriteria.length > 0 && criterionAssessments.every((entry) => entry.outcome === "accepted");
  const verificationPassed = verificationAssessments.length > 0 && verificationAssessments.every((entry) => entry.outcome === "accepted");
  const hasFailedVerification = verificationAssessments.some((entry) => entry.claimStatus === "failed");
  const continuationKind = continuation.kind;
  const actionableAttentionRequests: PrpNormalizedAttentionRequest[] = [];
  const ignoredAttentionRequests = [...normalizedSignals.ignoredAttentionRequests];
  for (const request of normalizedSignals.actionableAttentionRequests) {
    if (request.ownerClass === "agent" && request.targetAgentId) {
      const targetQuery = input.db.select({ id: agents.id }).from(agents).where(and(
        eq(agents.id, request.targetAgentId),
        eq(agents.companyId, input.companyId),
      )).limit(1);
      const targets = await (input.lockSources ? targetQuery.for("share", { noWait: true }) : targetQuery);
      if (targets.length === 0) {
        ignoredAttentionRequests.push({
          sourceIndex: request.sourceIndex,
          sourceKind: request.sourceKind,
          summary: request.summary,
          disposition: "rejected",
          reasonCode: "attention_request_target_agent_unavailable",
        });
        continue;
      }
    }
    actionableAttentionRequests.push(request);
  }

  return {
    objectiveClaimSatisfied: claim.objectiveSatisfied === true,
    objectiveSatisfied: claim.objectiveSatisfied === true && allCriteriaSatisfied,
    allCriteriaSatisfied,
    verificationPassed,
    hasFailedVerification,
    hasBlockingRemainingWork:
      remaining.some((entry) => entry.blocksCompletion === true)
      || (reported === "done" && Object.keys(blocker).length > 0),
    reportedDisposition: reported as NativeEvidenceAssessment["reportedDisposition"],
    summary: typeof input.result.summary === "string" ? input.result.summary : "Native run completed.",
    contractRevisionMatches,
    criterionAssessments,
    verificationAssessments,
    verificationCaveats: verificationAssessments
      .filter((entry) => entry.claimStatus === "not_run")
      .map((entry) => ({
        commandOrCheck: entry.commandOrCheck,
        reasonCode: entry.reportedReasonCode,
        detail: entry.detail,
      })),
    acceptedEvidenceRefs,
    missingRequirements,
    rejectedEvidence,
    unverifiableEvidence,
    blocker: reported === "blocked" && typeof blocker.unblockAction === "string"
      ? {
          unblockAction: blocker.unblockAction,
          boardOwned: ["board", "user"].includes(String(blockerOwner.kind)) || blockerOwner.owner === "board",
          ownerKind: nativeBlockerOwnerKind(blockerOwner),
          scope: blocker.scope === "task_wide" ? "task_wide" : "current_track",
        }
      : null,
    continuation: reported === "yielded"
      && ["same_agent", "retry", "delegated_issue", "response_wake", "monitor", "case_handoff"].includes(String(continuationKind))
      && text(continuation.summary)
      && text(continuation.idempotencyKey)
      ? {
          kind: continuationKind as NativeEvidenceAssessment["continuation"] extends infer T ? T extends { kind: infer K } ? K : never : never,
          summary: text(continuation.summary)!,
          idempotencyKey: text(continuation.idempotencyKey)!,
          ...(continuationKind === "case_handoff" && text(continuation.handoffId) ? { handoffId: text(continuation.handoffId)! } : {}),
        }
      : null,
    attentionRequests: actionableAttentionRequests,
    ignoredAttentionRequests,
  };
}