Share the external Runner lifecycle and add strict Muse contracts (#16444)

Add the external Runner foundation with distinct Dot/Grok V6, canonical home-mode V7 and Muse V8 contracts.

Keep generated schemas and profiles synchronized. Bound default question-answer command IDs in a disjoint namespace while preserving existing IDs and durable replay. Preserve canonical migration history and add normally generated, replay-safe Muse schema milestones with literal preview-history fixtures.

Refs #16444.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

scripts/smoke/muse-qualification-report.mjs

@@ -0,0 +1,121 @@
+/** Secret-free interpretation of a bounded personal-Muse probe, not a Product E2E campaign. */
+export const MUSE_QUALIFICATION_GRADER = "paperclip.muse-qualification.v4";
+const HOUR = 3_600_000;
+const DAY = 24 * HOUR;
+const timestamp = value => typeof value === "string" ? Date.parse(value) : NaN;
+const nonnegative = value => Number.isSafeInteger(value) && value >= 0;
+const sha = value => typeof value === "string" && /^[a-f0-9]{40,64}$/.test(value);
+const duration = (from, to) => {
+  const value = timestamp(to) - timestamp(from);
+  return Number.isFinite(value) && value >= 0 ? value : null;
+};
+const distribution = values => {
+  const sorted = values.filter(value => value !== null).sort((a, b) => a - b);
+  const percentile = fraction => sorted.length ? sorted[Math.max(0, Math.ceil(sorted.length * fraction) - 1)] : null;
+  return { count: sorted.length, p50Ms: percentile(0.5), p95Ms: percentile(0.95), maxMs: sorted.at(-1) ?? null };
+};
+
+/** Explicitly project safe facts; never spread retained evidence or print task/account/session identifiers. */
+export function qualificationReport(evidence, now = Date.now()) {
+  const run = evidence.qualification ?? {};
+  const provenance = evidence.provenance ?? {};
+  const start = timestamp(run.startedAt), expiry = timestamp(run.expiresAt);
+  const bounded = Number.isFinite(start) && expiry - start === DAY && run.requestedPollIntervalMs === 5000;
+  const samples = Array.isArray(evidence.samples) ? evidence.samples : [];
+  const uniqueSlots = new Set(samples.map(sample => sample.slot)).size === samples.length;
+  const ordered = sample => {
+    const times = [sample.queuedAt, sample.offeredAt, sample.claimedAt, sample.nativeAcceptedAt, sample.acceptedResultAt, sample.finalizedAt].map(timestamp);
+    return times.every((value, index) => Number.isFinite(value) && value >= start && value <= expiry && (index === 0 || value >= times[index - 1]));
+  };
+  const successful = samples.filter(sample => ordered(sample) && sample.unattended === true && sample.attempts === 1
+    && sample.nativeSucceeded === true && sha(sample.documentSha256));
+  const cadence = evidence.cadence ?? {};
+  const intervalsValid = nonnegative(cadence.observedIntervals) && cadence.observedIntervals > 0
+    && nonnegative(cadence.withinSevenSeconds) && cadence.withinSevenSeconds <= cadence.observedIntervals;
+  const fraction = intervalsValid ? cadence.withinSevenSeconds / cadence.observedIntervals : null;
+  const cadenceComplete = bounded && cadence.complete === true && cadence.missingIntervals === 0
+    && nonnegative(cadence.maxUnobservedGapMs) && cadence.maxUnobservedGapMs <= 60_000
+    && timestamp(cadence.from) <= start + 60_000 && timestamp(cadence.to) >= expiry - 60_000
+    && intervalsValid && cadence.observedIntervals >= Math.floor(DAY / 7000);
+  const windows = Array.isArray(evidence.idleWindows) ? evidence.idleWindows : [];
+  const idleVerified = windows.some(window => duration(window.from, window.to) >= 2 * HOUR
+    && timestamp(window.from) >= start && timestamp(window.to) <= expiry
+    && window.coverageComplete === true && window.healthy === true
+    && window.activeAssignments === 0 && window.unconsumedInputs === 0
+    && !samples.some(sample => timestamp(sample.queuedAt) < timestamp(window.to)
+      && (!Number.isFinite(timestamp(sample.finalizedAt)) || timestamp(sample.finalizedAt) > timestamp(window.from))));
+  const cleanup = evidence.cleanup ?? {};
+  const authorityRevoked = cleanup.authorityRevoked === true;
+  const detectorRemovalRequested = cleanup.detectorRemovalRequested === true;
+  const detectorRemovalReported = cleanup.detectorRemoved === true;
+  const detectorEvidenceSource = ["authenticated_worker_report", "operator_runtime_attestation"].includes(cleanup.detectorEvidenceSource) ? cleanup.detectorEvidenceSource : null;
+  const detectorEvidenceRecordedAt = timestamp(cleanup.detectorEvidenceRecordedAt), detectorObservationAt = timestamp(cleanup.detectorObservationAt);
+  const detectorEvidenceTimely = detectorEvidenceRecordedAt >= expiry && detectorEvidenceRecordedAt <= now && detectorObservationAt >= expiry && detectorObservationAt <= detectorEvidenceRecordedAt;
+  const stopped = detectorEvidenceSource !== null && detectorEvidenceTimely && authorityRevoked && detectorRemovalRequested && detectorRemovalReported
+    && timestamp(run.deadlineEnforcedAt) >= expiry && timestamp(run.deadlineEnforcedAt) <= expiry + 60_000;
+  const noUncertainty = cleanup.nativeEffectsUnknown === false && cleanup.workerStopUnconfirmed === false;
+  const sourceKnown = sha(provenance.coreRevision) && sha(provenance.runnerRevision)
+    && provenance.protocolVersion === 1 && provenance.profile === "muse-personal"
+    && ["local", "cloud", "self-hosted"].includes(provenance.environment);
+  const live = provenance.mode === "live";
+  const reasons = [];
+  if (!bounded) reasons.push("invalid_duration_or_requested_cadence");
+  if (!sourceKnown) reasons.push("missing_source_provenance");
+  if (!live) reasons.push("not_live_personal_muse");
+  if (!bounded || now < expiry) reasons.push("twenty_four_hours_not_elapsed");
+  if (!stopped) reasons.push("deadline_cleanup_not_verified");
+  if (!uniqueSlots || samples.some(sample => !Number.isInteger(sample.slot) || sample.slot < 0 || sample.slot >= 24)) reasons.push("invalid_or_duplicate_slots");
+  if (successful.length < 20 || successful.length !== samples.length) reasons.push("insufficient_unattended_native_document_samples");
+  if (!cadenceComplete) reasons.push("incomplete_receiver_cadence_evidence");
+  if (fraction === null || fraction < 0.9) reasons.push("five_second_cadence_not_qualified");
+  if (!idleVerified) reasons.push("sustained_idle_not_verified");
+  if (!noUncertainty) reasons.push("unresolved_remote_work");
+  if ((evidence.interventions ?? []).length) reasons.push("intervened_samples");
+  const workflowNames = ["researchDocument", "clarification", "followUp", "freshConversationProactive"];
+  const workflows = Object.fromEntries(workflowNames.map(name => [name, evidence.workflows?.[name]?.verified === true]));
+  const passed = reasons.length === 0;
+  return {
+    schema: MUSE_QUALIFICATION_GRADER,
+    provenance: { coreRevision: sha(provenance.coreRevision) ? provenance.coreRevision : null,
+      runnerRevision: sha(provenance.runnerRevision) ? provenance.runnerRevision : null,
+      protocolVersion: provenance.protocolVersion === 1 ? 1 : null, mode: live ? "live" : "synthetic_or_unspecified",
+      environment: ["local", "cloud", "self-hosted"].includes(provenance.environment) ? provenance.environment : null },
+    startedAt: Number.isFinite(start) ? new Date(start).toISOString() : null,
+    expiresAt: Number.isFinite(expiry) ? new Date(expiry).toISOString() : null,
+    elapsedHours: Number.isFinite(start) ? Math.max(0, now - start) / HOUR : null,
+    samples: samples.map(sample => ({ slot: Number.isInteger(sample.slot) ? sample.slot : null,
+      queueToOfferMs: duration(sample.queuedAt, sample.offeredAt), queueToClaimMs: duration(sample.queuedAt, sample.claimedAt),
+      queueToNativeAcceptMs: duration(sample.queuedAt, sample.nativeAcceptedAt),
+      queueToAcceptedResultMs: duration(sample.queuedAt, sample.acceptedResultAt), fullResponseMs: duration(sample.queuedAt, sample.finalizedAt),
+      nativeSucceeded: sample.nativeSucceeded === true, unattended: sample.unattended === true,
+      attempts: nonnegative(sample.attempts) ? sample.attempts : null, documentSha256: sha(sample.documentSha256) ? sample.documentSha256 : null })),
+    successfulNativeTasks: successful.length,
+    queueToClaim: distribution(samples.map(sample => duration(sample.queuedAt, sample.claimedAt))),
+    fullResponse: distribution(samples.map(sample => duration(sample.queuedAt, sample.finalizedAt))),
+    receiverIntervals: intervalsValid ? cadence.observedIntervals : null, fractionWithinSevenSeconds: fraction,
+    requestedPollIntervalMs: 5000, cadenceEvidenceComplete: cadenceComplete, sustainedIdleVerified: idleVerified,
+    authorityRevoked, detectorRemovalRequested, detectorRemovalReported, detectorEvidenceSource,
+    deadlineCleanupVerified: stopped, unresolvedRemoteWork: !noUncertainty,
+    usage: null, cost: null, usageAccounting: "unavailable", workflows,
+    qualificationPassed: passed, releaseQualified: passed && Object.values(workflows).every(Boolean), reasons,
+    evidenceBoundary: `Authenticated unattended round trips and native receipts; detector removal is ${detectorEvidenceSource === "operator_runtime_attestation" ? "operator-attested from supported-runtime observation" : detectorEvidenceSource === "authenticated_worker_report" ? "authenticated worker-reported" : "unconfirmed"}, not independently confirmed by the provider. Private Muse tool activity is not independently observed.`,
+  };
+}
+
+/** Merge persisted replica buckets before measuring gaps; bucket totals alone hide cross-replica gaps. */
+export function qualificationCadence(evidence) {
+  const start = timestamp(evidence.startedAt), expiry = timestamp(evidence.expiresAt);
+  const contacts = Array.isArray(evidence.contacts) ? evidence.contacts : [];
+  const times = [...new Set(contacts.flatMap(bucket => bucket.timestamps ?? []).map(timestamp)
+    .filter(at => Number.isFinite(at) && at >= start && at <= expiry))].sort((a, b) => a - b);
+  const gaps = times.slice(1).map((at, index) => at - times[index]);
+  const missing = contacts.filter(bucket => bucket.incomplete || bucket.contacts !== bucket.timestamps?.length).length;
+  const boundaryGap = times.length ? Math.max(0, times[0] - start, expiry - times.at(-1)) : Infinity;
+  const largestGap = gaps.reduce((maximum, gap) => Math.max(maximum, gap), boundaryGap);
+  return { from: times.length ? new Date(times[0]).toISOString() : null,
+    to: times.length ? new Date(times.at(-1)).toISOString() : null,
+    observedIntervals: gaps.length, withinSevenSeconds: gaps.filter(gap => gap <= 7000).length,
+    missingIntervals: missing + (evidence.cadenceEvidenceComplete === true ? 0 : 1),
+    maxUnobservedGapMs: Number.isFinite(largestGap) ? largestGap : null,
+    complete: evidence.cadenceEvidenceComplete === true && missing === 0 && times.length > 1 };
+}

scripts/smoke/muse-qualification-report.test.mjs

@@ -0,0 +1,131 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { qualificationReport, qualificationCadence } from "./muse-qualification-report.mjs";
+const HOUR = 3_600_000, start = Date.UTC(2099, 0, 1), expiry = start + 24 * HOUR;
+const iso = ms => new Date(ms).toISOString();
+function evidence() {
+  const samples = Array.from({ length: 24 }, (_, slot) => slot).filter(slot => slot < 6 || slot >= 9).map(slot => {
+    const queued = start + slot * HOUR;
+    return { slot, queuedAt: iso(queued), offeredAt: iso(queued + 100), claimedAt: iso(queued + 2000),
+      nativeAcceptedAt: iso(queued + 2100), acceptedResultAt: iso(queued + 9000), finalizedAt: iso(queued + 10000),
+      nativeSucceeded: true, unattended: true, attempts: 1, documentSha256: "b".repeat(64) };
+  });
+  return { qualification: { startedAt: iso(start), expiresAt: iso(expiry), deadlineEnforcedAt: iso(expiry + 1000), requestedPollIntervalMs: 5000 },
+    provenance: { coreRevision: "a".repeat(40), runnerRevision: "b".repeat(40), protocolVersion: 1, mode: "live", profile: "muse-personal", environment: "local" },
+    samples, cadence: { from: iso(start), to: iso(expiry), observedIntervals: 17279, withinSevenSeconds: 17000, missingIntervals: 0, maxUnobservedGapMs: 0, complete: true },
+    idleWindows: [{ from: iso(start + 6 * HOUR), to: iso(start + 9 * HOUR), coverageComplete: true, healthy: true, activeAssignments: 0, unconsumedInputs: 0 }],
+    cleanup: { authorityRevoked: true, detectorRemovalRequested: true, detectorRemoved: true, detectorEvidenceSource: "authenticated_worker_report", detectorEvidenceRecordedAt: iso(expiry + 1000), detectorObservationAt: iso(expiry + 1000), workerStopUnconfirmed: false, nativeEffectsUnknown: false }, interventions: [],
+    workflows: Object.fromEntries(["researchDocument", "clarification", "followUp", "freshConversationProactive"].map(key => [key, { verified: true }])) };
+}
+test("complete live evidence reports claim and full-response latency separately", () => {
+  const result = qualificationReport(evidence(), expiry + 2000);
+  assert.equal(result.qualificationPassed, true); assert.equal(result.releaseQualified, true);
+  assert.equal(result.successfulNativeTasks, 21); assert.equal(result.queueToClaim.p95Ms, 2000);
+  assert.equal(result.fullResponse.p95Ms, 10000); assert.equal(result.cost, null);
+});
+test("synthetic evidence and old sixty-second cadence cannot qualify", () => {
+  for (const change of [e => e.provenance.mode = "synthetic", e => e.qualification.requestedPollIntervalMs = 60000,
+    e => { e.cadence.observedIntervals = 1440; e.cadence.withinSevenSeconds = 0; }]) {
+    const e = evidence(); change(e); assert.equal(qualificationReport(e, expiry + 2000).qualificationPassed, false);
+  }
+});
+test("missing cadence tails, idle gaps, claims, native completion and cleanup each fail closed", () => {
+  const changes = [e => e.cadence.complete = false, e => e.cadence.missingIntervals = 1,
+    e => e.cadence.to = iso(expiry - HOUR), e => e.cadence.maxUnobservedGapMs = 61000,
+    e => e.idleWindows[0].coverageComplete = false, e => e.idleWindows[0].activeAssignments = 1,
+    e => delete e.samples[0].claimedAt, e => e.samples[0].nativeSucceeded = false,
+    e => e.samples[0].unattended = false, e => e.samples[0].attempts = 2,
+    e => e.samples[0].slot = e.samples[1].slot, e => e.samples[0].acceptedResultAt = iso(expiry + 1),
+    e => e.samples[0].finalizedAt = iso(start + 7 * HOUR), e => e.cleanup.workerStopUnconfirmed = true,
+    e => e.cleanup.nativeEffectsUnknown = true, e => e.qualification.deadlineEnforcedAt = iso(expiry + HOUR),
+    e => e.interventions.push({ kind: "manual_chat" }), e => e.provenance.runnerRevision = "unknown"];
+  for (const change of changes) { const e = evidence(); change(e); assert.equal(qualificationReport(e, expiry + 2000).qualificationPassed, false, String(change)); }
+  assert.equal(qualificationReport(evidence(), start + HOUR).qualificationPassed, false);
+});
+test("successful soak alone does not imply product journey qualification", () => {
+  const e = evidence(); e.workflows.clarification.verified = false;
+  const result = qualificationReport(e, expiry + 2000);
+  assert.equal(result.qualificationPassed, true); assert.equal(result.releaseQualified, false);
+});
+test("public projection does not copy credentials, private text or identity fields", () => {
+  const e = evidence(); e.accessToken = "private-secret"; e.samples[0].issueId = "private-task";
+  e.provenance.privateConversation = "private-conversation";
+  const output = JSON.stringify(qualificationReport(e, expiry + 2000));
+  for (const secret of ["private-secret", "private-task", "private-conversation"]) assert.equal(output.includes(secret), false);
+});
+
+test("cadence merges replica timestamps and exposes missing persisted contacts", () => {
+  const input = { startedAt: iso(start), expiresAt: iso(start + 20000), cadenceEvidenceComplete: true,
+    contacts: [
+      { contacts: 3, timestamps: [iso(start), iso(start + 10000), iso(start + 20000)], incomplete: false },
+      { contacts: 2, timestamps: [iso(start + 5000), iso(start + 15000)], incomplete: false },
+    ] };
+  assert.deepEqual(qualificationCadence(input), { from: iso(start), to: iso(start + 20000), observedIntervals: 4,
+    withinSevenSeconds: 4, missingIntervals: 0, maxUnobservedGapMs: 5000, complete: true });
+  input.contacts[1].contacts = 3;
+  assert.equal(qualificationCadence(input).complete, false);
+  assert.equal(qualificationCadence(input).missingIntervals, 1);
+  input.contacts[1].contacts = 2; input.cadenceEvidenceComplete = false;
+  assert.equal(qualificationCadence(input).complete, false);
+});
+
+
+test("a receiver outage fails qualification despite otherwise regular five-second polling", () => {
+  const e = evidence();
+  const outageStart = start + 12 * HOUR;
+  const timestamps = Array.from({ length: 24 * HOUR / 5000 + 1 }, (_, index) => start + index * 5000)
+    .filter(at => at <= outageStart || at >= outageStart + 10 * 60_000).map(iso);
+  e.cadence = qualificationCadence({ startedAt: iso(start), expiresAt: iso(expiry), cadenceEvidenceComplete: true,
+    contacts: [{ contacts: timestamps.length, timestamps, incomplete: false }] });
+  assert.equal(e.cadence.maxUnobservedGapMs, 10 * 60_000);
+  assert.ok(e.cadence.withinSevenSeconds / e.cadence.observedIntervals > 0.99);
+  assert.ok(e.cadence.observedIntervals >= Math.floor(24 * HOUR / 7000));
+  const report = qualificationReport(e, expiry + 2000);
+  assert.equal(report.qualificationPassed, false);
+  assert.equal(report.releaseQualified, false);
+  assert.ok(report.reasons.includes("incomplete_receiver_cadence_evidence"));
+});
+
+test("deadline cleanup remains incomplete until detector removal is requested and reported", () => {
+  for (const flags of [{}, { detectorRemovalRequested: false, detectorRemoved: false },
+    { detectorRemovalRequested: true, detectorRemoved: false },
+    { detectorRemovalRequested: false, detectorRemoved: true },
+    { detectorRemovalRequested: "true", detectorRemoved: true },
+    { detectorRemovalRequested: true, detectorRemoved: 1 }]) {
+    const e = evidence();
+    delete e.cleanup.detectorRemovalRequested; delete e.cleanup.detectorRemoved;
+    Object.assign(e.cleanup, flags);
+    const report = qualificationReport(e, expiry + 2000);
+    assert.equal(report.qualificationPassed, false);
+    assert.equal(report.releaseQualified, false); assert.equal(report.deadlineCleanupVerified, false);
+    assert.equal(report.authorityRevoked, true);
+    assert.equal(report.detectorRemovalRequested, flags.detectorRemovalRequested === true);
+    assert.equal(report.detectorRemovalReported, flags.detectorRemoved === true);
+    assert.ok(report.reasons.includes("deadline_cleanup_not_verified"));
+  }
+  const complete = qualificationReport(evidence(), expiry + 2000);
+  assert.equal(complete.schema, "paperclip.muse-qualification.v4");
+  assert.equal(complete.qualificationPassed, true); assert.equal(complete.releaseQualified, true);
+  assert.equal(complete.deadlineCleanupVerified, true);
+  assert.equal(complete.detectorRemovalRequested, true); assert.equal(complete.detectorRemovalReported, true);
+  assert.match(complete.evidenceBoundary, /worker-reported, not independently confirmed by the provider/);
+});
+
+
+test("deadline cleanup requires trusted timely provenance and distinguishes operator observation", () => {
+  for (const source of [undefined, "disabled", "public_activity", "operator_summary"]) {
+    const e = evidence(); e.cleanup.detectorEvidenceSource = source;
+    assert.equal(qualificationReport(e, expiry + 2000).qualificationPassed, false);
+  }
+  for (const field of ["detectorEvidenceRecordedAt", "detectorObservationAt"]) {
+    const e = evidence(); e.cleanup[field] = iso(expiry - 1);
+    assert.equal(qualificationReport(e, expiry + 2000).deadlineCleanupVerified, false);
+  }
+  const e = evidence(); e.cleanup.detectorEvidenceSource = "operator_runtime_attestation";
+  const report = qualificationReport(e, expiry + 2000);
+  assert.equal(report.qualificationPassed, true); assert.equal(report.detectorEvidenceSource, "operator_runtime_attestation");
+  assert.match(report.evidenceBoundary, /operator-attested from supported-runtime observation/);
+  assert.doesNotMatch(report.evidenceBoundary, /removal is authenticated worker-reported/);
+  e.cleanup.nativeEffectsUnknown = true;
+  assert.equal(qualificationReport(e, expiry + 2000).qualificationPassed, false);
+});