diff --git a/packages/paperclip-runner/vitest.config.ts b/packages/paperclip-runner/vitest.config.ts index 4c5817aec31..488531c4144 100644 --- a/packages/paperclip-runner/vitest.config.ts +++ b/packages/paperclip-runner/vitest.config.ts @@ -19,7 +19,11 @@ export default defineConfig({ ? { reporters: [...configDefaults.reporters, resolve(packageRoot, "../../scripts/ci-profile/vitest-reporter.mjs")] } : {}), environment: "node", - ...(process.env.GITHUB_ACTIONS === "true" ? { maxWorkers: Math.min(4, availableParallelism()) } : {}), + // Hermetic files also launch Runner and sidecar processes. Reserve CPU + // capacity for their startup within the original readiness deadlines. + ...(process.env.GITHUB_ACTIONS === "true" + ? { maxWorkers: Math.min(4, Math.max(1, Math.floor(availableParallelism() / 2))) } + : {}), sequence: { sequencer: RunnerSequencer }, include: [ "src/**/*.test.ts", diff --git a/tests/e2e/cloud-auth.spec.ts b/tests/e2e/cloud-auth.spec.ts index e18084feef0..34b902dcf29 100644 --- a/tests/e2e/cloud-auth.spec.ts +++ b/tests/e2e/cloud-auth.spec.ts @@ -1,6 +1,8 @@ import { randomUUID } from "node:crypto"; import { expect, test } from "@playwright/test"; +test.use({ trace: "retain-on-failure" }); + // The tenant UI and task database are real. Cloud is an external dependency: // simulate its entry endpoint and independent session states at the HTTP edge. for (const cloudOrigin of ["https://my.paperclip.app", "https://my-staging.paperclip.app"]) { diff --git a/tests/runner-e2e/product-regression-testops-catalog.test.ts b/tests/runner-e2e/product-regression-testops-catalog.test.ts index aecd0524cdb..27429cdc003 100644 --- a/tests/runner-e2e/product-regression-testops-catalog.test.ts +++ b/tests/runner-e2e/product-regression-testops-catalog.test.ts @@ -1,6 +1,14 @@ import { expect, it } from "vitest"; import { selectRunnerExecutions, parseRunnerSelectors } from "./selectors.js"; -import { productRegressionCases } from "./product-regression-testops.mjs"; +import { productRegressionCases, productReportingModels, productReportingMatcherKinds, projectProductResults } from "./product-regression-testops.mjs"; +import { gradeFirstTask, type FirstTaskEvidence } from "./first-task-scoring.js"; +import { projectFirstTaskResult } from "./first-task-flow.js"; +import { gradeContinuation } from "./continuation-scoring.js"; +import type { ContinuationCase } from "./continuation-cases.js"; +import { gradeTaskWorkspaces } from "./task-workspaces-scoring.js"; +import { gradeRepositoryProof } from "./repository-verification.js"; +import type { Matcher, RunnerE2EResult } from "./types.js"; +import { createHash } from "node:crypto"; it("runs every standing Product regression case through its existing full-stack fixture", () => { const executions = selectRunnerExecutions(parseRunnerSelectors( @@ -14,5 +22,80 @@ it("runs every standing Product regression case through its existing full-stack expect(execution.profile.id).toBe(row.profile); expect(execution.environment.id).toBe("local"); expect(execution.requiredCredentials).not.toContain("CODEX_AUTH_JSON"); + expect(productReportingModels).toContain(execution.profile.model); + expect(execution.profile.model).toBe(row.model); + for (const matcher of execution.task.buildMatchers("report-contract-control", execution)) { + expect(productReportingMatcherKinds).toContain(matcher.kind); + } + // Incomplete synthetic observations exercise the actual pure graders' output + // contracts. Their failures must survive reporting; these are not live runs. + let checks: Array<{ id: string; passed: boolean }> = []; + let firstTask: FirstTaskEvidence | undefined; + if (row.suite === "first-task") { + firstTask = { + caseId: row.name, nonce: "report-control", onboardingIssueId: "issue", agentId: "agent", initialTaskIds: ["issue"], + instructions: [], configuredModel: execution.profile.model ?? null, observedModels: [], checks: [], + checkpoints: [{ id: "opening", at: "2026-11-04T00:00:00Z", phase: "opening", issueId: "issue", + tasks: [], agents: [], comments: [], interactions: [], documents: [], runs: [] }], + }; + checks = gradeFirstTask(firstTask); + } + if (row.suite === "continuation") checks = gradeContinuation({ id: row.name as ContinuationCase, + fact: "fact", marker: "marker", old: "old", injected: "injected", childTitle: "child", checkpoints: [], runtimeMode: "native" }); + if (row.suite === "task-workspaces") checks = gradeTaskWorkspaces({ nonce: "control", native: true, daytona: false, + initial: null, checkpoints: [], projectCount: 0, runCount: 0, restarted: false, pendingInteractions: 0, + activeOperations: 0, activeRecovery: false, artifact: null, remoteLeaseRunIds: [] }); + if (row.suite === "configured-repository") checks = gradeRepositoryProof({ + conversion: false, expectedLegacyAdapter: "codex_local", originalAdapter: "paperclip_runner", finalAdapter: "paperclip_runner", + configurationBefore: "", configurationAfter: "", instructionsBefore: "", instructionsAfter: "", permissionsPreserved: false, + expectedModel: execution.profile.model!, actualModel: null, actualProvider: null, expectedProvider: "codex", environmentRetained: false, + issueDone: false, runCount: 0, committedRuns: 0, sourceBytes: "", currentSourceBytes: null, expectedCommit: "", + downloaded: null, downloadVerified: false, visibleDelivery: false, pendingInteractions: 0, + }); + const matcherResults = [ + ...execution.task.buildMatchers("report-contract-control", execution).map(matcher => ({ matcher, passed: false, detail: "contract control" })), + ...checks.map(check => ({ matcher: { kind: "json_path", path: check.id, expected: true } satisfies Matcher, passed: check.passed, detail: "contract control" })), + ]; + expect(matcherResults.length).toBeGreaterThan(0); + const sourceSha = "b".repeat(40); + const raw: RunnerE2EResult = { schema: "paperclip.runner-e2e.result/v2", executionId: row.executionId, + suiteId: row.suite, caseId: row.name, profileId: row.profile, environmentId: "local", + source: { sha: sourceSha, ref: null, workflowRunUrl: null }, firstTask, + attempt: 1, status: "failed", failureClass: "candidate_failure", cleanup: "passed", model: execution.profile.model, + provider: execution.profile.provider, runtimeMode: "native", startedAt: "2026-11-04T00:00:00Z", + finishedAt: "2026-11-04T00:00:01Z", durationMs: 1000, matcherResults }; + const project = (result: RunnerE2EResult) => projectProductResults({ schema: "paperclip.runner-e2e.campaign/v2", campaignId: "gha-123-1", + source: { sha: sourceSha }, expected: [row.executionId], results: [projectFirstTaskResult(result)] }, + { runId: "123", attempt: "1", workflowSha: "a".repeat(40), cases: [row] }, sourceSha); + const projected = project(raw); + expect(projected.summary).toMatchObject({ selected: 1, passed: 0, failed: 1 }); + expect(projected.records[1]!.value.checks).toHaveLength(matcherResults.length); + expect(projected.records[1]!.value.checks?.map(check => check.passed)).toEqual(matcherResults.map(check => check.passed)); + if (firstTask) { + const alias = "gpt-5.6-sol-2026-11-01", secret = "sk-proj-syntheticcredentialabcdefghijklmnop"; + for (const [configuredModel, observedModels] of [[row.model, [alias]], [null, []], [row.model, [secret]]] as const) { + const reported = observedModels[0] ?? configuredModel ?? "provider-default (unreported)"; + const output = project({ ...raw, model: reported, firstTask: { ...firstTask, configuredModel, observedModels: [...observedModels] } }); + expect(output.summary).toMatchObject({ passed: 0, failed: 1 }); + expect(output.records[1]!.value.model).toBe(configuredModel); + expect(output.records[1]!.value.declaredModel).toBe(row.model); + expect(output.records[1]!.value.modelSource).toBe(configuredModel == null ? "not_recorded" : "first_task_configuration"); + expect(output.records[1]!.value.reportedModelSha256).toBe(createHash("sha256").update(reported).digest("hex")); + expect(JSON.stringify(output)).not.toContain(reported); + } + for (const configuredModel of ["gpt-6.1-sol", secret]) { + expect(() => project({ ...raw, model: alias, firstTask: { ...firstTask, configuredModel, observedModels: [alias] } })) + .toThrow("product_regression_testops_metadata_invalid"); + } + // An error before the first observe() leaves no firstTaskEvidence. The + // production constructor then reports execution.profile.model, which is + // declared intent, not a captured agent-configuration receipt. + const early = project({ ...raw, firstTask: undefined, model: execution.profile.model }); + expect(early.summary).toMatchObject({ passed: 0, failed: 1 }); + expect(early.records[1]!.value.model).toBeNull(); + expect(early.records[1]!.value.modelSource).toBe("not_recorded"); + expect(early.records[1]!.value.declaredModel).toBe(row.model); + expect(early.records[1]!.value.reportedModelSha256).toBe(createHash("sha256").update(row.model).digest("hex")); + } } }); diff --git a/tests/runner-e2e/product-regression-testops-cli.mjs b/tests/runner-e2e/product-regression-testops-cli.mjs index a9d506ff3f5..92da5762da8 100644 --- a/tests/runner-e2e/product-regression-testops-cli.mjs +++ b/tests/runner-e2e/product-regression-testops-cli.mjs @@ -132,7 +132,8 @@ export async function productTestOps(command, argument, env = process.env, exec const api = await testopsClient(env.ALLURE_TOKEN, fetchImpl); await verifyProductJob(api, selection, env); const summary = await readJson(env.EXPORT_SUMMARY, 16_384); - // Check server-side identity and count. A successful upload is not a pass. + // Observe one upload with at most 55 seconds of poll waits, matching the + // mature chat lane. This budget never retries or extends a product scenario. let observed, lastVerification; for (let attempt = 0; attempt < 12; attempt++) { observed = await api(`/api/testresult?${new URLSearchParams({ launchId: String(selection.launchId), page: '0', size: '100' })}`); @@ -150,7 +151,7 @@ export async function productTestOps(command, argument, env = process.env, exec // their run binding and verdict. Observe this one upload, without rerunning // the product or accepting unknown/partial results. if (attempt === 11) { if (lastVerification) throw lastVerification; fail('upload_incomplete'); } - await wait(2_000); + await wait(5_000); } } diff --git a/tests/runner-e2e/product-regression-testops.d.mts b/tests/runner-e2e/product-regression-testops.d.mts index 097acf7e860..4a1fd9eb3e4 100644 --- a/tests/runner-e2e/product-regression-testops.d.mts +++ b/tests/runner-e2e/product-regression-testops.d.mts @@ -3,8 +3,17 @@ export interface ProductRegressionCase { suite: string; name: string; profile: string; + model: string; fullName: string; executionId: string; } export const productRegressionCases: readonly ProductRegressionCase[]; +export const productReportingModels: readonly string[]; +export const productReportingMatcherKinds: readonly string[]; +export function projectProductResults(campaign: unknown, selection: unknown, sourceSha: string): { + summary: { selected: number; passed: number; failed: number; broken: number }; + records: Array<{ file: string; value: { model?: string | null; declaredModel?: string; reportedModelSha256?: string | null; + modelSource?: 'first_task_configuration' | 'execution_profile' | 'not_recorded'; + checks?: Array<{ name: string; ordinal: number; passed: boolean; pathSha256: string | null }> } }>; +}; diff --git a/tests/runner-e2e/product-regression-testops.mjs b/tests/runner-e2e/product-regression-testops.mjs index 6c672f0aafa..dab6baa6d1b 100644 --- a/tests/runner-e2e/product-regression-testops.mjs +++ b/tests/runner-e2e/product-regression-testops.mjs @@ -25,14 +25,22 @@ export const productRegressionCases = [ [168, 'agent-chat', 'new-session'], ].map(([id, suite, name, profile = 'runner-codex']) => Object.freeze({ id, suite, name, profile, + model: suite === 'task-workspaces' ? 'openrouter/deepseek/deepseek-v4-flash-0731' + : suite === 'configured-repository' ? 'gpt-6.1-sol' : 'gpt-5.6-sol', fullName: `paperclip.Product E2E.${suite}.${name}`, executionId: `${suite}.${profile}.local.${name}`, })); const fail = code => { throw new Error(`product_regression_testops_${code}`); }; const sha = value => createHash('sha256').update(value).digest('hex'); -const identifier = value => typeof value === 'string' && /^[a-zA-Z0-9._:-]{1,200}$/.test(value); -const modelIdentifier = value => typeof value === 'string' && /^[a-zA-Z0-9][a-zA-Z0-9._:/-]{0,199}$/.test(value); +// Closed transmission contracts, checked against all 20 catalog cells. Dynamic +// matcher paths stay in the linked source artifact; only their digest leaves it. +export const productReportingModels = [...new Set(productRegressionCases.map(row => row.model))]; +export const productReportingMatcherKinds = ['message_exact', 'message_contains', 'message_occurrences', 'message_regex', + 'message_ordered', 'issue_status', 'run_status', 'runtime_mode', 'environment', 'file_exists', 'file_exact', + 'file_contains', 'artifact_exists', 'artifact_exact', 'json_path', 'json_schema']; +const failureClasses = ['candidate_failure', 'provider_variance', 'transient_infrastructure', 'permanent_infrastructure', + 'secret_leak', 'cleanup_failure', 'setup_blocked']; export const endpoint = 'https://paperclip.testops.cloud'; export const workflowUrl = 'https://github.com/paperclipai/paperclip/actions/workflows/runner-full-stack-e2e.yml'; // Official GitHub/Allure job metadata uses these exact aliases for workflow @@ -106,6 +114,14 @@ export function projectProductResults(campaign, selection, sourceSha) { || !['passed', 'failed'].includes(result.status) || (!missingArtifact && !Array.isArray(result.matcherResults)))) fail('result_binding'); const checks = result?.matcherResults ?? []; if (checks.length > 300 || checks.some(check => typeof check.passed !== 'boolean')) fail('checks_invalid'); + // First-task's producer reports an observed alias in result.model. Its + // independently captured agent configuration is the configuration receipt. + const firstTask = selected.suite === 'first-task'; + const configuredModel = (firstTask ? result?.firstTask?.configuredModel : result?.model) ?? null; + if ((configuredModel != null && configuredModel !== selected.model) + || (result?.failureClass != null && !failureClasses.includes(result.failureClass)) + || (result && !['passed', 'failed', 'not_started'].includes(result.cleanup)) + || checks.some(check => !productReportingMatcherKinds.includes(check.matcher?.kind))) fail('metadata_invalid'); const passed = result?.status === 'passed' && result.cleanup === 'passed' && checks.length > 0 && checks.every(check => check.passed); const infrastructure = !result || ['permanent_infrastructure', 'transient_infrastructure', 'setup_blocked'].includes(result.failureClass); const status = passed ? 'passed' : infrastructure ? 'broken' : 'failed'; @@ -115,12 +131,15 @@ export function projectProductResults(campaign, selection, sourceSha) { workflowSha: selection.workflowSha, sourceSha, status, originalStatus: result?.status ?? null, attempt: result?.attempt ?? null, cleanup: result?.cleanup ?? 'not_started', profile: selected.profile, environment: 'local', - model: modelIdentifier(result?.model) ? result.model : null, + declaredModel: selected.model, model: configuredModel, + modelSource: configuredModel == null ? 'not_recorded' : firstTask ? 'first_task_configuration' : 'execution_profile', + reportedModelSha256: firstTask && typeof result?.model === 'string' ? sha(result.model) : null, suiteDefinitionHash: /^[a-f0-9]{64}$/.test(result?.suiteDefinitionHash ?? '') ? result.suiteDefinitionHash : null, - failureClass: identifier(result?.failureClass) ? result.failureClass : null, + failureClass: result?.failureClass ?? null, durationMs: Number.isFinite(result?.durationMs) && result.durationMs >= 0 ? result.durationMs : null, - checks: checks.map(check => ({ - name: identifier(check.matcher?.path) ? check.matcher.path : identifier(check.matcher?.kind) ? check.matcher.kind : 'assertion', + checks: checks.map((check, index) => ({ + name: check.matcher.kind, ordinal: index + 1, + pathSha256: typeof check.matcher.path === 'string' ? sha(check.matcher.path) : null, passed: check.passed, })), retainedAttempts: (accounting?.attempts ?? []).filter(row => row.executionId === selected.executionId).map(row => ({ @@ -142,12 +161,13 @@ export function projectProductResults(campaign, selection, sourceSha) { labels: [{ name: 'AS_ID', value: String(selected.id) }, { name: 'parentSuite', value: 'Product E2E' }, { name: 'suite', value: selected.suite }, { name: 'framework', value: 'Paperclip Evals' }], parameters: [{ name: 'profile', value: selected.profile }, { name: 'environment', value: 'local / GitHub Actions' }, - { name: 'model', value: provenance.model ?? 'Not recorded' }, { name: 'sourceSha', value: sourceSha }], + { name: 'declaredModel', value: selected.model }, { name: 'configuredModel', value: provenance.model ?? 'Not recorded' }, + { name: 'sourceSha', value: sourceSha }], links: [{ name: 'Full-stack execution and retained diagnostics', type: 'custom', url: `https://github.com/paperclipai/paperclip/actions/runs/${selection.runId}` }, { name: 'Evaluated source', type: 'custom', url: `https://github.com/paperclipai/paperclip/commit/${sourceSha}` }], attachments: [{ name: 'Bound execution provenance and assertions', source: attachment, type: 'application/json' }], - steps: provenance.checks.map(check => ({ name: check.name, status: check.passed ? 'passed' : 'failed', stage: 'finished' })), + steps: provenance.checks.map(check => ({ name: `${check.ordinal}. ${check.name}`, status: check.passed ? 'passed' : 'failed', stage: 'finished' })), }; if (result && !missingArtifact) for (const [key, date] of [['start', result.startedAt], ['stop', result.finishedAt]]) { const timestamp = Date.parse(date); diff --git a/tests/runner-e2e/product-regression-testops.test.mjs b/tests/runner-e2e/product-regression-testops.test.mjs index fcf46039387..386ffba8092 100644 --- a/tests/runner-e2e/product-regression-testops.test.mjs +++ b/tests/runner-e2e/product-regression-testops.test.mjs @@ -1,6 +1,7 @@ import test from 'node:test'; import assert from 'node:assert/strict'; -import { mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; +import { mkdtemp, mkdir, readFile, rm, writeFile, lstat } from 'node:fs/promises'; +import { createHash } from 'node:crypto'; import { tmpdir } from 'node:os'; import path from 'node:path'; import { productRegressionCases, selectProductRegression, validateSelection, projectProductResults, workflowUrl, workflowUrls } from './product-regression-testops.mjs'; @@ -19,7 +20,7 @@ const selection = cases => ({ schema: 'paperclip.product-regression-testops-sele const result = (overrides = {}) => ({ executionId: selected.executionId, suiteId: selected.suite, caseId: selected.name, profileId: selected.profile, environmentId: 'local', source: { sha: targetSha }, attempt: 1, - status: 'passed', cleanup: 'passed', model: 'gpt-6.1-sol', suiteDefinitionHash: 'c'.repeat(64), + status: 'passed', cleanup: 'passed', model: 'gpt-5.6-sol', suiteDefinitionHash: 'c'.repeat(64), matcherResults: [{ matcher: { kind: 'json_path', path: 'observed.outputVerified', expected: true }, passed: true }], ...overrides, }); @@ -114,7 +115,7 @@ test('authoritative normalized accounting preserves earlier failed attempt verdi assert.deepEqual(projected.records[1].value.attemptAccounting, { scope: 'retained_attempts', missingAttempts: 0, excludedArtifacts: 0 }); assert.throws(() => projectProductResults({ ...measured, accounting: { ...measured.accounting, attempts: [{ ...measured.accounting.attempts[0], status: 'unknown' }] } }, selection([selected]), targetSha)); const openCode = productRegressionCases.find(row => row.id === 54); - const model = 'openrouter/deepseek/deepseek-v4.1-flash-1017'; + const model = 'openrouter/deepseek/deepseek-v4-flash-0731'; const record = result({ executionId: openCode.executionId, suiteId: openCode.suite, caseId: openCode.name, profileId: openCode.profile, model }); const openCodeCampaign = { ...campaign([record]), expected: [openCode.executionId] }; assert.equal(projectProductResults(openCodeCampaign, selection([openCode]), targetSha).records[1].value.model, model); @@ -123,13 +124,31 @@ test('authoritative normalized accounting preserves earlier failed attempt verdi test('the transmission projection excludes raw errors, provider IDs, narratives and unexpected fields', () => { const dirty = result({ error: 'secret-error', runIds: ['private-provider-session'], detail: 'private-context', usage: { credential: 'private-token' }, matcherResults: [{ passed: true, detail: 'secret-detail', - matcher: { kind: 'contains', expected: 'fixture-answer-secret', path: 'unsafe narrative with secrets' } }] }); + matcher: { kind: 'message_contains', expected: 'fixture-answer-secret', path: 'unsafe narrative with secrets' } }] }); const text = JSON.stringify(projectProductResults(campaign([dirty]), selection([selected]), targetSha)); for (const secret of ['secret-error', 'private-provider-session', 'private-context', 'private-token', 'secret-detail', 'fixture-answer-secret', 'unsafe narrative']) { assert.equal(text.includes(secret), false, secret); } }); +test('secret-shaped and unknown metadata cannot cross the reporting boundary', () => { + const sentinel = 'sk-proj-syntheticcredentialabcdefghijklmnop'; + for (const mutation of [{ model: sentinel }, { model: 'gpt-6.1-sol' }, { failureClass: sentinel }, { cleanup: sentinel }, + { matcherResults: [{ passed: true, matcher: { kind: sentinel } }] }]) { + assert.throws(() => projectProductResults(campaign([result(mutation)]), selection([selected]), targetSha), + error => error.message === 'product_regression_testops_metadata_invalid'); + } + const projected = projectProductResults(campaign([result({ matcherResults: [ + { passed: false, matcher: { kind: 'json_path', path: sentinel } }, + ] })]), selection([selected]), targetSha); + assert.equal(JSON.stringify(projected).includes(sentinel), false); + assert.equal(projected.summary.passed, 0); + assert.equal(projected.records[1].value.checks[0].name, 'json_path'); + assert.equal(projected.records[1].value.checks[0].ordinal, 1); + assert.equal(projected.records[1].value.checks[0].passed, false); + assert.equal(projected.records[1].value.checks[0].pathSha256, createHash('sha256').update(sentinel).digest('hex')); +}); + test('a TestOps job run must belong to this workflow, source launch and GitHub execution', async () => { const values = { '/api/jobrun/456': { id: 456, launchId: 789, job: { id: 50, type: 'github' }, url: 'https://github.com/paperclipai/paperclip/actions/runs/123' }, '/api/job/50': { id: 50, type: 'github', url: workflowUrl }, '/api/launch/789': { id: 789, projectId: 1 } }; @@ -193,11 +212,28 @@ test('the separated selection, retained-report projection and server verificatio await mkdir(path.join(runtime.REPORT_DIR, 'normalized'), { recursive: true }); await writeFile(path.join(runtime.REPORT_DIR, 'normalized', 'normalized-results.json'), JSON.stringify(campaign())); assert.equal(await productTestOps('project', selectionFile, runtime, exec, fetchImpl), 0); + const unsafeRuntime = { ...runtime, ALLURE_RESULTS: path.join(root, 'unsafe-results'), EXPORT_SUMMARY: path.join(root, 'unsafe-export.json') }; + for (const mutation of [{ model: 'sk-proj-syntheticcredentialabcdefghijklmnop' }, + { cleanup: 'sk-proj-syntheticcredentialabcdefghijklmnop' }]) { + await writeFile(path.join(runtime.REPORT_DIR, 'normalized', 'normalized-results.json'), JSON.stringify(campaign([result(mutation)]))); + await assert.rejects(productTestOps('project', selectionFile, unsafeRuntime, exec, fetchImpl), + error => error.message === 'product_regression_testops_metadata_invalid'); + await assert.rejects(lstat(unsafeRuntime.ALLURE_RESULTS), { code: 'ENOENT' }); + await assert.rejects(lstat(unsafeRuntime.EXPORT_SUMMARY), { code: 'ENOENT' }); + } + await writeFile(path.join(runtime.REPORT_DIR, 'normalized', 'normalized-results.json'), JSON.stringify(campaign())); assert.equal(await productTestOps('verify', selectionFile, runtime, exec, fetchImpl), 0); uploadedStatus = 'unknown'; let readinessWaits = 0; assert.equal(await productTestOps('verify', selectionFile, runtime, exec, fetchImpl, async () => { readinessWaits++; uploadedStatus = 'passed'; }), 0); assert.equal(readinessWaits, 1); + uploadedStatus = 'unknown'; + let virtualMs = 0; + assert.equal(await productTestOps('verify', selectionFile, runtime, exec, fetchImpl, async ms => { + virtualMs += ms; + if (virtualMs >= 40_000) uploadedStatus = 'passed'; + }), 0); + assert.equal(virtualMs, 40_000); const failureOutput = path.join(root, 'failed-selection-output'); const badPlan = async (command, args) => args[1] === 'plan' ? (await writeFile(args.at(-1), JSON.stringify({ version: '1.0', tests: [{ id: '999', selector: selected.fullName }] })), { stdout: '' }) @@ -211,7 +247,10 @@ test('the separated selection, retained-report projection and server verificatio { ...runtime, ALLURE_JOB_RUN_ID: '', TESTOPS_CASE: 'unknown' }, async () => { opened = true; }, fetchImpl), /manual_selection_invalid/); assert.equal(opened, false); uploadedStatus = 'failed'; - await assert.rejects(productTestOps('verify', selectionFile, runtime, exec, fetchImpl, async () => {}), /uploaded_status_mismatch/); + let deadlineMs = 0; + await assert.rejects(productTestOps('verify', selectionFile, runtime, exec, fetchImpl, + async ms => { deadlineMs += ms; }), /uploaded_status_mismatch/); + assert.equal(deadlineMs, 55_000); await writeFile(selectionFile, JSON.stringify({ ...receipt, targetSha: workflowSha })); await assert.rejects(productTestOps('project', selectionFile, runtime, exec, fetchImpl), /selection_hash/); }); diff --git a/ui/src/lib/issueDetailCache.ts b/ui/src/lib/issueDetailCache.ts index 8fca3079c20..9cf08dbc216 100644 --- a/ui/src/lib/issueDetailCache.ts +++ b/ui/src/lib/issueDetailCache.ts @@ -116,12 +116,21 @@ export function seedIssueDetailCache( return merged; } -/** Seed saved projections without replacing lists already read or refreshed. */ +/** Retain authenticated core aliases and saved projections without replacing + * data already read or refreshed. Core entries never fill full-detail keys. */ export function seedIssueDetailProjections( queryClient: QueryClient, issue: Issue, issueRef?: string | null, ) { + // Canonical navigation must not discard metadata just fetched through a UUID. + // Only the response's own refs are aliases; an arbitrary requested ref is not. + for (const ref of getIssueDetailCacheRefs(issue)) { + const key = [...queryKeys.issues.detail(ref), "core"]; + if (queryClient.getQueryData(key) === undefined) { + queryClient.setQueryData(key, issue); + } + } const planKey = [...queryKeys.issues.documents(issue.id), "plan"]; if (issue.planDocument !== undefined && queryClient.getQueryData(planKey) === undefined) { queryClient.setQueryData(planKey, issue.planDocument); diff --git a/ui/src/lib/issueDetailQuery.test.tsx b/ui/src/lib/issueDetailQuery.test.tsx index 313d011dd30..ada28a2b173 100644 --- a/ui/src/lib/issueDetailQuery.test.tsx +++ b/ui/src/lib/issueDetailQuery.test.tsx @@ -7,11 +7,12 @@ import { QueryClient, QueryClientProvider, useQuery, useQueryClient } from "@tan import { afterEach, describe, expect, it, vi } from "vitest"; import { issuesApi } from "@/api/issues"; import { queryKeys } from "@/lib/queryKeys"; -import { getIssueDetailQueryOptions } from "./issueDetailCache"; +import { getIssueDetailQueryOptions, seedIssueDetailProjections } from "./issueDetailCache"; vi.mock("@/api/issues", () => ({ issuesApi: { get: vi.fn(), + getCore: vi.fn(), }, })); @@ -75,6 +76,19 @@ function IssueDetailQueryHarness({ return
{query.data?.description ?? "EMPTY"}
; } +function IssueCoreQueryHarness({ issueRef }: { issueRef: string }) { + const queryClient = useQueryClient(); + const query = useQuery({ + queryKey: [...queryKeys.issues.detail(issueRef), "core"], + queryFn: async ({ signal }) => { + const core = await issuesApi.getCore(issueRef, { signal }); + seedIssueDetailProjections(queryClient, core, issueRef); + return core; + }, + }); + return
{query.data?.title ?? "EMPTY"} / {query.data?.companyId ?? "UNKNOWN"} / {query.fetchStatus}
; +} + async function flush() { // Multiple act cycles to allow React Query to process the async queryFn for (let i = 0; i < 5; i++) { @@ -89,6 +103,52 @@ describe("getIssueDetailQueryOptions", () => { vi.clearAllMocks(); }); + it("retains authenticated task metadata across UUID-to-identifier navigation while canonical core refresh is pending", async () => { + const container = document.createElement("div"); + document.body.appendChild(container); + const root = createRoot(container); + const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + const issue = makeIssue({ id: "c1267799-90d5-4cb3-8311-38bd480dd76e", title: "Organization-bound task link", companyId: "owning-company" }); + let release!: (value: Issue) => void; + const canonical = new Promise((resolve) => { release = resolve; }); + vi.mocked(issuesApi.getCore).mockImplementation(async (ref) => ref === issue.id ? issue : canonical); + const render = async (ref: string) => { + await act(async () => root.render()); + await flush(); + }; + try { + await render(issue.id); + expect(container.textContent).toContain(issue.title); + await render(issue.identifier!); + expect(issuesApi.getCore).toHaveBeenCalledWith(issue.identifier, { signal: expect.any(AbortSignal) }); + expect(container.textContent).toContain(issue.title); + expect(container.textContent).toContain("owning-company"); + expect(container.textContent).toContain("fetching"); + expect(queryClient.getQueryData(queryKeys.issues.detail(issue.identifier!))).toBeUndefined(); + expect(queryClient.getQueryData([...queryKeys.issues.detail("unrelated-task"), "core"])).toBeUndefined(); + } finally { + release(issue); + await act(async () => root.unmount()); + queryClient.clear(); + container.remove(); + } + }); + + it("does not replace an existing canonical core receipt or fill full-detail keys", () => { + const queryClient = new QueryClient(); + const issue = makeIssue(); + const newer = makeIssue({ title: "Newer saved title", updatedAt: new Date("2026-11-04T00:00:00Z") }); + const key = [...queryKeys.issues.detail(issue.identifier!), "core"]; + queryClient.setQueryData(key, newer); + seedIssueDetailProjections(queryClient, issue, "unrelated-task"); + expect(queryClient.getQueryData(key)).toBe(newer); + expect(queryClient.getQueryData([...queryKeys.issues.detail(issue.id), "core"])).toEqual(issue); + expect(queryClient.getQueryData(queryKeys.issues.detail(issue.id))).toBeUndefined(); + expect(queryClient.getQueryData(queryKeys.issues.detail(issue.identifier!))).toBeUndefined(); + expect(queryClient.getQueryData([...queryKeys.issues.detail("unrelated-task"), "core"])).toBeUndefined(); + queryClient.clear(); + }); + it("treats cached issue data as placeholder and still fetches full detail", async () => { const container = document.createElement("div"); document.body.appendChild(container);