From cb4b55dd58e10e204ea5ee578f0268c11e0c59e4 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 21 Oct 2026 08:00:17 -0500 Subject: [PATCH] test: reject inherited and untrusted qualification metadata Co-Authored-By: Paperclip --- .../dependable-qualification.test.ts | 23 +++++++++++++++++-- tests/runner-e2e/dependable-qualification.ts | 8 ++----- tests/runner-e2e/result-validation.ts | 2 +- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/tests/runner-e2e/dependable-qualification.test.ts b/tests/runner-e2e/dependable-qualification.test.ts index 26135010af9..b756b400b7d 100644 --- a/tests/runner-e2e/dependable-qualification.test.ts +++ b/tests/runner-e2e/dependable-qualification.test.ts @@ -130,8 +130,8 @@ it("rejects unexpected credential-bearing metadata before retained report proces const { manifestVerified: _manifest, secretScanPassed: _scan, ...evidence } = row.evidence; const result = { ...row.result, dependableWork: { campaignId: row.campaignId, cell: row.cell, phase: row.phase, evidence } }; expect(() => validateRetainedRunnerResult(result)).not.toThrow(); - expect(() => validateRetainedRunnerResult({ ...result, dependableWork: { ...result.dependableWork, credential: "not-allowed" } })).toThrow("credential"); - expect(() => validateRetainedRunnerResult({ ...result, dependableWork: { ...result.dependableWork, evidence: { ...evidence, callbackUrl: "https://invalid.example" } } })).toThrow("callbackUrl"); + expect(() => validateRetainedRunnerResult({ ...result, dependableWork: { ...result.dependableWork, credential: "not-allowed" } })).toThrow("result.dependableWork"); + expect(() => validateRetainedRunnerResult({ ...result, dependableWork: { ...result.dependableWork, evidence: { ...evidence, callbackUrl: "https://invalid.example" } } })).toThrow("result.dependableWork.evidence"); }); it("makes the actual report command fail an incomplete checkpoint while retaining an earlier failure", async () => { @@ -161,3 +161,22 @@ it("makes the actual report command fail an incomplete checkpoint while retainin expect(await readFile(path.join(output, "summary.md"), "utf8")).toContain("checkpoint: incomplete"); } finally { await rm(root, { recursive: true, force: true }); } }); + +it.each(["constructor", "toString", "__proto__", "secret-value-in-a-field-name"])("rejects unexpected metadata keys without echoing %s", key => { + const row = attempt(DEPENDABLE_CELLS[0]!, 1); + const { manifestVerified: _manifest, secretScanPassed: _scan, ...evidence } = row.evidence; + const receipt = { campaignId: row.campaignId, cell: row.cell, phase: row.phase, evidence }; + for (const dependableWork of [{ ...receipt, [key]: "untrusted" }, { ...receipt, evidence: { ...evidence, [key]: "untrusted" } }]) { + let error: unknown; + try { validateRetainedRunnerResult({ ...row.result, dependableWork }); } catch (caught) { error = caught; } + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).toContain("result.dependableWork"); + expect((error as Error).message).not.toContain(key); + } +}); + +it("keeps a complete rotation for valid UTC dates before the Unix epoch", () => { + const dates = ["1969-12-26", "1969-12-27", "1969-12-28", "1969-12-29", "1969-12-30", "1969-12-31"]; + expect(dates.every(date => dependableDailySelection(date).length === 4)).toBe(true); + expect([...new Set(dates.flatMap(dependableDailySelection))].sort()).toEqual([...DEPENDABLE_CELLS].sort()); +}); diff --git a/tests/runner-e2e/dependable-qualification.ts b/tests/runner-e2e/dependable-qualification.ts index ed7fbbb24b9..00eb2fcc597 100644 --- a/tests/runner-e2e/dependable-qualification.ts +++ b/tests/runner-e2e/dependable-qualification.ts @@ -34,7 +34,6 @@ export function dependableRequiredChecks(cell: DependableCell) { return [...COMMON_CHECKS, ...JOURNEY_CHECKS[journey]].map(id => `dependable.${id}`); } -/** Metadata is recorded by the Product E2E observer, never taken from agent prose. */ export interface DependableAttempt { campaignId: string; cell: DependableCell; @@ -57,7 +56,6 @@ export interface DependableQualification { revision: string; startedAt: string; candidatePassesPerCell: 3; - /** Captured effective configuration, including model, environment and auth method. */ settingsHashes: Record; models: Record; instructionsHash: string; @@ -98,7 +96,6 @@ function attemptProblems(attempt: DependableAttempt, config: DependableQualifica return problems; } -/** A checkpoint over existing attempt records; it neither runs work nor retries it. */ export function gradeDependableQualification(config: DependableQualification, attempts: readonly DependableAttempt[]) { if (!sha.test(config.revision) || time(config.startedAt) === null || config.candidatePassesPerCell !== 3 || !digest.test(config.definitionHash) || !digest.test(config.instructionsHash) || @@ -138,12 +135,12 @@ export function gradeDependableQualification(config: DependableQualification, at incompleteAttempts: observations.filter(row => !row.qualified).length }; } -/** Four explicit cells daily; the complete 24-cell set rotates every six UTC days. */ export function dependableDailySelection(utcDate: string) { if (!/^\d{4}-\d{2}-\d{2}$/.test(utcDate) || new Date(utcDate).toISOString().slice(0, 10) !== utcDate) throw new Error("Daily rotation requires an exact UTC date"); const day = Math.floor(Date.parse(utcDate) / 86_400_000); - const journey = DEPENDABLE_JOURNEYS[day % DEPENDABLE_JOURNEYS.length]!; + const length = DEPENDABLE_JOURNEYS.length; + const journey = DEPENDABLE_JOURNEYS[((day % length) + length) % length]!; return DEPENDABLE_CELLS.filter(cell => cell.endsWith(`.${journey}`)); } @@ -154,7 +151,6 @@ export function dependableDailyExecutions(utcDate: string, availableIds: readonl return ids; } -/** Integrates with the ordinary report's entire retained pool, before best-attempt selection. */ export function gradeRetainedDependableResults(config: DependableQualification, entries: readonly { result: RunnerE2EResult; evidence: { files?: string[]; leaks?: unknown[]; missing?: unknown[] } | null; diff --git a/tests/runner-e2e/result-validation.ts b/tests/runner-e2e/result-validation.ts index 9eb22164149..adc3e1576ba 100644 --- a/tests/runner-e2e/result-validation.ts +++ b/tests/runner-e2e/result-validation.ts @@ -60,7 +60,7 @@ const shape = }; const closedShape = (fields: Record): Rule => (value, at) => { shape(fields)(value, at); - for (const key of Object.keys(value as object)) if (!(key in fields)) invalid(`${at}.${key}`); + for (const key of Object.keys(value as object)) if (!Object.hasOwn(fields, key)) invalid(`${at} (unexpected field)`); }; const date: Rule = (value, at) => { string(value, at);