Curio/tests/golden/run-content-eval.ts

/**
 * Content verification eval — T1 cascade accuracy.
 *
 * Tests the T1 verifier against cases where content is known to be good or bad.
 * Measures false-pass rate (bad content slips through) and false-fail rate
 * (good content incorrectly rejected).
 *
 * CI gate: false-pass rate ≤ 15%, false-fail rate ≤ 15%.
 */

import { generateObject } from 'ai';
import { VerifyOutputSchema } from '../../src/schemas/verification';
import type { VerifyOutput } from '../../src/schemas/verification';
import { prompts } from '../../src/lib/llm/prompts';
import { llmClient } from '../../src/lib/llm/client';
import { CONTENT_CASES, CONTENT_VERIFY_THRESHOLDS } from './content-cases';
import type { ContentCase } from './content-cases';

// ── Prompt builder (mirrors verify-content.ts T1 logic) ──────────────────────

function buildT1Prompt(c: ContentCase): string {
  const sourcesText = c.sourceChunks.map((s) => `[${s.docRef}]\n${s.text}`).join('\n\n---\n\n');

  return `Source chunks:
${sourcesText}

Segment text to verify:
"""
${c.segmentText}
"""

Checkpoint prompt (blind self-solve — answer from sources only, NOT from segment):
${c.checkpointPrompt}

Fact-check the segment and independently solve the checkpoint.`;
}

// ── Single verification call ──────────────────────────────────────────────────

async function verify(
  c: ContentCase,
): Promise<{ output: VerifyOutput | null; error?: string; latencyMs: number }> {
  const start = Date.now();
  try {
    const { object } = await generateObject({
      model: llmClient.grader,
      schema: VerifyOutputSchema,
      system: prompts.VERIFY_CONTENT_T1.template,
      prompt: buildT1Prompt(c),
    });
    return { output: object, latencyMs: Date.now() - start };
  } catch (err) {
    return {
      output: null,
      error: err instanceof Error ? err.message : String(err),
      latencyMs: Date.now() - start,
    };
  }
}

// ── Runner ────────────────────────────────────────────────────────────────────

async function main(): Promise<void> {
  console.log(`\nCurio — content verification eval (${CONTENT_CASES.length} cases)\n`);
  console.log(`  Verifier: ${process.env.LLM_GRADER_PROVIDER ?? 'openai'} / ${process.env.LLM_GRADER_MODEL ?? 'gpt-4o-mini'} (T1)`);
  console.log(
    `  Thresholds: false-pass ≤ ${(CONTENT_VERIFY_THRESHOLDS.maxFalsePassRate * 100).toFixed(0)}%,` +
    ` false-fail ≤ ${(CONTENT_VERIFY_THRESHOLDS.maxFalseFailRate * 100).toFixed(0)}%\n`,
  );

  const results: Array<{
    case: ContentCase;
    output: VerifyOutput | null;
    error?: string;
    isFalsePass: boolean;
    isFalseFail: boolean;
    latencyMs: number;
  }> = [];

  for (const c of CONTENT_CASES) {
    process.stdout.write(`  [${c.id}] "${c.label}"…\r`);
    const { output, error, latencyMs } = await verify(c);

    const verdict = output?.verdict ?? null;
    const isFalsePass = c.isBadContent && verdict === 'pass';
    const isFalseFail = c.isGoodContent && verdict === 'fail';

    results.push({ case: c, output, error, isFalsePass, isFalseFail, latencyMs });

    const flag = error ? '  ERR ' : isFalsePass ? '✗ FP  ' : isFalseFail ? '✗ FF  ' : '✓     ';
    const ms = `${latencyMs}ms`.padStart(6);
    console.log(
      `  [${c.id}]  ${flag}  verdict=${verdict ?? 'null'}  expected=${c.expectedT1Verdict}  ${ms}  "${c.label}"`,
    );

    if (error) console.log(`    Error: ${error}`);
    if (output && output.claims.some((cl) => cl.status !== 'supported')) {
      const unsupported = output.claims.filter((cl) => cl.status !== 'supported');
      console.log(`    Unsupported claims: ${unsupported.map((cl) => `"${cl.text.slice(0, 60)}…"`).join(', ')}`);
    }
  }

  // ── Metrics ──────────────────────────────────────────────────────────────────

  const ran = results.filter((r) => !r.error);
  const errors = results.filter((r) => r.error);
  const badCases = ran.filter((r) => r.case.isBadContent);
  const goodCases = ran.filter((r) => r.case.isGoodContent);
  const falsePasses = ran.filter((r) => r.isFalsePass);
  const falseFails = ran.filter((r) => r.isFalseFail);

  const falsePassRate = badCases.length > 0 ? falsePasses.length / badCases.length : 0;
  const falseFailRate = goodCases.length > 0 ? falseFails.length / goodCases.length : 0;

  console.log('\n── Results ─────────────────────────────────────────────────────────────');
  console.log(`  Cases run:    ${ran.length} / ${CONTENT_CASES.length} (${errors.length} errors)`);
  console.log(`  False-pass:   ${falsePasses.length}/${badCases.length} = ${(falsePassRate * 100).toFixed(1)}%  (bad content passing — max ${(CONTENT_VERIFY_THRESHOLDS.maxFalsePassRate * 100).toFixed(0)}%)`);
  console.log(`  False-fail:   ${falseFails.length}/${goodCases.length} = ${(falseFailRate * 100).toFixed(1)}%  (good content rejected — max ${(CONTENT_VERIFY_THRESHOLDS.maxFalseFailRate * 100).toFixed(0)}%)`);

  let failed = false;

  if (falsePassRate > CONTENT_VERIFY_THRESHOLDS.maxFalsePassRate) {
    console.log(`\n✗ FAIL: false-pass rate ${(falsePassRate * 100).toFixed(1)}% exceeds threshold`);
    failed = true;
  }

  if (falseFailRate > CONTENT_VERIFY_THRESHOLDS.maxFalseFailRate) {
    console.log(`\n✗ FAIL: false-fail rate ${(falseFailRate * 100).toFixed(1)}% exceeds threshold`);
    failed = true;
  }

  if (!failed) {
    console.log('\n✓ PASS: content verification quality within thresholds\n');
  }

  process.exit(failed ? 1 : 0);
}

main().catch((err) => {
  console.error('Eval runner crashed:', err);
  process.exit(1);
});