Curio/tests/golden/run-grading-eval.ts

/**
 * Golden eval harness — response grading accuracy.
 *
 * Runs real-model grading calls against the frozen labeled cases in grading-cases.ts.
 * Measures false-fail rate and exits non-zero if it exceeds FALSE_FAIL_THRESHOLD.
 *
 * Usage:
 *   pnpm test:golden
 *
 * Required env: OPENAI_API_KEY (grader is openai/gpt-4o-mini by default)
 * Optional env: ANTHROPIC_API_KEY (only if LLM_GRADER_PROVIDER=anthropic)
 *
 * CI gate: exits 1 if false-fail rate > 10% OR accuracy < 70%.
 */

import { generateObject } from 'ai';
import { CASES, FALSE_FAIL_THRESHOLD } from './grading-cases';
import type { GoldenCase } from './grading-cases';
import { GradeOutputSchema } from '../../src/schemas/grading';
import type { GradeOutput } from '../../src/schemas/grading';
import { prompts } from '../../src/lib/llm/prompts';
import { llmClient } from '../../src/lib/llm/client';

// ── Types ────────────────────────────────────────────────────────────────────

interface CaseResult {
  case: GoldenCase;
  actual: GradeOutput | null;
  error?: string;
  /** verdict matched expected */
  verdictMatch: boolean;
  /** isCorrect case graded as misconception */
  isFalseFail: boolean;
  /** !isCorrect case graded as mastered */
  isFalsePass: boolean;
  latencyMs: number;
}

// ── Prompt builder (mirrors grader.ts logic) ─────────────────────────────────

function buildPrompt(c: GoldenCase): string {
  const rubricLines = c.rubric
    .map((r) => `  - [${r.id}] ${r.criterion} (weight: ${r.weight.toFixed(2)})`)
    .join('\n');

  const miscLines =
    c.misconceptions.length > 0
      ? c.misconceptions.map((m) => `  - tag: "${m.tag}"\n    signature: ${m.signature}`).join('\n')
      : '  (none)';

  return `Checkpoint prompt: ${c.checkpointPrompt}

Reference answer: ${c.referenceAnswer}

Rubric criteria:
${rubricLines}

Known misconceptions for this concept:
${miscLines}

Learner's response:
"""
${c.learnerResponse}
"""`;
}

// ── Runner ────────────────────────────────────────────────────────────────────

async function runCase(c: GoldenCase): Promise<CaseResult> {
  const start = Date.now();
  let actual: GradeOutput | null = null;
  let error: string | undefined;

  try {
    const { object } = await generateObject({
      model: llmClient.grader,
      schema: GradeOutputSchema,
      system: prompts.GRADE_RESPONSE.template,
      prompt: buildPrompt(c),
    });
    actual = object;
  } catch (err) {
    error = err instanceof Error ? err.message : String(err);
  }

  const latencyMs = Date.now() - start;
  const verdictMatch = actual?.verdict === c.expectedVerdict;
  const isFalseFail = c.isCorrect && actual?.verdict === 'misconception';
  const isFalsePass = !c.isCorrect && actual?.verdict === 'mastered';

  return { case: c, actual, error, verdictMatch, isFalseFail, isFalsePass, latencyMs };
}

function printResult(r: CaseResult, idx: number, total: number): void {
  const status = r.error
    ? '  ERROR '
    : r.isFalseFail
      ? '✗ FF    '
      : r.isFalsePass
        ? '✗ FP    '
        : r.verdictMatch
          ? '✓       '
          : '✗       ';

  const expected = r.case.expectedVerdict.padEnd(12);
  const actual = r.actual ? r.actual.verdict.padEnd(12) : 'null        ';
  const ms = `${r.latencyMs}ms`.padStart(6);

  console.log(
    `  [${String(idx + 1).padStart(2)}/${total}] ${status}  expected=${expected} actual=${actual} ${ms}  "${r.case.label}"`,
  );

  if (r.error) {
    console.log(`           Error: ${r.error}`);
  }
}

async function main(): Promise<void> {
  console.log(`\nCurio — golden grading eval (${CASES.length} cases)\n`);
  console.log(`  Grader: ${process.env.LLM_GRADER_PROVIDER ?? 'openai'} / ${process.env.LLM_GRADER_MODEL ?? 'gpt-4o-mini'}`);
  console.log(`  False-fail threshold: ${(FALSE_FAIL_THRESHOLD * 100).toFixed(0)}%\n`);

  const results: CaseResult[] = [];

  for (let i = 0; i < CASES.length; i++) {
    const c = CASES[i];
    process.stdout.write(`  running [${i + 1}/${CASES.length}] "${c.label}"…\r`);
    const result = await runCase(c);
    results.push(result);
    printResult(result, i, CASES.length);
  }

  // ── Metrics ──────────────────────────────────────────────────────────────────

  const ran = results.filter((r) => !r.error);
  const errors = results.filter((r) => r.error);
  const verdictMatches = ran.filter((r) => r.verdictMatch).length;
  const falseFails = ran.filter((r) => r.isFalseFail);
  const falsePasses = ran.filter((r) => r.isFalsePass);
  const correctCases = ran.filter((r) => r.case.isCorrect);

  const accuracy = ran.length > 0 ? verdictMatches / ran.length : 0;
  const falseFailRate = correctCases.length > 0 ? falseFails.length / correctCases.length : 0;

  console.log('\n── Results ─────────────────────────────────────────────────────────────');
  console.log(`  Cases run:         ${ran.length} / ${CASES.length} (${errors.length} errors)`);
  console.log(`  Verdict accuracy:  ${verdictMatches}/${ran.length} = ${(accuracy * 100).toFixed(1)}%`);
  console.log(`  False-fail rate:   ${falseFails.length}/${correctCases.length} = ${(falseFailRate * 100).toFixed(1)}%  (threshold: ${(FALSE_FAIL_THRESHOLD * 100).toFixed(0)}%)`);
  console.log(`  False-pass rate:   ${falsePasses.length}/${ran.filter((r) => !r.case.isCorrect).length}`);

  if (falseFails.length > 0) {
    console.log('\n── False-fails (correct learner marked wrong) ───────────────────────────');
    for (const ff of falseFails) {
      console.log(`  [${ff.case.id}] "${ff.case.label}"`);
      console.log(`    Response: "${ff.case.learnerResponse.slice(0, 80)}…"`);
      console.log(`    Verdict: ${ff.actual?.verdict}  Evidence: "${ff.actual?.evidence_span}"`);
    }
  }

  if (errors.length > 0) {
    console.log('\n── Errors ───────────────────────────────────────────────────────────────');
    for (const r of errors) {
      console.log(`  [${r.case.id}] ${r.error}`);
    }
  }

  // ── CI gate ───────────────────────────────────────────────────────────────────

  const ACCURACY_FLOOR = 0.70;
  let failed = false;

  if (falseFailRate > FALSE_FAIL_THRESHOLD) {
    console.log(
      `\n✗ FAIL: false-fail rate ${(falseFailRate * 100).toFixed(1)}% exceeds threshold ${(FALSE_FAIL_THRESHOLD * 100).toFixed(0)}%`,
    );
    failed = true;
  }

  if (accuracy < ACCURACY_FLOOR) {
    console.log(
      `\n✗ FAIL: verdict accuracy ${(accuracy * 100).toFixed(1)}% below floor ${(ACCURACY_FLOOR * 100).toFixed(0)}%`,
    );
    failed = true;
  }

  if (!failed) {
    console.log('\n✓ PASS: grading quality within thresholds\n');
  }

  // Persist run to DB when PERSIST_EVAL_RUNS=1 (CI/admin use; skip locally by default)
  if (process.env.PERSIST_EVAL_RUNS === '1') {
    try {
      const { insertEvalRun } = await import('../../src/lib/db/queries');
      await insertEvalRun({
        suite: 'grading',
        falseFailRate,
        accuracy,
        total: CASES.length,
        passed: Math.round(accuracy * CASES.length),
        modelVersion: process.env.LLM_GRADER_MODEL ?? 'unknown',
        gitSha: process.env.GIT_SHA,
      });
      console.log('[eval] Run persisted to eval_run table.');
    } catch (err) {
      console.warn('[eval] Failed to persist run (non-fatal):', err);
    }
  }

  process.exit(failed ? 1 : 0);
}

main().catch((err) => {
  console.error('Eval runner crashed:', err);
  process.exit(1);
});