feat: initial commit — Curio mastery tutor
Full Next.js App Router application with: - AI-graded lesson checkpoints (BKT/Elo mastery tracking) - Auth.js v5: credentials, Google, GitHub, generic OIDC - Anonymous-session-first with migrate-on-signin - Admin panel: users, blueprints, reports, site settings - Password reset + email verification (nodemailer/SMTP) - Site config: require_auth + signups_enabled flags - server-only guards on all DB/generation/verification modules - PostgreSQL 16 + pgvector, Redis cache, Drizzle ORM - 271 unit tests (Vitest), golden-eval harness, Playwright e2e stubs Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,196 @@
|
||||
/**
|
||||
* Golden eval harness — response grading accuracy.
|
||||
*
|
||||
* Runs real-model grading calls against the frozen labeled cases in grading-cases.ts.
|
||||
* Measures false-fail rate and exits non-zero if it exceeds FALSE_FAIL_THRESHOLD.
|
||||
*
|
||||
* Usage:
|
||||
* pnpm test:golden
|
||||
*
|
||||
* Required env: OPENAI_API_KEY (grader is openai/gpt-4o-mini by default)
|
||||
* Optional env: ANTHROPIC_API_KEY (only if LLM_GRADER_PROVIDER=anthropic)
|
||||
*
|
||||
* CI gate: exits 1 if false-fail rate > 10% OR accuracy < 70%.
|
||||
*/
|
||||
|
||||
import { generateObject } from 'ai';
|
||||
import { CASES, FALSE_FAIL_THRESHOLD } from './grading-cases';
|
||||
import type { GoldenCase } from './grading-cases';
|
||||
import { GradeOutputSchema } from '../../src/schemas/grading';
|
||||
import type { GradeOutput } from '../../src/schemas/grading';
|
||||
import { prompts } from '../../src/lib/llm/prompts';
|
||||
import { llmClient } from '../../src/lib/llm/client';
|
||||
|
||||
// ── Types ────────────────────────────────────────────────────────────────────
|
||||
|
||||
interface CaseResult {
|
||||
case: GoldenCase;
|
||||
actual: GradeOutput | null;
|
||||
error?: string;
|
||||
/** verdict matched expected */
|
||||
verdictMatch: boolean;
|
||||
/** isCorrect case graded as misconception */
|
||||
isFalseFail: boolean;
|
||||
/** !isCorrect case graded as mastered */
|
||||
isFalsePass: boolean;
|
||||
latencyMs: number;
|
||||
}
|
||||
|
||||
// ── Prompt builder (mirrors grader.ts logic) ─────────────────────────────────
|
||||
|
||||
function buildPrompt(c: GoldenCase): string {
|
||||
const rubricLines = c.rubric
|
||||
.map((r) => ` - [${r.id}] ${r.criterion} (weight: ${r.weight.toFixed(2)})`)
|
||||
.join('\n');
|
||||
|
||||
const miscLines =
|
||||
c.misconceptions.length > 0
|
||||
? c.misconceptions.map((m) => ` - tag: "${m.tag}"\n signature: ${m.signature}`).join('\n')
|
||||
: ' (none)';
|
||||
|
||||
return `Checkpoint prompt: ${c.checkpointPrompt}
|
||||
|
||||
Reference answer: ${c.referenceAnswer}
|
||||
|
||||
Rubric criteria:
|
||||
${rubricLines}
|
||||
|
||||
Known misconceptions for this concept:
|
||||
${miscLines}
|
||||
|
||||
Learner's response:
|
||||
"""
|
||||
${c.learnerResponse}
|
||||
"""`;
|
||||
}
|
||||
|
||||
// ── Runner ────────────────────────────────────────────────────────────────────
|
||||
|
||||
async function runCase(c: GoldenCase): Promise<CaseResult> {
|
||||
const start = Date.now();
|
||||
let actual: GradeOutput | null = null;
|
||||
let error: string | undefined;
|
||||
|
||||
try {
|
||||
const { object } = await generateObject({
|
||||
model: llmClient.grader,
|
||||
schema: GradeOutputSchema,
|
||||
system: prompts.GRADE_RESPONSE.template,
|
||||
prompt: buildPrompt(c),
|
||||
});
|
||||
actual = object;
|
||||
} catch (err) {
|
||||
error = err instanceof Error ? err.message : String(err);
|
||||
}
|
||||
|
||||
const latencyMs = Date.now() - start;
|
||||
const verdictMatch = actual?.verdict === c.expectedVerdict;
|
||||
const isFalseFail = c.isCorrect && actual?.verdict === 'misconception';
|
||||
const isFalsePass = !c.isCorrect && actual?.verdict === 'mastered';
|
||||
|
||||
return { case: c, actual, error, verdictMatch, isFalseFail, isFalsePass, latencyMs };
|
||||
}
|
||||
|
||||
function printResult(r: CaseResult, idx: number, total: number): void {
|
||||
const status = r.error
|
||||
? ' ERROR '
|
||||
: r.isFalseFail
|
||||
? '✗ FF '
|
||||
: r.isFalsePass
|
||||
? '✗ FP '
|
||||
: r.verdictMatch
|
||||
? '✓ '
|
||||
: '✗ ';
|
||||
|
||||
const expected = r.case.expectedVerdict.padEnd(12);
|
||||
const actual = r.actual ? r.actual.verdict.padEnd(12) : 'null ';
|
||||
const ms = `${r.latencyMs}ms`.padStart(6);
|
||||
|
||||
console.log(
|
||||
` [${String(idx + 1).padStart(2)}/${total}] ${status} expected=${expected} actual=${actual} ${ms} "${r.case.label}"`,
|
||||
);
|
||||
|
||||
if (r.error) {
|
||||
console.log(` Error: ${r.error}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
console.log(`\nCurio — golden grading eval (${CASES.length} cases)\n`);
|
||||
console.log(` Grader: ${process.env.LLM_GRADER_PROVIDER ?? 'openai'} / ${process.env.LLM_GRADER_MODEL ?? 'gpt-4o-mini'}`);
|
||||
console.log(` False-fail threshold: ${(FALSE_FAIL_THRESHOLD * 100).toFixed(0)}%\n`);
|
||||
|
||||
const results: CaseResult[] = [];
|
||||
|
||||
for (let i = 0; i < CASES.length; i++) {
|
||||
const c = CASES[i];
|
||||
process.stdout.write(` running [${i + 1}/${CASES.length}] "${c.label}"…\r`);
|
||||
const result = await runCase(c);
|
||||
results.push(result);
|
||||
printResult(result, i, CASES.length);
|
||||
}
|
||||
|
||||
// ── Metrics ──────────────────────────────────────────────────────────────────
|
||||
|
||||
const ran = results.filter((r) => !r.error);
|
||||
const errors = results.filter((r) => r.error);
|
||||
const verdictMatches = ran.filter((r) => r.verdictMatch).length;
|
||||
const falseFails = ran.filter((r) => r.isFalseFail);
|
||||
const falsePasses = ran.filter((r) => r.isFalsePass);
|
||||
const correctCases = ran.filter((r) => r.case.isCorrect);
|
||||
|
||||
const accuracy = ran.length > 0 ? verdictMatches / ran.length : 0;
|
||||
const falseFailRate = correctCases.length > 0 ? falseFails.length / correctCases.length : 0;
|
||||
|
||||
console.log('\n── Results ─────────────────────────────────────────────────────────────');
|
||||
console.log(` Cases run: ${ran.length} / ${CASES.length} (${errors.length} errors)`);
|
||||
console.log(` Verdict accuracy: ${verdictMatches}/${ran.length} = ${(accuracy * 100).toFixed(1)}%`);
|
||||
console.log(` False-fail rate: ${falseFails.length}/${correctCases.length} = ${(falseFailRate * 100).toFixed(1)}% (threshold: ${(FALSE_FAIL_THRESHOLD * 100).toFixed(0)}%)`);
|
||||
console.log(` False-pass rate: ${falsePasses.length}/${ran.filter((r) => !r.case.isCorrect).length}`);
|
||||
|
||||
if (falseFails.length > 0) {
|
||||
console.log('\n── False-fails (correct learner marked wrong) ───────────────────────────');
|
||||
for (const ff of falseFails) {
|
||||
console.log(` [${ff.case.id}] "${ff.case.label}"`);
|
||||
console.log(` Response: "${ff.case.learnerResponse.slice(0, 80)}…"`);
|
||||
console.log(` Verdict: ${ff.actual?.verdict} Evidence: "${ff.actual?.evidence_span}"`);
|
||||
}
|
||||
}
|
||||
|
||||
if (errors.length > 0) {
|
||||
console.log('\n── Errors ───────────────────────────────────────────────────────────────');
|
||||
for (const r of errors) {
|
||||
console.log(` [${r.case.id}] ${r.error}`);
|
||||
}
|
||||
}
|
||||
|
||||
// ── CI gate ───────────────────────────────────────────────────────────────────
|
||||
|
||||
const ACCURACY_FLOOR = 0.70;
|
||||
let failed = false;
|
||||
|
||||
if (falseFailRate > FALSE_FAIL_THRESHOLD) {
|
||||
console.log(
|
||||
`\n✗ FAIL: false-fail rate ${(falseFailRate * 100).toFixed(1)}% exceeds threshold ${(FALSE_FAIL_THRESHOLD * 100).toFixed(0)}%`,
|
||||
);
|
||||
failed = true;
|
||||
}
|
||||
|
||||
if (accuracy < ACCURACY_FLOOR) {
|
||||
console.log(
|
||||
`\n✗ FAIL: verdict accuracy ${(accuracy * 100).toFixed(1)}% below floor ${(ACCURACY_FLOOR * 100).toFixed(0)}%`,
|
||||
);
|
||||
failed = true;
|
||||
}
|
||||
|
||||
if (!failed) {
|
||||
console.log('\n✓ PASS: grading quality within thresholds\n');
|
||||
}
|
||||
|
||||
process.exit(failed ? 1 : 0);
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error('Eval runner crashed:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user