feat: initial commit — Curio mastery tutor

Full Next.js App Router application with: - AI-graded lesson checkpoints (BKT/Elo mastery tracking) - Auth.js v5: credentials, Google, GitHub, generic OIDC - Anonymous-session-first with migrate-on-signin - Admin panel: users, blueprints, reports, site settings - Password reset + email verification (nodemailer/SMTP) - Site config: require_auth + signups_enabled flags - server-only guards on all DB/generation/verification modules - PostgreSQL 16 + pgvector, Redis cache, Drizzle ORM - 271 unit tests (Vitest), golden-eval harness, Playwright e2e stubs Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-21 22:25:43 +02:00
commit 4e38b5a791
194 changed files with 30789 additions and 0 deletions
@@ -0,0 +1,145 @@
+/**
+ * Misconception hit-rate eval.
+ *
+ * Measures: given a response exhibiting a known misconception, does the
+ * grader return the correct tag when the library is provided?
+ *
+ * Two conditions run per case:
+ *   WITH library  — grader receives the misconception library (expected: hit the tag)
+ *   WITHOUT library — grader receives empty library (baseline: "novel" or "none")
+ *
+ * CI gate: WITH-library hit rate must exceed HIT_RATE_THRESHOLD.
+ */
+
+import { generateObject } from 'ai';
+import { GradeOutputSchema } from '../../src/schemas/grading';
+import type { GradeOutput } from '../../src/schemas/grading';
+import { prompts } from '../../src/lib/llm/prompts';
+import { llmClient } from '../../src/lib/llm/client';
+import {
+  MISCONCEPTION_CASES,
+  HIT_RATE_THRESHOLD,
+  BASELINE_HIT_RATE,
+} from './misconception-cases';
+import type { MisconceptionCase } from './misconception-cases';
+
+// ── Prompt builder ────────────────────────────────────────────────────────────
+
+function buildPrompt(
+  c: MisconceptionCase,
+  library: Array<{ tag: string; signature: string }>,
+): string {
+  const rubricLines = c.rubric
+    .map((r) => `  - [${r.id}] ${r.criterion} (weight: ${r.weight.toFixed(2)})`)
+    .join('\n');
+
+  const miscLines =
+    library.length > 0
+      ? library.map((m) => `  - tag: "${m.tag}"\n    signature: ${m.signature}`).join('\n')
+      : '  (none)';
+
+  return `Checkpoint prompt: ${c.checkpointPrompt}
+
+Reference answer: ${c.referenceAnswer}
+
+Rubric criteria:
+${rubricLines}
+
+Known misconceptions for this concept:
+${miscLines}
+
+Learner's response:
+"""
+${c.learnerResponse}
+"""`;
+}
+
+// ── Single grading call ───────────────────────────────────────────────────────
+
+async function grade(
+  c: MisconceptionCase,
+  library: Array<{ tag: string; signature: string }>,
+): Promise<{ grade: GradeOutput | null; error?: string; latencyMs: number }> {
+  const start = Date.now();
+  try {
+    const { object } = await generateObject({
+      model: llmClient.grader,
+      schema: GradeOutputSchema,
+      system: prompts.GRADE_RESPONSE.template,
+      prompt: buildPrompt(c, library),
+    });
+    return { grade: object, latencyMs: Date.now() - start };
+  } catch (err) {
+    return {
+      grade: null,
+      error: err instanceof Error ? err.message : String(err),
+      latencyMs: Date.now() - start,
+    };
+  }
+}
+
+// ── Runner ────────────────────────────────────────────────────────────────────
+
+async function main(): Promise<void> {
+  console.log(`\nCurio — misconception hit-rate eval (${MISCONCEPTION_CASES.length} cases × 2 conditions)\n`);
+  console.log(`  Grader: ${process.env.LLM_GRADER_PROVIDER ?? 'openai'} / ${process.env.LLM_GRADER_MODEL ?? 'gpt-4o-mini'}`);
+  console.log(`  Hit-rate threshold: ${(HIT_RATE_THRESHOLD * 100).toFixed(0)}%\n`);
+
+  let withHits = 0;
+  let baselineHits = 0;
+  let errors = 0;
+
+  for (const c of MISCONCEPTION_CASES) {
+    process.stdout.write(`  [${c.id}] "${c.label}"…\r`);
+
+    const [withResult, baselineResult] = await Promise.all([
+      grade(c, c.library),
+      grade(c, []),
+    ]);
+
+    const withTag = withResult.grade?.misconception_tag;
+    const baselineTag = baselineResult.grade?.misconception_tag;
+
+    const withHit = withTag === c.targetTag;
+    const baselineHit = baselineTag === c.targetTag;
+
+    if (withResult.error || baselineResult.error) errors++;
+    if (withHit) withHits++;
+    if (baselineHit) baselineHits++;
+
+    const withStatus = withResult.error ? '  ERR ' : withHit ? '✓ hit ' : '✗ miss';
+    const bStatus = baselineResult.error ? '  ERR ' : baselineHit ? '✓ hit ' : '✗ miss';
+
+    console.log(
+      `  [${c.id}]  WITH=${withStatus} (tag="${withTag ?? 'null'}")  BASE=${bStatus} (tag="${baselineTag ?? 'null'}")`,
+    );
+  }
+
+  const n = MISCONCEPTION_CASES.length;
+  const withRate = n > 0 ? withHits / n : 0;
+  const baseRate = n > 0 ? baselineHits / n : 0;
+  const lift = withRate - baseRate;
+
+  console.log('\n── Results ─────────────────────────────────────────────────────────────');
+  console.log(`  Cases:            ${n} (${errors} errors)`);
+  console.log(`  WITH library:     ${withHits}/${n} = ${(withRate * 100).toFixed(1)}%  (threshold: ${(HIT_RATE_THRESHOLD * 100).toFixed(0)}%)`);
+  console.log(`  Baseline (no lib): ${baselineHits}/${n} = ${(baseRate * 100).toFixed(1)}%  (expected: ${(BASELINE_HIT_RATE * 100).toFixed(0)}%)`);
+  console.log(`  Lift:             +${(lift * 100).toFixed(1)}pp`);
+
+  const failed = withRate < HIT_RATE_THRESHOLD;
+
+  if (failed) {
+    console.log(
+      `\n✗ FAIL: hit rate ${(withRate * 100).toFixed(1)}% below threshold ${(HIT_RATE_THRESHOLD * 100).toFixed(0)}%`,
+    );
+  } else {
+    console.log('\n✓ PASS: misconception hit rate within threshold\n');
+  }
+
+  process.exit(failed ? 1 : 0);
+}
+
+main().catch((err) => {
+  console.error('Eval runner crashed:', err);
+  process.exit(1);
+});