feat: initial commit — Curio mastery tutor
Full Next.js App Router application with: - AI-graded lesson checkpoints (BKT/Elo mastery tracking) - Auth.js v5: credentials, Google, GitHub, generic OIDC - Anonymous-session-first with migrate-on-signin - Admin panel: users, blueprints, reports, site settings - Password reset + email verification (nodemailer/SMTP) - Site config: require_auth + signups_enabled flags - server-only guards on all DB/generation/verification modules - PostgreSQL 16 + pgvector, Redis cache, Drizzle ORM - 271 unit tests (Vitest), golden-eval harness, Playwright e2e stubs Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,142 @@
|
||||
/**
|
||||
* Content verification eval — T1 cascade accuracy.
|
||||
*
|
||||
* Tests the T1 verifier against cases where content is known to be good or bad.
|
||||
* Measures false-pass rate (bad content slips through) and false-fail rate
|
||||
* (good content incorrectly rejected).
|
||||
*
|
||||
* CI gate: false-pass rate ≤ 15%, false-fail rate ≤ 15%.
|
||||
*/
|
||||
|
||||
import { generateObject } from 'ai';
|
||||
import { VerifyOutputSchema } from '../../src/schemas/verification';
|
||||
import type { VerifyOutput } from '../../src/schemas/verification';
|
||||
import { prompts } from '../../src/lib/llm/prompts';
|
||||
import { llmClient } from '../../src/lib/llm/client';
|
||||
import { CONTENT_CASES, CONTENT_VERIFY_THRESHOLDS } from './content-cases';
|
||||
import type { ContentCase } from './content-cases';
|
||||
|
||||
// ── Prompt builder (mirrors verify-content.ts T1 logic) ──────────────────────
|
||||
|
||||
function buildT1Prompt(c: ContentCase): string {
|
||||
const sourcesText = c.sourceChunks.map((s) => `[${s.docRef}]\n${s.text}`).join('\n\n---\n\n');
|
||||
|
||||
return `Source chunks:
|
||||
${sourcesText}
|
||||
|
||||
Segment text to verify:
|
||||
"""
|
||||
${c.segmentText}
|
||||
"""
|
||||
|
||||
Checkpoint prompt (blind self-solve — answer from sources only, NOT from segment):
|
||||
${c.checkpointPrompt}
|
||||
|
||||
Fact-check the segment and independently solve the checkpoint.`;
|
||||
}
|
||||
|
||||
// ── Single verification call ──────────────────────────────────────────────────
|
||||
|
||||
async function verify(
|
||||
c: ContentCase,
|
||||
): Promise<{ output: VerifyOutput | null; error?: string; latencyMs: number }> {
|
||||
const start = Date.now();
|
||||
try {
|
||||
const { object } = await generateObject({
|
||||
model: llmClient.grader,
|
||||
schema: VerifyOutputSchema,
|
||||
system: prompts.VERIFY_CONTENT_T1.template,
|
||||
prompt: buildT1Prompt(c),
|
||||
});
|
||||
return { output: object, latencyMs: Date.now() - start };
|
||||
} catch (err) {
|
||||
return {
|
||||
output: null,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
latencyMs: Date.now() - start,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ── Runner ────────────────────────────────────────────────────────────────────
|
||||
|
||||
async function main(): Promise<void> {
|
||||
console.log(`\nCurio — content verification eval (${CONTENT_CASES.length} cases)\n`);
|
||||
console.log(` Verifier: ${process.env.LLM_GRADER_PROVIDER ?? 'openai'} / ${process.env.LLM_GRADER_MODEL ?? 'gpt-4o-mini'} (T1)`);
|
||||
console.log(
|
||||
` Thresholds: false-pass ≤ ${(CONTENT_VERIFY_THRESHOLDS.maxFalsePassRate * 100).toFixed(0)}%,` +
|
||||
` false-fail ≤ ${(CONTENT_VERIFY_THRESHOLDS.maxFalseFailRate * 100).toFixed(0)}%\n`,
|
||||
);
|
||||
|
||||
const results: Array<{
|
||||
case: ContentCase;
|
||||
output: VerifyOutput | null;
|
||||
error?: string;
|
||||
isFalsePass: boolean;
|
||||
isFalseFail: boolean;
|
||||
latencyMs: number;
|
||||
}> = [];
|
||||
|
||||
for (const c of CONTENT_CASES) {
|
||||
process.stdout.write(` [${c.id}] "${c.label}"…\r`);
|
||||
const { output, error, latencyMs } = await verify(c);
|
||||
|
||||
const verdict = output?.verdict ?? null;
|
||||
const isFalsePass = c.isBadContent && verdict === 'pass';
|
||||
const isFalseFail = c.isGoodContent && verdict === 'fail';
|
||||
|
||||
results.push({ case: c, output, error, isFalsePass, isFalseFail, latencyMs });
|
||||
|
||||
const flag = error ? ' ERR ' : isFalsePass ? '✗ FP ' : isFalseFail ? '✗ FF ' : '✓ ';
|
||||
const ms = `${latencyMs}ms`.padStart(6);
|
||||
console.log(
|
||||
` [${c.id}] ${flag} verdict=${verdict ?? 'null'} expected=${c.expectedT1Verdict} ${ms} "${c.label}"`,
|
||||
);
|
||||
|
||||
if (error) console.log(` Error: ${error}`);
|
||||
if (output && output.claims.some((cl) => cl.status !== 'supported')) {
|
||||
const unsupported = output.claims.filter((cl) => cl.status !== 'supported');
|
||||
console.log(` Unsupported claims: ${unsupported.map((cl) => `"${cl.text.slice(0, 60)}…"`).join(', ')}`);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Metrics ──────────────────────────────────────────────────────────────────
|
||||
|
||||
const ran = results.filter((r) => !r.error);
|
||||
const errors = results.filter((r) => r.error);
|
||||
const badCases = ran.filter((r) => r.case.isBadContent);
|
||||
const goodCases = ran.filter((r) => r.case.isGoodContent);
|
||||
const falsePasses = ran.filter((r) => r.isFalsePass);
|
||||
const falseFails = ran.filter((r) => r.isFalseFail);
|
||||
|
||||
const falsePassRate = badCases.length > 0 ? falsePasses.length / badCases.length : 0;
|
||||
const falseFailRate = goodCases.length > 0 ? falseFails.length / goodCases.length : 0;
|
||||
|
||||
console.log('\n── Results ─────────────────────────────────────────────────────────────');
|
||||
console.log(` Cases run: ${ran.length} / ${CONTENT_CASES.length} (${errors.length} errors)`);
|
||||
console.log(` False-pass: ${falsePasses.length}/${badCases.length} = ${(falsePassRate * 100).toFixed(1)}% (bad content passing — max ${(CONTENT_VERIFY_THRESHOLDS.maxFalsePassRate * 100).toFixed(0)}%)`);
|
||||
console.log(` False-fail: ${falseFails.length}/${goodCases.length} = ${(falseFailRate * 100).toFixed(1)}% (good content rejected — max ${(CONTENT_VERIFY_THRESHOLDS.maxFalseFailRate * 100).toFixed(0)}%)`);
|
||||
|
||||
let failed = false;
|
||||
|
||||
if (falsePassRate > CONTENT_VERIFY_THRESHOLDS.maxFalsePassRate) {
|
||||
console.log(`\n✗ FAIL: false-pass rate ${(falsePassRate * 100).toFixed(1)}% exceeds threshold`);
|
||||
failed = true;
|
||||
}
|
||||
|
||||
if (falseFailRate > CONTENT_VERIFY_THRESHOLDS.maxFalseFailRate) {
|
||||
console.log(`\n✗ FAIL: false-fail rate ${(falseFailRate * 100).toFixed(1)}% exceeds threshold`);
|
||||
failed = true;
|
||||
}
|
||||
|
||||
if (!failed) {
|
||||
console.log('\n✓ PASS: content verification quality within thresholds\n');
|
||||
}
|
||||
|
||||
process.exit(failed ? 1 : 0);
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error('Eval runner crashed:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user