Curio/tests/golden/misconception-cases.ts

/**
 * Frozen cases for misconception-hit-rate eval.
 *
 * Each case: learner response exhibits a specific misconception tag.
 * The grader should return that tag (hit) vs "novel"/"none" (miss).
 *
 * Baseline comparison: grader WITHOUT misconception library → measures
 * how many it gets as "novel". With library: should hit the known tag.
 */

export interface MisconceptionCase {
  id: string;
  label: string;
  checkpointPrompt: string;
  referenceAnswer: string;
  rubric: Array<{ id: string; criterion: string; weight: number }>;
  /** The target misconception — what the grader SHOULD return as misconception_tag. */
  targetTag: string;
  /** The full library entry — used to run WITH-library condition. */
  library: Array<{ tag: string; signature: string }>;
  learnerResponse: string;
}

export const MISCONCEPTION_CASES: MisconceptionCase[] = [
  {
    id: 'mc-hit-001',
    label: 'closure-is-object — should tag correctly with library',
    checkpointPrompt: 'In your own words, explain what a closure is.',
    referenceAnswer: 'A closure is a function that retains access to its lexical scope after the outer function returns.',
    rubric: [
      { id: 'r1', criterion: 'Mentions scope retention', weight: 0.6 },
      { id: 'r2', criterion: 'Notes outer function lifetime', weight: 0.4 },
    ],
    targetTag: 'closure-is-object',
    library: [
      {
        tag: 'closure-is-object',
        signature: 'Learner describes a closure as an object, class instance, or data container rather than as a function with scope access',
      },
      {
        tag: 'closure-requires-return',
        signature: 'Learner states that a closure only exists or "activates" when the inner function is explicitly returned from the outer function',
      },
    ],
    learnerResponse:
      'A closure is basically a JavaScript object that wraps a function together with some private state. It is similar to creating an instance of a class.',
  },

  {
    id: 'mc-hit-002',
    label: 'async-timing — should tag correctly with library',
    checkpointPrompt: 'Why does this loop log "3" three times? for (var i=0; i<3; i++) setTimeout(()=>console.log(i), 0)',
    referenceAnswer: 'var is function-scoped; all callbacks share one i. By the time they fire, i is 3. Fix with let.',
    rubric: [
      { id: 'r1', criterion: 'Identifies var scoping as cause', weight: 0.7 },
      { id: 'r2', criterion: 'Notes final value of i', weight: 0.3 },
    ],
    targetTag: 'async-timing',
    library: [
      {
        tag: 'async-timing',
        signature: 'Learner attributes the bug to setTimeout being asynchronous or running after the loop, without mentioning that var scoping is the actual cause',
      },
      {
        tag: 'var-block-scoped',
        signature: 'Learner claims var is block-scoped like let, or confuses var and let scoping behavior',
      },
    ],
    learnerResponse:
      'Because setTimeout runs asynchronously, the callback does not execute until after the loop has already completed. By that time i is already 3, so all three logs show 3.',
  },

  {
    id: 'mc-hit-003',
    label: 'privacy-by-naming — should tag correctly with library',
    checkpointPrompt: 'How do closures enforce data privacy? Give an example.',
    referenceAnswer: 'The private variable lives in the outer function scope; only the returned methods can access it through the closure. No external code can reach it.',
    rubric: [
      { id: 'r1', criterion: 'Variable in outer scope, not accessible externally', weight: 0.6 },
      { id: 'r2', criterion: 'Returned functions are the only access path', weight: 0.4 },
    ],
    targetTag: 'privacy-by-naming',
    library: [
      {
        tag: 'privacy-by-naming',
        signature: 'Learner claims data privacy is achieved by naming convention (prefixing with underscore, using ALL_CAPS) rather than by scoping variables inside a closure',
      },
      {
        tag: 'closure-is-object',
        signature: 'Learner describes a closure as an object with private properties',
      },
    ],
    learnerResponse:
      'You can achieve data privacy in JavaScript by prefixing your variables with an underscore, like _count. This is a convention that tells other developers the variable is private and should not be touched.',
  },

  {
    id: 'mc-hit-004',
    label: 'shared-global — should tag correctly with library',
    checkpointPrompt: 'Why do two calls to makeCounter() produce independent counters?',
    referenceAnswer: 'Each call creates a new execution context with its own count. Returned functions close over their own separate count.',
    rubric: [
      { id: 'r1', criterion: 'Each call creates new scope', weight: 0.5 },
      { id: 'r2', criterion: 'Each closure has own count', weight: 0.5 },
    ],
    targetTag: 'shared-global',
    library: [
      {
        tag: 'shared-global',
        signature: 'Learner believes that because both counters come from the same function definition, they share a single count variable in memory',
      },
    ],
    learnerResponse:
      'Actually if you call makeCounter twice, both counters will share the same count variable because they both reference the same function definition. You would need to store count somewhere unique to avoid this.',
  },
];

/** Fraction of cases where grader returns correct tag. Must beat baseline. */
export const HIT_RATE_THRESHOLD = 0.65;

/** Baseline: expected hit rate without any misconception library (grader returns "novel"). */
export const BASELINE_HIT_RATE = 0.0;