import { generateObject } from "ai"; import { z } from "zod"; import { resolveModel, type AiConfig } from "../factory"; const PantryScanSchema = z.object({ items: z.array( z.object({ rawName: z.string(), estimatedQuantity: z.string().optional(), unit: z.string().optional(), }) ).max(30), }); export type PantryScanResult = z.infer; export async function scanPantryPhoto( imageBase64: string, mimeType: "image/jpeg" | "image/png" | "image/webp", config?: AiConfig, ): Promise { const model = resolveModel(config); const { object } = await generateObject({ model, schema: PantryScanSchema, system: "You identify food items visible in a photo of a pantry, fridge, or shelf, or a single unbarcoded item (e.g. fresh produce). " + "List each distinct item once with a short, common ingredient name. Only include an estimated quantity or unit when it can be " + "reasonably judged from the image (e.g. '3' apples, '1' bunch). Do not guess brand names or nutrition facts.", messages: [ { role: "user", content: [ { type: "image", image: imageBase64, mediaType: mimeType }, { type: "text", text: "Identify the food items in this photo." }, ], }, ], }); return object; }