import { generateObject } from "ai"; import { z } from "zod"; import { resolveModel, type AiConfig } from "../factory"; const PhotoRecognitionSchema = z.object({ found: z.boolean().describe("false if the image does not show a dish, drink, or recipe (random photo, no visible food)"), recipeType: z.enum(["dish", "drink"]).describe("\"drink\" only for a beverage/cocktail — \"dish\" for anything else, including if unsure."), title: z.string().describe("Best guess at the dish or drink's name"), visibleIngredients: z.array(z.string()).describe("Ingredients visibly identifiable in the photo"), description: z.string().describe("What's visible — presentation, garnish, cooking state, texture, any visible labels/text — detailed enough that someone who never saw the photo could write the full recipe from this alone"), }); export type PhotoRecognition = z.infer; /** Step 1 of the photo-import flow: a vision model looks at the photo and * describes what it sees. Deliberately recognition-only — the actual recipe * (quantities, steps, timing) is reconstructed afterwards by a text model in * `generateRecipeFromRecognition`, the same split pantry photo-scan uses * between "what's in this photo" and downstream structuring. */ export async function recognizePhoto( imageBase64: string, mimeType: "image/jpeg" | "image/png" | "image/webp", config?: AiConfig, ): Promise { const model = resolveModel(config); const { object } = await generateObject({ model, schema: PhotoRecognitionSchema, system: "You are a food-recognition specialist. Look at the image and describe the dish or drink it shows in enough visual detail — ingredients, presentation, cooking state — that someone who never saw the photo could write a full recipe from your description alone. If the image doesn't show food, a drink, or a recipe, set found to false and leave the other fields minimal.", messages: [ { role: "user", content: [ { type: "image", image: imageBase64, mediaType: mimeType }, { type: "text", text: "What dish or drink is shown in this photo?" }, ], }, ], }); return object; }