feat: split photo-import into vision recognition + text generation

The photo-import flow used one vision-capable model to both read the photo and structure the full recipe (quantities, steps, timing) in a single call. Split it into two: a vision model recognizes what's in the picture, then a text model reconstructs the recipe from that description — same pattern the pantry photo-scan already uses for recognition, and lets structuring use whichever model is actually configured for text generation. Still counts as one AI-quota unit. v0.37.0
2026-07-17 16:12:39 +02:00
parent 5763bd3318
commit f0340859fa
12 changed files with 157 additions and 59 deletions
@@ -0,0 +1,44 @@
+import { generateObject } from "ai";
+import { z } from "zod";
+import { resolveModel, type AiConfig } from "../factory";
+
+const PhotoRecognitionSchema = z.object({
+  found: z.boolean().describe("false if the image does not show a dish, drink, or recipe (random photo, no visible food)"),
+  recipeType: z.enum(["dish", "drink"]).describe("\"drink\" only for a beverage/cocktail — \"dish\" for anything else, including if unsure."),
+  title: z.string().describe("Best guess at the dish or drink's name"),
+  visibleIngredients: z.array(z.string()).describe("Ingredients visibly identifiable in the photo"),
+  description: z.string().describe("What's visible — presentation, garnish, cooking state, texture, any visible labels/text — detailed enough that someone who never saw the photo could write the full recipe from this alone"),
+});
+
+export type PhotoRecognition = z.infer<typeof PhotoRecognitionSchema>;
+
+/** Step 1 of the photo-import flow: a vision model looks at the photo and
+ * describes what it sees. Deliberately recognition-only — the actual recipe
+ * (quantities, steps, timing) is reconstructed afterwards by a text model in
+ * `generateRecipeFromRecognition`, the same split pantry photo-scan uses
+ * between "what's in this photo" and downstream structuring. */
+export async function recognizePhoto(
+  imageBase64: string,
+  mimeType: "image/jpeg" | "image/png" | "image/webp",
+  config?: AiConfig,
+): Promise<PhotoRecognition> {
+  const model = resolveModel(config);
+
+  const { object } = await generateObject({
+    model,
+    schema: PhotoRecognitionSchema,
+    system:
+      "You are a food-recognition specialist. Look at the image and describe the dish or drink it shows in enough visual detail — ingredients, presentation, cooking state — that someone who never saw the photo could write a full recipe from your description alone. If the image doesn't show food, a drink, or a recipe, set found to false and leave the other fields minimal.",
+    messages: [
+      {
+        role: "user",
+        content: [
+          { type: "image", image: imageBase64, mediaType: mimeType },
+          { type: "text", text: "What dish or drink is shown in this photo?" },
+        ],
+      },
+    ],
+  });
+
+  return object;
+}