feat: split photo-import into vision recognition + text generation

The photo-import flow used one vision-capable model to both read the
photo and structure the full recipe (quantities, steps, timing) in a
single call. Split it into two: a vision model recognizes what's in
the picture, then a text model reconstructs the recipe from that
description — same pattern the pantry photo-scan already uses for
recognition, and lets structuring use whichever model is actually
configured for text generation. Still counts as one AI-quota unit.

v0.37.0

CHANGELOG.md

@@ -2,6 +2,11 @@
 
 All notable changes to Epicure are documented here. This file is mirrored in-app at `/changelog` (and in the admin dashboard) via `apps/web/lib/changelog.ts` — update both together.
 
+## 0.37.0 — 2026-07-17 12:30
+
+### Fixed
+- Generating a recipe from a photo now uses two separate AI steps — a vision model recognizes what's in the picture, then a text model writes the recipe from that — instead of one model doing both, so each step can use the model actually suited to it.
+
 ## 0.36.0 — 2026-07-17 12:00
 
 ### Added

apps/web/app/api/v1/ai/import-photo/route.ts

@@ -28,19 +28,23 @@ export async function POST(req: NextRequest) {
   const userId = session!.user.id;
   const locale = (session!.user as { locale?: string }).locale ?? "en";
 
-  const configResult = await resolveAiConfigOrError(() => getModelConfigForUseCase(userId, "vision"));
-  if (!configResult.ok) return configResult.response;
-  const aiConfig = configResult.data;
+  const visionConfigResult = await resolveAiConfigOrError(() => getModelConfigForUseCase(userId, "vision"));
+  if (!visionConfigResult.ok) return visionConfigResult.response;
+  const visionConfig = visionConfigResult.data;
 
   // Fall back to vision-capable defaults if no explicit model configured
-  if (!aiConfig.model) {
-    if (aiConfig.provider === "openai") aiConfig.model = "gpt-4o";
-    else if (aiConfig.provider === "anthropic") aiConfig.model = "claude-sonnet-4-6";
+  if (!visionConfig.model) {
+    if (visionConfig.provider === "openai") visionConfig.model = "gpt-4o";
+    else if (visionConfig.provider === "anthropic") visionConfig.model = "claude-sonnet-4-6";
   }
 
+  const textConfigResult = await resolveAiConfigOrError(() => getModelConfigForUseCase(userId, "text"));
+  if (!textConfigResult.ok) return textConfigResult.response;
+  const textConfig = textConfigResult.data;
+
   const result = await withAiQuota(userId, session!.user.tier as "free" | "pro", () =>
-    importFromPhoto(parsed.data.imageBase64, parsed.data.mimeType, aiConfig, locale),
-    { skipQuota: aiConfig.isByok }
+    importFromPhoto(parsed.data.imageBase64, parsed.data.mimeType, visionConfig, textConfig, locale),
+    { skipQuota: visionConfig.isByok && textConfig.isByok }
   );
   if (!result.ok) return result.response;
   const recipe = result.data;

apps/web/components/recipe/photo-import-button.tsx

@@ -13,6 +13,8 @@ export function PhotoImportButton() {
   const router = useRouter();
   const fileRef = useRef<HTMLInputElement>(null);
   const [loading, setLoading] = useState(false);
+  const [stage, setStage] = useState<"recognizing" | "generating">("recognizing");
+  const stageTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
 
   function handleClick() {
     fileRef.current?.click();
@@ -23,6 +25,12 @@ export function PhotoImportButton() {
     if (!file) return;
 
     setLoading(true);
+    setStage("recognizing");
+    // The photo-import flow is two sequential AI calls (vision recognition,
+    // then text generation) behind a single request/response — there's no
+    // real progress signal to react to, so this just approximates the
+    // handoff point for the loading copy.
+    stageTimerRef.current = setTimeout(() => setStage("generating"), 5000);
 
     try {
       const base64 = await new Promise<string>((resolve, reject) => {
@@ -57,6 +65,7 @@ export function PhotoImportButton() {
     } catch (err) {
       toast.error(err instanceof Error ? err.message : t("photoImportFailed"));
     } finally {
+      if (stageTimerRef.current) clearTimeout(stageTimerRef.current);
       setLoading(false);
       // Reset input so the same file can be re-selected
       if (fileRef.current) fileRef.current.value = "";
@@ -80,7 +89,11 @@ export function PhotoImportButton() {
         )}
         {t("importFromPhoto")}
       </Button>
-      <FakeProgressBar active={loading} durationMs={12000} label={loading ? t("analyzingPhoto") : undefined} />
+      <FakeProgressBar
+        active={loading}
+        durationMs={12000}
+        label={loading ? (stage === "recognizing" ? t("recognizingPhoto") : t("writingRecipe")) : undefined}
+      />
     </div>
   );
 }

apps/web/lib/ai/features/generate-recipe-from-recognition.ts

@@ -0,0 +1,47 @@
+import { generateObject } from "ai";
+import { z } from "zod";
+import { resolveModel, type AiConfig } from "../factory";
+import { dietaryTagsSchema, ingredientSchema, stepSchema } from "./recipe-schema";
+import type { PhotoRecognition } from "./recognize-photo";
+
+const RecipeOutputSchema = z.object({
+  title: z.string(),
+  description: z.string().optional(),
+  baseServings: z.number().int().min(1).max(100),
+  prepMins: z.number().int().min(0).optional(),
+  cookMins: z.number().int().min(0).optional(),
+  difficulty: z.enum(["easy", "medium", "hard"]),
+  dietaryTags: dietaryTagsSchema,
+  ingredients: z.array(ingredientSchema(z.number())),
+  steps: z.array(stepSchema),
+});
+
+export type GeneratedRecipeFromRecognition = z.infer<typeof RecipeOutputSchema>;
+
+/** Step 2 of the photo-import flow: a text model reconstructs the full recipe
+ * (quantities, steps, timing) from what the vision model recognized in
+ * `recognizePhoto` — it never sees the photo itself. */
+export async function generateRecipeFromRecognition(
+  recognition: PhotoRecognition,
+  config?: AiConfig & { language?: string },
+): Promise<GeneratedRecipeFromRecognition> {
+  const model = resolveModel(config);
+  const lang = config?.language ?? "en";
+  const langInstruction = lang !== "en" ? ` Write the entire recipe (title, description, ingredient names, step instructions) in ${lang}.` : "";
+  const drinkInstruction = recognition.recipeType === "drink"
+    ? " This is a drink: never set cookMins, and default baseServings to 1 unless the description implies more than one serving."
+    : "";
+
+  const { object } = await generateObject({
+    model,
+    schema: RecipeOutputSchema,
+    system:
+      `You are a professional chef and culinary writer. A vision model has already identified what's in a photo — reconstruct the complete, precise recipe implied by that description. For ingredients: quantity must be a number only (e.g. 0.25, 1.5, 2), unit is a separate string. Never combine quantity and unit into one string.${drinkInstruction}${langInstruction}`,
+    prompt: `Photo shows: "${recognition.title}"\nVisible ingredients: ${recognition.visibleIngredients.join(", ") || "none clearly visible"}\nDescription: ${recognition.description}\n\nWrite the complete recipe for this ${recognition.recipeType}.`,
+  });
+
+  if (recognition.recipeType === "drink") {
+    return { ...object, cookMins: undefined };
+  }
+  return object;
+}

apps/web/lib/ai/features/import-photo.ts

@@ -1,54 +1,30 @@
-import { generateObject } from "ai";
-import { z } from "zod";
-import { resolveModel, type AiConfig } from "../factory";
-import { dietaryTagsSchema, ingredientSchema, stepSchema } from "./recipe-schema";
+import type { AiConfig } from "../factory";
+import { recognizePhoto } from "./recognize-photo";
+import { generateRecipeFromRecognition, type GeneratedRecipeFromRecognition } from "./generate-recipe-from-recognition";
 
-const ImportedRecipeSchema = z.object({
-  found: z.boolean().describe("false if the image does not contain a recognizable recipe (e.g. random photo, no visible ingredients/instructions)"),
-  recipeType: z.enum(["dish", "drink"]).describe("\"drink\" only for a beverage/cocktail (no cooking step) — \"dish\" for anything else, including if unsure."),
-  title: z.string(),
-  description: z.string().optional(),
-  baseServings: z.number().int().min(1).optional(),
-  prepMins: z.number().int().min(0).optional(),
-  cookMins: z.number().int().min(0).optional(),
-  difficulty: z.enum(["easy", "medium", "hard"]).optional(),
-  dietaryTags: dietaryTagsSchema.optional(),
-  ingredients: z.array(ingredientSchema(z.string())),
-  steps: z.array(stepSchema),
-});
-
-export type ImportedRecipe = z.infer<typeof ImportedRecipeSchema>;
-
-const LANG: Record<string, string> = { en: "English", fr: "French" };
+export type ImportedRecipe =
+  | { found: false; recipeType: "dish" | "drink"; title: string; ingredients: []; steps: [] }
+  | ({ found: true; recipeType: "dish" | "drink" } & GeneratedRecipeFromRecognition);
 
+/** Orchestrates the two-step photo-import flow: a vision model recognizes
+ * what's in the photo (`recognizePhoto`), then a text model reconstructs the
+ * full structured recipe from that recognition (`generateRecipeFromRecognition`).
+ * Splitting the call lets each step use the model best suited to it — vision
+ * for the photo, text for structuring — instead of one model doing both. */
 export async function importFromPhoto(
   imageBase64: string,
   mimeType: "image/jpeg" | "image/png" | "image/webp",
-  config?: AiConfig,
+  visionConfig: AiConfig | undefined,
+  textConfig: AiConfig | undefined,
   locale?: string,
 ): Promise<ImportedRecipe> {
-  const model = resolveModel(config);
-  const lang = LANG[locale ?? "en"] ?? "English";
-  const langInstruction = lang !== "English" ? ` Write the entire recipe (title, description, ingredient names, step instructions) in ${lang}, regardless of the language used in the image.` : "";
+  const recognition = await recognizePhoto(imageBase64, mimeType, visionConfig);
 
-  const { object } = await generateObject({
-    model,
-    schema: ImportedRecipeSchema,
-    system:
-      `You are a recipe extraction specialist. Extract the complete recipe from the provided image. Be precise with ingredient quantities and cooking instructions. If the image does not contain a recognizable recipe (no visible ingredients or cooking instructions), set found to false and leave the other fields minimal/empty. Set recipeType to "drink" only for a beverage/cocktail with no cooking involved — never set cookMins for a drink.${langInstruction}`,
-    messages: [
-      {
-        role: "user",
-        content: [
-          { type: "image", image: imageBase64, mediaType: mimeType },
-          { type: "text", text: "Extract the complete recipe from this image." },
-        ],
-      },
-    ],
-  });
-
-  if (object.recipeType === "drink") {
-    return { ...object, cookMins: undefined };
+  if (!recognition.found) {
+    return { found: false, recipeType: recognition.recipeType, title: recognition.title, ingredients: [], steps: [] };
   }
-  return object;
+
+  const generated = await generateRecipeFromRecognition(recognition, { ...textConfig, language: locale ?? "en" });
+
+  return { found: true, recipeType: recognition.recipeType, ...generated };
 }

apps/web/lib/ai/features/recognize-photo.ts

@@ -0,0 +1,44 @@
+import { generateObject } from "ai";
+import { z } from "zod";
+import { resolveModel, type AiConfig } from "../factory";
+
+const PhotoRecognitionSchema = z.object({
+  found: z.boolean().describe("false if the image does not show a dish, drink, or recipe (random photo, no visible food)"),
+  recipeType: z.enum(["dish", "drink"]).describe("\"drink\" only for a beverage/cocktail — \"dish\" for anything else, including if unsure."),
+  title: z.string().describe("Best guess at the dish or drink's name"),
+  visibleIngredients: z.array(z.string()).describe("Ingredients visibly identifiable in the photo"),
+  description: z.string().describe("What's visible — presentation, garnish, cooking state, texture, any visible labels/text — detailed enough that someone who never saw the photo could write the full recipe from this alone"),
+});
+
+export type PhotoRecognition = z.infer<typeof PhotoRecognitionSchema>;
+
+/** Step 1 of the photo-import flow: a vision model looks at the photo and
+ * describes what it sees. Deliberately recognition-only — the actual recipe
+ * (quantities, steps, timing) is reconstructed afterwards by a text model in
+ * `generateRecipeFromRecognition`, the same split pantry photo-scan uses
+ * between "what's in this photo" and downstream structuring. */
+export async function recognizePhoto(
+  imageBase64: string,
+  mimeType: "image/jpeg" | "image/png" | "image/webp",
+  config?: AiConfig,
+): Promise<PhotoRecognition> {
+  const model = resolveModel(config);
+
+  const { object } = await generateObject({
+    model,
+    schema: PhotoRecognitionSchema,
+    system:
+      "You are a food-recognition specialist. Look at the image and describe the dish or drink it shows in enough visual detail — ingredients, presentation, cooking state — that someone who never saw the photo could write a full recipe from your description alone. If the image doesn't show food, a drink, or a recipe, set found to false and leave the other fields minimal.",
+    messages: [
+      {
+        role: "user",
+        content: [
+          { type: "image", image: imageBase64, mediaType: mimeType },
+          { type: "text", text: "What dish or drink is shown in this photo?" },
+        ],
+      },
+    ],
+  });
+
+  return object;
+}

apps/web/lib/changelog.ts

@@ -1,5 +1,5 @@
 // Mirrors CHANGELOG.md at the repo root — update both together.
-export const APP_VERSION = "0.36.0";
+export const APP_VERSION = "0.37.0";
 
 export type ChangelogEntry = {
   version: string;
@@ -11,6 +11,13 @@ export type ChangelogEntry = {
 };
 
 export const CHANGELOG: ChangelogEntry[] = [
+  {
+    version: "0.37.0",
+    date: "2026-07-17 12:30",
+    fixed: [
+      "Generating a recipe from a photo now uses two separate AI steps — a vision model recognizes what's in the picture, then a text model writes the recipe from that — instead of one model doing both, so each step can use the model actually suited to it.",
+    ],
+  },
   {
     version: "0.36.0",
     date: "2026-07-17 12:00",

apps/web/lib/openapi.ts

@@ -301,7 +301,7 @@ export function generateOpenApiSpec(): object {
   }));
 
   registry.registerPath({ method: "post", path: "/api/v1/ai/generate-from-idea", summary: "Generate a full recipe from a short title/idea", description: "Rate-limited: 10 req/min. Consumes AI quota.", security, request: { body: { content: { "application/json": { schema: z.object({ title: z.string().min(1).max(200), provider: z.enum(["openai", "anthropic", "openrouter", "ollama"]).optional(), model: z.string().optional() }) } }, required: true } }, responses: { 200: { description: "Created recipe id", content: { "application/json": { schema: z.object({ id: z.string() }) } } }, 400: { description: "Validation error", content: { "application/json": { schema: ApiErrorRef } } }, 429: { description: "Rate limited or AI quota exhausted", content: { "application/json": { schema: ApiErrorRef } } } } });
-  registry.registerPath({ method: "post", path: "/api/v1/ai/import-photo", summary: "Import a recipe from a photo using AI vision", description: "Rate-limited: 5 req/min. Consumes AI quota. Written in the caller's app language, regardless of the language shown in the photo.", security, request: { body: { content: { "application/json": { schema: z.object({ imageBase64: z.string().max(14_000_000), mimeType: z.enum(["image/jpeg", "image/png", "image/webp"]) }) } }, required: true } }, responses: { 200: { description: "Created recipe id", content: { "application/json": { schema: z.object({ id: z.string() }) } } }, 400: { description: "Validation error", content: { "application/json": { schema: ApiErrorRef } } }, 422: { description: "No recipe recognized in the photo", content: { "application/json": { schema: ApiErrorRef } } }, 429: { description: "Rate limited or AI quota exhausted", content: { "application/json": { schema: ApiErrorRef } } } } });
+  registry.registerPath({ method: "post", path: "/api/v1/ai/import-photo", summary: "Import a recipe from a photo using AI vision", description: "Rate-limited: 5 req/min. Consumes AI quota. Two AI calls internally — a vision model recognizes the photo, then a text model writes the structured recipe — but only counts as one quota unit. Written in the caller's app language, regardless of the language shown in the photo.", security, request: { body: { content: { "application/json": { schema: z.object({ imageBase64: z.string().max(14_000_000), mimeType: z.enum(["image/jpeg", "image/png", "image/webp"]) }) } }, required: true } }, responses: { 200: { description: "Created recipe id", content: { "application/json": { schema: z.object({ id: z.string() }) } } }, 400: { description: "Validation error", content: { "application/json": { schema: ApiErrorRef } } }, 422: { description: "No recipe recognized in the photo", content: { "application/json": { schema: ApiErrorRef } } }, 429: { description: "Rate limited or AI quota exhausted", content: { "application/json": { schema: ApiErrorRef } } } } });
   registry.registerPath({ method: "post", path: "/api/v1/ai/substitute", summary: "Suggest ingredient substitutions", description: "Rate-limited: 10 req/min. Consumes AI quota.", security, request: { body: { content: { "application/json": { schema: z.object({ ingredient: z.string().min(1).max(200), recipeTitle: z.string().max(200).optional(), provider: z.enum(["openai", "anthropic", "openrouter", "ollama"]).optional(), model: z.string().optional() }) } }, required: true } }, responses: { 200: { description: "Substitutions", content: { "application/json": { schema: z.object({ substitutions: z.array(z.record(z.string(), z.unknown())) }) } } }, 400: { description: "Validation error", content: { "application/json": { schema: ApiErrorRef } } }, 429: { description: "Rate limited or AI quota exhausted", content: { "application/json": { schema: ApiErrorRef } } } } });
   registry.registerPath({ method: "post", path: "/api/v1/ai/scale", summary: "Scale a recipe's ingredients to a target serving count", description: "Rate-limited: 20 req/min. Consumes AI quota.", security, request: { body: { content: { "application/json": { schema: z.object({ recipeId: z.string(), targetServings: z.number().int().min(1).max(100) }) } }, required: true } }, responses: { 200: { description: "Scaled ingredients", content: { "application/json": { schema: z.object({ ingredients: z.array(z.record(z.string(), z.unknown())) }) } } }, 400: { description: "Validation error", content: { "application/json": { schema: ApiErrorRef } } }, 404: { description: "Not found", content: { "application/json": { schema: ApiErrorRef } } }, 429: { description: "Rate limited or AI quota exhausted", content: { "application/json": { schema: ApiErrorRef } } } } });
   registry.registerPath({ method: "post", path: "/api/v1/ai/adapt/{id}", summary: "Adapt a recipe to exclude ingredients or meet extra constraints", description: "Consumes AI quota. Creates a new private draft recipe.", security, request: { params: idParam, body: { content: { "application/json": { schema: z.object({ excludeIngredients: z.array(z.string()).default([]), extraConstraints: z.string().max(500).optional(), provider: z.enum(["openai", "anthropic", "openrouter", "ollama"]).optional(), model: z.string().optional() }) } }, required: true } }, responses: { 200: { description: "New recipe id and adaptation notes", content: { "application/json": { schema: z.object({ id: z.string(), adaptationNotes: z.string().optional() }) } } }, 400: { description: "Validation error or no constraint provided", content: { "application/json": { schema: ApiErrorRef } } }, 404: { description: "Not found", content: { "application/json": { schema: ApiErrorRef } } }, 429: { description: "AI quota exhausted", content: { "application/json": { schema: ApiErrorRef } } } } });

apps/web/messages/en.json

@@ -187,7 +187,8 @@
     "versionRestored": "Recipe restored to version {version}",
     "versionDetailLoading": "Loading...",
     "versionDetailSummary": "{ingredients, plural, one {1 ingredient} other {{ingredients} ingredients}}, {steps, plural, one {1 step} other {{steps} steps}}",
-    "analyzingPhoto": "Analyzing photo…",
+    "recognizingPhoto": "Recognizing what's in the photo…",
+    "writingRecipe": "Writing the recipe…",
     "pairMealTooltip": "Pair meal",
     "historyTooltip": "History",
     "translateTooltip": "Translate",

apps/web/messages/fr.json

@@ -187,7 +187,8 @@
     "versionRestored": "Recette restaurée à la version {version}",
     "versionDetailLoading": "Chargement...",
     "versionDetailSummary": "{ingredients, plural, one {1 ingrédient} other {{ingredients} ingrédients}}, {steps, plural, one {1 étape} other {{steps} étapes}}",
-    "analyzingPhoto": "Analyse de la photo…",
+    "recognizingPhoto": "Identification du contenu de la photo…",
+    "writingRecipe": "Rédaction de la recette…",
     "pairMealTooltip": "Accorder un plat",
     "historyTooltip": "Historique",
     "translateTooltip": "Traduire",

apps/web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@epicure/web",
-  "version": "0.36.0",
+  "version": "0.37.0",
   "private": true,
   "scripts": {
     "dev": "next dev",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "epicure",
-  "version": "0.36.0",
+  "version": "0.37.0",
   "private": true,
   "scripts": {
     "dev": "pnpm --filter web dev",