diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c6028b..c4d57d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ All notable changes to Epicure are documented here. This file is mirrored in-app at `/changelog` (and in the admin dashboard) via `apps/web/lib/changelog.ts` — update both together. +## 0.37.0 — 2026-07-17 12:30 + +### Fixed +- Generating a recipe from a photo now uses two separate AI steps — a vision model recognizes what's in the picture, then a text model writes the recipe from that — instead of one model doing both, so each step can use the model actually suited to it. + ## 0.36.0 — 2026-07-17 12:00 ### Added diff --git a/apps/web/app/api/v1/ai/import-photo/route.ts b/apps/web/app/api/v1/ai/import-photo/route.ts index 60f07f1..1591137 100644 --- a/apps/web/app/api/v1/ai/import-photo/route.ts +++ b/apps/web/app/api/v1/ai/import-photo/route.ts @@ -28,19 +28,23 @@ export async function POST(req: NextRequest) { const userId = session!.user.id; const locale = (session!.user as { locale?: string }).locale ?? "en"; - const configResult = await resolveAiConfigOrError(() => getModelConfigForUseCase(userId, "vision")); - if (!configResult.ok) return configResult.response; - const aiConfig = configResult.data; + const visionConfigResult = await resolveAiConfigOrError(() => getModelConfigForUseCase(userId, "vision")); + if (!visionConfigResult.ok) return visionConfigResult.response; + const visionConfig = visionConfigResult.data; // Fall back to vision-capable defaults if no explicit model configured - if (!aiConfig.model) { - if (aiConfig.provider === "openai") aiConfig.model = "gpt-4o"; - else if (aiConfig.provider === "anthropic") aiConfig.model = "claude-sonnet-4-6"; + if (!visionConfig.model) { + if (visionConfig.provider === "openai") visionConfig.model = "gpt-4o"; + else if (visionConfig.provider === "anthropic") visionConfig.model = "claude-sonnet-4-6"; } + const textConfigResult = await resolveAiConfigOrError(() => getModelConfigForUseCase(userId, "text")); + if (!textConfigResult.ok) return textConfigResult.response; + const textConfig = textConfigResult.data; + const result = await withAiQuota(userId, session!.user.tier as "free" | "pro", () => - importFromPhoto(parsed.data.imageBase64, parsed.data.mimeType, aiConfig, locale), - { skipQuota: aiConfig.isByok } + importFromPhoto(parsed.data.imageBase64, parsed.data.mimeType, visionConfig, textConfig, locale), + { skipQuota: visionConfig.isByok && textConfig.isByok } ); if (!result.ok) return result.response; const recipe = result.data; diff --git a/apps/web/components/recipe/photo-import-button.tsx b/apps/web/components/recipe/photo-import-button.tsx index d3db79f..bd99059 100644 --- a/apps/web/components/recipe/photo-import-button.tsx +++ b/apps/web/components/recipe/photo-import-button.tsx @@ -13,6 +13,8 @@ export function PhotoImportButton() { const router = useRouter(); const fileRef = useRef(null); const [loading, setLoading] = useState(false); + const [stage, setStage] = useState<"recognizing" | "generating">("recognizing"); + const stageTimerRef = useRef | null>(null); function handleClick() { fileRef.current?.click(); @@ -23,6 +25,12 @@ export function PhotoImportButton() { if (!file) return; setLoading(true); + setStage("recognizing"); + // The photo-import flow is two sequential AI calls (vision recognition, + // then text generation) behind a single request/response — there's no + // real progress signal to react to, so this just approximates the + // handoff point for the loading copy. + stageTimerRef.current = setTimeout(() => setStage("generating"), 5000); try { const base64 = await new Promise((resolve, reject) => { @@ -57,6 +65,7 @@ export function PhotoImportButton() { } catch (err) { toast.error(err instanceof Error ? err.message : t("photoImportFailed")); } finally { + if (stageTimerRef.current) clearTimeout(stageTimerRef.current); setLoading(false); // Reset input so the same file can be re-selected if (fileRef.current) fileRef.current.value = ""; @@ -80,7 +89,11 @@ export function PhotoImportButton() { )} {t("importFromPhoto")} - + ); } diff --git a/apps/web/lib/ai/features/generate-recipe-from-recognition.ts b/apps/web/lib/ai/features/generate-recipe-from-recognition.ts new file mode 100644 index 0000000..5fa555f --- /dev/null +++ b/apps/web/lib/ai/features/generate-recipe-from-recognition.ts @@ -0,0 +1,47 @@ +import { generateObject } from "ai"; +import { z } from "zod"; +import { resolveModel, type AiConfig } from "../factory"; +import { dietaryTagsSchema, ingredientSchema, stepSchema } from "./recipe-schema"; +import type { PhotoRecognition } from "./recognize-photo"; + +const RecipeOutputSchema = z.object({ + title: z.string(), + description: z.string().optional(), + baseServings: z.number().int().min(1).max(100), + prepMins: z.number().int().min(0).optional(), + cookMins: z.number().int().min(0).optional(), + difficulty: z.enum(["easy", "medium", "hard"]), + dietaryTags: dietaryTagsSchema, + ingredients: z.array(ingredientSchema(z.number())), + steps: z.array(stepSchema), +}); + +export type GeneratedRecipeFromRecognition = z.infer; + +/** Step 2 of the photo-import flow: a text model reconstructs the full recipe + * (quantities, steps, timing) from what the vision model recognized in + * `recognizePhoto` — it never sees the photo itself. */ +export async function generateRecipeFromRecognition( + recognition: PhotoRecognition, + config?: AiConfig & { language?: string }, +): Promise { + const model = resolveModel(config); + const lang = config?.language ?? "en"; + const langInstruction = lang !== "en" ? ` Write the entire recipe (title, description, ingredient names, step instructions) in ${lang}.` : ""; + const drinkInstruction = recognition.recipeType === "drink" + ? " This is a drink: never set cookMins, and default baseServings to 1 unless the description implies more than one serving." + : ""; + + const { object } = await generateObject({ + model, + schema: RecipeOutputSchema, + system: + `You are a professional chef and culinary writer. A vision model has already identified what's in a photo — reconstruct the complete, precise recipe implied by that description. For ingredients: quantity must be a number only (e.g. 0.25, 1.5, 2), unit is a separate string. Never combine quantity and unit into one string.${drinkInstruction}${langInstruction}`, + prompt: `Photo shows: "${recognition.title}"\nVisible ingredients: ${recognition.visibleIngredients.join(", ") || "none clearly visible"}\nDescription: ${recognition.description}\n\nWrite the complete recipe for this ${recognition.recipeType}.`, + }); + + if (recognition.recipeType === "drink") { + return { ...object, cookMins: undefined }; + } + return object; +} diff --git a/apps/web/lib/ai/features/import-photo.ts b/apps/web/lib/ai/features/import-photo.ts index f28c55c..e47f38e 100644 --- a/apps/web/lib/ai/features/import-photo.ts +++ b/apps/web/lib/ai/features/import-photo.ts @@ -1,54 +1,30 @@ -import { generateObject } from "ai"; -import { z } from "zod"; -import { resolveModel, type AiConfig } from "../factory"; -import { dietaryTagsSchema, ingredientSchema, stepSchema } from "./recipe-schema"; +import type { AiConfig } from "../factory"; +import { recognizePhoto } from "./recognize-photo"; +import { generateRecipeFromRecognition, type GeneratedRecipeFromRecognition } from "./generate-recipe-from-recognition"; -const ImportedRecipeSchema = z.object({ - found: z.boolean().describe("false if the image does not contain a recognizable recipe (e.g. random photo, no visible ingredients/instructions)"), - recipeType: z.enum(["dish", "drink"]).describe("\"drink\" only for a beverage/cocktail (no cooking step) — \"dish\" for anything else, including if unsure."), - title: z.string(), - description: z.string().optional(), - baseServings: z.number().int().min(1).optional(), - prepMins: z.number().int().min(0).optional(), - cookMins: z.number().int().min(0).optional(), - difficulty: z.enum(["easy", "medium", "hard"]).optional(), - dietaryTags: dietaryTagsSchema.optional(), - ingredients: z.array(ingredientSchema(z.string())), - steps: z.array(stepSchema), -}); - -export type ImportedRecipe = z.infer; - -const LANG: Record = { en: "English", fr: "French" }; +export type ImportedRecipe = + | { found: false; recipeType: "dish" | "drink"; title: string; ingredients: []; steps: [] } + | ({ found: true; recipeType: "dish" | "drink" } & GeneratedRecipeFromRecognition); +/** Orchestrates the two-step photo-import flow: a vision model recognizes + * what's in the photo (`recognizePhoto`), then a text model reconstructs the + * full structured recipe from that recognition (`generateRecipeFromRecognition`). + * Splitting the call lets each step use the model best suited to it — vision + * for the photo, text for structuring — instead of one model doing both. */ export async function importFromPhoto( imageBase64: string, mimeType: "image/jpeg" | "image/png" | "image/webp", - config?: AiConfig, + visionConfig: AiConfig | undefined, + textConfig: AiConfig | undefined, locale?: string, ): Promise { - const model = resolveModel(config); - const lang = LANG[locale ?? "en"] ?? "English"; - const langInstruction = lang !== "English" ? ` Write the entire recipe (title, description, ingredient names, step instructions) in ${lang}, regardless of the language used in the image.` : ""; + const recognition = await recognizePhoto(imageBase64, mimeType, visionConfig); - const { object } = await generateObject({ - model, - schema: ImportedRecipeSchema, - system: - `You are a recipe extraction specialist. Extract the complete recipe from the provided image. Be precise with ingredient quantities and cooking instructions. If the image does not contain a recognizable recipe (no visible ingredients or cooking instructions), set found to false and leave the other fields minimal/empty. Set recipeType to "drink" only for a beverage/cocktail with no cooking involved — never set cookMins for a drink.${langInstruction}`, - messages: [ - { - role: "user", - content: [ - { type: "image", image: imageBase64, mediaType: mimeType }, - { type: "text", text: "Extract the complete recipe from this image." }, - ], - }, - ], - }); - - if (object.recipeType === "drink") { - return { ...object, cookMins: undefined }; + if (!recognition.found) { + return { found: false, recipeType: recognition.recipeType, title: recognition.title, ingredients: [], steps: [] }; } - return object; + + const generated = await generateRecipeFromRecognition(recognition, { ...textConfig, language: locale ?? "en" }); + + return { found: true, recipeType: recognition.recipeType, ...generated }; } diff --git a/apps/web/lib/ai/features/recognize-photo.ts b/apps/web/lib/ai/features/recognize-photo.ts new file mode 100644 index 0000000..3207bad --- /dev/null +++ b/apps/web/lib/ai/features/recognize-photo.ts @@ -0,0 +1,44 @@ +import { generateObject } from "ai"; +import { z } from "zod"; +import { resolveModel, type AiConfig } from "../factory"; + +const PhotoRecognitionSchema = z.object({ + found: z.boolean().describe("false if the image does not show a dish, drink, or recipe (random photo, no visible food)"), + recipeType: z.enum(["dish", "drink"]).describe("\"drink\" only for a beverage/cocktail — \"dish\" for anything else, including if unsure."), + title: z.string().describe("Best guess at the dish or drink's name"), + visibleIngredients: z.array(z.string()).describe("Ingredients visibly identifiable in the photo"), + description: z.string().describe("What's visible — presentation, garnish, cooking state, texture, any visible labels/text — detailed enough that someone who never saw the photo could write the full recipe from this alone"), +}); + +export type PhotoRecognition = z.infer; + +/** Step 1 of the photo-import flow: a vision model looks at the photo and + * describes what it sees. Deliberately recognition-only — the actual recipe + * (quantities, steps, timing) is reconstructed afterwards by a text model in + * `generateRecipeFromRecognition`, the same split pantry photo-scan uses + * between "what's in this photo" and downstream structuring. */ +export async function recognizePhoto( + imageBase64: string, + mimeType: "image/jpeg" | "image/png" | "image/webp", + config?: AiConfig, +): Promise { + const model = resolveModel(config); + + const { object } = await generateObject({ + model, + schema: PhotoRecognitionSchema, + system: + "You are a food-recognition specialist. Look at the image and describe the dish or drink it shows in enough visual detail — ingredients, presentation, cooking state — that someone who never saw the photo could write a full recipe from your description alone. If the image doesn't show food, a drink, or a recipe, set found to false and leave the other fields minimal.", + messages: [ + { + role: "user", + content: [ + { type: "image", image: imageBase64, mediaType: mimeType }, + { type: "text", text: "What dish or drink is shown in this photo?" }, + ], + }, + ], + }); + + return object; +} diff --git a/apps/web/lib/changelog.ts b/apps/web/lib/changelog.ts index 19b08bf..8708ffb 100644 --- a/apps/web/lib/changelog.ts +++ b/apps/web/lib/changelog.ts @@ -1,5 +1,5 @@ // Mirrors CHANGELOG.md at the repo root — update both together. -export const APP_VERSION = "0.36.0"; +export const APP_VERSION = "0.37.0"; export type ChangelogEntry = { version: string; @@ -11,6 +11,13 @@ export type ChangelogEntry = { }; export const CHANGELOG: ChangelogEntry[] = [ + { + version: "0.37.0", + date: "2026-07-17 12:30", + fixed: [ + "Generating a recipe from a photo now uses two separate AI steps — a vision model recognizes what's in the picture, then a text model writes the recipe from that — instead of one model doing both, so each step can use the model actually suited to it.", + ], + }, { version: "0.36.0", date: "2026-07-17 12:00", diff --git a/apps/web/lib/openapi.ts b/apps/web/lib/openapi.ts index 9c4133f..a5d0de1 100644 --- a/apps/web/lib/openapi.ts +++ b/apps/web/lib/openapi.ts @@ -301,7 +301,7 @@ export function generateOpenApiSpec(): object { })); registry.registerPath({ method: "post", path: "/api/v1/ai/generate-from-idea", summary: "Generate a full recipe from a short title/idea", description: "Rate-limited: 10 req/min. Consumes AI quota.", security, request: { body: { content: { "application/json": { schema: z.object({ title: z.string().min(1).max(200), provider: z.enum(["openai", "anthropic", "openrouter", "ollama"]).optional(), model: z.string().optional() }) } }, required: true } }, responses: { 200: { description: "Created recipe id", content: { "application/json": { schema: z.object({ id: z.string() }) } } }, 400: { description: "Validation error", content: { "application/json": { schema: ApiErrorRef } } }, 429: { description: "Rate limited or AI quota exhausted", content: { "application/json": { schema: ApiErrorRef } } } } }); - registry.registerPath({ method: "post", path: "/api/v1/ai/import-photo", summary: "Import a recipe from a photo using AI vision", description: "Rate-limited: 5 req/min. Consumes AI quota. Written in the caller's app language, regardless of the language shown in the photo.", security, request: { body: { content: { "application/json": { schema: z.object({ imageBase64: z.string().max(14_000_000), mimeType: z.enum(["image/jpeg", "image/png", "image/webp"]) }) } }, required: true } }, responses: { 200: { description: "Created recipe id", content: { "application/json": { schema: z.object({ id: z.string() }) } } }, 400: { description: "Validation error", content: { "application/json": { schema: ApiErrorRef } } }, 422: { description: "No recipe recognized in the photo", content: { "application/json": { schema: ApiErrorRef } } }, 429: { description: "Rate limited or AI quota exhausted", content: { "application/json": { schema: ApiErrorRef } } } } }); + registry.registerPath({ method: "post", path: "/api/v1/ai/import-photo", summary: "Import a recipe from a photo using AI vision", description: "Rate-limited: 5 req/min. Consumes AI quota. Two AI calls internally — a vision model recognizes the photo, then a text model writes the structured recipe — but only counts as one quota unit. Written in the caller's app language, regardless of the language shown in the photo.", security, request: { body: { content: { "application/json": { schema: z.object({ imageBase64: z.string().max(14_000_000), mimeType: z.enum(["image/jpeg", "image/png", "image/webp"]) }) } }, required: true } }, responses: { 200: { description: "Created recipe id", content: { "application/json": { schema: z.object({ id: z.string() }) } } }, 400: { description: "Validation error", content: { "application/json": { schema: ApiErrorRef } } }, 422: { description: "No recipe recognized in the photo", content: { "application/json": { schema: ApiErrorRef } } }, 429: { description: "Rate limited or AI quota exhausted", content: { "application/json": { schema: ApiErrorRef } } } } }); registry.registerPath({ method: "post", path: "/api/v1/ai/substitute", summary: "Suggest ingredient substitutions", description: "Rate-limited: 10 req/min. Consumes AI quota.", security, request: { body: { content: { "application/json": { schema: z.object({ ingredient: z.string().min(1).max(200), recipeTitle: z.string().max(200).optional(), provider: z.enum(["openai", "anthropic", "openrouter", "ollama"]).optional(), model: z.string().optional() }) } }, required: true } }, responses: { 200: { description: "Substitutions", content: { "application/json": { schema: z.object({ substitutions: z.array(z.record(z.string(), z.unknown())) }) } } }, 400: { description: "Validation error", content: { "application/json": { schema: ApiErrorRef } } }, 429: { description: "Rate limited or AI quota exhausted", content: { "application/json": { schema: ApiErrorRef } } } } }); registry.registerPath({ method: "post", path: "/api/v1/ai/scale", summary: "Scale a recipe's ingredients to a target serving count", description: "Rate-limited: 20 req/min. Consumes AI quota.", security, request: { body: { content: { "application/json": { schema: z.object({ recipeId: z.string(), targetServings: z.number().int().min(1).max(100) }) } }, required: true } }, responses: { 200: { description: "Scaled ingredients", content: { "application/json": { schema: z.object({ ingredients: z.array(z.record(z.string(), z.unknown())) }) } } }, 400: { description: "Validation error", content: { "application/json": { schema: ApiErrorRef } } }, 404: { description: "Not found", content: { "application/json": { schema: ApiErrorRef } } }, 429: { description: "Rate limited or AI quota exhausted", content: { "application/json": { schema: ApiErrorRef } } } } }); registry.registerPath({ method: "post", path: "/api/v1/ai/adapt/{id}", summary: "Adapt a recipe to exclude ingredients or meet extra constraints", description: "Consumes AI quota. Creates a new private draft recipe.", security, request: { params: idParam, body: { content: { "application/json": { schema: z.object({ excludeIngredients: z.array(z.string()).default([]), extraConstraints: z.string().max(500).optional(), provider: z.enum(["openai", "anthropic", "openrouter", "ollama"]).optional(), model: z.string().optional() }) } }, required: true } }, responses: { 200: { description: "New recipe id and adaptation notes", content: { "application/json": { schema: z.object({ id: z.string(), adaptationNotes: z.string().optional() }) } } }, 400: { description: "Validation error or no constraint provided", content: { "application/json": { schema: ApiErrorRef } } }, 404: { description: "Not found", content: { "application/json": { schema: ApiErrorRef } } }, 429: { description: "AI quota exhausted", content: { "application/json": { schema: ApiErrorRef } } } } }); diff --git a/apps/web/messages/en.json b/apps/web/messages/en.json index ae09682..bdaa040 100644 --- a/apps/web/messages/en.json +++ b/apps/web/messages/en.json @@ -187,7 +187,8 @@ "versionRestored": "Recipe restored to version {version}", "versionDetailLoading": "Loading...", "versionDetailSummary": "{ingredients, plural, one {1 ingredient} other {{ingredients} ingredients}}, {steps, plural, one {1 step} other {{steps} steps}}", - "analyzingPhoto": "Analyzing photo…", + "recognizingPhoto": "Recognizing what's in the photo…", + "writingRecipe": "Writing the recipe…", "pairMealTooltip": "Pair meal", "historyTooltip": "History", "translateTooltip": "Translate", diff --git a/apps/web/messages/fr.json b/apps/web/messages/fr.json index e76277a..c24a54e 100644 --- a/apps/web/messages/fr.json +++ b/apps/web/messages/fr.json @@ -187,7 +187,8 @@ "versionRestored": "Recette restaurée à la version {version}", "versionDetailLoading": "Chargement...", "versionDetailSummary": "{ingredients, plural, one {1 ingrédient} other {{ingredients} ingrédients}}, {steps, plural, one {1 étape} other {{steps} étapes}}", - "analyzingPhoto": "Analyse de la photo…", + "recognizingPhoto": "Identification du contenu de la photo…", + "writingRecipe": "Rédaction de la recette…", "pairMealTooltip": "Accorder un plat", "historyTooltip": "Historique", "translateTooltip": "Traduire", diff --git a/apps/web/package.json b/apps/web/package.json index 476e35a..910e869 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -1,6 +1,6 @@ { "name": "@epicure/web", - "version": "0.36.0", + "version": "0.37.0", "private": true, "scripts": { "dev": "next dev", diff --git a/package.json b/package.json index 7551d5a..d3267cb 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "epicure", - "version": "0.36.0", + "version": "0.37.0", "private": true, "scripts": { "dev": "pnpm --filter web dev",