feat: split photo-import into vision recognition + text generation
The photo-import flow used one vision-capable model to both read the photo and structure the full recipe (quantities, steps, timing) in a single call. Split it into two: a vision model recognizes what's in the picture, then a text model reconstructs the recipe from that description — same pattern the pantry photo-scan already uses for recognition, and lets structuring use whichever model is actually configured for text generation. Still counts as one AI-quota unit. v0.37.0
This commit is contained in:
Arnaud
@@ -0,0 +1,47 @@
|
||||
import { generateObject } from "ai";
|
||||
import { z } from "zod";
|
||||
import { resolveModel, type AiConfig } from "../factory";
|
||||
import { dietaryTagsSchema, ingredientSchema, stepSchema } from "./recipe-schema";
|
||||
import type { PhotoRecognition } from "./recognize-photo";
|
||||
|
||||
const RecipeOutputSchema = z.object({
|
||||
title: z.string(),
|
||||
description: z.string().optional(),
|
||||
baseServings: z.number().int().min(1).max(100),
|
||||
prepMins: z.number().int().min(0).optional(),
|
||||
cookMins: z.number().int().min(0).optional(),
|
||||
difficulty: z.enum(["easy", "medium", "hard"]),
|
||||
dietaryTags: dietaryTagsSchema,
|
||||
ingredients: z.array(ingredientSchema(z.number())),
|
||||
steps: z.array(stepSchema),
|
||||
});
|
||||
|
||||
export type GeneratedRecipeFromRecognition = z.infer<typeof RecipeOutputSchema>;
|
||||
|
||||
/** Step 2 of the photo-import flow: a text model reconstructs the full recipe
|
||||
* (quantities, steps, timing) from what the vision model recognized in
|
||||
* `recognizePhoto` — it never sees the photo itself. */
|
||||
export async function generateRecipeFromRecognition(
|
||||
recognition: PhotoRecognition,
|
||||
config?: AiConfig & { language?: string },
|
||||
): Promise<GeneratedRecipeFromRecognition> {
|
||||
const model = resolveModel(config);
|
||||
const lang = config?.language ?? "en";
|
||||
const langInstruction = lang !== "en" ? ` Write the entire recipe (title, description, ingredient names, step instructions) in ${lang}.` : "";
|
||||
const drinkInstruction = recognition.recipeType === "drink"
|
||||
? " This is a drink: never set cookMins, and default baseServings to 1 unless the description implies more than one serving."
|
||||
: "";
|
||||
|
||||
const { object } = await generateObject({
|
||||
model,
|
||||
schema: RecipeOutputSchema,
|
||||
system:
|
||||
`You are a professional chef and culinary writer. A vision model has already identified what's in a photo — reconstruct the complete, precise recipe implied by that description. For ingredients: quantity must be a number only (e.g. 0.25, 1.5, 2), unit is a separate string. Never combine quantity and unit into one string.${drinkInstruction}${langInstruction}`,
|
||||
prompt: `Photo shows: "${recognition.title}"\nVisible ingredients: ${recognition.visibleIngredients.join(", ") || "none clearly visible"}\nDescription: ${recognition.description}\n\nWrite the complete recipe for this ${recognition.recipeType}.`,
|
||||
});
|
||||
|
||||
if (recognition.recipeType === "drink") {
|
||||
return { ...object, cookMins: undefined };
|
||||
}
|
||||
return object;
|
||||
}
|
||||
@@ -1,54 +1,30 @@
|
||||
import { generateObject } from "ai";
|
||||
import { z } from "zod";
|
||||
import { resolveModel, type AiConfig } from "../factory";
|
||||
import { dietaryTagsSchema, ingredientSchema, stepSchema } from "./recipe-schema";
|
||||
import type { AiConfig } from "../factory";
|
||||
import { recognizePhoto } from "./recognize-photo";
|
||||
import { generateRecipeFromRecognition, type GeneratedRecipeFromRecognition } from "./generate-recipe-from-recognition";
|
||||
|
||||
const ImportedRecipeSchema = z.object({
|
||||
found: z.boolean().describe("false if the image does not contain a recognizable recipe (e.g. random photo, no visible ingredients/instructions)"),
|
||||
recipeType: z.enum(["dish", "drink"]).describe("\"drink\" only for a beverage/cocktail (no cooking step) — \"dish\" for anything else, including if unsure."),
|
||||
title: z.string(),
|
||||
description: z.string().optional(),
|
||||
baseServings: z.number().int().min(1).optional(),
|
||||
prepMins: z.number().int().min(0).optional(),
|
||||
cookMins: z.number().int().min(0).optional(),
|
||||
difficulty: z.enum(["easy", "medium", "hard"]).optional(),
|
||||
dietaryTags: dietaryTagsSchema.optional(),
|
||||
ingredients: z.array(ingredientSchema(z.string())),
|
||||
steps: z.array(stepSchema),
|
||||
});
|
||||
|
||||
export type ImportedRecipe = z.infer<typeof ImportedRecipeSchema>;
|
||||
|
||||
const LANG: Record<string, string> = { en: "English", fr: "French" };
|
||||
export type ImportedRecipe =
|
||||
| { found: false; recipeType: "dish" | "drink"; title: string; ingredients: []; steps: [] }
|
||||
| ({ found: true; recipeType: "dish" | "drink" } & GeneratedRecipeFromRecognition);
|
||||
|
||||
/** Orchestrates the two-step photo-import flow: a vision model recognizes
|
||||
* what's in the photo (`recognizePhoto`), then a text model reconstructs the
|
||||
* full structured recipe from that recognition (`generateRecipeFromRecognition`).
|
||||
* Splitting the call lets each step use the model best suited to it — vision
|
||||
* for the photo, text for structuring — instead of one model doing both. */
|
||||
export async function importFromPhoto(
|
||||
imageBase64: string,
|
||||
mimeType: "image/jpeg" | "image/png" | "image/webp",
|
||||
config?: AiConfig,
|
||||
visionConfig: AiConfig | undefined,
|
||||
textConfig: AiConfig | undefined,
|
||||
locale?: string,
|
||||
): Promise<ImportedRecipe> {
|
||||
const model = resolveModel(config);
|
||||
const lang = LANG[locale ?? "en"] ?? "English";
|
||||
const langInstruction = lang !== "English" ? ` Write the entire recipe (title, description, ingredient names, step instructions) in ${lang}, regardless of the language used in the image.` : "";
|
||||
const recognition = await recognizePhoto(imageBase64, mimeType, visionConfig);
|
||||
|
||||
const { object } = await generateObject({
|
||||
model,
|
||||
schema: ImportedRecipeSchema,
|
||||
system:
|
||||
`You are a recipe extraction specialist. Extract the complete recipe from the provided image. Be precise with ingredient quantities and cooking instructions. If the image does not contain a recognizable recipe (no visible ingredients or cooking instructions), set found to false and leave the other fields minimal/empty. Set recipeType to "drink" only for a beverage/cocktail with no cooking involved — never set cookMins for a drink.${langInstruction}`,
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "image", image: imageBase64, mediaType: mimeType },
|
||||
{ type: "text", text: "Extract the complete recipe from this image." },
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
if (object.recipeType === "drink") {
|
||||
return { ...object, cookMins: undefined };
|
||||
if (!recognition.found) {
|
||||
return { found: false, recipeType: recognition.recipeType, title: recognition.title, ingredients: [], steps: [] };
|
||||
}
|
||||
return object;
|
||||
|
||||
const generated = await generateRecipeFromRecognition(recognition, { ...textConfig, language: locale ?? "en" });
|
||||
|
||||
return { found: true, recipeType: recognition.recipeType, ...generated };
|
||||
}
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
import { generateObject } from "ai";
|
||||
import { z } from "zod";
|
||||
import { resolveModel, type AiConfig } from "../factory";
|
||||
|
||||
const PhotoRecognitionSchema = z.object({
|
||||
found: z.boolean().describe("false if the image does not show a dish, drink, or recipe (random photo, no visible food)"),
|
||||
recipeType: z.enum(["dish", "drink"]).describe("\"drink\" only for a beverage/cocktail — \"dish\" for anything else, including if unsure."),
|
||||
title: z.string().describe("Best guess at the dish or drink's name"),
|
||||
visibleIngredients: z.array(z.string()).describe("Ingredients visibly identifiable in the photo"),
|
||||
description: z.string().describe("What's visible — presentation, garnish, cooking state, texture, any visible labels/text — detailed enough that someone who never saw the photo could write the full recipe from this alone"),
|
||||
});
|
||||
|
||||
export type PhotoRecognition = z.infer<typeof PhotoRecognitionSchema>;
|
||||
|
||||
/** Step 1 of the photo-import flow: a vision model looks at the photo and
|
||||
* describes what it sees. Deliberately recognition-only — the actual recipe
|
||||
* (quantities, steps, timing) is reconstructed afterwards by a text model in
|
||||
* `generateRecipeFromRecognition`, the same split pantry photo-scan uses
|
||||
* between "what's in this photo" and downstream structuring. */
|
||||
export async function recognizePhoto(
|
||||
imageBase64: string,
|
||||
mimeType: "image/jpeg" | "image/png" | "image/webp",
|
||||
config?: AiConfig,
|
||||
): Promise<PhotoRecognition> {
|
||||
const model = resolveModel(config);
|
||||
|
||||
const { object } = await generateObject({
|
||||
model,
|
||||
schema: PhotoRecognitionSchema,
|
||||
system:
|
||||
"You are a food-recognition specialist. Look at the image and describe the dish or drink it shows in enough visual detail — ingredients, presentation, cooking state — that someone who never saw the photo could write a full recipe from your description alone. If the image doesn't show food, a drink, or a recipe, set found to false and leave the other fields minimal.",
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "image", image: imageBase64, mediaType: mimeType },
|
||||
{ type: "text", text: "What dish or drink is shown in this photo?" },
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
return object;
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
// Mirrors CHANGELOG.md at the repo root — update both together.
|
||||
export const APP_VERSION = "0.36.0";
|
||||
export const APP_VERSION = "0.37.0";
|
||||
|
||||
export type ChangelogEntry = {
|
||||
version: string;
|
||||
@@ -11,6 +11,13 @@ export type ChangelogEntry = {
|
||||
};
|
||||
|
||||
export const CHANGELOG: ChangelogEntry[] = [
|
||||
{
|
||||
version: "0.37.0",
|
||||
date: "2026-07-17 12:30",
|
||||
fixed: [
|
||||
"Generating a recipe from a photo now uses two separate AI steps — a vision model recognizes what's in the picture, then a text model writes the recipe from that — instead of one model doing both, so each step can use the model actually suited to it.",
|
||||
],
|
||||
},
|
||||
{
|
||||
version: "0.36.0",
|
||||
date: "2026-07-17 12:00",
|
||||
|
||||
@@ -301,7 +301,7 @@ export function generateOpenApiSpec(): object {
|
||||
}));
|
||||
|
||||
registry.registerPath({ method: "post", path: "/api/v1/ai/generate-from-idea", summary: "Generate a full recipe from a short title/idea", description: "Rate-limited: 10 req/min. Consumes AI quota.", security, request: { body: { content: { "application/json": { schema: z.object({ title: z.string().min(1).max(200), provider: z.enum(["openai", "anthropic", "openrouter", "ollama"]).optional(), model: z.string().optional() }) } }, required: true } }, responses: { 200: { description: "Created recipe id", content: { "application/json": { schema: z.object({ id: z.string() }) } } }, 400: { description: "Validation error", content: { "application/json": { schema: ApiErrorRef } } }, 429: { description: "Rate limited or AI quota exhausted", content: { "application/json": { schema: ApiErrorRef } } } } });
|
||||
registry.registerPath({ method: "post", path: "/api/v1/ai/import-photo", summary: "Import a recipe from a photo using AI vision", description: "Rate-limited: 5 req/min. Consumes AI quota. Written in the caller's app language, regardless of the language shown in the photo.", security, request: { body: { content: { "application/json": { schema: z.object({ imageBase64: z.string().max(14_000_000), mimeType: z.enum(["image/jpeg", "image/png", "image/webp"]) }) } }, required: true } }, responses: { 200: { description: "Created recipe id", content: { "application/json": { schema: z.object({ id: z.string() }) } } }, 400: { description: "Validation error", content: { "application/json": { schema: ApiErrorRef } } }, 422: { description: "No recipe recognized in the photo", content: { "application/json": { schema: ApiErrorRef } } }, 429: { description: "Rate limited or AI quota exhausted", content: { "application/json": { schema: ApiErrorRef } } } } });
|
||||
registry.registerPath({ method: "post", path: "/api/v1/ai/import-photo", summary: "Import a recipe from a photo using AI vision", description: "Rate-limited: 5 req/min. Consumes AI quota. Two AI calls internally — a vision model recognizes the photo, then a text model writes the structured recipe — but only counts as one quota unit. Written in the caller's app language, regardless of the language shown in the photo.", security, request: { body: { content: { "application/json": { schema: z.object({ imageBase64: z.string().max(14_000_000), mimeType: z.enum(["image/jpeg", "image/png", "image/webp"]) }) } }, required: true } }, responses: { 200: { description: "Created recipe id", content: { "application/json": { schema: z.object({ id: z.string() }) } } }, 400: { description: "Validation error", content: { "application/json": { schema: ApiErrorRef } } }, 422: { description: "No recipe recognized in the photo", content: { "application/json": { schema: ApiErrorRef } } }, 429: { description: "Rate limited or AI quota exhausted", content: { "application/json": { schema: ApiErrorRef } } } } });
|
||||
registry.registerPath({ method: "post", path: "/api/v1/ai/substitute", summary: "Suggest ingredient substitutions", description: "Rate-limited: 10 req/min. Consumes AI quota.", security, request: { body: { content: { "application/json": { schema: z.object({ ingredient: z.string().min(1).max(200), recipeTitle: z.string().max(200).optional(), provider: z.enum(["openai", "anthropic", "openrouter", "ollama"]).optional(), model: z.string().optional() }) } }, required: true } }, responses: { 200: { description: "Substitutions", content: { "application/json": { schema: z.object({ substitutions: z.array(z.record(z.string(), z.unknown())) }) } } }, 400: { description: "Validation error", content: { "application/json": { schema: ApiErrorRef } } }, 429: { description: "Rate limited or AI quota exhausted", content: { "application/json": { schema: ApiErrorRef } } } } });
|
||||
registry.registerPath({ method: "post", path: "/api/v1/ai/scale", summary: "Scale a recipe's ingredients to a target serving count", description: "Rate-limited: 20 req/min. Consumes AI quota.", security, request: { body: { content: { "application/json": { schema: z.object({ recipeId: z.string(), targetServings: z.number().int().min(1).max(100) }) } }, required: true } }, responses: { 200: { description: "Scaled ingredients", content: { "application/json": { schema: z.object({ ingredients: z.array(z.record(z.string(), z.unknown())) }) } } }, 400: { description: "Validation error", content: { "application/json": { schema: ApiErrorRef } } }, 404: { description: "Not found", content: { "application/json": { schema: ApiErrorRef } } }, 429: { description: "Rate limited or AI quota exhausted", content: { "application/json": { schema: ApiErrorRef } } } } });
|
||||
registry.registerPath({ method: "post", path: "/api/v1/ai/adapt/{id}", summary: "Adapt a recipe to exclude ingredients or meet extra constraints", description: "Consumes AI quota. Creates a new private draft recipe.", security, request: { params: idParam, body: { content: { "application/json": { schema: z.object({ excludeIngredients: z.array(z.string()).default([]), extraConstraints: z.string().max(500).optional(), provider: z.enum(["openai", "anthropic", "openrouter", "ollama"]).optional(), model: z.string().optional() }) } }, required: true } }, responses: { 200: { description: "New recipe id and adaptation notes", content: { "application/json": { schema: z.object({ id: z.string(), adaptationNotes: z.string().optional() }) } } }, 400: { description: "Validation error or no constraint provided", content: { "application/json": { schema: ApiErrorRef } } }, 404: { description: "Not found", content: { "application/json": { schema: ApiErrorRef } } }, 429: { description: "AI quota exhausted", content: { "application/json": { schema: ApiErrorRef } } } } });
|
||||
|
||||
Reference in New Issue
Block a user