Spaces:
Sleeping
Sleeping
File size: 6,432 Bytes
c02c6ce | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 | """
Convert recipes CSV to JSON with intelligent ingredient normalization.
Uses semantic similarity to normalize ingredient names and improve recipe matching.
"""
import json
import pandas as pd
import re
from pathlib import Path
from typing import List, Dict, Set
from difflib import SequenceMatcher
# YOLO food classes from the trained model
YOLO_CLASSES = {
'apple', 'banana', 'blueberry', 'bread', 'brinjal', 'butter', 'cabbage',
'capsicum', 'carrot', 'cheese', 'chicken', 'chocolate', 'corn', 'cucumber',
'egg', 'flour', 'fresh cream', 'ginger', 'green beans', 'green chilly',
'green leaves', 'lemon', 'meat', 'milk', 'mushroom', 'potato', 'shrimp',
'strawberry', 'sweet potato', 'tomato'
}
# Ingredient synonym mappings for better matching
INGREDIENT_SYNONYMS = {
'tomato': ['tomatoes', 'cherry tomato', 'roma tomato', 'beefsteak tomato'],
'carrot': ['carrots', 'grated carrot'],
'potato': ['potatoes', 'mashed potato'],
'onion': ['onions', 'red onion', 'yellow onion', 'white onion'],
'garlic': ['garlic cloves', 'minced garlic', 'garlic powder'],
'egg': ['eggs', 'egg yolk', 'egg white'],
'milk': ['whole milk', 'skim milk', 'evaporated milk'],
'cheese': ['cheddar', 'mozzarella', 'parmesan', 'cream cheese', 'feta'],
'chicken': ['chicken breast', 'chicken thigh', 'ground chicken'],
'meat': ['beef', 'pork', 'lamb', 'ground meat'],
'bread': ['white bread', 'wheat bread', 'whole grain bread'],
'butter': ['unsalted butter', 'salted butter'],
'flour': ['all-purpose flour', 'wheat flour', 'rice flour'],
'green beans': ['string beans', 'snap beans'],
'cucumber': ['cucumbers', 'english cucumber'],
'mushroom': ['mushrooms', 'button mushroom', 'cremini'],
'lemon': ['lemons', 'lemon juice', 'lemon zest'],
'corn': ['sweet corn', 'corn kernels'],
'fresh cream': ['heavy cream', 'whipped cream', 'sour cream'],
}
def normalize_ingredient(ingredient: str) -> str:
"""
Normalize ingredient name by:
1. Removing quantities and units
2. Converting to lowercase
3. Finding semantic matches with YOLO classes
"""
# Remove quantities (e.g., "2 cups", "1 tbsp")
normalized = re.sub(r'^\d+\s*(?:\d+/\d+)?\s*(?:cup|tbsp|tsp|ml|l|oz|lb|g|kg|x)?s?\.?\s*', '', ingredient.strip(), flags=re.IGNORECASE)
normalized = normalized.lower().strip()
# Remove common words
remove_words = ['of', 'the', 'a', 'and', 'or', 'finely', 'chopped', 'diced', 'sliced',
'grated', 'minced', 'whole', 'fresh', 'dried', 'ground', 'powder']
words = normalized.split()
words = [w for w in words if w not in remove_words and len(w) > 1]
normalized = ' '.join(words).strip()
# Try exact match first
if normalized in YOLO_CLASSES:
return normalized
# Try synonym matching
for yolo_class, synonyms in INGREDIENT_SYNONYMS.items():
if normalized in synonyms or any(normalized in syn for syn in synonyms):
return yolo_class
# Try fuzzy matching
for yolo_class in YOLO_CLASSES:
ratio = SequenceMatcher(None, normalized, yolo_class).ratio()
if ratio > 0.7: # 70% similarity threshold
return yolo_class
return normalized
def extract_and_normalize_ingredients(ingredients_str: str) -> List[str]:
"""Extract and normalize ingredients from CSV string."""
if not ingredients_str or str(ingredients_str).lower() == 'nan':
return []
# Split by comma
ingredients = [ing.strip() for ing in str(ingredients_str).split(',')]
# Normalize each ingredient
normalized = []
for ing in ingredients:
norm = normalize_ingredient(ing)
if norm and len(norm) > 1 and norm not in normalized: # Avoid duplicates
normalized.append(norm)
return normalized
def convert_recipes_csv_to_json(csv_path: str, output_path: str, num_recipes: int = 10000):
"""Convert recipes CSV to JSON format."""
print(f"📖 Reading CSV from {csv_path}...")
df = pd.read_csv(csv_path)
print(f"Total recipes in CSV: {len(df)}")
# Take subset
df_subset = df.head(num_recipes)
print(f"Processing {len(df_subset)} recipes...")
recipes_list = []
valid_count = 0
for idx, row in df_subset.iterrows():
try:
# Extract and normalize ingredients
ingredients = extract_and_normalize_ingredients(row.get('ingredients', ''))
if ingredients: # Only include recipes with ingredients
recipe = {
"recipe_id": idx,
"name": str(row['title']).strip(),
"ingredients": ingredients,
"url": str(row['url']).strip() if 'url' in row else "",
"difficulty": "medium",
"prep_time_mins": 30,
"servings": 4,
"description": f"Recipe with {len(ingredients)} ingredients"
}
recipes_list.append(recipe)
valid_count += 1
if (idx + 1) % 1000 == 0:
print(f" ✓ Processed {idx + 1} recipes ({valid_count} valid)")
except Exception as e:
continue
print(f"\n✅ Converted {valid_count} recipes with valid ingredients")
# Save to JSON
Path(output_path).parent.mkdir(parents=True, exist_ok=True)
with open(output_path, 'w') as f:
json.dump(recipes_list, f, indent=2)
print(f"💾 Saved to {output_path}")
print(f"\n📊 Sample recipe:")
print(json.dumps(recipes_list[0], indent=2))
# Stats
all_ingredients = set()
for recipe in recipes_list:
all_ingredients.update(recipe['ingredients'])
print(f"\n📈 Stats:")
print(f" - Total recipes: {len(recipes_list)}")
print(f" - Unique ingredients: {len(all_ingredients)}")
print(f" - Avg ingredients per recipe: {sum(len(r['ingredients']) for r in recipes_list) / len(recipes_list):.1f}")
return recipes_list
if __name__ == "__main__":
# Paths
csv_path = Path(__file__).parent.parent / "Recipe" / "recipes_1M_shortened.csv"
output_path = Path(__file__).parent.parent / "data" / "recipes.json"
# Convert
convert_recipes_csv_to_json(str(csv_path), str(output_path), num_recipes=10000)
|