modified translation module to be able to reduce the number of request to API

.gitignore

@@ -12,4 +12,8 @@ __pycache__/
 env/
 
 translated_df.csv
-before_translate*
+before_translate*
+
+*translated.xlsx
+*translation_log.jsonl
+*translations.json

app.py

@@ -8,7 +8,7 @@ import io
 import pandas as pd
 from config.logger import logger
 
-def run_script(shop_id, environment, image_process_flag, spreadsheet_format_version, program_command):
+def run_script(shop_id, environment, image_process_flag, spreadsheet_format_version, program_command, batch_num):
     """main.py を実行し、ログを UI に表示"""
     log_output = []
     
@@ -21,7 +21,7 @@ def run_script(shop_id, environment, image_process_flag, spreadsheet_format_vers
         if program_command == 'main':
             command = ["python", "main.py", environment, image_process_flag, spreadsheet_format_version]
         elif program_command == 'translate':
-            command = ["python", "translate_exe.py", spreadsheet_format_version]
+            command = ["python", "translate_exe.py", batch_num]
         else:
             pass
         
@@ -56,13 +56,13 @@ def run_script(shop_id, environment, image_process_flag, spreadsheet_format_vers
 
     yield "\n".join(log_output)
 
-def download_csv(shop_id_input):
-    df = pd.read_csv("translated_df.csv")
-    output_file = "translated_df.csv"
-    df.to_csv(output_file, index=False, encoding="utf-8-sig")
-    if os.path.exists(output_file):
-        logger.info('file exist')
-    return output_file
+# def download_csv(shop_id_input):
+#     df = pd.read_csv("translated_df.csv")
+#     output_file = "translated_df.csv"
+#     df.to_csv(output_file, index=False, encoding="utf-8-sig")
+#     if os.path.exists(output_file):
+#         logger.info('file exist')
+#     return output_file
 
 # Gradio UI の定義
 with gr.Blocks() as demo:
@@ -72,6 +72,7 @@ with gr.Blocks() as demo:
     environment_input = gr.Radio(["登録しない", "開発", "本番"], label="Select Environment", value="開発")
     image_copy_input = gr.Radio(["実行", "スキップ"], label="S3の画像コピー処理", value="実行")
     spreadsheet_format_version_input = gr.Radio(["v1", "v2"], label="スプレッドシートのフォーマットバージョン", value="v2")
+    batch_num = gr.Textbox(label="AI翻訳のバッチ数", placeholder="例: 50")
     
     log_output = gr.Textbox(label="ログ", interactive=False, lines=15)
 
@@ -87,7 +88,8 @@ with gr.Blocks() as demo:
             environment_input,
             image_copy_input,
             spreadsheet_format_version_input,
-            gr.State("main")
+            gr.State("main"),
+            batch_num
             ],
             outputs=[log_output]
         )
@@ -98,7 +100,8 @@ with gr.Blocks() as demo:
             environment_input,
             image_copy_input,
             spreadsheet_format_version_input,
-            gr.State("translate")
+            gr.State("translate"),
+            batch_num
             ],
             outputs=[log_output]
         )

domain/dish.py

@@ -4,7 +4,7 @@ from config.variables import *
 from decimal import Decimal
 
 class Dish:
-    def __init__(self, spreadsheet_format_version):
+    def __init__(self, spreadsheet_format_version="v1"):
         self.spreadsheet_format_version = spreadsheet_format_version
         return
     

domain/option.py

@@ -5,7 +5,7 @@ from decimal import Decimal
 from domain.dish import Dish
 
 class Option:
-    def __init__(self, spreadsheet_format_version):
+    def __init__(self, spreadsheet_format_version="v1"):
         self.spreadsheet_format_version = spreadsheet_format_version
         return
     

domain/translator.py

@@ -32,3 +32,6 @@ class OpenAITranslator:
         )
         response = completion.choices[0].message.content
         return response
+    
+    def create_description(self, text, target_language):
+        return

requirements.txt

@@ -6,3 +6,4 @@ openpyxl
 gradio
 google-genai
 openai
+xlsxwriter

services/s3_manager.py

@@ -46,6 +46,26 @@ class S3Manager:
         except Exception as e:
             logger.error(f"翻訳結果のアップロード失敗: {file_key}: {str(e)}", exc_info=True)
 
+
+    def upload_excel_to_s3(self, dishes_df: pd.DataFrame, options_df: pd.DataFrame, file_key: str):
+        """
+        2つのDataFrameを1つのExcelファイルにまとめ、S3にアップロードする
+        """
+        try:
+            buffer = BytesIO()
+
+            # ExcelWriterで複数シート作成
+            with pd.ExcelWriter(buffer, engine='xlsxwriter') as writer:
+                dishes_df.to_excel(writer, sheet_name="料理", index=False)
+                options_df.to_excel(writer, sheet_name="オプション", index=False)
+
+            buffer.seek(0)
+
+            # S3にアップロード
+            self.s3.put_object(Bucket=self.bucket_name, Key=file_key, Body=buffer.getvalue())
+            logger.info(f"ExcelファイルをS3にアップロード成功: s3://{self.bucket_name}/{file_key}")
+        except Exception as e:
+            logger.error(f"Excelファイルのアップロード失敗: {file_key}: {str(e)}", exc_info=True)
         
     def delete_existing_image_files(self, target_bucket, target_prefix):
         # copy_images_by_dish_idを実行する前に、target_prefix内のフォルダとファイルを一旦全て削除する

translate_exe.py

@@ -1,38 +1,276 @@
 import pandas as pd
+import json
+import openai
+from typing import Dict, List, Tuple
+import time
 import os
-from domain.translator import GeminiTranslator, OpenAITranslator
-from services.translate_manager import DishTranslator
+import sys
 from config.variables import *
+from domain.dish import Dish
+from domain.option import Option
 from logging import INFO, DEBUG
 from services.s3_manager import S3Manager
 logger.setLevel(INFO)
 pd.set_option('display.max_columns', None)
 
-from domain.dish import Dish
-# spreadsheet_format_version = sys.argv[3]
-spreadsheet_format_version = "v1"
-dish_inst = Dish(spreadsheet_format_version)
-df = dish_inst.get_df(excel)
-# df = dish_inst.get_df(excel).head(20)
-# df = pd.read_csv("/Users/keisukeogawa/Downloads/personal/AtoX/translate_tool_internal/before_translate2.csv")
-
-# gemini_api_key = os.getenv("GEMINI_API_KEY")
-# translator = GeminiTranslator(gemini_api_key)
-openai_api_key = os.getenv("OPENAI_API_KEY")
-translator = OpenAITranslator(openai_api_key)
-
-# DishTranslatorのインスタンスを作成
-df_translator = DishTranslator(df, translator)
-
-# DataFrameの翻訳を実行
-translated_df = df_translator.translate_columns()
-translated_df.to_csv('translated_df.csv', index=False, encoding='utf-8-sig')
-logger.info("exported translated_df.csv")
-# logger.info(translated_df)
-
-# CSVとしてS3に保存
-bucket_name = "operation-menu-boy"
-OUTPUT_FILE_KEY = f"{shop_id}/翻訳結果/{shop_id}_translated.csv"
-
-s3_manager = S3Manager(bucket_name)
-s3_manager.upload_df_to_s3(translated_df, OUTPUT_FILE_KEY)
+# Replace with your actual OpenAI API key
+openai.api_key = os.getenv("OPENAI_API_KEY")
+
+# File paths
+OUTPUT_FILE = "translated.xlsx"
+
+def main():
+    # Load Excel sheets
+    logger.info("Loading Excel file...")
+    try:
+        # localで実行したい場合：
+        # INPUT_FILE = "/Users/keisukeogawa/Downloads/personal/AtoX/translate_tool_internal/before_translate3.xlsx"
+        # dishes_df = pd.read_excel(INPUT_FILE, sheet_name="料理")
+        # options_df = pd.read_excel(INPUT_FILE, sheet_name="オプション")
+        dish_inst = Dish()
+        dishes_df = dish_inst.get_df(excel)
+        option_inst = Option()
+        options_df = option_inst.get_option_df(excel)
+        options_df = change_column_name(options_df)
+        logger.info(f"Loaded successfully: {len(dishes_df)} dishes and {len(options_df)} options")
+    except Exception as e:
+        logger.info(f"Error loading Excel file: {e}")
+        return
+
+    # Target columns for translation (Japanese columns only)
+    dish_translation_targets = [
+        "料理名", "カテゴリ名", "見出し名", "リード文", "説明文"
+    ]
+
+    option_translation_targets = [
+        "オプション名", "タイトル名", "リード文"
+    ]
+    
+    # Target languages and their corresponding column suffixes
+    languages = {
+        "英語": "English",
+        # "韓国語": "Korean",
+        # "繁体字": "Traditional Chinese",
+        # "簡体字": "Simplified Chinese"
+    }
+    
+    # Extract unique Japanese texts to translate
+    dish_texts = extract_unique_texts(dishes_df, dish_translation_targets, "料理")
+    option_texts = extract_unique_texts(options_df, option_translation_targets, "オプション")
+    unique_texts = list(set(dish_texts + option_texts))
+    logger.info(f"Found {len(unique_texts)} unique texts to translate")
+
+    # import pdb;pdb.set_trace()
+    
+    # Translate all texts at once with a single API call
+    # translations = load_translations_from_json("translations.json")
+    translations = batch_translate_texts(unique_texts, languages)
+    save_translations_to_json(translations)
+    
+    # Update dataframes with translations
+    update_dataframes_with_translations(dishes_df, translations, dish_translation_targets, languages, "料理")
+    update_dataframes_with_translations(options_df, translations, option_translation_targets, languages, "オプション")
+
+    # Save the translated data back to Excel
+    save_to_excel(dishes_df, options_df)
+    logger.info(f"Translations completed and saved to {OUTPUT_FILE}")
+
+    # Save excel file to S3
+    save_to_excel_and_upload_to_s3(dishes_df, options_df, shop_id)
+
+def change_column_name(df: pd.DataFrame):
+    df.columns = [col.replace("（", "(").replace("）", ")") for col in df.columns]
+    rename_map = {
+        'オプションタイトル(日本語)': 'オプション名(日本語)',
+        'オプションタイトル(英語)': 'オプション名(英語)',
+        'オプションタイトル(韓国語)': 'オプション名(韓国語)',
+        'オプションタイトル(繁体字)': 'オプション名(繁体字)',
+        'オプションタイトル(簡体字)': 'オプション名(簡体字)',
+        'オプションバリュー(日本語)': 'タイトル名(日本語)',
+        'オプションバリュー(英語)': 'タイトル名(英語)',
+        'オプションバリュー(韓国語)': 'タイト���名(韓国語)',
+        'オプションバリュー(繁体字)': 'タイトル名(繁体字)',
+        'オプションバリュー(簡体字)': 'タイトル名(簡体字)',
+        'オプションバリューリード文(日本語)': 'リード文(日本語)',
+        'オプションバリューリード文(英語)': 'リード文(英語)',
+        'オプションバリューリード文(韓国語)': 'リード文(韓国語)',
+        'オプションバリューリード文(繁体字)': 'リード文(繁体字)',
+        'オプションバリューリード文(簡体字)': 'リード文(簡体字)'
+    }
+    df = df.rename(columns=rename_map)
+    df = df.rename(columns=rename_map)
+    return df
+
+def extract_unique_texts(df: pd.DataFrame, targets: List[str], which_column: str) -> List[str]:
+    """Extract all unique Japanese texts that need translation."""
+    unique_texts = set()
+    
+    # Process both dataframes
+    for column in targets:
+        column_ja = column + "(日本語)"
+        if column_ja in df.columns:
+            # Add non-empty text values to the set
+            unique_texts.update([
+                text for text in df[column_ja].dropna().unique() 
+                if isinstance(text, str) and text.strip()
+            ])
+
+    logger.info(f"extracted unique text for {which_column}")
+    
+    return list(unique_texts)
+
+
+def batch_translate_texts(texts: List[str], target_languages: Dict[str, str]) -> Dict[str, Dict[str, str]]:
+    if not texts:
+        return {}
+
+    translations = {}
+    batch_size = int(sys.argv[1]) # 1回のリクエストで翻訳する件数（多すぎるとエラーになる可能性）
+
+    for i in range(0, len(texts), batch_size):
+        batch = texts[i:i + batch_size]
+        logger.info(f"Translating batch {i+1} - {i+len(batch)} / {len(texts)} ...")
+
+        # , Korean, Traditional Chinese, and Simplified Chinese
+        system_message = """
+        You are a professional translator for food menus. Please translate the given Japanese texts to English. Format your response as a JSON object with each original Japanese text
+        as a key, and for each key provide an object with translations to all requested languages.
+
+        For single words or short phrases, capitalize the first letter in English translations.
+        Don't include quotation marks in your translations.
+        Provide exactly one translation per text and language.
+        """
+
+        user_message = f"""
+        Please translate the following Japanese texts to {', '.join(target_languages.values())}:
+
+        {json.dumps(batch, ensure_ascii=False, indent=2)}
+
+        Format your response as a valid JSON object with this structure:
+        {{
+            "Japanese text 1": {{
+                "English": "English translation",
+            }},
+            ...
+        }}
+        """
+        logger.info(f"prompt start:\n{user_message}\n:end")
+                # "Korean": "Korean translation",
+                # "Traditional Chinese": "Traditional Chinese translation",
+                # "Simplified Chinese": "Simplified Chinese translation"
+
+        try:
+            logger.info(f"Sending request for batch {i+1} - {i+len(batch)}...")
+
+            start_time = time.time()  # 計測開始
+
+            response = openai.chat.completions.create(
+                model="gpt-4o-mini",
+                response_format={"type": "json_object"},
+                messages=[
+                    {"role": "system", "content": system_message},
+                    {"role": "user", "content": user_message}
+                ],
+                temperature=0.3,
+            )
+        
+            end_time = time.time()  # 計測終了
+            logger.info(f"Translation completed in {(end_time - start_time) // 60} minutes.")
+
+            response_content = response.choices[0].message.content
+            batch_translations = json.loads(response_content)
+
+            # ログファイルにリクエスト・レスポンスを保存（デバッグ用）
+            with open("translation_log.jsonl", "a", encoding="utf-8") as log_file:
+                log_file.write(json.dumps({
+                    "batch_start": i+1,
+                    "batch_end": i+len(batch),
+                    "request": batch,
+                    "response": batch_translations
+                }, ensure_ascii=False) + "\n")
+
+            # 結果を統合
+            translations.update(batch_translations)
+
+        except Exception as e:
+            logger.info(f"Error during translation batch {i+1} - {i+len(batch)}: {e}")
+            continue
+
+    logger.info("All translations completed.")
+    return translations
+
+
+def update_dataframes_with_translations(
+    df: pd.DataFrame,
+    translations: Dict[str, Dict[str, str]], 
+    target_columns: List[str], 
+    languages: Dict[str, str],
+    which_column: str
+):
+    """Update the dataframes with the translations."""
+    if not translations:
+        logger.info("No translations to apply")
+        return
+    
+    # Process both dataframes
+    for jp_column in target_columns:
+        jp_column_par = jp_column + "(日本語)"
+        if jp_column_par not in df.columns:
+            logger.info(f"jp {jp_column_par} not found in df columns")
+            continue
+            
+        # Update each language column
+        for jp_lang, en_lang in languages.items():
+            target_column = f"{jp_column}({jp_lang})"
+            
+            # Skip if target column doesn't exist
+            if target_column not in df.columns:
+                logger.info(f"target {target_column} not found in df columns")
+                continue
+
+            df[target_column] = df[target_column].astype(object)
+            
+            # Apply translations
+            for i, value in enumerate(df[jp_column_par]):
+                if isinstance(value, str) and value.strip() and value in translations:
+                    df.at[i, target_column] = translations[value].get(en_lang, "")
+    
+    logger.info(f"Applied translations to dataframes for {which_column}")
+
+def save_translations_to_json(translations: Dict[str, Dict[str, str]], filename: str = "translations.json"):
+    """Save translated contents in local PC as a json file"""
+    try:
+        with open(filename, "w", encoding="utf-8") as f:
+            json.dump(translations, f, ensure_ascii=False, indent=2)
+        logger.info(f"Translations saved to {filename}")
+    except Exception as e:
+        logger.info(f"Failed to save translations to JSON: {e}")
+
+
+def save_to_excel(dishes_df: pd.DataFrame, options_df: pd.DataFrame):
+    """Save the updated dataframes back to Excel."""
+    try:
+        with pd.ExcelWriter(OUTPUT_FILE) as writer:
+            dishes_df.to_excel(writer, sheet_name="料理", index=False)
+            options_df.to_excel(writer, sheet_name="オプション", index=False)
+        logger.info(f"Successfully saved to {OUTPUT_FILE}")
+    except Exception as e:
+        logger.info(f"Error saving Excel file: {e}")
+
+def save_to_excel_and_upload_to_s3(dishes_df, options_df, shop_id, bucket_name="operation-menu-boy"):
+    file_key = f"{shop_id}/翻訳結果/{shop_id}_translated.xlsx"
+    s3_manager = S3Manager(bucket_name)
+    s3_manager.upload_excel_to_s3(dishes_df, options_df, file_key)
+
+def load_translations_from_json(filename: str) -> Dict[str, Dict[str, str]]:
+    try:
+        with open(filename, "r", encoding="utf-8") as f:
+            translations = json.load(f)
+        logger.info(f"Loaded translations from {filename}")
+        return translations
+    except Exception as e:
+        logger.info(f"Failed to load translations from JSON: {e}")
+        return {}
+
+if __name__ == "__main__":
+    main()

translate_exe_old.py

@@ -0,0 +1,38 @@
+import pandas as pd
+import os
+from domain.translator import GeminiTranslator, OpenAITranslator
+from services.translate_manager import DishTranslator
+from config.variables import *
+from logging import INFO, DEBUG
+from services.s3_manager import S3Manager
+logger.setLevel(INFO)
+pd.set_option('display.max_columns', None)
+
+from domain.dish import Dish
+# spreadsheet_format_version = sys.argv[3]
+# spreadsheet_format_version = "v1"
+# dish_inst = Dish(spreadsheet_format_version)
+# df = dish_inst.get_df(excel)
+# df = dish_inst.get_df(excel).head(20)
+df = pd.read_csv("/Users/keisukeogawa/Downloads/personal/AtoX/translate_tool_internal/before_translate2.csv")
+
+# gemini_api_key = os.getenv("GEMINI_API_KEY")
+# translator = GeminiTranslator(gemini_api_key)
+openai_api_key = os.getenv("OPENAI_API_KEY")
+translator = OpenAITranslator(openai_api_key)
+
+# DishTranslatorのインスタンスを作成
+df_translator = DishTranslator(df, translator)
+
+# DataFrameの翻訳を実行
+translated_df = df_translator.translate_columns()
+translated_df.to_csv('translated_df.csv', index=False, encoding='utf-8-sig')
+logger.info("exported translated_df.csv")
+# logger.info(translated_df)
+
+# CSVとしてS3に保存
+bucket_name = "operation-menu-boy"
+OUTPUT_FILE_KEY = f"{shop_id}/翻訳結果/{shop_id}_translated.csv"
+
+s3_manager = S3Manager(bucket_name)
+s3_manager.upload_df_to_s3(translated_df, OUTPUT_FILE_KEY)