Update article_generator.py

article_generator.py

@@ -59,12 +59,16 @@ state_file = "state.json"
 def save_state(state):
     with open(state_file, "w", encoding="utf-8") as f:
         json.dump(state, f, ensure_ascii=False, indent=4)
+    print("State saved. Current index:", state.get('current_index', 'Not available'))  # インデックス情報をログに出力
 
 # 状態をロードする関数
 def load_state():
     if os.path.exists(state_file):
         with open(state_file, "r", encoding="utf-8") as f:
-            return json.load(f)
+            state = json.load(f)
+            print("State loaded. Current index:", state.get('current_index', 'Not available'))  # インデックス情報をログに出力
+            return state
+    print("No state file found.")
     return None
 
 # 状態をクリアする関数
@@ -74,6 +78,7 @@ def clear_state():
     global executed_instructions, research_results
     executed_instructions = []
     research_results = []
+    print("State cleared.")
     return "状態がクリアされました"
 
 # 見出しを処理する関数
@@ -89,22 +94,25 @@ def perform_initial_tavily_search(h2_texts, h3_texts):
     tavily_search_tool = EnhancedTavilySearchTool()
     queries = []
     
-    for h2_text in h2_texts:
-        h3_for_this_h2 = [h3 for h3 in h3_texts if h3.startswith(f"{h2_texts.index(h2_text)+1}-")]
+    for idx, h2_text in enumerate(h2_texts):  # インデックスの取得方法を改善
+        h3_for_this_h2 = [h3 for h3 in h3_texts if h3.startswith(f"{idx+1}-")]
         query = f"{h2_text} {' '.join(h3_for_this_h2)}"
         queries.append(query)
     
-    response = tavily_search_tool.search(queries)  # 一回のリクエストで全てのクエリを処理
-    return {query: response[i] for i, query in enumerate(queries)}  # 結果を適切にマッピング
+    print("Performing Tavily search with queries:", queries)  # デバッグ情報追加
+    response = tavily_search_tool.search(queries)
+    return {query: response[i] for i, query in enumerate(queries)}
 
 # キャッシュされたTavilyデータを保存する関数
 def save_preloaded_tavily_data(data):
     with open("preloaded_tavily_data.json", "w", encoding="utf-8") as f:
         json.dump(data, f, ensure_ascii=False, indent=4)
+    print("Preloaded Tavily data saved.")
 
 # キャッシュされたTavilyデータをロードする関数
 def load_preloaded_tavily_data():
     with open("preloaded_tavily_data.json", "r", encoding="utf-8") as f:
+        print("Preloaded Tavily data loaded.")
         return json.load(f)
 
 # PlanAndExecuteエージェントをセットアップする関数
@@ -123,10 +131,94 @@ def setup_plan_and_execute_agent():
     executor = load_agent_executor(llm, tools, verbose=True)
 
     agent = PlanAndExecute(planner=planner, executor=executor, verbose=True)
+    print("PlanAndExecute agent setup complete.")
     return agent
 
+# GPT-4を使用してテキストを生成するヘルパー関数
+def generate_text_with_gpt4(prompt):
+    response = openai.ChatCompletion.create(
+        model="gpt-4o",
+        messages=[{"role": "system", "content": "以下についての詳細な情報をまとめ、適宜箇所書き、もしくは表を使ってオリジナルの内容にしてください。"},
+                  {"role": "user", "content": prompt}],
+        temperature=0.7,
+        max_tokens=500
+    )
+    return response.choices[0]["message"]["content"].strip()
+
+# 記事のセクションをGPT-4で拡張する関数
+def expand_section_with_gpt4(h2_text, h3_texts, preloaded_data):
+    prompts = []
+    h3_to_text = {}
+    for h3_text in h3_texts:
+        key = f"{h2_text} {h3_text}"
+        if key in preloaded_data:
+            context = preloaded_data[key]
+            prompt = f"「{h3_text}」について詳しく説明してください。こちらが背景情報です：\n{context}"
+            prompts.append(prompt)
+            h3_to_text[h3_text] = prompt  # プロンプトではなく後で置き換えるテキストを格納するための準備
+        else:
+            prompt = f"「{h3_text}」について詳しく説明してください。"
+            prompts.append(prompt)
+            h3_to_text[h3_text] = prompt
+
+    if not prompts:  # promptsが空の場合
+        print("No prompts to process.")
+        return []
+
+    expanded_texts = []
+    # ThreadPoolExecutorのmax_workersに最小値を設定
+    with ThreadPoolExecutor(max_workers=max(1, len(prompts))) as executor:
+        future_to_prompt = {executor.submit(generate_text_with_gpt4, prompt): h3_text for prompt, h3_text in zip(prompts, h3_texts)}
+        for future in as_completed(future_to_prompt):
+            h3_text = future_to_prompt[future]
+            try:
+                expanded_text = future.result()
+                expanded_texts.append(expanded_text)
+                h3_to_text[h3_text] = expanded_text  # 実際に生成されたテキストを保存
+            except Exception as e:
+                error_message = f"Error generating text for {h3_text}: {str(e)}"
+                print(error_message)
+                expanded_texts.append("Error in text generation.")
+
+    return h3_to_text
+
+# 記事を拡張する関数
+def process_standalone_h2(soup):
+    h2_elements = soup.find_all('h2')
+    for h2 in h2_elements:
+        if not h2.find_next_sibling(lambda tag: tag.name == 'h3'):
+            # 'まとめ'のような<h3>タグがないセクションを処理
+            preloaded_data = load_preloaded_tavily_data()
+            key = f"{h2.get_text()}"
+            context = preloaded_data.get(key, "このセクションに関する具体的な情報はありません。")
+            prompt = f"「{h2.get_text()}」について詳しく説明してください。こちらが背景情報です：\n{context}"
+            expanded_text = generate_text_with_gpt4(prompt)
+            new_paragraph = soup.new_tag('p')
+            new_paragraph.string = expanded_text
+            h2.insert_after(new_paragraph)
+
+def generate_expanded_article(article_html, h3_to_text):
+    print("記事を拡張中...")
+    soup = BeautifulSoup(article_html, 'html.parser')
+    process_standalone_h2(soup)  # 独立した<h2>セクションを処理
+
+    h2_elements = soup.find_all('h2')
+    for h2 in h2_elements:
+        if h2.get_text().strip() == "まとめ":
+            continue  # "まとめ"セクションは拡張しない
+
+        h3_elements = h2.find_next_siblings('h3')
+        for h3 in h3_elements:
+            if h3.get_text() in h3_to_text:
+                new_paragraph = soup.new_tag('p')
+                new_paragraph.string = h3_to_text[h3.get_text()]
+                h3.insert_after(new_paragraph)
+
+    return str(soup)
+
 # 記事を生成する関数
 def generate_article(editable_output2):
+    print("Starting article generation...")
     # 途中から再開する場合のために状態を読み込み
     state = load_state() or {'executed_instructions': [], 'research_results': [], 'current_index': 0}
     executed_instructions = state['executed_instructions']
@@ -143,6 +235,7 @@ def generate_article(editable_output2):
     h3_texts = [h3.get_text() for h3 in soup.find_all('h3')]
 
     # 初期のTavily検索
+    print("Performing initial Tavily search...")
     cached_responses = perform_initial_tavily_search(h2_texts, h3_texts)
     save_preloaded_tavily_data(cached_responses)
 
@@ -159,6 +252,8 @@ def generate_article(editable_output2):
                 research_results.append(response)
                 save_state({'executed_instructions': executed_instructions, 'research_results': research_results, 'current_index': h2_texts.index(h2_text) + 1})
 
+    print("Tavily search complete.")
+
     system_message = {
         "role": "system",
         "content": "あなたはプロのライターです。すべての回答を日本語でお願いします。"
@@ -215,6 +310,7 @@ def generate_article(editable_output2):
             "content": f"{i+1}/{len(split_instructions)}: {split_instruction}"
         }
         try:
+            print(f"Sending instruction chunk {i+1} of {len(split_instructions)} to GPT-4...")
             response = openai.ChatCompletion.create(
                 model="gpt-4-turbo",
                 messages=[system_message, user_message],
@@ -237,18 +333,25 @@ def generate_article(editable_output2):
 
     final_result = "\n".join(results)
 
+    # 生成された初期記事を拡張
+    h3_to_text = expand_section_with_gpt4(final_result, h3_texts, cached_responses)
+    expanded_article = generate_expanded_article(final_result, h3_to_text)
+
     with open("output3.txt", "w", encoding="utf-8") as f:
-        f.write(final_result)
+        f.write(expanded_article)
 
-    print(final_result)  # ログに最終結果を出力
+    print("Article generation complete. Output saved to output3.txt.")
+    print(expanded_article)  # ログに最終結果を出力
 
     # 生成が完了したら状態ファイルを削除
     if os.path.exists("state.json"):
         os.remove("state.json")
+        print("State file removed.")
 
-    return final_result
+    return expanded_article
 
 def continue_generate_article():
+    print("Continuing article generation...")
     state = load_state()
     if not state:
         return "再開する状態がありません。"
@@ -270,6 +373,7 @@ def continue_generate_article():
             "content": f"{i+1}/{len(split_instructions)}: {split_instructions[i]}"
         }
         try:
+            print(f"Sending instruction chunk {i+1} of {len(split_instructions)} to GPT-4...")
             response = openai.ChatCompletion.create(
                 model="gpt-4-turbo",
                 messages=[system_message, user_message],
@@ -292,13 +396,19 @@ def continue_generate_article():
 
     final_result = "\n".join(results)
 
+    # 生成された初期記事を拡張
+    h3_to_text = expand_section_with_gpt4(final_result, h3_texts, cached_responses)
+    expanded_article = generate_expanded_article(final_result, h3_to_text)
+
     with open("output3.txt", "w", encoding="utf-8") as f:
-        f.write(final_result)
+        f.write(expanded_article)
 
-    print(final_result)  # ログに最終結果を出力
+    print("Article continuation complete. Output saved to output3.txt.")
+    print(expanded_article)  # ログに最終結果を出力
 
     # 生成が完了したら状態ファイルを削除
     if os.path.exists("state.json"):
         os.remove("state.json")
+        print("State file removed.")
 
-    return final_result
+    return expanded_article