Upload BBS2ChatGPT.ipynb

BBS2ChatGPT.ipynb

@@ -0,0 +1,268 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "USVNvArSTCIw"
+      },
+      "outputs": [],
+      "source": [
+        "# ChatGPT APIを用いた掲示板ログ解析スクリプト\n",
+        "\n",
+        "# 必要なライブラリのインストール\n",
+        "!pip install --upgrade requests sentence-transformers tiktoken tenacity\n",
+        "\n",
+        "# 必要なライブラリのインポート\n",
+        "import os\n",
+        "import glob\n",
+        "import json\n",
+        "import requests\n",
+        "from requests.exceptions import HTTPError\n",
+        "from sentence_transformers import SentenceTransformer, util\n",
+        "from typing import List, Optional\n",
+        "from tenacity import retry, stop_after_attempt, wait_exponential\n",
+        "\n",
+        "# クラス定義: OpenAIClient\n",
+        "class OpenAIClient:\n",
+        "    \"\"\"OpenAI APIとの通信を管理するクラス\"\"\"\n",
+        "\n",
+        "    def __init__(self, api_key: str):\n",
+        "        self.api_key = api_key\n",
+        "        self.endpoint = \"https://api.openai.com/v1/chat/completions\"\n",
+        "\n",
+        "    def count_tokens(self, text: str) -> int:\n",
+        "        \"\"\"テキストのトークン数を計算\"\"\"\n",
+        "        import tiktoken  # トークン計算用ライブラリ\n",
+        "        encoding = tiktoken.get_encoding(\"cl100k_base\")\n",
+        "        return len(encoding.encode(text))\n",
+        "\n",
+        "    @retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4, max=10))\n",
+        "    def generate_response(self, messages: List[dict], max_tokens: int = 1000) -> str:\n",
+        "        \"\"\"ChatGPT APIを使用して応答を生成（エラー時は自動リトライ）\"\"\"\n",
+        "        headers = {\n",
+        "            \"Authorization\": f\"Bearer {self.api_key}\",\n",
+        "            \"Content-Type\": \"application/json\"\n",
+        "        }\n",
+        "\n",
+        "        payload = {\n",
+        "            \"model\": \"gpt-4\",\n",
+        "            \"messages\": messages,\n",
+        "            \"max_tokens\": max_tokens,\n",
+        "            \"temperature\": 0.7\n",
+        "        }\n",
+        "\n",
+        "        try:\n",
+        "            response = requests.post(self.endpoint, headers=headers, json=payload)\n",
+        "            response.raise_for_status()\n",
+        "            return response.json()[\"choices\"][0][\"message\"][\"content\"]\n",
+        "        except HTTPError as http_err:\n",
+        "            print(f\"HTTPエラーが発生しました: {http_err}\")\n",
+        "            raise\n",
+        "        except Exception as e:\n",
+        "            print(f\"予期せぬエラーが発生しました: {e}\")\n",
+        "            raise\n",
+        "\n",
+        "# クラス定義: FileHandler\n",
+        "class FileHandler:\n",
+        "    \"\"\"ファイル操作を管理するクラス\"\"\"\n",
+        "\n",
+        "    @staticmethod\n",
+        "    def delete_existing_files():\n",
+        "        \"\"\"既存のJSONファイルとマークダウンファイルを削除\"\"\"\n",
+        "        file_patterns = ['./*.json', './*.md']\n",
+        "        for pattern in file_patterns:\n",
+        "            files_to_delete = glob.glob(pattern)\n",
+        "            for file_path in files_to_delete:\n",
+        "                try:\n",
+        "                    os.remove(file_path)\n",
+        "                    print(f\"削除しました: {file_path}\")\n",
+        "                except Exception as e:\n",
+        "                    print(f\"{file_path} の削除中にエラーが発生しました: {e}\")\n",
+        "\n",
+        "    @staticmethod\n",
+        "    def upload_files(file_type: str) -> List[str]:\n",
+        "        \"\"\"ファイルをアップロードし、パスのリストを返す\"\"\"\n",
+        "        print(f\"{file_type}ファイルをアップロードしてください（複数選択可）\")\n",
+        "        try:\n",
+        "            from google.colab import files  # Google Colab環境用\n",
+        "            uploaded = files.upload()\n",
+        "            file_paths = []\n",
+        "            for filename in uploaded.keys():\n",
+        "                file_path = os.path.abspath(filename)\n",
+        "                file_paths.append(file_path)\n",
+        "                print(f\"ファイル {filename} をアップロードしました（パス: {file_path}）\")\n",
+        "            return file_paths\n",
+        "        except Exception as e:\n",
+        "            print(f\"ファイルアップロード中にエラーが発生しました: {e}\")\n",
+        "            return []\n",
+        "\n",
+        "    @staticmethod\n",
+        "    def read_markdown_file(file_path: str) -> Optional[List[str]]:\n",
+        "        \"\"\"マークダウンファイルを読み込み、行のリストを返す\"\"\"\n",
+        "        try:\n",
+        "            with open(file_path, 'r', encoding='utf-8') as file:\n",
+        "                return file.readlines()\n",
+        "        except Exception as e:\n",
+        "            print(f\"マークダウンファイル読み取り中にエラーが発生しました: {e}\")\n",
+        "            return None\n",
+        "\n",
+        "# クラス定義: PostProcessor\n",
+        "class PostProcessor:\n",
+        "    \"\"\"投稿データの処理を管理するクラス\"\"\"\n",
+        "\n",
+        "    def __init__(self, model_name: str = \"multi-qa-mpnet-base-dot-v1\"):\n",
+        "        self.model = SentenceTransformer(model_name)\n",
+        "\n",
+        "    def extract_theme_line(self, markdown_lines: List[str]) -> Optional[str]:\n",
+        "        \"\"\"マークダウンから「テーマ」行を抽出\"\"\"\n",
+        "        for line in markdown_lines:\n",
+        "            if \"**テーマ**\" in line:\n",
+        "                parts = line.split(\":\")\n",
+        "                if len(parts) > 1:\n",
+        "                    return parts[1].strip()\n",
+        "        print(\"「**テーマ**」行が見つかりませんでした。\")\n",
+        "        return None\n",
+        "\n",
+        "    def split_json_by_posts(self, file_paths: List[str]) -> List[str]:\n",
+        "        \"\"\"JSONファイルから投稿データを抽出し、個別の投稿に分割\"\"\"\n",
+        "        all_posts = []\n",
+        "        for file_path in file_paths:\n",
+        "            try:\n",
+        "                with open(file_path, 'r', encoding='utf-8') as file:\n",
+        "                    data = json.load(file)\n",
+        "                    posts = data.get('posts', [])\n",
+        "                    split_posts = [json.dumps(post, ensure_ascii=False) for post in posts]\n",
+        "                    all_posts.extend(split_posts)\n",
+        "            except Exception as e:\n",
+        "                print(f\"JSONファイル処理中にエラーが発生しました: {e}\")\n",
+        "        return all_posts\n",
+        "\n",
+        "    def filter_relevant_posts(self, user_prompt: str, posts: List[str], threshold: float = 0.8) -> List[str]:\n",
+        "        \"\"\"投稿データから関連性の高いものをフィルタリング（閾値を動的に調整）\"\"\"\n",
+        "        while threshold >= 0.1:\n",
+        "            try:\n",
+        "                prompt_embedding = self.model.encode(user_prompt, convert_to_tensor=True)\n",
+        "                post_embeddings = self.model.encode(posts, convert_to_tensor=True)\n",
+        "                cosine_scores = util.pytorch_cos_sim(prompt_embedding, post_embeddings)\n",
+        "                relevant_posts = [posts[i] for i, score in enumerate(cosine_scores[0]) if score >= threshold]\n",
+        "\n",
+        "                if relevant_posts:\n",
+        "                    print(f\"関連性のある投稿が見つかりました。使用された閾値: {threshold}\")\n",
+        "                    return relevant_posts\n",
+        "\n",
+        "                threshold -= 0.1\n",
+        "                print(f\"閾値を下げて再試行します。現在の閾値: {threshold:.1f}\")\n",
+        "            except Exception as e:\n",
+        "                print(f\"関連性判定中にエラーが発生しました: {e}\")\n",
+        "                return []\n",
+        "\n",
+        "        print(\"関連性のある投稿が見つかりませんでした。\")\n",
+        "        return []\n",
+        "\n",
+        "def main():\n",
+        "    \"\"\"メイン処理関数。ユーザー入力やアップロードされたファイルの処理。\"\"\"\n",
+        "    try:\n",
+        "        api_key = input(\"OpenAI APIキーを入力してください: \")\n",
+        "\n",
+        "        # 各クラスの初期化\n",
+        "        openai_client = OpenAIClient(api_key)\n",
+        "        file_handler = FileHandler()\n",
+        "        post_processor = PostProcessor()\n",
+        "\n",
+        "        print(\"\\n既存JSONおよびMarkdownファイル削除中...\")\n",
+        "        file_handler.delete_existing_files()\n",
+        "\n",
+        "        print(\"\\nプロンプトとして使用するマークダウンファイルをアップロードしてください。\")\n",
+        "        markdown_files = file_handler.upload_files(\"マークダウン\")\n",
+        "        if not markdown_files or len(markdown_files) != 1:\n",
+        "            raise ValueError(\"マークダウンファイルが正しくアップロードされませんでした。\")\n",
+        "\n",
+        "        markdown_content = file_handler.read_markdown_file(markdown_files[0])\n",
+        "        if not markdown_content:\n",
+        "            raise ValueError(\"マークダウンファイルが空です。\")\n",
+        "\n",
+        "        user_prompt_theme = post_processor.extract_theme_line(markdown_content)\n",
+        "        if not user_prompt_theme:\n",
+        "            raise ValueError(\"テーマ行が見つかりませんでした。\")\n",
+        "\n",
+        "        user_prompt_full = \"\".join(markdown_content)\n",
+        "\n",
+        "        print(\"\\n解析対象となるJSONファイルをアップロードしてください。\")\n",
+        "        json_files = file_handler.upload_files(\"JSON\")\n",
+        "        if not json_files:\n",
+        "            raise ValueError(\"JSONファイルが正しくアップロードされませんでした。\")\n",
+        "\n",
+        "        posts = post_processor.split_json_by_posts(json_files)\n",
+        "        if not posts:\n",
+        "            raise ValueError(\"有効な投稿データが見つかりませんでした。\")\n",
+        "\n",
+        "        relevant_posts = post_processor.filter_relevant_posts(user_prompt_theme.strip(), posts)\n",
+        "        if not relevant_posts:\n",
+        "            raise ValueError(\"関連性のある投稿データが見つかりませんでした。\")\n",
+        "\n",
+        "        # 関連性の高い投稿をJSONファイルとして保存\n",
+        "        with open(\"relevant_posts.json\", \"w\", encoding=\"utf-8\") as json_file:\n",
+        "            json.dump({\"relevant_posts\": [json.loads(post) for post in relevant_posts]},\n",
+        "                     json_file,\n",
+        "                     ensure_ascii=False,\n",
+        "                     indent=2)\n",
+        "        print(\"\\n関連性の高い投稿を 'relevant_posts.json' に保存しました。\")\n",
+        "\n",
+        "        combined_posts = \"\\n\".join(relevant_posts)\n",
+        "\n",
+        "        # トークン数計算と調整\n",
+        "        total_tokens = openai_client.count_tokens(user_prompt_full + combined_posts)\n",
+        "\n",
+        "        # トークン数制限対応\n",
+        "        if total_tokens > 4096:\n",
+        "            print(f\"トークン数が上限（4096）を超えています。投稿データを切り詰めます。\")\n",
+        "            truncation_point = int(4096 * len(combined_posts) / total_tokens)\n",
+        "            combined_posts = combined_posts[:truncation_point]\n",
+        "\n",
+        "        # プロンプトと投稿データを結合\n",
+        "        prompt = f\"{user_prompt_full}\\n\\n関連性が高い投稿データ:\\n{combined_posts}\"\n",
+        "\n",
+        "        # ChatGPT API呼び出しと結果表示\n",
+        "        try:\n",
+        "            response = openai_client.generate_response(\n",
+        "                messages=[{\"role\": \"user\", \"content\": prompt}],\n",
+        "                max_tokens=1000\n",
+        "            )\n",
+        "\n",
+        "            print(\"\\n### ChatGPT解析結果 (マークダウン形式):\\n\")\n",
+        "            print(response)\n",
+        "\n",
+        "            # 結果をファイルに保存\n",
+        "            with open(\"chatgpt_analysis_result.md\", \"w\", encoding=\"utf-8\") as result_file:\n",
+        "                result_file.write(response)\n",
+        "            print(\"\\n解析結果が 'chatgpt_analysis_result.md' に保存されました。\")\n",
+        "\n",
+        "        except Exception as e:\n",
+        "            print(f\"ChatGPT API呼び出し中にエラーが発生しました: {e}\")\n",
+        "\n",
+        "    except Exception as e:\n",
+        "        print(f\"エラーが発生しました: {e}\")\n",
+        "\n",
+        "if __name__ == \"__main__\":\n",
+        "    main()\n"
+      ]
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "gpuType": "T4",
+      "provenance": []
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "name": "python"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}