Spaces:

fudii0921
/

co_agent

Sleeping

App Files Files Community

fudii0921 commited on May 27, 2025

Commit

c7ff072

verified ·

1 Parent(s): 19b3605

Update app.py

Browse files

Files changed (1) hide show

app.py +102 -25

app.py CHANGED Viewed

@@ -15,13 +15,18 @@ co_rerank = cohere.ClientV2(os.environ.get("COHERE_API_KEY"))
 vectored = None
-#dataid = requests.get("https://www.ryhintl.com/dbjson/getjson?sqlcmd=select * from company_matters")
-#data_str = dataid.content.decode('utf-8')
-#data = json.loads(data_str)
-# デコード関数
 '''def decode_text(data):
     for item in data:
         try:
             # latin1 でデコードし、utf-8に変換
@@ -29,19 +34,62 @@ vectored = None
         except UnicodeDecodeError as e:
             print(f"エラー: {e}")
             item['text'] = "[デコード失敗]"
-    return data'''
-#raw_documents = [{"title": "会議議事録", "url": os.environ.get("URL")+"output_cm.html"}]
-raw_documents = [{"title": "生成AI", "url": "https://ja.wikipedia.org/wiki/生成的人工知能"}]
-'''raw_documents = [
     {"title": "バグダードの戦い", "url": "https://ja.wikipedia.org/wiki/バグダードの戦い"},
     {"title": "2006年トリノオリンピック", "url": "https://ja.wikipedia.org/wiki/2006年トリノオリンピック"},
     {"title": "ドレッドノート_(戦艦)", "url": "https://ja.wikipedia.org/wiki/ドレッドノート_(戦艦)"},
-    {"title": "生成AI", "url": "https://ja.wikipedia.org/wiki/生成的人工知能"}
-]'''
 # あなたのクラスとロジックをここに統合します
 class Vectorstore:
@@ -81,7 +129,7 @@ class Vectorstore:
         """
         Embeds the document chunks using the Cohere API.
         """
-        #print("Embedding document chunks...")
         batch_size = 90
         self.docs_len = len(self.docs)
@@ -95,17 +143,32 @@ class Vectorstore:
                 embedding_types=["float"]
             ).embeddings.float
             self.docs_embs.extend(docs_embs_batch)
-            #print(docs_embs_batch)
-    def index(self):
         print("Indexing document chunks...")
         self.idx = hnswlib.Index(space="ip", dim=1024)
-        self.idx.init_index(max_elements=self.docs_len, ef_construction=512, M=128)  # Increased M
         self.idx.add_items(self.docs_embs, list(range(len(self.docs_embs))))
-        #print(f"Indexing complete with {self.idx.get_current_count()} document chunks.")
-    def retrieve(self, query: str):
-        print("Retrieving document chunks...")
         query_emb = co_embed.embed(
             texts=[query],
             model="embed-multilingual-v3.0",
@@ -113,10 +176,24 @@ class Vectorstore:
             embedding_types=["float"]
         ).embeddings.float
-        self.idx.set_ef(100)  # Set higher ef for query
-        doc_ids = self.idx.knn_query(query_emb, k=self.retrieve_top_k)[0]  # Retrieve IDs safely
-        docs_retrieved = [self.docs[doc_id]["data"] for doc_id in doc_ids]
         return docs_retrieved
 if not vectored == "vectored":
@@ -130,8 +207,8 @@ vectorstore = Vectorstore(raw_documents)
 # Gradioの関数
 def search(query):
     results = vectorstore.retrieve(query)
-    #decoded_data = decode_text(results)
-    #print("decoded_data:",decoded_data)
     return "\n\n".join([f"**Title**: {r['title']}\n**Text**: {r['text']}\n**URL**: {r['url']}" for r in decoded_data])
 # Gradioインターフェース

 vectored = None
+dataid = requests.get("https://www.ryhintl.com/dbjson/getjson?sqlcmd=select * from company_matters")
+data_str = dataid.content.decode('utf-8')
+data = json.loads(data_str)
 '''def decode_text(data):
+    for item in data:
+        item['text'] = item['text'].encode('latin1').decode('utf-8')  # latin1でエンコード、UTF-8でデコード
+    return data'''
+# デコード関数
+def decode_text(data):
     for item in data:
         try:
             # latin1 でデコードし、utf-8に変換
         except UnicodeDecodeError as e:
             print(f"エラー: {e}")
             item['text'] = "[デコード失敗]"
+    return data
+'''def upload_html_to_server(file_path, url):
+    with open(file_path, "rb") as file:
+        # ファイルをアップロードする
+        response = requests.post(url, files={"file": file})
+        return response.status_code, response.text
+def upload_text_to_server(text, url):
+    # テキストを送信
+    response = requests.post(url, data={"content": text})
+    return response.status_code, response.text
+# 自然言語にする関数
+def json_to_text(data):
+    final_context = ""
+    for item in data:
+        text = (
+            f"タイトル: {item['caption']}\n"
+            f"表題: {item['title']}\n"
+            f"詳細: {item['content']}\n"
+            f"日付: {item['date']}\n"
+        )
+        final_context += text + "\n"  # 結果を連結
+    #last_content = "<html>\n<head>\n<title>会議議事録</title>\n</head>\n<body>\n"+final_context+"</body>\n</html>"
+    last_content = final_context
+    file_name = "output_cm.html"
+    with open(file_name, "w", encoding="utf-8") as file:
+        file.write(last_content)
+    # サーバーURLを指定
+    server_url = "https://www.ryhintl.com/company_matters.php"  # 実際のアップロード先のURLに置き換えてね
+    # アップロード実行
+    status, response_text = upload_text_to_server(last_content, server_url)
+    print(f"アップロードのステータス: {status}")
+    print(f"レスポンス: {response_text}")
+    return final_context
+# 実行例
+result = json_to_text(data)
+'''
+raw_documents = [
     {"title": "バグダードの戦い", "url": "https://ja.wikipedia.org/wiki/バグダードの戦い"},
     {"title": "2006年トリノオリンピック", "url": "https://ja.wikipedia.org/wiki/2006年トリノオリンピック"},
     {"title": "ドレッドノート_(戦艦)", "url": "https://ja.wikipedia.org/wiki/ドレッドノート_(戦艦)"},
+    {"title": "えひめ丸事故", "url": "https://ja.wikipedia.org/wiki/えひめ丸事故"},
+    {"title": "会議議事録", "url": "https://www.ryhintl.com/reqfiles/company_matters/cm_output.html"}
+]
+#raw_documents = [{"title": "会議議事録", "url": "https://www.ryhintl.com/reqfiles/company_matters/cm_output.html"}]
 # あなたのクラスとロジックをここに統合します
 class Vectorstore:
         """
         Embeds the document chunks using the Cohere API.
         """
+        print("Embedding document chunks...")
         batch_size = 90
         self.docs_len = len(self.docs)
                 embedding_types=["float"]
             ).embeddings.float
             self.docs_embs.extend(docs_embs_batch)
+            print(docs_embs_batch)
+    def index(self) -> None:
+        """
+        Indexes the document chunks for efficient retrieval.
+        """
         print("Indexing document chunks...")
         self.idx = hnswlib.Index(space="ip", dim=1024)
+        self.idx.init_index(max_elements=self.docs_len, ef_construction=512, M=64)
         self.idx.add_items(self.docs_embs, list(range(len(self.docs_embs))))
+        print(f"Indexing complete with {self.idx.get_current_count()} document chunks.")
+    def retrieve(self, query: str) -> List[Dict[str, str]]:
+        """
+        Retrieves document chunks based on the given query.
+        Parameters:
+        query (str): The query to retrieve document chunks for.
+        Returns:
+        List[Dict[str, str]]: A list of dictionaries representing the retrieved document chunks, with 'title', 'text', and 'url' keys.
+        """
+        # Dense retrieval
         query_emb = co_embed.embed(
             texts=[query],
             model="embed-multilingual-v3.0",
             embedding_types=["float"]
         ).embeddings.float
+        doc_ids = self.idx.knn_query(query_emb, k=self.retrieve_top_k)[0][0]
+        # Reranking
+        docs_to_rerank = [self.docs[doc_id]["data"] for doc_id in doc_ids]
+        yaml_docs = [yaml.dump(doc, sort_keys=False) for doc in docs_to_rerank]
+        rerank_results = co_rerank.rerank(
+            query=query,
+            documents=yaml_docs,
+            model="rerank-v3.5", # Pass a dummy string
+            top_n=self.rerank_top_k
+        )
+        doc_ids_reranked = [doc_ids[result.index] for result in rerank_results.results]
+        docs_retrieved = []
+        for doc_id in doc_ids_reranked:
+            docs_retrieved.append(self.docs[doc_id]["data"])
         return docs_retrieved
 if not vectored == "vectored":
 # Gradioの関数
 def search(query):
     results = vectorstore.retrieve(query)
+    decoded_data = decode_text(results)
+    print("decoded_data:",decoded_data)
     return "\n\n".join([f"**Title**: {r['title']}\n**Text**: {r['text']}\n**URL**: {r['url']}" for r in decoded_data])
 # Gradioインターフェース