Spaces:
Paused
Paused
lanny xu
commited on
Commit
·
94a7032
1
Parent(s):
ddd99a5
delete vectara
Browse files- document_processor.py +3 -0
- entity_extractor.py +3 -0
- evaluate_retrieval.py +21 -4
- graph_indexer.py +11 -0
- graph_retriever.py +3 -0
- knowledge_graph.py +3 -0
- local_llm_rag.py +14 -2
- retrieval_evaluation.py +7 -1
- routers_and_graders.py +3 -0
- test_reranking.py +7 -1
- workflow_nodes.py +9 -0
document_processor.py
CHANGED
|
@@ -5,6 +5,9 @@
|
|
| 5 |
|
| 6 |
try:
|
| 7 |
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
|
|
|
|
|
|
|
|
|
| 8 |
except ImportError:
|
| 9 |
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
| 10 |
|
|
|
|
| 5 |
|
| 6 |
try:
|
| 7 |
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
| 8 |
+
except ImportError:
|
| 9 |
+
try:
|
| 10 |
+
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
| 11 |
except ImportError:
|
| 12 |
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
| 13 |
|
entity_extractor.py
CHANGED
|
@@ -10,6 +10,9 @@ import aiohttp
|
|
| 10 |
import json
|
| 11 |
try:
|
| 12 |
from langchain_core.prompts import PromptTemplate
|
|
|
|
|
|
|
|
|
|
| 13 |
except ImportError:
|
| 14 |
from langchain.prompts import PromptTemplate
|
| 15 |
|
|
|
|
| 10 |
import json
|
| 11 |
try:
|
| 12 |
from langchain_core.prompts import PromptTemplate
|
| 13 |
+
except ImportError:
|
| 14 |
+
try:
|
| 15 |
+
from langchain_core.prompts import PromptTemplate
|
| 16 |
except ImportError:
|
| 17 |
from langchain.prompts import PromptTemplate
|
| 18 |
|
evaluate_retrieval.py
CHANGED
|
@@ -18,14 +18,31 @@ load_dotenv()
|
|
| 18 |
from main import AdaptiveRAGSystem
|
| 19 |
from document_processor import DocumentProcessor
|
| 20 |
from retrieval_evaluation import RetrievalEvaluator, RetrievalResult, RetrievalTestSet
|
| 21 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 22 |
|
| 23 |
# 导入LangChain相关模块
|
| 24 |
from langchain_community.vectorstores import FAISS, Chroma
|
| 25 |
from langchain_community.retrievers import BM25Retriever
|
| 26 |
-
|
| 27 |
-
from langchain.retrievers import
|
| 28 |
-
from langchain.retrievers
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
|
| 30 |
|
| 31 |
class AdaptiveRAGRetriever:
|
|
|
|
| 18 |
from main import AdaptiveRAGSystem
|
| 19 |
from document_processor import DocumentProcessor
|
| 20 |
from retrieval_evaluation import RetrievalEvaluator, RetrievalResult, RetrievalTestSet
|
| 21 |
+
try:
|
| 22 |
+
from langchain_core.documents import Document
|
| 23 |
+
except ImportError:
|
| 24 |
+
try:
|
| 25 |
+
from langchain_core.documents import Document
|
| 26 |
+
except ImportError:
|
| 27 |
+
from langchain.schema import Document
|
| 28 |
|
| 29 |
# 导入LangChain相关模块
|
| 30 |
from langchain_community.vectorstores import FAISS, Chroma
|
| 31 |
from langchain_community.retrievers import BM25Retriever
|
| 32 |
+
try:
|
| 33 |
+
from langchain.retrievers import EnsembleRetriever
|
| 34 |
+
from langchain.retrievers import ContextualCompressionRetriever
|
| 35 |
+
from langchain.retrievers.document_compressors import LLMChainExtractor
|
| 36 |
+
except ImportError:
|
| 37 |
+
try:
|
| 38 |
+
from langchain_core.retrievers import EnsembleRetriever
|
| 39 |
+
from langchain_core.retrievers import ContextualCompressionRetriever
|
| 40 |
+
from langchain.retrievers.document_compressors import LLMChainExtractor
|
| 41 |
+
except ImportError:
|
| 42 |
+
print("Warning: Could not import advanced retriever components. Some features may be limited.")
|
| 43 |
+
EnsembleRetriever = None
|
| 44 |
+
ContextualCompressionRetriever = None
|
| 45 |
+
LLMChainExtractor = None
|
| 46 |
|
| 47 |
|
| 48 |
class AdaptiveRAGRetriever:
|
graph_indexer.py
CHANGED
|
@@ -7,9 +7,20 @@ from typing import List, Dict, Optional
|
|
| 7 |
import asyncio
|
| 8 |
try:
|
| 9 |
from langchain_core.documents import Document
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 10 |
except ImportError:
|
| 11 |
from langchain.schema import Document
|
| 12 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
from entity_extractor import EntityExtractor, EntityDeduplicator
|
| 14 |
from knowledge_graph import KnowledgeGraph, CommunitySummarizer
|
| 15 |
|
|
|
|
| 7 |
import asyncio
|
| 8 |
try:
|
| 9 |
from langchain_core.documents import Document
|
| 10 |
+
except ImportError:
|
| 11 |
+
try:
|
| 12 |
+
from langchain_core.documents import Document
|
| 13 |
+
except ImportError:
|
| 14 |
+
try:
|
| 15 |
+
from langchain_core.documents import Document
|
| 16 |
except ImportError:
|
| 17 |
from langchain.schema import Document
|
| 18 |
|
| 19 |
+
try:
|
| 20 |
+
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
| 21 |
+
except ImportError:
|
| 22 |
+
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
| 23 |
+
|
| 24 |
from entity_extractor import EntityExtractor, EntityDeduplicator
|
| 25 |
from knowledge_graph import KnowledgeGraph, CommunitySummarizer
|
| 26 |
|
graph_retriever.py
CHANGED
|
@@ -6,6 +6,9 @@ GraphRAG检索器
|
|
| 6 |
from typing import List, Dict, Set, Tuple
|
| 7 |
try:
|
| 8 |
from langchain_core.prompts import PromptTemplate
|
|
|
|
|
|
|
|
|
|
| 9 |
except ImportError:
|
| 10 |
from langchain.prompts import PromptTemplate
|
| 11 |
|
|
|
|
| 6 |
from typing import List, Dict, Set, Tuple
|
| 7 |
try:
|
| 8 |
from langchain_core.prompts import PromptTemplate
|
| 9 |
+
except ImportError:
|
| 10 |
+
try:
|
| 11 |
+
from langchain_core.prompts import PromptTemplate
|
| 12 |
except ImportError:
|
| 13 |
from langchain.prompts import PromptTemplate
|
| 14 |
|
knowledge_graph.py
CHANGED
|
@@ -17,6 +17,9 @@ except ImportError:
|
|
| 17 |
|
| 18 |
try:
|
| 19 |
from langchain_core.prompts import PromptTemplate
|
|
|
|
|
|
|
|
|
|
| 20 |
except ImportError:
|
| 21 |
from langchain.prompts import PromptTemplate
|
| 22 |
from langchain_community.chat_models import ChatOllama
|
|
|
|
| 17 |
|
| 18 |
try:
|
| 19 |
from langchain_core.prompts import PromptTemplate
|
| 20 |
+
except ImportError:
|
| 21 |
+
try:
|
| 22 |
+
from langchain_core.prompts import PromptTemplate
|
| 23 |
except ImportError:
|
| 24 |
from langchain.prompts import PromptTemplate
|
| 25 |
from langchain_community.chat_models import ChatOllama
|
local_llm_rag.py
CHANGED
|
@@ -19,7 +19,10 @@ _set_env("NOMIC_API_KEY")
|
|
| 19 |
# Ollama model name
|
| 20 |
local_llm = "mistral"
|
| 21 |
|
| 22 |
-
|
|
|
|
|
|
|
|
|
|
| 23 |
from langchain_community.document_loaders import WebBaseLoader
|
| 24 |
from langchain_community.vectorstores import Chroma
|
| 25 |
from langchain_nomic.embeddings import NomicEmbeddings
|
|
@@ -48,7 +51,10 @@ retriever = vectorstore.as_retriever()
|
|
| 48 |
|
| 49 |
### Router
|
| 50 |
|
| 51 |
-
|
|
|
|
|
|
|
|
|
|
| 52 |
from langchain_community.chat_models import ChatOllama
|
| 53 |
from langchain_core.output_parsers import JsonOutputParser
|
| 54 |
|
|
@@ -161,6 +167,12 @@ class GraphState(TypedDict):
|
|
| 161 |
generation: str
|
| 162 |
documents: List[str]
|
| 163 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 164 |
from langchain.schema import Document
|
| 165 |
|
| 166 |
|
|
|
|
| 19 |
# Ollama model name
|
| 20 |
local_llm = "mistral"
|
| 21 |
|
| 22 |
+
try:
|
| 23 |
+
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
| 24 |
+
except ImportError:
|
| 25 |
+
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
| 26 |
from langchain_community.document_loaders import WebBaseLoader
|
| 27 |
from langchain_community.vectorstores import Chroma
|
| 28 |
from langchain_nomic.embeddings import NomicEmbeddings
|
|
|
|
| 51 |
|
| 52 |
### Router
|
| 53 |
|
| 54 |
+
try:
|
| 55 |
+
from langchain_core.prompts import PromptTemplate
|
| 56 |
+
except ImportError:
|
| 57 |
+
from langchain.prompts import PromptTemplate
|
| 58 |
from langchain_community.chat_models import ChatOllama
|
| 59 |
from langchain_core.output_parsers import JsonOutputParser
|
| 60 |
|
|
|
|
| 167 |
generation: str
|
| 168 |
documents: List[str]
|
| 169 |
|
| 170 |
+
try:
|
| 171 |
+
from langchain_core.documents import Document
|
| 172 |
+
except ImportError:
|
| 173 |
+
try:
|
| 174 |
+
from langchain_core.documents import Document
|
| 175 |
+
except ImportError:
|
| 176 |
from langchain.schema import Document
|
| 177 |
|
| 178 |
|
retrieval_evaluation.py
CHANGED
|
@@ -8,7 +8,13 @@ import json
|
|
| 8 |
import numpy as np
|
| 9 |
from typing import List, Dict, Tuple, Any, Optional, Union
|
| 10 |
from dataclasses import dataclass, asdict
|
| 11 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 12 |
from sklearn.metrics import ndcg_score, precision_score, recall_score, f1_score
|
| 13 |
from sentence_transformers import SentenceTransformer, util
|
| 14 |
import matplotlib.pyplot as plt
|
|
|
|
| 8 |
import numpy as np
|
| 9 |
from typing import List, Dict, Tuple, Any, Optional, Union
|
| 10 |
from dataclasses import dataclass, asdict
|
| 11 |
+
try:
|
| 12 |
+
from langchain_core.documents import Document
|
| 13 |
+
except ImportError:
|
| 14 |
+
try:
|
| 15 |
+
from langchain_core.documents import Document
|
| 16 |
+
except ImportError:
|
| 17 |
+
from langchain.schema import Document
|
| 18 |
from sklearn.metrics import ndcg_score, precision_score, recall_score, f1_score
|
| 19 |
from sentence_transformers import SentenceTransformer, util
|
| 20 |
import matplotlib.pyplot as plt
|
routers_and_graders.py
CHANGED
|
@@ -5,6 +5,9 @@
|
|
| 5 |
|
| 6 |
try:
|
| 7 |
from langchain_core.prompts import PromptTemplate
|
|
|
|
|
|
|
|
|
|
| 8 |
except ImportError:
|
| 9 |
from langchain.prompts import PromptTemplate
|
| 10 |
|
|
|
|
| 5 |
|
| 6 |
try:
|
| 7 |
from langchain_core.prompts import PromptTemplate
|
| 8 |
+
except ImportError:
|
| 9 |
+
try:
|
| 10 |
+
from langchain_core.prompts import PromptTemplate
|
| 11 |
except ImportError:
|
| 12 |
from langchain.prompts import PromptTemplate
|
| 13 |
|
test_reranking.py
CHANGED
|
@@ -10,7 +10,13 @@ sys.path.append(os.path.dirname(__file__))
|
|
| 10 |
|
| 11 |
from document_processor import DocumentProcessor
|
| 12 |
from reranker import *
|
| 13 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14 |
import time
|
| 15 |
|
| 16 |
|
|
|
|
| 10 |
|
| 11 |
from document_processor import DocumentProcessor
|
| 12 |
from reranker import *
|
| 13 |
+
try:
|
| 14 |
+
from langchain_core.documents import Document
|
| 15 |
+
except ImportError:
|
| 16 |
+
try:
|
| 17 |
+
from langchain_core.documents import Document
|
| 18 |
+
except ImportError:
|
| 19 |
+
from langchain.schema import Document
|
| 20 |
import time
|
| 21 |
|
| 22 |
|
workflow_nodes.py
CHANGED
|
@@ -8,6 +8,12 @@ from typing import List
|
|
| 8 |
from typing_extensions import TypedDict
|
| 9 |
try:
|
| 10 |
from langchain_core.documents import Document
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11 |
except ImportError:
|
| 12 |
from langchain.schema import Document
|
| 13 |
from langchain_community.chat_models import ChatOllama
|
|
@@ -15,6 +21,9 @@ from langchain_core.output_parsers import StrOutputParser
|
|
| 15 |
from langchain_community.tools.tavily_search import TavilySearchResults
|
| 16 |
try:
|
| 17 |
from langchain_core.prompts import PromptTemplate
|
|
|
|
|
|
|
|
|
|
| 18 |
except ImportError:
|
| 19 |
from langchain.prompts import PromptTemplate
|
| 20 |
|
|
|
|
| 8 |
from typing_extensions import TypedDict
|
| 9 |
try:
|
| 10 |
from langchain_core.documents import Document
|
| 11 |
+
except ImportError:
|
| 12 |
+
try:
|
| 13 |
+
from langchain_core.documents import Document
|
| 14 |
+
except ImportError:
|
| 15 |
+
try:
|
| 16 |
+
from langchain_core.documents import Document
|
| 17 |
except ImportError:
|
| 18 |
from langchain.schema import Document
|
| 19 |
from langchain_community.chat_models import ChatOllama
|
|
|
|
| 21 |
from langchain_community.tools.tavily_search import TavilySearchResults
|
| 22 |
try:
|
| 23 |
from langchain_core.prompts import PromptTemplate
|
| 24 |
+
except ImportError:
|
| 25 |
+
try:
|
| 26 |
+
from langchain_core.prompts import PromptTemplate
|
| 27 |
except ImportError:
|
| 28 |
from langchain.prompts import PromptTemplate
|
| 29 |
|