Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
|
@@ -2,11 +2,10 @@ from transformers import T5Tokenizer, T5ForConditionalGeneration
|
|
| 2 |
from langchain.llms import HuggingFacePipeline
|
| 3 |
from langchain.prompts import PromptTemplate
|
| 4 |
from langchain.chains import RetrievalQA
|
| 5 |
-
from
|
| 6 |
-
from
|
| 7 |
-
from
|
| 8 |
from langchain.text_splitter import CharacterTextSplitter
|
| 9 |
-
from langchain_community.document_loaders import WikipediaLoader
|
| 10 |
from transformers import pipeline
|
| 11 |
|
| 12 |
# Load T5-small model and tokenizer
|
|
@@ -26,19 +25,16 @@ text_generation_pipeline = pipeline(
|
|
| 26 |
# Create a LangChain LLM from the pipeline
|
| 27 |
llm = HuggingFacePipeline(pipeline=text_generation_pipeline)
|
| 28 |
|
| 29 |
-
# Load and process documents
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
# Load content from Wikipedia
|
| 34 |
-
loader = WikipediaLoader(query="Artificial neuron", load_max_docs=1)
|
| 35 |
documents = loader.load()
|
| 36 |
-
|
| 37 |
text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
|
| 38 |
texts = text_splitter.split_documents(documents)
|
| 39 |
|
| 40 |
-
# Create embeddings
|
| 41 |
-
embeddings = HuggingFaceEmbeddings()
|
|
|
|
|
|
|
| 42 |
db = Chroma.from_documents(texts, embeddings)
|
| 43 |
|
| 44 |
# Create a retriever
|
|
|
|
| 2 |
from langchain.llms import HuggingFacePipeline
|
| 3 |
from langchain.prompts import PromptTemplate
|
| 4 |
from langchain.chains import RetrievalQA
|
| 5 |
+
from langchain.embeddings import HuggingFaceEmbeddings
|
| 6 |
+
from langchain.vectorstores import Chroma
|
| 7 |
+
from langchain.document_loaders import TextLoader
|
| 8 |
from langchain.text_splitter import CharacterTextSplitter
|
|
|
|
| 9 |
from transformers import pipeline
|
| 10 |
|
| 11 |
# Load T5-small model and tokenizer
|
|
|
|
| 25 |
# Create a LangChain LLM from the pipeline
|
| 26 |
llm = HuggingFacePipeline(pipeline=text_generation_pipeline)
|
| 27 |
|
| 28 |
+
# Load and process documents from a local file
|
| 29 |
+
loader = TextLoader("NeuralNetworkWikipedia.txt")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 30 |
documents = loader.load()
|
|
|
|
| 31 |
text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
|
| 32 |
texts = text_splitter.split_documents(documents)
|
| 33 |
|
| 34 |
+
# Create embeddings using a smaller model
|
| 35 |
+
embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
|
| 36 |
+
|
| 37 |
+
# Create vector store
|
| 38 |
db = Chroma.from_documents(texts, embeddings)
|
| 39 |
|
| 40 |
# Create a retriever
|