FAISS db store

fast_app_cz(obsolete).py

@@ -1,110 +0,0 @@
-from dotenv import load_dotenv
-import os
-import json
-from fastapi import FastAPI, Request, Form, Response
-from fastapi.responses import HTMLResponse
-from fastapi.templating import Jinja2Templates
-from fastapi.staticfiles import StaticFiles
-from fastapi.encoders import jsonable_encoder
-from langchain.llms import CTransformers
-
-from langchain.vectorstores import Chroma
-from langchain.text_splitter import RecursiveCharacterTextSplitter
-
-from langchain.chains import RetrievalQA
-from langchain.document_loaders import TextLoader, PyPDFLoader, DirectoryLoader
-from langchain.llms import OpenAI
-from langchain import PromptTemplate
-from langchain.embeddings import OpenAIEmbeddings, HuggingFaceEmbeddings
-
-app = FastAPI()
-load_dotenv()  
-openai_api_key = os.environ.get("OPENAI_API_KEY")
-templates = Jinja2Templates(directory="templates")
-app.mount("/static", StaticFiles(directory="static"), name="static")
-# embedding_model = "Seznam/simcse-dist-mpnet-czeng-cs-en"
-embedding_model = "Seznam/simcse-dist-mpnet-paracrawl-cs-en"
-persist_directory = "stores/seznampara_ul_512"
-
-llm = OpenAI(openai_api_key=openai_api_key)
-# llm = "model\dolphin-2.6-mistral-7b.Q4_K_S.gguf"
-# llm = "neural-chat-7b-v3-1.Q4_K_M.gguf"
-
-
-"""
-### - Local LLM settings - ###
-
-config = {
-    "max_new_tokens": 1024,
-    "repetition_penalty": 1.1,
-    "temperature": 0.1,
-    "top_k": 50,
-    "top_p": 0.9,
-    "stream": True,
-    "threads": int(os.cpu_count() / 2),
-}
-
-llm = CTransformers(
-    model=llm, model_type="mistral", lib="avx2", **config  # for CPU use
-)
-
-### - Local LLM settings end - ###
-"""
-
-prompt_template = """Use the following pieces of information to answer the user's question.
-If you don't know the answer, just say that you don't know, don't try to make up an answer.
-
-Context: {context}
-Question: {question}
-
-Only return the helpful answer below and nothing else.
-Helpful answer:
-"""
-
-prompt = PromptTemplate(
-    template=prompt_template, input_variables=["context", "question"]
-)
-
-print("\n Prompt ready... \n\n")
-
-
-model_name = embedding_model
-model_kwargs = {"device": "cpu"}
-encode_kwargs = {"normalize_embeddings": False}
-embedding = HuggingFaceEmbeddings(
-    model_name=model_name, model_kwargs=model_kwargs, encode_kwargs=encode_kwargs
-)
-vectordb = Chroma(persist_directory=persist_directory, embedding_function=embedding)
-retriever = vectordb.as_retriever(search_kwargs={"k": 3})
-
-print("\n Retrieval Ready....\n\n")
-
-
-@app.get("/", response_class=HTMLResponse)
-def read_item(request: Request):
-    return templates.TemplateResponse("index.html", {"request": request})
-
-
-@app.post("/get_response")
-async def get_response(query: str = Form(...)):
-
-    chain_type_kwargs = {"prompt": prompt}
-    qa_chain = RetrievalQA.from_chain_type(
-        llm=llm,
-        chain_type="stuff",
-        retriever=retriever,
-        return_source_documents=True,
-        chain_type_kwargs=chain_type_kwargs,
-        verbose=True,
-    )
-    response = qa_chain(query)
-    print(response)
-    answer = response["result"]
-    source_document = response["source_documents"][0].page_content
-    doc = response["source_documents"][0].metadata["source"]
-    response_data = jsonable_encoder(
-        json.dumps({"answer": answer, "source_document": source_document, "doc": doc})
-    )
-
-    res = Response(response_data)
-    return res

ingest(obsolete).py

@@ -1,59 +0,0 @@
-from langchain.vectorstores import Chroma
-from langchain.text_splitter import RecursiveCharacterTextSplitter
-
-from langchain.document_loaders import (
-    PyPDFLoader,
-    DirectoryLoader,
-    UnstructuredFileLoader,
-)
-from langchain.document_loaders.csv_loader import CSVLoader
-from langchain.embeddings import (
-    OpenAIEmbeddings,
-    HuggingFaceBgeEmbeddings,
-    HuggingFaceEmbeddings,
-    HuggingFaceInstructEmbeddings,
-)
-
-
-persist_directory = "stores/test_512"
-data = "data\czech"
-chunk = 512
-overlap = 128
-# embedding_model = "Seznam/simcse-dist-mpnet-czeng-cs-en"
-embedding_model = "Seznam/simcse-dist-mpnet-paracrawl-cs-en"
-
-model_name = embedding_model
-model_kwargs = {"device": "cpu"}
-encode_kwargs = {"normalize_embeddings": False}
-embedding = HuggingFaceEmbeddings(
-    model_name=model_name, model_kwargs=model_kwargs, encode_kwargs=encode_kwargs
-)
-
-"""
-loader = CSVLoader(
-    file_path="data/emails.csv",
-    encoding="utf-8",
-    csv_args={
-        "delimiter": ";",
-    },
-)
-
-"""
-
-loader = DirectoryLoader(data, show_progress=True)
-
-
-documents = loader.load()
-text_splitter = RecursiveCharacterTextSplitter(
-    chunk_size=chunk,
-    chunk_overlap=overlap,
-)
-texts = text_splitter.split_documents(documents)
-vectordb = Chroma.from_documents(
-    documents=texts,
-    embedding=embedding,
-    persist_directory=persist_directory,
-    collection_metadata={"hnsw:space": "cosine"},
-)
-
-print("\n Vector Store Created.......\n\n")

ingest.py

@@ -90,41 +90,3 @@ class Ingest:
         vectordb.save_local(self.czech_store)
 
         print("\n Czech vector Store Created.......\n\n")
-
-
-"""       
-    
-    
-    
-openai_api_key = "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"
-persist_directory = "stores/store_512"
-data = "data/"
-chunk = 512
-overlap = 256
-
-embedding = OpenAIEmbeddings(
-    openai_api_key=openai_api_key,
-    model="text-embedding-3-large",
-    #    model_kwargs={"device": "cpu"},
-)
-
-loader = DirectoryLoader(
-    data, glob="**/*.pdf", show_progress=True, loader_cls=PyPDFLoader
-)
-documents = loader.load()
-text_splitter = RecursiveCharacterTextSplitter(
-    chunk_size=chunk,
-    chunk_overlap=overlap,
-)
-texts = text_splitter.split_documents(documents)
-
-vectordb = Chroma.from_documents(
-    documents=texts,
-    embedding=embedding,
-    persist_directory=persist_directory,
-    collection_metadata={"hnsw:space": "cosine"},
-)
-
-print("\n Vector Store Created.......\n\n")
-
-"""