added ctransformers

.env.example

@@ -2,6 +2,7 @@
 # LLM_MODEL_TYPE=gpt4all-j
 # LLM_MODEL_TYPE=gpt4all
 # LLM_MODEL_TYPE=llamacpp
+# LLM_MODEL_TYPE=ctransformers
 # LLM_MODEL_TYPE=huggingface
 # LLM_MODEL_TYPE=mosaicml
 # LLM_MODEL_TYPE=stablelm
@@ -71,6 +72,9 @@ GPT4ALL_DOWNLOAD_LINK=https://gpt4all.io/models/ggml-nous-gpt4-vicuna-13b.bin
 LLAMACPP_MODEL_PATH="../models/llama-2-7b-chat.ggmlv3.q4_K_M.bin"
 LLAMACPP_DOWNLOAD_LINK=https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGML/resolve/main/llama-2-7b-chat.ggmlv3.q4_K_M.bin
 
+CTRANSFORMERS_MODEL_PATH="../models/llama-2-7b-chat.ggmlv3.q4_K_M.bin"
+CTRANSFORMERS_DOWNLOAD_LINK=https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGML/resolve/main/llama-2-7b-chat.ggmlv3.q4_K_M.bin
+
 # Index for AI Books PDF files - chunk_size=1024 chunk_overlap=512
 # CHROMADB_INDEX_PATH="./data/chromadb_1024_512/"
 FAISS_INDEX_PATH="./data/ai_books/"

Makefile

@@ -55,9 +55,9 @@ install:
 	
 install-extra:
 	CXX=g++-11  CC=gcc-11 pip install -U -r requirements_extra.txt
-	pip show langchain llama-cpp-python transformers
+	pip show llama-cpp-python ctransformers
 	
 install-extra-mac:
 	# brew install llvm libomp
 	CXX=/usr/local/opt/llvm/bin/clang++ CC=/usr/local/opt/llvm/bin/clang pip install -U -r requirements_extra.txt
-	pip show langchain llama-cpp-python transformers
+	pip show llama-cpp-python ctransformers

app_modules/llm_loader.py

@@ -9,7 +9,13 @@ from langchain import HuggingFaceTextGenInference
 from langchain.callbacks.base import BaseCallbackHandler
 from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
 from langchain.chat_models import ChatOpenAI
-from langchain.llms import GPT4All, HuggingFacePipeline, LlamaCpp, OpenLLM
+from langchain.llms import (
+    CTransformers,
+    GPT4All,
+    HuggingFacePipeline,
+    LlamaCpp,
+    OpenLLM,
+)
 from langchain.schema import LLMResult
 from transformers import (
     AutoConfig,
@@ -189,6 +195,15 @@ class LLMLoader:
                     verbose=True,
                     use_mlock=True,
                 )
+            elif self.llm_model_type == "ctransformers":
+                MODEL_PATH = ensure_model_is_downloaded(self.llm_model_type)
+                config = {
+                    "max_new_tokens": self.max_tokens_limit,
+                    "repetition_penalty": 1.1,
+                }
+                self.llm = CTransformers(
+                    model=MODEL_PATH, model_type="llama", config=config
+                )
             elif self.llm_model_type == "hftgi":
                 HFTGI_SERVER_URL = os.environ.get("HFTGI_SERVER_URL")
                 self.max_tokens_limit = 4096

app_modules/utils.py

@@ -132,6 +132,9 @@ def ensure_model_is_downloaded(llm_model_type):
     elif llm_model_type == "llamacpp":
         local_path = os.environ.get("LLAMACPP_MODEL_PATH")
         url = os.environ.get("LLAMACPP_DOWNLOAD_LINK")
+    elif llm_model_type == "ctransformers":
+        local_path = os.environ.get("CTRANSFORMERS_MODEL_PATH")
+        url = os.environ.get("CTRANSFORMERS_DOWNLOAD_LINK")
     else:
         raise ValueError(f"wrong model typle: {llm_model_type}")
 

requirements_extra.txt

@@ -1,3 +1,4 @@
 llama-cpp-python
 pyllamacpp
 chromadb
+ctransformers