upload backend

conversadocs/bones.py

@@ -7,7 +7,7 @@ from langchain.memory import ConversationBufferMemory
 from langchain.chat_models import ChatOpenAI
 from langchain.embeddings import HuggingFaceEmbeddings
 from langchain import HuggingFaceHub
-from langchain.llms import LlamaCpp
+from conversadocs.llamacppmodels import LlamaCpp #from langchain.llms import LlamaCpp
 from huggingface_hub import hf_hub_download
 import param
 import os
@@ -23,37 +23,28 @@ from langchain.document_loaders import (
     UnstructuredWordDocumentLoader,
     PyPDFLoader,
 )
+import gc
+gc.collect()
+torch.cuda.empty_cache()
 
 #YOUR_HF_TOKEN = os.getenv("My_hf_token")
-llm_api=HuggingFaceHub(
-    huggingfacehub_api_token=os.getenv("My_hf_token"),
-    repo_id="tiiuae/falcon-7b-instruct",
-    model_kwargs={
-        "temperature":0.2,
-        "max_new_tokens":500,
-        "top_k":50,
-        "top_p":0.95,
-        "repetition_penalty":1.2,
-        },), #ChatOpenAI(model_name=llm_name, temperature=0)
 
+EXTENSIONS = {
+    ".txt": (TextLoader, {"encoding": "utf8"}),
+    ".pdf": (PyPDFLoader, {}),
+    ".doc": (UnstructuredWordDocumentLoader, {}),
+    ".docx": (UnstructuredWordDocumentLoader, {}),
+    ".enex": (EverNoteLoader, {}),
+    ".epub": (UnstructuredEPubLoader, {}),
+    ".html": (UnstructuredHTMLLoader, {}),
+    ".md": (UnstructuredMarkdownLoader, {}),
+    ".odt": (UnstructuredODTLoader, {}),
+    ".ppt": (UnstructuredPowerPointLoader, {}),
+    ".pptx": (UnstructuredPowerPointLoader, {}),
+}
 
 #alter
 def load_db(files):
-    EXTENSIONS = {
-        ".txt": (TextLoader, {"encoding": "utf8"}),
-        ".pdf": (PyPDFLoader, {}),
-        ".doc": (UnstructuredWordDocumentLoader, {}),
-        ".docx": (UnstructuredWordDocumentLoader, {}),
-        ".enex": (EverNoteLoader, {}),
-        ".epub": (UnstructuredEPubLoader, {}),
-        ".html": (UnstructuredHTMLLoader, {}),
-        ".md": (UnstructuredMarkdownLoader, {}),
-        ".odt": (UnstructuredODTLoader, {}),
-        ".ppt": (UnstructuredPowerPointLoader, {}),
-        ".pptx": (UnstructuredPowerPointLoader, {}),
-    }
-
-
 
     # select extensions loader
     documents = []
@@ -102,14 +93,14 @@ class DocChat(param.Parameterized):
     answer = param.String("")
     db_query  = param.String("")
     db_response = param.List([])
-    llm = llm_api[0]
     k_value = param.Integer(3)
-
+    llm = None
 
     def __init__(self,  **params):
         super(DocChat, self).__init__( **params)
-        self.loaded_file = "demo_docs/demo.txt"
+        self.loaded_file = ["demo_docs/demo.txt"]
         self.db = load_db(self.loaded_file)
+        self.change_llm("TheBloke/Llama-2-7B-Chat-GGML", "llama-2-7b-chat.ggmlv3.q5_1.bin", max_tokens=256, temperature=0.2, top_p=0.95, top_k=50, repeat_penalty=1.2, k=3)
         self.qa = q_a(self.db, "stuff", self.k_value, self.llm)
         
 
@@ -141,7 +132,8 @@ class DocChat(param.Parameterized):
         try:
           result = self.qa({"question": query, "chat_history": self.chat_history})
         except:
-          self.default_falcon_model()
+          print("Error not get response from model, reloaded default llama-2 7B config")
+          self.change_llm("TheBloke/Llama-2-7B-Chat-GGML", "llama-2-7b-chat.ggmlv3.q5_1.bin", max_tokens=256, temperature=0.2, top_p=0.95, top_k=50, repeat_penalty=1.2, k=3)
           self.qa = q_a(self.db, "stuff", k_max, self.llm)
           result = self.qa({"question": query, "chat_history": self.chat_history})
         
@@ -151,17 +143,48 @@ class DocChat(param.Parameterized):
         self.answer = result['answer']
         return self.answer
 
-    def change_llm(self, repo_, file_, max_tokens=16, temperature=0.2, top_p=0.95, top_k=50, repeat_penalty=1.2, k=3):
+    def summarize(self, chunk_size=2000, chunk_overlap=100):
+        # load docs
+        documents = []
+        for file in self.loaded_file:
+          ext = "." + file.rsplit(".", 1)[-1]
+          if ext in EXTENSIONS:
+              loader_class, loader_args = EXTENSIONS[ext]
+              loader = loader_class(file, **loader_args)
+              documents.extend(loader.load_and_split())
+
+        if documents == []:
+            return "Error in summarization"
+
+        # split documents
+        text_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=chunk_size, 
+            chunk_overlap=chunk_overlap,
+            separators=["\n\n", "\n", "(?<=\. )", " ", ""]
+            )
+        docs = text_splitter.split_documents(documents)
+        # summarize
+        from langchain.chains.summarize import load_summarize_chain
+        chain = load_summarize_chain(self.llm, chain_type='map_reduce', verbose=True)
+        return chain.run(docs)
+
+    def change_llm(self, repo_, file_, max_tokens=256, temperature=0.2, top_p=0.95, top_k=50, repeat_penalty=1.2, k=3):
 
         if torch.cuda.is_available():
             try:
               model_path = hf_hub_download(repo_id=repo_, filename=file_)
+              
+              self.qa = None
+              self.llm = None
+              gc.collect()
+              torch.cuda.empty_cache()
+              gpu_llm_layers = 35 if not '70B' in repo_.upper() else 25 # fix for 70B
 
               self.llm = LlamaCpp(
                   model_path=model_path,
-                  n_ctx=1000,
+                  n_ctx=4096,
                   n_batch=512,
-                  n_gpu_layers=35,
+                  n_gpu_layers=gpu_llm_layers, 
                   max_tokens=max_tokens,
                   verbose=False,
                   temperature=temperature,
@@ -173,14 +196,20 @@ class DocChat(param.Parameterized):
               self.k_value = k
               return f"Loaded {file_} [GPU INFERENCE]"
             except:
-              return "No valid model"
+              self.change_llm("TheBloke/Llama-2-7B-Chat-GGML", "llama-2-7b-chat.ggmlv3.q5_1.bin", max_tokens=256, temperature=0.2, top_p=0.95, top_k=50, repeat_penalty=1.2, k=3)
+              return "No valid model | Reloaded Reloaded default llama-2 7B config"
         else:
             try:
               model_path = hf_hub_download(repo_id=repo_, filename=file_)
+              
+              self.qa = None
+              self.llm = None
+              gc.collect()
+              torch.cuda.empty_cache()
 
               self.llm = LlamaCpp(
                   model_path=model_path,
-                  n_ctx=1000,
+                  n_ctx=2048,
                   n_batch=8,
                   max_tokens=max_tokens,
                   verbose=False,
@@ -193,10 +222,20 @@ class DocChat(param.Parameterized):
               self.k_value = k
               return f"Loaded {file_} [CPU INFERENCE SLOW]"
             except:
-              return "No valid model"
+              self.change_llm("TheBloke/Llama-2-7B-Chat-GGML", "llama-2-7b-chat.ggmlv3.q5_1.bin", max_tokens=256, temperature=0.2, top_p=0.95, top_k=50, repeat_penalty=1.2, k=3)
+              return "No valid model | Reloaded default llama-2 7B config"
 
-    def default_falcon_model(self):
-      self.llm = llm_api[0]
+    def default_falcon_model(self, HF_TOKEN):
+      self.llm = llm_api=HuggingFaceHub(
+          huggingfacehub_api_token=HF_TOKEN,
+          repo_id="tiiuae/falcon-7b-instruct",
+          model_kwargs={
+              "temperature":0.2,
+              "max_new_tokens":500,
+              "top_k":50,
+              "top_p":0.95,
+              "repetition_penalty":1.2,
+              },)
       self.qa = q_a(self.db, "stuff", self.k_value, self.llm)
       return "Loaded model Falcon 7B-instruct [API FAST INFERENCE]"
 
@@ -204,7 +243,7 @@ class DocChat(param.Parameterized):
         self.llm = ChatOpenAI(temperature=0, openai_api_key=API_KEY, model_name='gpt-3.5-turbo')
         self.qa = q_a(self.db, "stuff", self.k_value, self.llm)
         API_KEY = ""
-        return "Loaded model OpenAI gpt-3.5-turbo [API FAST INFERENCE] | If there is no response from the API, Falcon 7B-instruct will be used."
+        return "Loaded model OpenAI gpt-3.5-turbo [API FAST INFERENCE]"
 
     @param.depends('db_query ', )
     def get_lquest(self):

conversadocs/llamacppmodels.py

@@ -0,0 +1,309 @@
+import logging
+from typing import Any, Dict, Iterator, List, Optional
+
+from pydantic import Field, root_validator
+
+from langchain.callbacks.manager import CallbackManagerForLLMRun
+from langchain.llms.base import LLM
+from langchain.schema.output import GenerationChunk
+
+logger = logging.getLogger(__name__)
+
+
+class LlamaCpp(LLM):
+    """llama.cpp model.
+
+    To use, you should have the llama-cpp-python library installed, and provide the
+    path to the Llama model as a named parameter to the constructor.
+    Check out: https://github.com/abetlen/llama-cpp-python
+
+    Example:
+        .. code-block:: python
+
+            from langchain.llms import LlamaCpp
+            llm = LlamaCpp(model_path="/path/to/llama/model")
+    """
+
+    client: Any  #: :meta private:
+    model_path: str
+    """The path to the Llama model file."""
+
+    lora_base: Optional[str] = None
+    """The path to the Llama LoRA base model."""
+
+    lora_path: Optional[str] = None
+    """The path to the Llama LoRA. If None, no LoRa is loaded."""
+
+    n_ctx: int = Field(512, alias="n_ctx")
+    """Token context window."""
+
+    n_parts: int = Field(-1, alias="n_parts")
+    """Number of parts to split the model into.
+    If -1, the number of parts is automatically determined."""
+
+    seed: int = Field(-1, alias="seed")
+    """Seed. If -1, a random seed is used."""
+
+    f16_kv: bool = Field(True, alias="f16_kv")
+    """Use half-precision for key/value cache."""
+
+    logits_all: bool = Field(False, alias="logits_all")
+    """Return logits for all tokens, not just the last token."""
+
+    vocab_only: bool = Field(False, alias="vocab_only")
+    """Only load the vocabulary, no weights."""
+
+    use_mlock: bool = Field(False, alias="use_mlock")
+    """Force system to keep model in RAM."""
+
+    n_threads: Optional[int] = Field(None, alias="n_threads")
+    """Number of threads to use.
+    If None, the number of threads is automatically determined."""
+
+    n_batch: Optional[int] = Field(8, alias="n_batch")
+    """Number of tokens to process in parallel.
+    Should be a number between 1 and n_ctx."""
+
+    n_gpu_layers: Optional[int] = Field(None, alias="n_gpu_layers")
+    """Number of layers to be loaded into gpu memory. Default None."""
+
+    suffix: Optional[str] = Field(None)
+    """A suffix to append to the generated text. If None, no suffix is appended."""
+
+    max_tokens: Optional[int] = 256
+    """The maximum number of tokens to generate."""
+
+    temperature: Optional[float] = 0.8
+    """The temperature to use for sampling."""
+
+    top_p: Optional[float] = 0.95
+    """The top-p value to use for sampling."""
+
+    logprobs: Optional[int] = Field(None)
+    """The number of logprobs to return. If None, no logprobs are returned."""
+
+    echo: Optional[bool] = False
+    """Whether to echo the prompt."""
+
+    stop: Optional[List[str]] = []
+    """A list of strings to stop generation when encountered."""
+
+    repeat_penalty: Optional[float] = 1.1
+    """The penalty to apply to repeated tokens."""
+
+    top_k: Optional[int] = 40
+    """The top-k value to use for sampling."""
+
+    last_n_tokens_size: Optional[int] = 64
+    """The number of tokens to look back when applying the repeat_penalty."""
+
+    use_mmap: Optional[bool] = True
+    """Whether to keep the model loaded in RAM"""
+
+    rope_freq_scale: float = 1.0
+    """Scale factor for rope sampling."""
+
+    rope_freq_base: float = 10000.0
+    """Base frequency for rope sampling."""
+
+    streaming: bool = True
+    """Whether to stream the results, token by token."""
+
+    verbose: bool = True
+    """Print verbose output to stderr."""
+
+    n_gqa: Optional[int] = None
+
+    @root_validator()
+    def validate_environment(cls, values: Dict) -> Dict:
+        """Validate that llama-cpp-python library is installed."""
+
+        
+        model_path = values["model_path"]
+        model_param_names = [
+            "n_gqa",
+            "rope_freq_scale",
+            "rope_freq_base",
+            "lora_path",
+            "lora_base",
+            "n_ctx",
+            "n_parts",
+            "seed",
+            "f16_kv",
+            "logits_all",
+            "vocab_only",
+            "use_mlock",
+            "n_threads",
+            "n_batch",
+            "use_mmap",
+            "last_n_tokens_size",
+            "verbose",
+        ]
+        model_params = {k: values[k] for k in model_param_names}
+
+        model_params['n_gqa'] = 8 if '70B' in model_path.upper() else None # (TEMPORARY) must be 8 for llama2 70b
+        # For backwards compatibility, only include if non-null.
+        if values["n_gpu_layers"] is not None:
+            model_params["n_gpu_layers"] = values["n_gpu_layers"]
+
+        try:
+            from llama_cpp import Llama
+
+            values["client"] = Llama(model_path, **model_params)
+        except ImportError:
+            raise ImportError(
+                "Could not import llama-cpp-python library. "
+                "Please install the llama-cpp-python library to "
+                "use this embedding model: pip install llama-cpp-python"
+            )
+        except Exception as e:
+            raise ValueError(
+                f"Could not load Llama model from path: {model_path}. "
+                f"Received error {e}"
+            )
+
+        return values
+
+    @property
+    def _default_params(self) -> Dict[str, Any]:
+        """Get the default parameters for calling llama_cpp."""
+        return {
+            "suffix": self.suffix,
+            "max_tokens": self.max_tokens,
+            "temperature": self.temperature,
+            "top_p": self.top_p,
+            "logprobs": self.logprobs,
+            "echo": self.echo,
+            "stop_sequences": self.stop,  # key here is convention among LLM classes
+            "repeat_penalty": self.repeat_penalty,
+            "top_k": self.top_k,
+        }
+
+    @property
+    def _identifying_params(self) -> Dict[str, Any]:
+        """Get the identifying parameters."""
+        return {**{"model_path": self.model_path}, **self._default_params}
+
+    @property
+    def _llm_type(self) -> str:
+        """Return type of llm."""
+        return "llamacpp"
+
+    def _get_parameters(self, stop: Optional[List[str]] = None) -> Dict[str, Any]:
+        """
+        Performs sanity check, preparing parameters in format needed by llama_cpp.
+
+        Args:
+            stop (Optional[List[str]]): List of stop sequences for llama_cpp.
+
+        Returns:
+            Dictionary containing the combined parameters.
+        """
+
+        # Raise error if stop sequences are in both input and default params
+        if self.stop and stop is not None:
+            raise ValueError("`stop` found in both the input and default params.")
+
+        params = self._default_params
+
+        # llama_cpp expects the "stop" key not this, so we remove it:
+        params.pop("stop_sequences")
+
+        # then sets it as configured, or default to an empty list:
+        params["stop"] = self.stop or stop or []
+
+        return params
+
+    def _call(
+        self,
+        prompt: str,
+        stop: Optional[List[str]] = None,
+        run_manager: Optional[CallbackManagerForLLMRun] = None,
+        **kwargs: Any,
+    ) -> str:
+        """Call the Llama model and return the output.
+
+        Args:
+            prompt: The prompt to use for generation.
+            stop: A list of strings to stop generation when encountered.
+
+        Returns:
+            The generated text.
+
+        Example:
+            .. code-block:: python
+
+                from langchain.llms import LlamaCpp
+                llm = LlamaCpp(model_path="/path/to/local/llama/model.bin")
+                llm("This is a prompt.")
+        """
+        if self.streaming:
+            # If streaming is enabled, we use the stream
+            # method that yields as they are generated
+            # and return the combined strings from the first choices's text:
+            combined_text_output = ""
+            for chunk in self._stream(
+                prompt=prompt, stop=stop, run_manager=run_manager, **kwargs
+            ):
+                combined_text_output += chunk.text
+            return combined_text_output
+        else:
+            params = self._get_parameters(stop)
+            params = {**params, **kwargs}
+            result = self.client(prompt=prompt, **params)
+            return result["choices"][0]["text"]
+
+    def _stream(
+        self,
+        prompt: str,
+        stop: Optional[List[str]] = None,
+        run_manager: Optional[CallbackManagerForLLMRun] = None,
+        **kwargs: Any,
+    ) -> Iterator[GenerationChunk]:
+        """Yields results objects as they are generated in real time.
+
+        It also calls the callback manager's on_llm_new_token event with
+        similar parameters to the OpenAI LLM class method of the same name.
+
+        Args:
+            prompt: The prompts to pass into the model.
+            stop: Optional list of stop words to use when generating.
+
+        Returns:
+            A generator representing the stream of tokens being generated.
+
+        Yields:
+            A dictionary like objects containing a string token and metadata.
+            See llama-cpp-python docs and below for more.
+
+        Example:
+            .. code-block:: python
+
+                from langchain.llms import LlamaCpp
+                llm = LlamaCpp(
+                    model_path="/path/to/local/model.bin",
+                    temperature = 0.5
+                )
+                for chunk in llm.stream("Ask 'Hi, how are you?' like a pirate:'",
+                        stop=["'","\n"]):
+                    result = chunk["choices"][0]
+                    print(result["text"], end='', flush=True)
+
+        """
+        params = {**self._get_parameters(stop), **kwargs}
+        result = self.client(prompt=prompt, stream=True, **params)
+        for part in result:
+            logprobs = part["choices"][0].get("logprobs", None)
+            chunk = GenerationChunk(
+                text=part["choices"][0]["text"],
+                generation_info={"logprobs": logprobs},
+            )
+            yield chunk
+            if run_manager:
+                run_manager.on_llm_new_token(
+                    token=chunk.text, verbose=self.verbose, log_probs=logprobs
+                )
+
+    def get_num_tokens(self, text: str) -> int:
+        tokenized_text = self.client.tokenize(text.encode("utf-8"))
+        return len(tokenized_text)

conversadocs/llm_chess.py

@@ -0,0 +1,101 @@
+import os
+import chess
+import chess.pgn
+
+# credit code https://github.com/notnil/chess-gpt
+def get_legal_moves(board):
+    """Returns a list of legal moves in UCI notation."""
+    return list(map(board.san, board.legal_moves))
+
+def init_game() -> tuple[chess.pgn.Game, chess.Board]:
+    """Initializes a new game."""
+    board = chess.Board()
+    game = chess.pgn.Game()
+    game.headers["White"] = "User"
+    game.headers["Black"] = "Chess-engine"
+    del game.headers["Event"]
+    del game.headers["Date"]
+    del game.headers["Site"]
+    del game.headers["Round"]
+    del game.headers["Result"]
+    game.setup(board)
+    return game, board
+
+def generate_prompt(game: chess.pgn.Game, board: chess.Board) -> str:
+
+    moves = get_legal_moves(board)
+    moves_str = ",".join(moves)
+    return f"""
+    The task is play a Chess game:
+    You are the Chess-engine playing a chess match against the user as black and trying to win.
+
+    The current FEN notation is:
+    {board.fen()}
+
+    The next valid moves are:
+    {moves_str}
+
+    Continue the game.
+    {str(game)[:-2]}"""
+
+def get_move(content, moves):
+    lines = content.splitlines()
+    for line in lines:
+      for lm in moves:
+        if lm in line:
+          return lm
+
+class ChessGame:
+    def __init__(self, docschatllm):
+      self.docschatllm = docschatllm
+
+    def start_game(self):
+        self.game, self.board = init_game()
+        self.game_cp, _ = init_game()
+        self.node = self.game
+        self.node_copy = self.game_cp
+
+        svg_board = chess.svg.board(self.board, size=350)
+        return svg_board, "Valid moves: "+",".join(get_legal_moves(self.board)) # display(self.board)
+
+    def user_move(self, move_input):
+        try:
+          self.board.push_san(move_input)
+        except ValueError:
+          print("Invalid move")
+          svg_board = chess.svg.board(self.board, size=350)
+          return svg_board, "Valid moves: "+",".join(get_legal_moves(self.board)), 'Invalid move'
+        self.node = self.node.add_variation(self.board.move_stack[-1])
+        self.node_copy = self.node_copy.add_variation(self.board.move_stack[-1])
+
+        if self.board.is_game_over():
+          svg_board = chess.svg.board(self.board, size=350)
+          return svg_board, ",".join(get_legal_moves(self.board)), 'GAME OVER'
+
+        prompt = generate_prompt(self.game, self.board)
+        print("Prompt: \n"+prompt)
+        print("#############")
+        for i in range(10): #tries
+          if i == 9:
+            svg_board = chess.svg.board(self.board, size=350)
+            return svg_board, ",".join(get_legal_moves(self.board)), "The model can't do a valid move"
+          try:
+            """Returns the move from the prompt."""
+            content = self.docschatllm.llm.predict(prompt) ### from selected model ###
+            #print(moves)
+            print("Response: \n"+content)
+            print("#############")
+
+            moves = get_legal_moves(self.board)
+            move = get_move(content, moves)
+            print(move)
+            print("#############")
+            self.board.push_san(move)
+            break
+          except:
+            prompt = prompt[1:]
+            print("attempt a move.")
+        self.node = self.node.add_variation(self.board.move_stack[-1])
+        self.node_copy = self.node_copy.add_variation(self.board.move_stack[-1])
+        svg_board = chess.svg.board(self.board, size=350)
+        return svg_board, "Valid moves: "+",".join(get_legal_moves(self.board)), ''