smol fixes

gradio_app.py

@@ -59,9 +59,9 @@ def generate_answer(request: str, max_tokens: int = 256, language: str = "en", c
     try:
         maxTokens = max_tokens if 16 <= max_tokens <= 256 else 64
         if isinstance(custom_prompt, str):
-            userPrompt = custom_prompt + "\n\nUSER: " + request + "\nASSISTANT: "
+            userPrompt = custom_prompt.replace("{prompt}", request)
         else:
-            userPrompt = prompt + "\n\nUSER: " + request + "\nASSISTANT: "
+            userPrompt = prompt.replace("{prompt}", request)
         logs += f"\nFinal prompt: {userPrompt}\n"
     except:
         return "Not enough data! Check that you passed all needed data.", logs
@@ -78,7 +78,8 @@ def generate_answer(request: str, max_tokens: int = 256, language: str = "en", c
             counter += 1
         logs += f"Final attempt: {counter}\n"
                 
-        if language in languages:
+        
+        if language in languages and language != "en":
             logs += f"\nTranslating from en to {language}"
             encoded_input = translator_tokenizer(text, return_tensors="pt")
             generated_tokens = translator_model.generate(

quart_app.py

@@ -1,72 +0,0 @@
-# Importing libraries
-from transformers import M2M100Tokenizer, M2M100ForConditionalGeneration
-from quart import Quart, request
-from llama_cpp import Llama
-import psutil
-
-# Initing things
-app = Quart(__name__)                                              # Quart app
-llm = Llama(model_path="./model.bin")                              # LLaMa model
-llama_model_name = "TheBloke/WizardLM-1.0-Uncensored-Llama2-13B-GGUF"                
-translator_tokenizer = M2M100Tokenizer.from_pretrained(            # tokenizer for translator
-    "facebook/m2m100_418M", cache_dir="translator/"
-)
-translator_model = M2M100ForConditionalGeneration.from_pretrained( # translator model
-    "facebook/m2m100_418M", cache_dir="translator/"
-)
-translator_model.eval()
-
-# Preparing things to work
-translator_tokenizer.src_lang = "en"
-
-# Loading prompt
-with open('system.prompt', 'r', encoding='utf-8') as f:
-    prompt = f.read()
-
-# Defining
-@app.post("/request")
-async def echo():
-    try:
-        data = await request.get_json()
-        maxTokens = data.get("max_tokens", 64)
-        if isinstance(data.get("system_prompt"), str):
-            userPrompt = data.get("system_prompt") + "\n\nUser: " + data['request'] + "\nAssistant: "
-        else:
-            userPrompt = prompt + "\n\nUser: " + data['request'] + "\nAssistant: "
-    except:
-        return {"error": "Not enough data", "output": "Oops! Error occured! If you're a developer, using this API, check 'error' key."}, 400
-    try:
-        output = llm(userPrompt, max_tokens=maxTokens, stop=["User:", "\n"], echo=False)
-        text = output["choices"][0]["text"]
-        # i allowed only certain languages:
-        # russian (ru), ukranian (uk), chinese (zh)
-        if isinstance(data.get("target_lang"), str) and data.get("target_lang").lower() in ["ru", "uk", "zh"]:
-            encoded_input = translator_tokenizer(output, return_tensors="pt")
-            generated_tokens = translator_model.generate(
-                **encoded_input, forced_bos_token_id=translator_tokenizer.get_lang_id(data.get("target_lang"))
-            )
-            translated_text = translator_tokenizer.batch_decode(
-                generated_tokens, skip_special_tokens=True
-            )[0]
-            return {"output": text, "translated_output": translated_text}
-            
-        return {"output": text}
-    except Exception as e:
-        print(e)
-        return {"error": str(e), "output": "Oops! Internal server error. Check the logs. If you're a developer, using this API, check 'error' key."}, 500
-
-@app.get("/")
-async def get():
-    return '''<style>a:visited{color:black;}</style>
-<h1>Hello, world!</h1>
-This is showcase how to make own server with Llama2 model.<br>
-I'm using here 7b model just for example. Also here's only CPU power.<br>
-But you can use GPU power as well!<br><br>
-<h1>How to GPU?</h1>
-Change <code>`CMAKE_ARGS="-DLLAMA_BLAS=ON -DLLAMA_BLAS_VENDOR=OpenBLAS`</code> in Dockerfile on <code>`CMAKE_ARGS="-DLLAMA_CUBLAS=on"`</code>. Also you can try <code>`DLLAMA_CLBLAST`</code> or <code>`DLLAMA_METAL`</code>.<br><br>
-<h1>How to test it on own machine?</h1>
-You can install Docker, build image and run it. I made <code>`run-docker.sh`</code> for ya. To stop container run <code>`docker ps`</code>, find name of container and run <code>`docker stop _dockerContainerName_`</code><br>
-Or you can once follow steps in Dockerfile and try it on your machine, not in Docker.<br>
-<br>''' + f"Memory used: {psutil.virtual_memory()[2]}<br>" + '''
-<script>document.write("<b>URL of space:</b> "+window.location.href);</script>''' + '''
-Powered by <a href="https://github.com/abetlen/llama-cpp-python">llama-cpp-python</a>, <a href="https://quart.palletsprojects.com/">Quart</a> and <a href="https://www.uvicorn.org/">Uvicorn</a>.<br><br>'''

system.prompt

@@ -1,5 +1,8 @@
 You're an AI assistant named Alex. You're friendly and respectful.
 You speak as briefly, clearly and to the point as possible.
-You know many languages, for example: Russian, English.
+You know many languages, for example: English, Russian.
 You don't have access to the internet, so rely on your knowledge.
-If you are not sure of your answer, say so, but try not to misinform the user.
+If you are not sure of your answer, say so, but try not to misinform the user.
+
+USER: {prompt}
+ASSISTANT: