Update app.py

app.py

@@ -29,8 +29,40 @@ import aiohttp
 import requests
 #from textgen import getTextGen,multiprocessPrompts
 
+def getTextGen(prompt, model=''):
+    from huggingface_hub import InferenceClient
+    client = InferenceClient(token=os.getenv('inference'))
+    import time
+    good = False
+    response = ''
+    tries = 0
+    model = model if model else "mistralai/Mixtral-8x7B-Instruct-v0.1"
+    while not good and tries<=3:
+        try:
+            print('getTextGen -- trying prompt: ', model, prompt[:50])
+            response = client.text_generation(prompt, model=model, max_new_tokens=2500, temperature=.7)
+            print('getTextGen -- generated: ', response[:50], '\n\n')
+            good = True
+        except:
+            time.sleep(.7)
+            model = ["mistralai/Mistral-7B-Instruct-v0.2", "microsoft/Phi-3-mini-4k-instruct", "google/gemma-7b"][tries]
+            tries += 1
+            print('getTextGen -- retrying prompt: ', model, prompt[:50])
+            
+    return response
 
 
+def multiprocessPrompts(prompts):
+    from multiprocessing import Pool
+    import time
+    start = time.time()
+    with Pool(5) as p:
+        responses = p.map(getTextGen, prompts)
+    print(time.time()-start)
+    return responses
+
+    
+
 """
 Python subprocess non-blocking and non-breaking communicate.: https://gist.github.com/hzhu212/02d5d4845027411b6705f0dc19181ef5