Update main.py

main.py

@@ -6,6 +6,7 @@ from pydantic import BaseModel
 import re
 import replicate
 import os
+import json
 from supabase import create_client, Client
 
 url: str = os.environ.get("DB_URL")
@@ -66,6 +67,32 @@ async def root(item: Item):
 
 
 
+# @app.post("/summarize-v2")
+# async def root(item: Item):
+#     try:
+#         article = extract_article_content(item.url)
+
+#         if len(article) == 0:
+#             return {'summary': ""}
+
+#         def event_generator():
+#             for event in replicate.stream("snowflake/snowflake-arctic-instruct", input={
+#                 "prompt": f"summarize this news article in {item.max_tokens} lines:" + article,
+#                 "temperature": 0.2,
+#                 "max_new_tokens" : 1000
+#             }):
+#                 # Yield the event as a string
+#                 yield str(event)
+#                 #print(str(event), end="")
+
+#         # Use StreamingResponse to stream the events
+#         return StreamingResponse(event_generator(), media_type='text/event-stream')
+
+#     except requests.RequestException as e:
+#         return {"error": str(e), "status_code": 500}
+
+
+
 @app.post("/summarize-v2")
 async def root(item: Item):
     try:
@@ -74,18 +101,42 @@ async def root(item: Item):
         if len(article) == 0:
             return {'summary': ""}
 
-        def event_generator():
-            for event in replicate.stream("snowflake/snowflake-arctic-instruct", input={
-                "prompt": f"summarize this news article in {item.max_tokens} lines:" + article,
-                "temperature": 0.2,
-                "max_new_tokens" : 1000
-            }):
-                # Yield the event as a string
-                yield str(event)
-                print(str(event), end="")
+
+        url =  'https://yashxx07-hf-llm-api.hf.space/api/v1/chat/completions'
+
+        headers = {  "content-type": "application/json" }
+        data = {
+          "model": "mixtral-8x7b",
+          "messages": [
+            {
+              "role": "user",
+              "content": "who are you?"
+            }
+          ],
+          "temperature": 0.5,
+          "top_p": 0.95,
+          "max_tokens": -1,
+          "use_cache": False,
+          "stream": True
+        }
+
+        def get_stream(url):
+            s = requests.Session()
+        
+            with s.post(url, headers=headers, stream=True, json=data ) as resp:
+        
+                for line in resp.iter_lines():
+                    if line:
+                        obj = json.loads(line[5:])
+                        try:
+                            ouput = obj["choices"][0]["delta"]["content"])
+                          #print(obj["choices"][0]["delta"]["content"])
+                            yield str(ouput)
+                        except:
+                            pass
 
         # Use StreamingResponse to stream the events
-        return StreamingResponse(event_generator(), media_type='text/event-stream')
+        return StreamingResponse(get_stream(), media_type='text/event-stream')
 
     except requests.RequestException as e:
         return {"error": str(e), "status_code": 500}