ai generator logic

.gitignore

@@ -0,0 +1 @@
+**/__pycache__/

.vscode/launch.json

@@ -0,0 +1,18 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [        
+        
+        {
+            "name": "Python: Module",
+            "type": "python",
+            "request": "launch",
+            "module": "uvicorn",
+            // "args": ["main:app","--reload"],
+            "args": ["main_v2:app","--reload"],
+            "console": "integratedTerminal"
+        }
+    ]
+}

Dockerfile

@@ -8,4 +8,4 @@ RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
 
 COPY . .
 
-CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]
+CMD ["uvicorn", "main_v2:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -1,5 +1,5 @@
 ---
-title: FASTAPI1
+title: MIXTRAL-FASTAPI
 emoji: 📚
 colorFrom: blue
 colorTo: red

main.py

@@ -5,9 +5,9 @@ import uvicorn
 
 
 app = FastAPI()
-
 client = InferenceClient("mistralai/Mixtral-8x7B-Instruct-v0.1")
 
+
 class Item(BaseModel):
     prompt: str
     history: list
@@ -46,9 +46,13 @@ def generate(item: Item):
 
     for response in stream:
         output += response.token.text
+        # print(response.token.text)
     return output
 
 @app.post("/generate/")
 async def generate_text(item: Item):
     return {"response": generate(item)}
 
+@app.get("/test-api/")
+async def testAPI():
+    return "API is running"

main_v2.py

@@ -0,0 +1,147 @@
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import BaseModel
+from huggingface_hub import InferenceClient
+import uvicorn
+from paho.mqtt import client as mqtt_client
+import uuid
+import time
+
+
+
+app = FastAPI()
+origins = [
+    "*",
+    # "http://localhost:8000",
+]
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=origins,
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+inferenceClient = InferenceClient(
+        model="mistralai/Mixtral-8x7B-Instruct-v0.1",
+        token="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"
+    )
+historyMap = {}
+
+broker_host = "broker.emqx.io"
+broker_port = 1883
+broker_topic_prefix = "/diopsoft/mixtral-fastapi"
+# mqttClient = None
+mqttClient = mqtt_client.Client(mqtt_client.CallbackAPIVersion.VERSION1, str(uuid.uuid1()))
+def connectMqttBroker():
+    try:
+        res = mqttClient.connect(broker_host, broker_port, 60)
+        print('MQTT connection status : ' + str(res))
+        mqttClient.loop_start()
+    except Exception as Argument:
+        # logging.getLogger().exception("Failed to connect to MQTT Broker")
+        # logging.getLogger().error(e)
+        traceback.print_exc()
+def on_connect(client, userdata, flags, rc):
+        if rc == 0:
+            print("Connected to MQTT Broker!")
+        else:
+            raise Exception("Failed to connect to MQTT Broker, return code " + rc)
+mqttClient.on_connect = on_connect
+# connectMqttBroker()
+
+
+class Item(BaseModel):
+    is_mock: bool = False
+    token: str = None
+    mqttTopic: str = None
+    prompt: str
+    history: list = None
+    # system_prompt: str = "Soyez un assistant utile, en français de préférence"
+    system_prompt: str = "Soyez un assistant utile"
+    temperature: float = 0.7
+    max_new_tokens: int = 1048
+    top_p: float = 0.9
+    repetition_penalty: float = 1.1
+
+def format_prompt(message, history):
+    prompt = "<s>"
+    if history:
+        for hist in history:
+            prompt += f"[INST] {hist['user_prompt']} [/INST]"
+            prompt += f" {hist['bot_response']}</s> "
+    prompt += f"[INST] {message} [/INST]"
+    return prompt
+
+def generate(item: Item):
+    temperature = float(item.temperature)
+    if temperature < 1e-2:
+        temperature = 1e-2
+    top_p = float(item.top_p)
+
+    generate_kwargs = dict(
+        temperature=temperature,
+        max_new_tokens=item.max_new_tokens,
+        top_p=top_p,
+        repetition_penalty=item.repetition_penalty,
+        do_sample=True,
+        seed=42,
+    )
+
+    if item.mqttTopic and not mqttClient.is_connected():
+        connectMqttBroker()
+
+    token = item.token
+    history = None
+    if token:
+        history = historyMap.get(token)
+
+    formatted_prompt = format_prompt(f"{item.system_prompt}, {item.prompt}", history)
+    if not item.is_mock:
+        stream = inferenceClient.text_generation(formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)
+    output = ""
+
+    if not item.is_mock:
+        for response in stream:
+            print(response.token.text)
+            output += response.token.text
+            if item.mqttTopic and mqttClient.is_connected():
+                # result = mqttClient.publish(broker_topic_prefix+'/'+item.mqttTopic, response.token.text)
+                result = mqttClient.publish(broker_topic_prefix+'/'+item.mqttTopic, output)
+                status = result[0]
+                if status == 0:
+                    print("Message sent to topic")
+                else:
+                    print("Failed to send message to topic")
+    else:
+        for i in range(15):
+            text = " generated text " + str(i)
+            time.sleep(0.5)
+            print(text)
+            output += text
+            if item.mqttTopic and mqttClient.is_connected():
+                # result = mqttClient.publish(broker_topic_prefix+'/'+item.mqttTopic, text)
+                result = mqttClient.publish(broker_topic_prefix+'/'+item.mqttTopic, output)
+                status = result[0]
+                if status == 0:
+                    print("Message sent to topic")
+                else:
+                    print("Failed to send message to topic")
+
+    if item.mqttTopic and mqttClient.is_connected():
+        mqttClient.unsubscribe(broker_topic_prefix+'/'+item.mqttTopic)
+
+    if token:
+        if not historyMap.get(token):
+            historyMap[token] = []
+        historyMap.get(token).append({ "user_prompt":item.prompt, "bot_response":output })
+
+    return output
+
+@app.post("/generate/")
+async def generate_text(item: Item):
+    return {"response": generate(item)}
+
+@app.get("/test-api/")
+async def testAPI():
+    return "API V2 is running"

requirements.txt

@@ -1,4 +1,5 @@
 fastapi
 uvicorn
 huggingface_hub
-pydantic
+pydantic
+paho-mqtt