init

Dockerfile

@@ -0,0 +1,22 @@
+# Use an official Python runtime as a parent image
+FROM python:3.10
+# Set Python to use unbuffered mode
+ENV PYTHONUNBUFFERED 1
+# Set the working directory in the container
+RUN mkdir /var/www
+ENV HOME /var/www
+WORKDIR /var/www
+
+# Copy the current directory contents into the container at /app
+COPY . /var/www
+
+RUN pip install -r requirements
+
+# Make port 5000 available to the world outside this container
+EXPOSE 7860
+
+# Define environment variable
+ENV FLASK_APP app.py
+
+# Run app.py when the container launches
+CMD flask run --host=0.0.0.0 --port=7860

app.py

@@ -0,0 +1,101 @@
+from flask import Flask, request, Response, jsonify
+from huggingface_hub import InferenceClient
+from flask_cors import CORS
+import json
+
+client = InferenceClient("mistralai/Mixtral-8x7B-Instruct-v0.1")
+
+summary_prompt = '<s>[INST]Ты ассистент. Отвечаешь на русском языке. Сформируй краткое изложение следующего текста: {}[/INST]'
+
+def format_prompt(message, history):
+  prompt = "<s>"
+  for user_prompt, bot_response in history:
+    prompt += f"[INST] {user_prompt} [/INST]"
+    prompt += f" {bot_response}</s> "
+  prompt += f"[INST] {message} [/INST]"
+  return prompt
+
+def generate(
+    prompt, history=[], temperature=0, max_new_tokens=2000, top_p=0.95, repetition_penalty=1.0,
+):
+    temperature = float(temperature)
+    if temperature < 1e-2:
+        temperature = 1e-2
+    top_p = float(top_p)
+
+    generate_kwargs = dict(
+        temperature=temperature,
+        max_new_tokens=max_new_tokens,
+        top_p=top_p,
+        repetition_penalty=repetition_penalty,
+        do_sample=True,
+        seed=42,
+    )
+
+    #formatted_prompt = format_prompt(prompt, history)
+
+    #stream = client.text_generation(prompt, **generate_kwargs, stream=True, details=False, return_full_text=False)
+    response = client.text_generation(prompt, **generate_kwargs, stream=False, details=False, return_full_text=False)
+    print(response)
+    return response.encode('utf-8')
+    #output = ""
+
+    #for response in stream:
+    #    yield response.token.text.encode('utf-8')
+
+app = Flask(__name__)
+CORS(app)
+
+@app.route('/health', methods=['GET'])
+def health():
+    return jsonify({"status": "ok"})
+
+@app.route('/completion', methods=['POST'])
+def completion_route():
+    data = request.get_json()
+    prompt = data.get('prompt', '')
+    #truncated_prompt = prompt[:32768]
+    return Response(generate(prompt[:52768]), content_type='text/plain; charset=utf-8', status=200, direct_passthrough=True)
+
+@app.route('/getsummary', methods=['POST'])
+def summary_route():
+    data = request.get_json()
+    text = data.get('text', '')
+    
+    summary_prompt = f'''<s>[INST]Ты ассистент. Отвечаешь на русском языке. Сформируй краткое изложение следующего текста: {text}[/INST]'''
+    response = generate(summary_prompt[:52000])
+    
+    return Response(json.dumps({'result': response}, ensure_ascii=False, sort_keys=False), content_type='text/json; charset=utf-8', status=200, direct_passthrough=False)
+
+@app.route('/cleantext', methods=['POST'])
+def summary_route():
+    data = request.get_json()
+    text = data.get('text', '')
+    
+    summary_prompt = f'''<s>[INST]Ты ассистент. Отвечаешь на русском языке. Сформируй краткое изложение следующего текста: {text}[/INST]'''
+    response = generate(summary_prompt[:52000])
+    
+    return Response(json.dumps({'result': response}, ensure_ascii=False, sort_keys=False), content_type='text/json; charset=utf-8', status=200, direct_passthrough=False)
+
+@app.route('/getfollowup', methods=['POST'])
+def summary_route():
+    data = request.get_json()
+    text = data.get('text', '')
+    
+    summary_prompt = f'''<s>[INST]Ты ассистент. Отвечаешь на русском языке. Сформируй краткое изложение следующего текста: {text}[/INST]'''
+    response = generate(summary_prompt[:52000])
+    
+    return Response(json.dumps({'result': response}, ensure_ascii=False, sort_keys=False), content_type='text/json; charset=utf-8', status=200, direct_passthrough=False)
+
+@app.route('/getagenda', methods=['POST'])
+def summary_route():
+    data = request.get_json()
+    text = data.get('text', '')
+    
+    summary_prompt = f'''<s>[INST]Ты ассистент. Отвечаешь на русском языке. Сформируй краткое изложение следующего текста: {text}[/INST]'''
+    response = generate(summary_prompt[:52000])
+    
+    return Response(json.dumps({'result': response}, ensure_ascii=False, sort_keys=False), content_type='text/json; charset=utf-8', status=200, direct_passthrough=False)
+
+if __name__ == '__main__':
+    app.run(debug=False, host='0.0.0.0', port=7860)

requirements

@@ -0,0 +1,3 @@
+flask
+flask-cors
+huggingface_hub