Add application file

app.py

@@ -1,13 +1,33 @@
 from fastapi import FastAPI
-from optimum.onnxruntime import ORTModelForSequenceClassification
 from transformers import AutoTokenizer
+from optimum.onnxruntime import ORTModelForSequenceClassification, ORTOptimizer, ORTQuantizer
+from optimum.onnxruntime.configuration import OptimizationConfig, AutoQuantizationConfig
+import torch
 import time
 
 # Load the tokenizer
 tokenizer = AutoTokenizer.from_pretrained("cardiffnlp/twitter-roberta-base-sentiment")
 
-# Load the quantized ONNX model from Hugging Face
-model = ORTModelForSequenceClassification.from_pretrained("minhdang/model_onnx", file_name="quantized_model.onnx")
+# Convert the model to ONNX and optimize it
+model_id = "cardiffnlp/twitter-roberta-base-sentiment"
+
+# Load and convert the model to ONNX
+onnx_model = ORTModelForSequenceClassification.from_pretrained(model_id, from_transformers=True)
+onnx_model.save_pretrained("./model_onnx")
+
+# Optimize the ONNX model
+optimizer = ORTOptimizer.from_pretrained(onnx_model)
+optimizer.optimize(
+    OptimizationConfig(optimization_level=99),  # Adjust optimization level as needed
+    save_dir="./model_onnx_optimized"
+)
+optimized_model = ORTModelForSequenceClassification.from_pretrained("./model_onnx_optimized", file_name="model_optimized.onnx")
+
+# Quantize the optimized ONNX model
+quantizer = ORTQuantizer.from_pretrained(optimized_model)
+quantization_config = AutoQuantizationConfig.avx512_vnni(is_static=False, per_channel=True)
+quantizer.quantize(save_dir="./model_onnx_quantized", quantization_config=quantization_config)
+quantized_model = ORTModelForSequenceClassification.from_pretrained("./model_onnx_quantized", file_name="model_quantized.onnx")
 
 app = FastAPI()
 
@@ -36,17 +56,18 @@ def analyze_sentiment(tweet: str):
     # Tokenize the input tweet
     inputs = tokenizer(tweet_proc, return_tensors="pt")
 
-    # Perform the inference with the ONNX model
-    outputs = model(**inputs)
+    # Perform the inference with the quantized ONNX model
+    with torch.no_grad():
+        outputs = quantized_model(**inputs)
     
     # Calculate the inference time
     inference_time = time.time() - start_time
 
     # Get the probabilities from the logits
-    probabilities = outputs.logits.softmax(dim=1)
+    probabilities = torch.softmax(outputs.logits, dim=1)
 
     # Get the label with the highest probability
-    max_prob, max_index = probabilities.max(dim=1)
+    max_prob, max_index = torch.max(probabilities, dim=1)
 
     # Map the labels to desired names
     label_map = {

requirements.txt

@@ -3,6 +3,6 @@ uvicorn
 transformers
 torch
 scipy
-
+optimum