from fastapi import FastAPI, File, UploadFile, HTTPException from pydantic import BaseModel from fastapi.staticfiles import StaticFiles from fastapi.responses import FileResponse from ml_service import MlProcessing from ml_service import process_single_comment from typing import List, Dict, Union import time from kafka import KafkaConsumer, KafkaProducer import asyncio import json import pandas as pd from datetime import datetime import spacy from simpletransformers.ner import NERModel import json import fasttext app = FastAPI() labels_file = f"ml_models/labels.json" ner_model_directory = f"ml_models/ner_model/" sentiment_model_file = f"ml_models/sentiment_model/model.ft" # LANGUAGE_MODEL = spacy.load('en_core_web_sm') # LABELS = ['O', 'B-AMENITIES', 'I-AMENITIES', 'I-CLEANLINESS', 'B-CLEANLINESS', 'I-COMMUNICATION', 'B-COMMUNICATION', 'B-CONDITION', 'I-CONDITION', 'I-CUSTOMER_SERVICE', 'B-CUSTOMER_SERVICE', 'B-EXTERIOR_LIGHTING', 'I-EXTERIOR_LIGHTING', 'B-FINANCIAL', 'I-FINANCIAL', 'B-INTERIOR_LIGHTING', 'I-INTERIOR_LIGHTING', 'B-INTERNET', 'I-INTERNET', 'B-LANDSCAPING_GROUNDS', 'I-LANDSCAPING_GROUNDS', 'B-MAINTENANCE_CLEANLINESS', 'I-MAINTENANCE_CLEANLINESS', 'I-MAINTENANCE_SERVICE', 'B-MAINTENANCE_SERVICE', 'B-MAINTENANCE_TIMELINESS', 'I-MAINTENANCE_TIMELINESS', 'B-MOVE_IN_QUALITY', 'I-MOVE_IN_QUALITY', 'I-NOISE', 'B-NOISE', 'B-PACKAGES_MAIL', 'I-PACKAGES_MAIL', 'B-PARKING', 'I-PARKING', 'I-PESTS', 'B-PESTS', 'B-PET_WASTE', 'I-PET_WASTE', 'I-SECURITY', 'B-SECURITY', 'B-SMOKE', 'I-SMOKE', 'B-TRASH', 'I-TRASH'] # SENTIMENT_MODEL = fasttext.load_model(sentiment_model_file) class ML_models: def __init__(self, sentiment_model_file): self.labels = ['O', 'B-AMENITIES', 'I-AMENITIES', 'I-CLEANLINESS', 'B-CLEANLINESS', 'I-COMMUNICATION', 'B-COMMUNICATION', 'B-CONDITION', 'I-CONDITION', 'I-CUSTOMER_SERVICE', 'B-CUSTOMER_SERVICE', 'B-EXTERIOR_LIGHTING', 'I-EXTERIOR_LIGHTING', 'B-FINANCIAL', 'I-FINANCIAL', 'B-INTERIOR_LIGHTING', 'I-INTERIOR_LIGHTING', 'B-INTERNET', 'I-INTERNET', 'B-LANDSCAPING_GROUNDS', 'I-LANDSCAPING_GROUNDS', 'B-MAINTENANCE_CLEANLINESS', 'I-MAINTENANCE_CLEANLINESS', 'I-MAINTENANCE_SERVICE', 'B-MAINTENANCE_SERVICE', 'B-MAINTENANCE_TIMELINESS', 'I-MAINTENANCE_TIMELINESS', 'B-MOVE_IN_QUALITY', 'I-MOVE_IN_QUALITY', 'I-NOISE', 'B-NOISE', 'B-PACKAGES_MAIL', 'I-PACKAGES_MAIL', 'B-PARKING', 'I-PARKING', 'I-PESTS', 'B-PESTS', 'B-PET_WASTE', 'I-PET_WASTE', 'I-SECURITY', 'B-SECURITY', 'B-SMOKE', 'I-SMOKE', 'B-TRASH', 'I-TRASH'] self.languate_model = spacy.load('en_core_web_sm') self.sentiment_model = fasttext.load_model(sentiment_model_file) print("Models loaded") ml_model = ML_models(sentiment_model_file) LANGUAGE_MODEL = ml_model.languate_model SENTIMENT_MODEL = ml_model.sentiment_model LABELS = ml_model.labels print("Models loading instances created...") class TextRatingRequest(BaseModel): text: str rating: int class AllTextRatingRequest(BaseModel): reviews : List[Dict[str, Union[str, dict, int]]] @app.post("/predict") def predict_single_review(text_rating: TextRatingRequest): text = text_rating.text rating = text_rating.rating skip = False raw_data = {"text": text, "star_rating": rating, "skip": skip} start_time = time.time() # ml = MlProcessing(comment_dict=raw_data) # processed_data = ml.main() # spans = processed_data.get('spans', list()) # has_sentiments = True # if not any(spans): # spans = [{'label': text, 'color': '', 'value': '', 'sentiment': '', 'score': ''}] # has_sentiments = False # processed_data['spans'] = spans try: processed_data, has_sentiments = process_single_comment(raw_data, LANGUAGE_MODEL, SENTIMENT_MODEL, LABELS ) except Exception as e: print("error during prediction: ", e) return {"error":e} end_time = time.time() print(f"Time taken to process the data : {end_time - start_time} seconds") return {"processed_data": processed_data, "has_sentiments":has_sentiments} @app.post("/predict_all") def predict_all_reviews(reviews: AllTextRatingRequest): reviews = reviews.reviews skip = False processed_data_list = list() start_time = time.time() for review in reviews: raw_data = {"text":review.get('text', str()), "star_rating":review.get('rating', 5), "skip":skip} try: processed_data, has_sentiments = process_single_comment(raw_data, LANGUAGE_MODEL, SENTIMENT_MODEL, LABELS ) except Exception as e: print("error during prediction: ", e) return {"error":e} processed_data_list.append({"processed_data":processed_data, "has_sentiments":has_sentiments}) end_time = time.time() print(f"Time taken to process the data : {end_time - start_time} seconds") return processed_data_list @app.post("/predict_file") def predict_file_responses(file: UploadFile = File(...)): if not file.filename.endswith(".csv"): raise HTTPException(status_code=400, detail="Only CSV files are allowed") try: df = pd.read_csv(file.file) df = df.fillna('') except Exception as e: raise HTTPException(status_code=400, detail=f"Error processing CSV file: {e}") processed_data_list = list() start_time = time.time() for index, row in df.iterrows(): try: text = row['ACTUAL REVIEW'] star_rating = row['STAR RATING'] review_id = row['REVIEWID'] raw_data = {"text":text, "star_rating":star_rating, "skip":False} try: processed_data, has_sentiments = process_single_comment(raw_data,LANGUAGE_MODEL, SENTIMENT_MODEL, LABELS ) except Exception as e: print("error during prediction: ", e) return {"error":e} now = datetime.now() print(f"Processed review with index {index} at time {now.time()}") processed_data_list.append({"processed_data":processed_data, "has_sentiments":has_sentiments, "review_id":review_id}) finally: pass end_time = time.time() print(f">>>>>>>>>>>>>>>>>>>>>>>> Processing time : {end_time - start_time} seconds >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>") return processed_data_list app.mount("/", StaticFiles(directory="static", html=True), name="static") @app.get("/") def index() -> FileResponse: return FileResponse(path="/static/index.html", media_type="text/html")