Update main.py

main.py

@@ -4,20 +4,22 @@ from fastapi.middleware.cors import CORSMiddleware
 import requests
 import pandas as pd
 import json
-import httpx,os,datetime
+import os,datetime
 import pandas as pd
 from sklearn.model_selection import train_test_split, GridSearchCV
 from sklearn.preprocessing import LabelEncoder
-from xgboost import XGBClassifier
 from sklearn.utils import resample
+from xgboost import XGBClassifier
 from sklearn.metrics import accuracy_score, classification_report
 from joblib import dump, load
 import numpy as np
+import requests
+import mysql.connector
+from mysql.connector import Error
 
 
 app = FastAPI()
 
-
 app.add_middleware(
     CORSMiddleware,
     allow_origins=["*"],
@@ -26,16 +28,20 @@ app.add_middleware(
     allow_headers=["*"],
 )
 
-        
-@app.get("/train_the_model_new_v2")
-async def train_the_model(Tenant: str):   
-        # Load the dataset
-        data = pd.read_csv(f"model/{Tenant}trainer_data_v1.csv")
-        print(data["customer_name"].count())
 
+@app.get("/trigger_the_model_trainer")
+async def train_the_model():     
+        # Load the dataset
+        #file_path = 'model/trainer_data.csv'  # Update to the correct file path 'model/trainer_data_new.csv'
+        #data = pd.read_csv(file_path)
+        csv_files = ['model/trainer_data.csv','model/trainer_data2.csv','model/trainer_data3.csv','model/trainer_data4.csv']
+        data_frames = [pd.read_csv(file) for file in csv_files]
+        
+        # Step 4: Concatenate all DataFrames into a single DataFrame
+        data = pd.concat(data_frames, ignore_index=True)
+        #data = data.iloc[0:50000]
         # Analyze class distribution
-        class_distribution = data['status.name'].value_counts()
-        bf = str(class_distribution)
+        class_distribution = data['status_name'].value_counts()
         print("Class Distribution before balancing:\n", class_distribution)
         
         # Get the size of the largest class to match other classes' sizes
@@ -43,7 +49,7 @@ async def train_the_model(Tenant: str):
         
         # Oversampling
         oversampled_data = pd.DataFrame()
-        for class_name, group in data.groupby('status.name'):
+        for class_name, group in data.groupby('status_name'):
             oversampled_group = resample(group,
                                          replace=True,  # Sample with replacement
                                          n_samples=max_class_size,  # to match majority class
@@ -51,31 +57,36 @@ async def train_the_model(Tenant: str):
             oversampled_data = pd.concat([oversampled_data, oversampled_group], axis=0)
         
         # Verify new class distribution
-        print("Class Distribution after oversampling:\n", oversampled_data['status.name'].value_counts())
+        print("Class Distribution after oversampling:\n", oversampled_data['status_name'].value_counts())
         
+        # Save the balanced dataset if needed
+        #oversampled_data.to_csv('model/trainer_data_balanced.csv', index=False)
+
+        data = pd.read_csv("model/trainer_data_new.csv")
+        print(data["customer_name"].count())
+    
+        data = pd.read_csv("model/trainer_data_balanced.csv")
+        print(data["customer_name"].count())
+
         data = oversampled_data
-        
-        # Select columns 'customer_email'
-        selected_columns = ['customer_name', 'customer_address', 'customer_phone',
-                            'cod', 'weight', 'origin_city.name',
-                            'destination_city.name','status.name','created_at']
+        print(data["customer_name"].count())
+        # Select columns
+        selected_columns = ['customer_name', 'customer_address', 'customer_phone_no',
+                            'weight','cod','pickup_address','client_number','destination_city',
+                            'status_name']
         
         # Handling missing values
         #data_filled = data[selected_columns].fillna('Missing')
         data_filled = data[selected_columns].dropna()
-        data_filled['customer_phone'] = data_filled['customer_phone'].astype(str)
-        data_filled['created_at'] = data_filled['created_at'].astype(str)
-        #data_filled = data_filled.drop(columns=['created_at'])
-
-        af = str(oversampled_data['status.name'].value_counts())
+        
         # Encoding categorical variables
         encoders = {col: LabelEncoder() for col in selected_columns if data_filled[col].dtype == 'object'}
         for col, encoder in encoders.items():
             data_filled[col] = encoder.fit_transform(data_filled[col])
         
         # Splitting the dataset
-        X = data_filled.drop('status.name', axis=1)
-        y = data_filled['status.name']
+        X = data_filled.drop('status_name', axis=1)
+        y = data_filled['status_name']
         X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
         
         # Parameters to use for the model
@@ -115,159 +126,173 @@ async def train_the_model(Tenant: str):
         classification_rep = classification_report(y_test, y_pred)
         
         # Save the model
-        model_filename = f'model/{Tenant}_curfox_xgb_model.joblib'
+        model_filename = 'model/transexpress_xgb_model.joblib'
         dump(xgb, model_filename)
         
         # Save the encoders
-        encoders_filename = f'model/{Tenant}_curfox_encoders.joblib'
+        encoders_filename = 'model/transexpress_encoders.joblib'
         dump(encoders, encoders_filename)
         
-        return accuracy,classification_rep,"Model trained with new data for :",model_filename,str(af),str(bf)
+        return accuracy,classification_rep,"Model trained with new data"
     
+@app.get("/trigger_the_data_fecher")
+async def get_data(page: str,paginate: str):
 
-@app.get("/trigger_the_data_fecher_for_me")
-async def continuous_function(page: int,paginate: int,Tenant: str):
-    print("data fetcher running.....")
-            
             
-    # Update the payload for each page
-    
-    #url = "https://dev3.api.curfox.parallaxtec.com/api/ml/order-list?sort=id&paginate="+str(paginate)+"&page="+str(page)
-    url = "https://v1.api.curfox.com/api/ml/order-list?sort=id&paginate="+str(paginate)+"&page="+str(page)
-    
-            
-    payload = {}
-    headers = {
-                    'Accept': 'application/json',
-                    'X-Tenant': Tenant #'royalexpress'
-                  }
-            
-    response = requests.request("GET", url, headers=headers, data=payload)
-            
-    # Sample JSON response
-    json_response = response.json()
-    # Extracting 'data' for conversion
-    data = json_response['data']
-    data_count = len(data)  
-    
-    df = pd.json_normalize(data)
-    
-            
-    df = df[df['status.name'].isin(['RETURN TO CLIENT', 'DELIVERED'])]
-    print("data collected from page : "+str(page))
-    #data.to_csv("new.csv")
-
-    try:
-        file_path = f'model/{Tenant}trainer_data_v1.csv'  # Replace with your file path
-        source_csv = pd.read_csv(file_path)
-        new_data = df
-        combined_df_final = pd.concat([source_csv,new_data], ignore_index=True)
-    
-        combined_df_final.to_csv(f"model/{Tenant}trainer_data_v1.csv")
-        print("data added")
-        message = "data added"
-    except:
-        
-        df.to_csv(f"model/{Tenant}trainer_data_v1.csv")
-        print("data created")
-        message = "data created"
-        
-    return {"message":message,"page_number":page,"data_count":data_count,'X-Tenant': Tenant}
-
-@app.get("/trigger_the_data_fecher")
-async def your_continuous_function(page: int,paginate: int,Tenant: str):
     print("data fetcher running.....")
             
+    # Initialize an empty DataFrame to store the combined data
+    combined_df = pd.DataFrame()
             
     # Update the payload for each page
+    url = "https://report.transexpress.lk/api/orders/delivery-success-rate/return-to-client-orders?page="+page+"&per_page="+paginate
     
-    #url = "https://dev3.api.curfox.parallaxtec.com/api/ml/order-list?sort=id&paginate="+str(paginate)+"&page="+str(page)
-    url = "https://v1.api.curfox.com/api/ml/order-list?sort=id&paginate="+str(paginate)+"&page="+str(page)
-    
-            
     payload = {}
     headers = {
-                    'Accept': 'application/json',
-                    'X-Tenant': Tenant #'royalexpress'
-                  }
+      'Cookie': 'development_trans_express_session=NaFDGzh5WQCFwiortxA6WEFuBjsAG9GHIQrbKZ8B'
+    }
             
     response = requests.request("GET", url, headers=headers, data=payload)
             
     # Sample JSON response
     json_response = response.json()
     # Extracting 'data' for conversion
-    data = json_response['data']
+    data = json_response["return_to_client_orders"]['data']
+
     data_count = len(data)  
     
     df = pd.json_normalize(data)
     
             
-    df = df[df['status.name'].isin(['RETURN TO CLIENT', 'DELIVERED'])]
-    print("data collected from page : "+str(page))
-    #data.to_csv("new.csv")
-
+    df['status_name'] = df['status_name'].replace('Partially Delivered', 'Delivered')
+    df['status_name'] = df['status_name'].replace('Received by Client', 'Returned to Client')
+    
+    print("data collected from page : "+page)
+    #return "done"
     try:
-        file_path = f'model/{Tenant}trainer_data_.csv'  # Replace with your file path
+        file_path = 'model/trainer_data5.csv'  # Replace with your file path
         source_csv = pd.read_csv(file_path)
         new_data = df
         combined_df_final = pd.concat([source_csv,new_data], ignore_index=True)
     
-        combined_df_final.to_csv(f"model/{Tenant}trainer_data_.csv")
+        combined_df_final.to_csv("model/trainer_data5.csv")
         print("data added")
     except:
         
-        df.to_csv(f"model/{Tenant}trainer_data_.csv")
+        df.to_csv("model/trainer_data5.csv")
         print("data created")
-        
-    return {"message":"done","page_number":page,"data_count":data_count,'X-Tenant': Tenant}
 
+    print({"page_number":page,"data_count":data_count})
+    return {"page_number":page,"data_count":data_count}
 
+@app.get("/get_module_versions")
+async def get_versions():
+    try: 
+        from pip._internal.operations import freeze
+    except ImportError: # pip < 10.0
+        from pip.operations import freeze
     
+    pkgs = freeze.freeze()
+    for pkg in pkgs: 
+        print(pkg)
+    return pkgs
+ 
 
 @app.get("/get_latest_model_updated_time")
-async def model_updated_time(Tenant: str):
-    import multiprocessing
-    
-    # Get the number of available CPU cores
-    available_cores = multiprocessing.cpu_count()
+async def model_updated_time():
     try:
-        m_time_encoder = os.path.getmtime(f'model/{Tenant}_curfox_encoders.joblib')
-        m_time_model = os.path.getmtime(f'model/{Tenant}_curfox_xgb_model.joblib')
-        return {
-                "Tenant":Tenant,
-                "base model created time ":datetime.datetime.fromtimestamp(m_time_encoder),
+        file_size = os.path.getsize("model/transexpress_xgb_model.joblib")
+        m_time_encoder = os.path.getmtime('model/transexpress_encoders.joblib')
+        m_time_model = os.path.getmtime('model/transexpress_xgb_model.joblib')
+        return {"base model created time ":datetime.datetime.fromtimestamp(m_time_encoder),
                 "last model updated time":datetime.datetime.fromtimestamp(m_time_model),
-                "Number of available CPU cores": available_cores
+                "The size of the file is bytes":file_size
                }
     except:
         return {"no model found so first trained the model using data fecther"}
 
 
 
+# Database connection parameters
+DB_HOST = 'trans-prod-clone-staging.mysql.database.azure.com'
+DB_PORT = 3306
+DB_DATABASE = 'defaultdb'
+DB_USERNAME = 'wwwdata'
+DB_PASSWORD = 'fcLa8F3sxgNYQ$K@%'
+# Connect to the database
 
+#calling this function for each request
+def fetch_customer_data(phone_number):
+                #local connection
+                connection = mysql.connector.connect(
+                                host=DB_HOST,
+                                port=DB_PORT,
+                                database=DB_DATABASE,
+                                user=DB_USERNAME,
+                                password=DB_PASSWORD
+                            )
+            #try:
+                if connection.is_connected():
+                    print("Connected to the database")
+        
+                    # SQL query
+                    query = """
+                    SELECT 
+                        orders.customer_name AS customer_name,
+                        orders.address AS customer_address,
+                        orders.phone_no AS customer_phone_no,
+                        primary_statuses.name AS status_name
+                    FROM 
+                        orders
+                    INNER JOIN 
+                        statuses ON orders.status_id = statuses.id
+                    INNER JOIN 
+                        primary_statuses ON statuses.name = primary_statuses.key
+                    WHERE orders.phone_no LIKE %s
+                    """
+        
+                    # Execute the query
+                    cursor = connection.cursor(dictionary=True)
+                    cursor.execute(query, (f"%{phone_number}%",))
+                    
+                    # Fetch results
+                    results = cursor.fetchall()
+                    #print("Results:", results)
+                    #close conection
+                    #if connection.is_connected():
+                    cursor.close()
+                    connection.close()
+                    print("Database connection closed")
+                    return results
+        
+            # except Error as e:
+            #     print(f"Error: {e}")
+            #     #close conection
+            #     #if connection.is_connected():
+            #     cursor.close()
+            #     connection.close()
+            #     print("Database connection closed")
 
-# Endpoint for making predictions
 
+# Endpoint for making predictions
 @app.post("/predict")
-def predict(
-    Tenant: str,
+async def predict(
+    date : str,
     customer_name: str,
     customer_address: str,
     customer_phone: str,
-    cod:str,
-    weight: str,
-    origin_city_name: str,
-    destination_city_name: str,
-    created_at: str,
-    customer_email: str,
+    weight: float,
+    cod: int,
     pickup_address: str,
-    origin_country: str
+    client_number:str,
+    destination_city:str
     ):
 
+
     try:
         # Load your trained model and encoders
-        xgb_model = load(f'model/{Tenant}_curfox_xgb_model.joblib')
-        encoders = load(f'model/{Tenant}_curfox_encoders.joblib')
+        xgb_model = load('model/transexpress_xgb_model.joblib')
+        encoders = load('model/transexpress_encoders.joblib')
     except:
         return {"no model found so first trained the model using data fecther"}
 
@@ -276,26 +301,20 @@ def predict(
     def safe_transform(encoder, column):
         classes = encoder.classes_
         return [encoder.transform([x])[0] if x in classes else -1 for x in column] 
-    # Function to handle unseen labels during encoding
-    def safe_transform(encoder, column):
-        classes = encoder.classes_
-        return [encoder.transform([x])[0] if x in classes else -1 for x in column] 
-        
         
-
+    # Convert input data to DataFrame
     input_data = {
         'customer_name': customer_name,
         'customer_address': customer_address,
-        'customer_phone': customer_phone, #'customer_email': customer_email,
+        'customer_phone_no': customer_phone,
+        'weight': float(weight),
         'cod': int(cod),
-        'weight': int(weight),
-        'origin_city.name':origin_city_name,
-        'destination_city.name':destination_city_name,
-        'created_at':created_at
+        'pickup_address':pickup_address,
+        'client_number':client_number,
+        'destination_city':destination_city
     }
     input_df = pd.DataFrame([input_data])
 
-
     # Encode categorical variables using the same encoders used during training
     for col in input_df.columns:
         if col in encoders:
@@ -304,12 +323,69 @@ def predict(
     # Predict and obtain probabilities
     pred = xgb_model.predict(input_df)
     pred_proba = xgb_model.predict_proba(input_df)
+    
+    import numpy as np
+    from urllib.parse import unquote
+    def extract_phone_numbers(customer_phone):
+        # Decode URL-encoded phone numbers
+        decoded_phone = unquote(customer_phone)
+        # Split into a list of phone numbers
+        phone_numbers = [phone.strip() for phone in decoded_phone.split('/')]
+        # Handle case where there is a single phone number
+        if len(phone_numbers) == 1 and phone_numbers[0]:
+            return phone_numbers
+        elif len(phone_numbers) == 0:
+            return []
+        return phone_numbers
+        
+    
+    def calculate_delivery_factor(phone_number):        
+        # Replace with the desired customer name and phone number
+        
+        #customer_phone_no = '0773224384'
+        json = fetch_customer_data(phone_number)
+        data = json
+        #print(url,data)
+        # Filter only relevant status names
+        valid_statuses = ['Failed to Deliver', 'Delivered', 'Returned to Client']
+        relevant_orders = [order for order in data if order['status_name'] in valid_statuses]
+    
+        if not relevant_orders:
+            base_probability = 0.50
+        else:
+            delivered_count = sum(1 for order in relevant_orders if order['status_name'] == 'Delivered')
+            total_orders_count = len(relevant_orders)
+    
+            base_probability = delivered_count / total_orders_count
+            base_probability = max(0.05, min(base_probability, 0.95))
+    
+        # Add a narrower random component
+        random_component = np.random.uniform(-0.05, 0.05)
+        adjusted_probability = base_probability + random_component
+    
+        return adjusted_probability
+    try:
+        
+        print(customer_phone)
+        phone_numbers = extract_phone_numbers(customer_phone)
+        print(phone_numbers, "api calling ......")
+        probability = calculate_delivery_factor(phone_numbers[0])
+        probability = round((probability * 100),2)
+        #probability = f"{probability:.2f}" probability = f"{float(probability):.2f}"
 
+        print(f"new model probability: {probability}")
+        predicted_status = "delivered"
+        
     # Output
-    predicted_status = "Unknown" if pred[0] == -1 else encoders['status.name'].inverse_transform([pred])[0]
-    probability = pred_proba[0][pred[0]] * 100 if pred[0] != -1 else "Unknown"
+    except Exception as e:
+        print(f"Error: {e}")
+        predicted_status = "Unknown" if pred[0] == -1 else encoders['status_name'].inverse_transform([pred])[0]
+        probability = pred_proba[0][pred[0]] * 100 if pred[0] != -1 else "Unknown"
+        print(str(predicted_status),probability)
+        if probability>98:
+           probability = probability-1
+        if predicted_status == "Returned to Client":
+           probability = 100 - probability
 
-    if predicted_status == "RETURN TO CLIENT":
-       probability = 100 - probability
 
-    return {"predicted_status":predicted_status,"Probability": round(probability,2),"Tenant_new":Tenant}
+    return {"Probability": round(probability,2),"predicted_status":predicted_status}