cousin method

back_end/models/embedding_model.py

@@ -1,3 +1,4 @@
+# back_end/models/embedding_model.py
 from sentence_transformers import SentenceTransformer
 
 # Load the pre-trained embedding model
@@ -5,5 +6,5 @@ model = SentenceTransformer('Alibaba-NLP/gte-base-en-v1.5', trust_remote_code=Tr
 
 def generate_embedding(text: str):
     """Generate a 768-dimensional embedding for the input text."""
-    embedding = model.encode(text).tolist()    # Convert NumPy array to list
+    embedding = model.encode(text).tolist()  # Convert NumPy array to list
     return embedding

back_end/models/logistic.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:380a06dab235d33a0ef39e31efc30e00acb036c3630e631069297d05f0844092
+size 6874

back_end/routers/embedding.py

@@ -1,14 +1,59 @@
 from fastapi import APIRouter, HTTPException
+import os
+import pickle
+
 from back_end.models.embedding_model import generate_embedding
 from back_end.schemas.request import TextRequest
+from sklearn.linear_model import LogisticRegression
+from scipy.spatial.distance import cosine
 
 router = APIRouter()
 
-@router.get("/generate_embedding/")
-def get_embedding(text: str):
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))  # Get the directory of the current file
+MODEL_PATH = os.path.join(BASE_DIR, "..", "models", "logistic.pkl")
+
+try:
+    with open(MODEL_PATH, "rb") as f:
+        logistic_model = pickle.load(f)
+except FileNotFoundError:
+    raise RuntimeError(f"Model file not found at {MODEL_PATH}")
+except pickle.UnpicklingError:
+    raise RuntimeError(f"Error unpickling model file at {MODEL_PATH}")
+
+
+@router.post("/generate_embedding/")
+def get_embedding(request: TextRequest):
     """Returns a 768-dimensional embedding for the given text."""
-    if not text:
+    if not request.text:
         raise HTTPException(status_code=400, detail="Text cannot be empty")
 
-    embedding = generate_embedding(text)
+    embedding = generate_embedding(request.text)
     return {"dimensions": len(embedding), "embedding": embedding}
+
+
+@router.post("/cosine_similarity/")
+def get_cosine_similarity(request: TextRequest):
+    """Returns the cosine similarity between two input texts."""
+    if not hasattr(request, 'text') or not hasattr(request, 'text2'):
+        raise HTTPException(status_code=400, detail="Both text inputs must be provided")
+
+    embedding1 = generate_embedding(request.text)
+    embedding2 = generate_embedding(request.text2)
+
+    similarity = 1 - cosine(embedding1, embedding2)
+    return {"cosine_similarity": similarity}
+
+
+@router.post("/logistic_prediction/")
+def get_logistic_prediction(request: TextRequest):
+    """Returns the prediction from the logistic regression model for the input text."""
+    if not request.text:
+        raise HTTPException(status_code=400, detail="Text cannot be empty")
+
+    embedding = generate_embedding(request.text)
+    try:
+        prediction = logistic_model.predict([embedding])[0]
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Model prediction failed: {str(e)}")
+
+    return {"prediction": prediction}

back_end/schemas/request.py

@@ -2,3 +2,6 @@ from pydantic import BaseModel
 
 class TextRequest(BaseModel):
     text: str
+    text2: str = None  # Optional for cosine similarity
+
+    

back_end/service/train_model.py

@@ -0,0 +1,50 @@
+import os
+import pickle
+import pandas as pd
+from sentence_transformers import SentenceTransformer
+from sklearn.linear_model import LogisticRegression
+from sklearn.model_selection import train_test_split
+from sklearn.metrics import accuracy_score
+
+# Define paths
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+DATA_PATH = "data/sms_process_data_main.xlsx"
+MODEL_PATH = "models/logistic.pkl"
+
+# Check if the dataset file exists
+print(DATA_PATH)
+if not os.path.exists(DATA_PATH):
+    raise FileNotFoundError(f"Dataset file not found at: {DATA_PATH}")
+
+# Load dataset
+df = pd.read_excel(DATA_PATH)  
+
+# Ensure the dataset has the required columns (adjust as necessary)
+if not {'text', 'label'}.issubset(df.columns):
+    raise ValueError("Dataset must contain 'text' and 'label' columns")
+
+# Load Sentence Transformer model
+embedding_model = SentenceTransformer('Alibaba-NLP/gte-base-en-v1.5', trust_remote_code=True)
+
+# Generate embeddings
+X = df['text'].apply(lambda x: embedding_model.encode(x).tolist()).tolist()
+y = df['label']
+
+# Train/test split
+X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
+
+# Train Logistic Regression model
+logistic_model = LogisticRegression(max_iter=1000)
+logistic_model.fit(X_train, y_train)
+
+# Evaluate the model
+y_pred = logistic_model.predict(X_test)
+accuracy = accuracy_score(y_test, y_pred)
+print(f"Model Accuracy: {accuracy:.4f}")
+
+# Save the model
+print("Saving model and embeddings...")
+with open(MODEL_PATH, 'wb') as f:
+    pickle.dump(logistic_model, f)
+
+print(f"Logistic model saved to {MODEL_PATH}")