Spaces:
Runtime error
Runtime error
File size: 2,501 Bytes
08efb1b 65f214d 08efb1b 65f214d 08efb1b 65f214d 08efb1b 65f214d 08efb1b 65f214d 08efb1b 65f214d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 | import pandas as pd
import numpy as np
from langchain_huggingface import HuggingFaceEndpointEmbeddings
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score
from sklearn.metrics.pairwise import cosine_similarity
from src.config import FAQ_CSV_PATH, HUGGINGFACEHUB_API_TOKEN
from src.rag_pipeline import get_answer
model = HuggingFaceEndpointEmbeddings(
model="sentence-transformers/all-MiniLM-L6-v2",
huggingfacehub_api_token=HUGGINGFACEHUB_API_TOKEN,
)
def evaluate_faq_model():
"""Evaluasi model chatbot FAQ menggunakan cosine similarity."""
df_test = pd.read_csv(FAQ_CSV_PATH, sep=None, engine="python")
y_true = [ans.strip().lower() for ans in df_test["Answer"].tolist()]
y_pred_gemini = [get_answer(q).strip().lower() for q in df_test["Question"].tolist()]
y_pred_rag = [get_answer(q, use_rag=True)['result'].strip().lower() for q in df_test["Question"].tolist()]
y_true_embed = model.embed_documents(y_true)
y_pred_gemini_embed = model.embed_documents(y_pred_gemini)
y_pred_rag_embed = model.embed_documents(y_pred_rag)
similarity_gemini = np.array([
cosine_similarity([true_emb], [pred_emb])[0, 0]
for true_emb, pred_emb in zip(y_true_embed, y_pred_gemini_embed)
])
similarity_rag = np.array([
cosine_similarity([true_emb], [pred_emb])[0, 0]
for true_emb, pred_emb in zip(y_true_embed, y_pred_rag_embed)
])
threshold = 0.8
y_eval_gemini = (similarity_gemini >= threshold).astype(int)
y_eval_rag = (similarity_rag >= threshold).astype(int)
metrics_gemini = {
"Model": "Gemini",
"Akurasi": accuracy_score(y_eval_gemini, np.ones_like(y_eval_gemini)),
"Presisi": precision_score(y_eval_gemini, np.ones_like(y_eval_gemini), zero_division=1),
"Recall": recall_score(y_eval_gemini, np.ones_like(y_eval_gemini), zero_division=1),
"F1-score": f1_score(y_eval_gemini, np.ones_like(y_eval_gemini), zero_division=1),
}
metrics_rag = {
"Model": "Gemini + RAG",
"Akurasi": accuracy_score(y_eval_rag, np.ones_like(y_eval_rag)),
"Presisi": precision_score(y_eval_rag, np.ones_like(y_eval_rag), zero_division=1),
"Recall": recall_score(y_eval_rag, np.ones_like(y_eval_rag), zero_division=1),
"F1-score": f1_score(y_eval_rag, np.ones_like(y_eval_rag), zero_division=1),
}
return pd.DataFrame([metrics_gemini, metrics_rag]), similarity_gemini, similarity_rag |