Update abc

abc

@@ -29,4 +29,98 @@ print("Cosine Similarity Matrix (rows: retrieved, columns: relevant):\n")
 for i, retrieved in enumerate(retrieved_chunks):
     for j, relevant in enumerate(relevant_chunks):
         score = cosine_sim_matrix[i][j].item()
-        print(f"Similarity between:\n  Retrieved: \"{retrieved}\"\n  Relevant : \"{relevant}\"\n  Score    : {score:.4f}\n")
+        print(f"Similarity between:\n  Retrieved: \"{retrieved}\"\n  Relevant : \"{relevant}\"\n  Score    : {score:.4f}\n")------
+        -----------------------------------------
+
+
+        import numpy as np
+from nltk.translate.bleu_score import sentence_bleu
+from rouge_score import rouge_scorer
+from sentence_transformers import SentenceTransformer
+from sklearn.metrics.pairwise import cosine_similarity
+from transformers import GPT2Tokenizer, GPT2LMHeadModel
+import torch
+
+#client = genai.Client(api_key=os.getenv("GEMINI_API_KEY"))
+
+# Load models
+embedding_model = SentenceTransformer('all-MiniLM-L6-v2')
+tokenizer = GPT2Tokenizer.from_pretrained("gpt2")
+perplexity_model = GPT2LMHeadModel.from_pretrained("gpt2")
+perplexity_model.eval()
+
+# Evaluation Metrics
+def bleu_rouge_score(reference, generated):
+    bleu = sentence_bleu([reference.split()], generated.split())
+    rouge = rouge_scorer.RougeScorer(['rougeL'], use_stemmer=True)
+    rougeL = rouge.score(reference, generated)['rougeL'].fmeasure
+    return {'bleu': bleu, 'rougeL': rougeL}
+
+def cosine_sim(reference, generated):
+    emb_ref = embedding_model.encode([reference])[0]
+    emb_gen = embedding_model.encode([generated])[0]
+    sim = cosine_similarity([emb_ref], [emb_gen])[0][0]
+    return sim
+
+def perplexity_score(text):
+    inputs = tokenizer(text, return_tensors="pt")
+    with torch.no_grad():
+        outputs = perplexity_model(**inputs, labels=inputs["input_ids"])
+        loss = outputs.loss
+    return torch.exp(loss).item()
+
+def precision_at_k(retrieved, relevant, k):
+    top_k = retrieved[:k]
+    correct = sum(1 for item in top_k if item in relevant)
+    return correct / k
+
+def recall_at_k(retrieved, relevant, k):
+    correct = sum(1 for item in retrieved[:k] if item in relevant)
+    return correct / len(relevant)
+
+def ndcg_at_k(retrieved, relevant, k):
+    def dcg(items):
+        return sum([1 / np.log2(i+2) if items[i] in relevant else 0 for i in range(len(items))])
+    ideal = dcg(relevant[:k])
+    actual = dcg(retrieved[:k])
+    return actual / ideal if ideal != 0 else 0
+
+def hit_at_k(retrieved, relevant, k):
+    top_k = retrieved[:k]
+    return int(any(item in relevant for item in top_k))
+
+# Main Evaluation
+def full_evaluation(reference, generated, retrieved, relevant_chunks):
+    return {
+        **bleu_rouge_score(reference, generated),
+        "cosine_similarity": cosine_sim(reference, generated),
+        "perplexity": perplexity_score(generated),
+        "precision@5": precision_at_k(retrieved, relevant_chunks, 5),
+        "recall@5": recall_at_k(retrieved, relevant_chunks, 5),
+        "ndcg@5": ndcg_at_k(retrieved, relevant_chunks, 5),
+        "hit@5": hit_at_k(retrieved, relevant_chunks, 5)
+    }
+
+# Sample Run
+if __name__ == "__main__":
+    reference_answer = "The Eiffel Tower is located in Paris."
+    generated_response = "Eiffel Tower stands in Paris."
+
+    retrieved_chunks = [
+        "The Eiffel Tower is a landmark in Paris.",
+        "Paris is the capital of France.",
+        "The Louvre is also in Paris.",
+        "Eiffel Tower was built in 1889.",
+        "It is a famous tourist spot."
+    ]
+
+    relevant_chunks = [
+        "The Eiffel Tower is a landmark in Paris.",
+        "Eiffel Tower was built in 1889."
+    ]
+
+    scores = full_evaluation(reference_answer, generated_response, retrieved_chunks, relevant_chunks)
+
+    for metric, score in scores.items():
+        print(f"{metric}: {score:.4f}" if isinstance(score, float) else f"{metric}: {score}")
+