Create llmeval.py

llmeval.py

@@ -0,0 +1,90 @@
+from groq import Groq
+import re
+import json
+
+AK="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"
+client = Groq(api_key=AK)
+
+
+
+
+class LLM_as_Evaluator():
+    
+    def __init__(self):
+        pass
+
+
+    def ___engine_core(self,messages):
+    
+        completion = client.chat.completions.create(
+            model="llama3-8b-8192",
+            messages=messages,
+            temperature=0.0,
+            max_completion_tokens=5000,
+            top_p=1,
+            stream=False,
+            stop=None,
+            )
+        actual_message=completion.choices[0].message.content
+        return actual_message
+        #cleaned_json=re.sub(r"```(?:json)?\s*(.*?)\s*```", r"\1", actual_message, flags=re.DOTALL).strip()
+        #is_json_like = cleaned_json.strip().startswith("{") and cleaned_json.strip().endswith("}")
+        #if is_json_like==True:
+            #return cleaned_json
+        #else:
+            #return "FATAL"
+        
+
+    def Paradigm_LLM_Evaluator(self,data_to_evaluate):
+
+        
+        SYSTEM='''
+Task:
+Evaluate the biological quality of a prompt-research data-response triplet on a 0–1 continuous scale.
+
+Goal:
+Assess:
+
+Whether the Prompt is clear, biologically specific, and aligned with the Research Data.
+
+Whether the Response is biologically relevant, mechanistically coherent, and experimentally actionable based on the Research Data.
+
+Scoring Guide (0–1 continuous scale):
+
+Score 1.0 if:
+
+Prompt is clear, biologically detailed, and correctly aligned to the research context.
+
+Response correctly identifies a biologically valid paradigm consistent with the Research Data.
+
+Lower scores if:
+
+The prompt is vague or misaligned.
+
+The response is biologically inaccurate, irrelevant, or mechanistically implausible.
+
+EXAMPLE:
+Input:
+    Prompt: Identify a paradigm explaining the functional impact of BRCA1 mutations in ovarian cancer, focusing on DNA repair mechanisms.
+    Research Data: BRCA1 loss-of-function mutations are associated with impaired homologous recombination repair, leading to genomic instability in ovarian epithelial cells.
+    Agent's Response: BRCA1 mutations inhibit non-homologous end joining, which causes increased apoptosis in neurons, suggesting a neurodegeneration model.
+    
+Your output must begin with Score: and contain only two fields: Score: and Reasoning:. No extra commentary, no markdown, no explanations before or after.:
+    Score: 0.3
+    Reasoning: The prompt and research data focus on ovarian cancer and homologous recombination, but the response incorrectly shifts to neurons and the wrong DNA repair pathway (non-homologous end joining instead of homologous recombination). Misalignment between response and biological context.
+
+
+Think step by step
+'''
+        messages=[
+            {"role":"system","content":SYSTEM},
+            {"role":"user","content":f"""
+            Prompt:{data_to_evaluate["prompt"]},
+            Reseaerch Data :{data_to_evaluate["research"]},
+            Agent's Response:{data_to_evaluate["response"]}
+            
+            """}
+        ]
+
+        evaluation_response=self.___engine_core(messages=messages)
+        return evaluation_response