minor updates

.gitignore

@@ -18,3 +18,7 @@ src/assets/model_counts.html
 
 generation_results/
 Hallucination Leaderboard Results
+dataset_stats.py
+
+get_comparison.py
+GPT-4-Turbo_v.s._GPT-4o.csv

requirements.txt

@@ -14,4 +14,9 @@ requests==2.28.2
 tqdm==4.65.0
 transformers==4.35.2
 tokenizers>=0.15.0
-sentence-transformers==2.2.2
+sentence-transformers==2.2.2
+google-generativeai
+replicate
+anthropic
+openai
+cohere

src/backend/evaluate_model.py

@@ -110,11 +110,14 @@ class Evaluator:
         
         source_summary_df = self.generated_summaries_df[["source", "summary"]]
 
-        # #update leaderboard_summaries.csv
-        # #first remove previous results for the current model
-        # existing_df = pd.read_csv(os.path.join(working_path, 'leaderboard_summaries.csv'), encoding='utf-8', sep="\t")
+        #update leaderboard_summaries.csv
+        #first remove previous results for the current model
+        # existing_df = pd.read_csv(os.path.join(working_path, 'leaderboard_summaries.csv'), encoding='utf-8')
         # mask = existing_df['model'] == self.model
         # existing_df = existing_df[~mask]
+        # print(existing_df.shape)
+        # summary_doc = set(existing_df['model'].values.tolist())
+        # print(summary_doc)
         # # get new result
         leaderboard_summaries_df = source_summary_df
         leaderboard_summaries_df.insert(2, "model", [self.model]*leaderboard_summaries_df.shape[0])
@@ -124,12 +127,22 @@ class Evaluator:
         # update leaderboard_summaries_with_scores.csv
         # BUG: get error when opening the file
         # existing_df = pd.read_csv(os.path.join(working_path, 'leaderboard_summaries_with_scores.csv'), 
-        #                         encoding='utf-8', sep=",", on_bad_lines='warn', quotechar='"', quoting=2)
+        #                         encoding='utf-8', sep=",", quotechar='"', quoting=2)
         # print(existing_df.shape)
+        # score_doc = set(existing_df['model'].values.tolist())
+        # print(score_doc)
         # mask = existing_df['model'] == self.model
         # existing_df = existing_df[~mask]
-        # get new result
+        # # get new result
         leaderboard_summaries_with_scores_df = pd.DataFrame.from_dict(self.eval_results)
         leaderboard_summaries_with_scores_df.insert(3, "model", [self.model]*leaderboard_summaries_with_scores_df.shape[0])
         leaderboard_summaries_with_scores_df.to_csv(os.path.join(working_path, 'leaderboard_summaries_with_scores.csv'), mode='a', index=False, header=False)
-        print('leaderboard_summaries_with_scores.csv has been updated')
+        print('leaderboard_summaries_with_scores.csv has been updated')
+
+        # for model in summary_doc:
+        #     if model not in score_doc:
+        #         print(f"{model} records missing in leaderboard_summaries_with_scores.csv")
+
+        # for model in score_doc:
+        #     if model not in summary_doc:
+        #         print(f"{model} records missing in leaderboard_summaries.csv")

src/backend/model_operations.py

@@ -13,18 +13,21 @@ from sentence_transformers import CrossEncoder
 import litellm
 # from litellm import completion
 from tqdm import tqdm
-from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig, AutoConfig
+from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig, AutoConfig, pipeline
 # from accelerate import PartialState
 # from accelerate.inference import prepare_pippy
 import torch
 import cohere
 from openai import OpenAI
+import anthropic
+import replicate
+# pip install -U google-generativeai
 import google.generativeai as genai
 
 import src.backend.util as util
 import src.envs as envs
 
-litellm.set_verbose=False
+litellm.set_verbose=True
 
 # Set up basic configuration for logging
 logging.basicConfig(level=logging.INFO,
@@ -123,15 +126,15 @@ class SummaryGenerator:
                         break
                     except Exception as e:
                         if 'Rate limit reached' in str(e):
-                            wait_time = 3660
+                            wait_time = 300
                             current_time = datetime.now().strftime('%H:%M:%S')
-                            print(f"Rate limit hit at {current_time}. Waiting for 1 hour before retrying...")
+                            print(f"Rate limit hit at {current_time}. Waiting for 5 minutes before retrying...")
                             time.sleep(wait_time)
                         elif 'is currently loading' in str(e):
                             wait_time = 200
                             print(f"Model is loading, wait for {wait_time}")
                             time.sleep(wait_time)
-                        elif '429 Resource has been exhausted' in str(e): # for gemini models
+                        elif '429' in str(e): # for gemini models
                             wait_time = 60
                             print(f"Quota has reached, wait for {wait_time}")
                             time.sleep(wait_time)
@@ -166,13 +169,14 @@ class SummaryGenerator:
     def generate_summary(self, system_prompt: str, user_prompt: str):
         # Using Together AI API
         using_together_api = False
-        together_ai_api_models = ['mixtral', 'dbrx', 'wizardlm', 'llama-3']
+        together_ai_api_models = ['mixtral', 'dbrx', 'wizardlm', 'llama-3', 'qwen'] #, 'mistralai'
         for together_ai_api_model in together_ai_api_models:
             if together_ai_api_model in self.model_id.lower():
                 using_together_api = True
                 break
         # if 'mixtral' in self.model_id.lower() or 'dbrx' in self.model_id.lower() or 'wizardlm' in self.model_id.lower(): # For mixtral and dbrx models, use Together AI API
         if using_together_api:
+            # print('using together api')
             # suffix = "completions" if ('mixtral' in self.model_id.lower() or 'base' in self.model_id.lower()) else "chat/completions"
             suffix = "chat/completions"
             url = f"https://api.together.xyz/v1/{suffix}"
@@ -184,14 +188,6 @@ class SummaryGenerator:
                 "temperature": 0.0,
                 # 'repetition_penalty': 1.1 if 'mixtral' in self.model_id.lower() else 1
             }
-            # if 'mixtral' in self.model_id.lower():
-            #     # payload['prompt'] = user_prompt
-            #     # payload['prompt'] = "Write a summary of the following passage:\nPassage:\n" + user_prompt.split('Passage:\n')[-1] + '\n\nSummary:'
-            #     payload['prompt'] = 'You must stick to the passage provided. Provide a concise summary of the following passage, covering the core pieces of information described:\nPassage:\n' + user_prompt.split('Passage:\n')[-1] + '\n\nSummary:'
-            #     print(payload)
-            # else:
-            #     payload['messages'] = [{"role": "system", "content": system_prompt},
-            #                             {"role": "user", "content": user_prompt}]
             payload['messages'] = [{"role": "system", "content": system_prompt},
                                         {"role": "user", "content": user_prompt}]
             headers = {
@@ -201,6 +197,7 @@ class SummaryGenerator:
             }
 
             response = requests.post(url, json=payload, headers=headers)
+            print(response)
             try:
                 result = json.loads(response.text)
                 # print(result)
@@ -219,14 +216,16 @@ class SummaryGenerator:
 
         # Using OpenAI API
         elif 'gpt' in self.model_id.lower():
-            response = litellm.completion(
+            client = OpenAI()
+            response = client.chat.completions.create(
                 model=self.model_id.replace('openai/',''),
                 messages=[{"role": "system", "content": system_prompt},
                         {"role": "user", "content": user_prompt}],
                 temperature=0.0,
                 max_tokens=250,
             )   
-            result = response['choices'][0]['message']['content']
+            # print(response)
+            result = response.choices[0].message.content
             print(result)
             return result
         
@@ -258,10 +257,11 @@ class SummaryGenerator:
                     "threshold": "BLOCK_NONE"
                 },
             ]
-            model = genai.GenerativeModel(model_name="gemini-1.5-pro-latest" if "gemini-1.5-pro" in self.model_id.lower() else self.model_id.lower().split('google/')[-1],
+            model = genai.GenerativeModel(model_name=self.model_id.lower().split('google/')[-1],
                               generation_config=generation_config,
                               system_instruction=system_prompt,
                               safety_settings=safety_settings)
+            # print(model)
             convo = model.start_chat(history=[])
             convo.send_message(user_prompt)
             # print(convo.last)
@@ -269,39 +269,116 @@ class SummaryGenerator:
             print(result)
             return result
 
+        elif 'snowflake' in self.model_id.lower():
+            print("using replicate")
+            input = {
+                "prompt": user_prompt,
+                "temperature": 0,
+                "max_new_tokens": 250,
+                "stop_sequences": "<|im_end|>",
+                "prompt_template": f"<|im_start|>system\n{system_prompt}<|im_end|>\n" + "<|im_start|>user\n{prompt}<|im_end|>\n\n<|im_start|>assistant\n",
+            }
+            response = replicate.run(
+                self.model_id.lower(),
+                input=input
+            )
+            if isinstance(response, list):
+                response = ''.join(response)
+                print(response)
+                print()
+            
+            return response
+
+        elif 'claude' in self.model_id.lower(): # using anthropic api
+            client = anthropic.Anthropic()
+            message = client.messages.create(
+                model=self.model_id.split('/')[-1],
+                max_tokens=250,
+                temperature=0,
+                system=system_prompt,
+                messages=[
+                    {
+                        "role": "user",
+                        "content": [
+                            {
+                                "type": "text",
+                                "text": user_prompt
+                            }
+                        ]
+                    }
+                ]
+            )
+            result = message.content[0].text
+            print(result)
+            return result
+                
         # Using HF API or download checkpoints
         elif self.local_model is None:
+            # response = litellm.completion(
+            #     model='command-r-plus' if 'command' in self.model else self.model,
+            #     messages=[{"role": "system", "content": system_prompt},
+            #                 {"role": "user", "content": user_prompt}],
+            #     temperature=0.0,
+            #     max_tokens=256,
+            #     api_base=self.api_base,
+            # )
+            # result = response['choices'][0]['message']['content']
+            # print(result)
+            # return result
             try: # try use HuggingFace API
-                
+                print('using huggingface api')
                 response = litellm.completion(
                     model='command-r-plus' if 'command' in self.model else self.model,
                     messages=[{"role": "system", "content": system_prompt},
                                 {"role": "user", "content": user_prompt}],
                     temperature=0.0,
-                    max_tokens=1024,
+                    max_tokens=250,
                     api_base=self.api_base,
                 )
                 result = response['choices'][0]['message']['content']
+                print(result)
                 return result
-            except: # fail to call api. run it locally.
-                self.tokenizer = AutoTokenizer.from_pretrained(self.model_id, trust_remote_code=True)
-                print("Tokenizer loaded")
-                self.local_model = AutoModelForCausalLM.from_pretrained(self.model_id, trust_remote_code=True, device_map="auto", torch_dtype="auto")
-                print("Local model loaded")
+            except Exception as e:
+                if 'Rate limit reached' in str(e):
+                    wait_time = 300
+                    current_time = datetime.now().strftime('%H:%M:%S')
+                    print(f"Rate limit hit at {current_time}. Waiting for 5 minutes before retrying...")
+                    time.sleep(wait_time)
+                else:
+                    self.tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-hf" if 'openelm' in self.model_id.lower() else self.model_id, trust_remote_code=True)
+                    print("Tokenizer loaded")
+                    self.local_model = AutoModelForCausalLM.from_pretrained(self.model_id, trust_remote_code=True, device_map="auto", torch_dtype="auto")
+                    print("Local model loaded")
         
         # Using local model
         if self.local_model: # cannot call API. using local model
-            messages=[
-                {"role": "system", "content": system_prompt}, # gemma-1.1 does not accept system role
-                {"role": "user", "content": user_prompt}
-            ],
-            prompt = self.tokenizer.apply_chat_template(messages,add_generation_prompt=True, tokenize=False)
+            if 'gemma' in self.model_id.lower() or 'mistral-7b' in self.model_id.lower():
+                messages=[
+                    # gemma-1.1, mistral-7b does not accept system role
+                    {"role": "user", "content": system_prompt + ' ' + user_prompt}
+                ]
+                prompt = self.tokenizer.apply_chat_template(messages,add_generation_prompt=True, tokenize=False)
+            
+            elif 'phi-2' in self.model_id.lower():
+                prompt = system_prompt + '\n' + user_prompt
+            
+            else:
+                messages=[
+                    {"role": "system", "content": system_prompt}, # gemma-1.1, mistral-7b does not accept system role
+                    {"role": "user", "content": user_prompt}
+                ]
+                prompt = self.tokenizer.apply_chat_template(messages,add_generation_prompt=True, tokenize=False)
             print(prompt)
+            print('-'*50)
             input_ids = self.tokenizer(prompt, return_tensors="pt").to('cuda')
             with torch.no_grad():
                 outputs = self.local_model.generate(**input_ids, max_new_tokens=250, do_sample=True, temperature=0.01, pad_token_id=self.tokenizer.eos_token_id)
             result = self.tokenizer.decode(outputs[0], skip_special_tokens=True)
-            result = result.replace(prompt[0], '')
+            if 'gemma-2' in self.model_id.lower():
+                result = result.split(user_prompt + '\nmodel')[-1].strip()
+            else:
+                result = result.replace(prompt.strip(), '')
+            
             print(result)
             return result
 
@@ -371,14 +448,12 @@ class EvaluationModel:
         summaries = []
         source_summary_pairs = util.create_pairs(summaries_df)
 
-        for doc, summary in tqdm(source_summary_pairs, desc="Evaluating hallucinations"):
+        for doc, summary in source_summary_pairs:
             if util.is_summary_valid(summary):
                 try:
                     # summary_pieces = summary.split('\n')
                     # summary = summary_pieces[0] if len(summary_pieces[0].strip()) > 0 else summary_pieces[1]
                     summary = summary.replace('<bos>','').replace('<eos>','')
-                    # print([doc, summary])
-                    # print(self.model.predict([doc, summary]))
                     score = self.model.predict([doc, summary])# [0]
                     if not isinstance(score, float):
                         try:
@@ -386,6 +461,12 @@ class EvaluationModel:
                         except:
                             logging.warning(f"Score type mismatch: Expected float, got {type(score)}.")
                             continue
+                        # print inconsistent summaries for checking
+                        if score < 0.5:
+                            print(doc)
+                            print('-'*10)
+                            print(summary)
+                            print('='*20)
                     hem_scores.append(score)
                     sources.append(doc)
                     summaries.append(summary)