Upload 4 files

scripts/evaluate_negative_rejection.py

@@ -15,10 +15,10 @@ def evaluate_negative_rejection(config):
     noise_rate = config['noise_rate']
     passage_num = config['passage_num']
     
-    if config['model_name'] in config['models']:
-        model = GroqClient(plm=config['model_name'])
+    if modelname in config['models']:
+        model = GroqClient(plm=modelname)
     else:
-        logging.warning(f"Skipping unknown model: {config['model_name']}")
+        logging.warning(f"Skipping unknown model: {modelname}")
         return
 
     # File paths

scripts/evaluate_noise_robustness.py

@@ -13,9 +13,10 @@ def evaluate_noise_robustness(config):
     result_path = config['result_path'] + 'Noise Robustness/'
     noise_rate = config['noise_rate']
     passage_num = config['passage_num']
+    model_name = config['model_name']
 
     # Iterate over each model specified in the config
-    filename = os.path.join(result_path, f'prediction_{config['model_name']}_noise_{noise_rate}_passage_{passage_num}.json')
+    filename = os.path.join(result_path, f'prediction_{model_name}_noise_{noise_rate}_passage_{passage_num}.json')
     ensure_directory_exists(filename)
 
     # Load existing results if file exists
@@ -56,7 +57,7 @@ def evaluate_noise_robustness(config):
     logging.info(f"score: {scores}")
     logging.info(f"Noise Robustness Accuracy: {accuracy:.2%}")
     
-    score_filename = os.path.join(result_path, f'scores_{config['model_name']}_noise_{noise_rate}_passage_{passage_num}.json')
+    score_filename = os.path.join(result_path, f'scores_{model_name}_noise_{noise_rate}_passage_{passage_num}.json')
     with open(score_filename, 'w') as f:
         json.dump(scores, f, ensure_ascii=False, indent=4)
 

scripts/get_factual_evaluation.py

@@ -13,9 +13,10 @@ def get_factual_evaluation(config):
     result_path = config['result_path'] + 'Counterfactual Robustness/'
     noise_rate = config['noise_rate']
     passage_num = config['passage_num']
+    model_name = config['model_name']
 
     # Iterate over each model specified in the config
-    filename = os.path.join(result_path, f'prediction_{config['model_name']}_noise_{noise_rate}_passage_{passage_num}.json')
+    filename = os.path.join(result_path, f'prediction_{model_name}_noise_{noise_rate}_passage_{passage_num}.json')
     ensure_directory_exists(filename)
 
     # Load existing results if file exists
@@ -61,7 +62,7 @@ def get_factual_evaluation(config):
     scores['correct_tt'] = correct_tt
 
     #logging.info(f"score: {scores}")
-    score_filename = os.path.join(result_path, f'scores_{config['model_name']}_noise_{noise_rate}_passage_{passage_num}.json')
+    score_filename = os.path.join(result_path, f'scores_{model_name}_noise_{noise_rate}_passage_{passage_num}.json')
     with open(score_filename, 'w') as f:
         json.dump(scores, f, ensure_ascii=False, indent=4)
 

scripts/get_prediction_result.py

@@ -12,16 +12,18 @@ logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(
 def get_prediction_result(config, data_file_name):
     results = []
     dataset = load_dataset(data_file_name)
+    modelname = config['model_name']
+
     # Create GroqClient instance for supported models
-    if config['model_name'] in config['models']:
-        model = GroqClient(plm=config['model_name'])
+    if modelname in config['models']:
+        model = GroqClient(plm=modelname)
     else:
-        logging.warning(f"Skipping unknown model: {config['model_name']}")
+        logging.warning(f"Skipping unknown model: {modelname}")
         return
     
     # Iterate through dataset and process queries
     for idx, instance in enumerate(dataset[:config['num_queries']], start=0):
-        logging.info(f"Executing Query {idx + 1} for Model: {config['model_name']}")
+        logging.info(f"Executing Query {idx + 1} for Model: {modelname}")
 
         query, ans, docs = process_data(instance, config['noise_rate'], config['passage_num'], data_file_name)