Upload 6 files

scripts/evaluate_factual_robustness.py

@@ -14,7 +14,7 @@ def evaluate_factual_robustness(config):
     noise_rate = config['noise_rate']
     passage_num = config['passage_num']
     
-    if config['model_name'] in config["models"]:
+    if config['model_name'] in config['models']:
         model = GroqClient(plm=config['model_name'])
     else:
         logging.warning(f"Skipping unknown model: {config['model_name']}")

scripts/evaluate_information_integration.py

@@ -10,12 +10,13 @@ logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(
 
 # Improved function to evaluate noise robustness
 def evaluate_information_integration(config):
-    result_path = config["result_path"] + 'Information Integration/'
+    result_path = config['result_path'] + 'Information Integration/'
     noise_rate = config['noise_rate']
     passage_num = config['passage_num']
+    model_name = config['model_name']
 
     # Iterate over each model specified in the config
-    filename = os.path.join(result_path, f'prediction_{config['model_name']}_noise_{noise_rate}_passage_{passage_num}.json')
+    filename = os.path.join(result_path, f'prediction_{model_name}_noise_{noise_rate}_passage_{passage_num}.json')
     ensure_directory_exists(filename)
 
     # Load existing results if file exists
@@ -28,7 +29,7 @@ def evaluate_information_integration(config):
                 data = json.loads(line)
                 useddata[data['id']] = data'''
 
-    results = get_prediction_result(config, config["integration_file_name"])  # Store results for this model
+    results = get_prediction_result(config, config['integration_file_name'])  # Store results for this model
 
     # Save results to a file
     with open(filename, 'w', encoding='utf-8') as f:
@@ -45,7 +46,7 @@ def evaluate_information_integration(config):
 
     # Save the final score file with tt and all_rate
     scores = {
-        'model': config['model_name'],
+        'model': model_name,
         'accuracy': accuracy,
         'noise_rate': noise_rate,
         'correct_count': correct_count,
@@ -56,7 +57,7 @@ def evaluate_information_integration(config):
     logging.info(f"Score: {scores}")
     logging.info(f"Information Integration Accuracy: {accuracy:.2%}")
     
-    score_filename = os.path.join(result_path, f'scores_{config['model_name']}_noise_{noise_rate}_passage_{passage_num}.json')
+    score_filename = os.path.join(result_path, f'scores_{model_name}_noise_{noise_rate}_passage_{passage_num}.json')
     with open(score_filename, 'w') as f:
         json.dump(scores, f, ensure_ascii=False, indent=4)
 

scripts/evaluate_negative_rejection.py

@@ -15,7 +15,7 @@ def evaluate_negative_rejection(config):
     noise_rate = config['noise_rate']
     passage_num = config['passage_num']
     
-    if config['model_name'] in config["models"]:
+    if config['model_name'] in config['models']:
         model = GroqClient(plm=config['model_name'])
     else:
         logging.warning(f"Skipping unknown model: {config['model_name']}")

scripts/evaluate_noise_robustness.py

@@ -10,7 +10,7 @@ logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(
 
 # Improved function to evaluate noise robustness
 def evaluate_noise_robustness(config):
-    result_path = config["result_path"] + 'Noise Robustness/'
+    result_path = config['result_path'] + 'Noise Robustness/'
     noise_rate = config['noise_rate']
     passage_num = config['passage_num']
 
@@ -28,7 +28,7 @@ def evaluate_noise_robustness(config):
                 data = json.loads(line)
                 useddata[data['id']] = data'''
 
-    results = get_prediction_result(config, config["robustness_file_name"])  # Store results for this model
+    results = get_prediction_result(config, config['robustness_file_name'])  # Store results for this model
 
     # Save results to a file
     with open(filename, 'w', encoding='utf-8') as f:

scripts/get_factual_evaluation.py

@@ -10,7 +10,7 @@ logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(
 
 # Improved function to evaluate noise robustness
 def get_factual_evaluation(config):
-    result_path = config["result_path"] + 'Counterfactual Robustness/'
+    result_path = config['result_path'] + 'Counterfactual Robustness/'
     noise_rate = config['noise_rate']
     passage_num = config['passage_num']
 
@@ -28,7 +28,7 @@ def get_factual_evaluation(config):
                 data = json.loads(line)
                 useddata[data['id']] = data'''
 
-    results = get_prediction_result(config, config["factual_file_name"])  # Store results for this model
+    results = get_prediction_result(config, config['factual_file_name'])  # Store results for this model
 
     # Save results to a file
     with open(filename, 'w', encoding='utf-8') as f:

scripts/get_prediction_result.py

@@ -13,7 +13,7 @@ def get_prediction_result(config, data_file_name):
     results = []
     dataset = load_dataset(data_file_name)
     # Create GroqClient instance for supported models
-    if config['model_name'] in config["models"]:
+    if config['model_name'] in config['models']:
         model = GroqClient(plm=config['model_name'])
     else:
         logging.warning(f"Skipping unknown model: {config['model_name']}")
@@ -26,7 +26,7 @@ def get_prediction_result(config, data_file_name):
         query, ans, docs = process_data(instance, config['noise_rate'], config['passage_num'], data_file_name)
 
         # Retry mechanism for prediction
-        for attempt in range(1, config["retry_attempts"] + 1):
+        for attempt in range(1, config['retry_attempts'] + 1):
             label, prediction, factlabel = predict(query, ans, docs, model, "Document:\n{DOCS} \n\nQuestion:\n{QUERY}", 0.7)
             if prediction:  # If response is not empty, break retry loop
                 break