text_files/scores.py

import re
import pandas as pd
import os

# Define a list of your log file names
log_files = [
    'training (2).txt',
    'training_log_1_18.txt',
    'training_log_17_27.txt',
    'training_log_21_30.txt'
    
]

# Create an empty list to store parsed BLEU data
bleu_data = []

# Regex to capture the Epoch number from training progress lines
epoch_pattern = re.compile(r"Epoch\s\[(\d+)/\d+],")

# Regex to capture the BLEU-4 score
bleu_pattern = re.compile(r"Validation BLEU-4:\s([\d.]+)")

current_epoch = None # Variable to keep track of the current epoch

print("Starting BLEU score parsing...")

# Loop through each log file
for file_name in log_files:
    if not os.path.exists(file_name):
        print(f"Warning: File not found - {file_name}. Skipping.")
        continue

    print(f"Processing {file_name} for BLEU scores...")
    with open(file_name, 'r', encoding='utf-8') as f: # Use UTF-8 encoding
        for line in f:
            # Check for epoch line first to update current_epoch
            epoch_match = epoch_pattern.search(line)
            if epoch_match:
                current_epoch = int(epoch_match.group(1))
            
            # Check for BLEU score line
            bleu_match = bleu_pattern.search(line)
            if bleu_match:
                bleu_score = float(bleu_match.group(1))
                
                # Only add if we have an associated epoch
                if current_epoch is not None:
                    bleu_data.append({
                        'Epoch': current_epoch,
                        'BLEU-4': bleu_score
                    })
                else:
                    print(f"Warning: Found BLEU score ({bleu_score}) without a preceding epoch in {file_name}. Skipping this entry.")

# Create a Pandas DataFrame from the parsed BLEU data
df_bleu = pd.DataFrame(bleu_data)

# Remove duplicate entries for the same epoch if multiple BLEU scores are logged per epoch
# (e.g., if validation runs multiple times, take the last one or average, here we take the last one)
df_bleu_unique = df_bleu.drop_duplicates(subset=['Epoch'], keep='last')

# Sort the data by Epoch
df_bleu_sorted = df_bleu_unique.sort_values(by=['Epoch']).reset_index(drop=True)

# Save the DataFrame to a CSV file
output_csv_file = 'bleu_metrics.csv'
df_bleu_sorted.to_csv(output_csv_file, index=False)

print(f"\nSuccessfully parsed BLEU scores and saved data to {output_csv_file}")
print("You can now import this CSV file into Power BI to create your BLEU score visualizations.")
print("\nFirst few rows of the generated BLEU CSV:")
print(df_bleu_sorted.head())